From 0406dbb6118b283cbf7f023d2cceab177cbca818 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:53:17 +0000 Subject: [PATCH 1/7] Add CLI acceptance on the real published 3.8 upgrade seed --- scripts/acceptance/command-keys/README.md | 57 +++++ .../acceptance/command-keys/physical_cli.py | 201 ++++++++++++++++++ .../command-keys/test_physical_cli.py | 91 ++++++++ 3 files changed, 349 insertions(+) create mode 100644 scripts/acceptance/command-keys/physical_cli.py create mode 100644 scripts/acceptance/command-keys/test_physical_cli.py diff --git a/scripts/acceptance/command-keys/README.md b/scripts/acceptance/command-keys/README.md index 56b6261..0975960 100644 --- a/scripts/acceptance/command-keys/README.md +++ b/scripts/acceptance/command-keys/README.md @@ -121,3 +121,60 @@ replacement, a forged codec, sealed-content writes, foreign-name survival and schema-only mutations that the predecessor's row-only comparison accepted. Modern and legacy zero-write checks use the same bounded typed logical snapshot, including catalog SQL, columns, indexes, triggers, views and all table rows. + +## Physical 3.8 upgrade + +`physical_cli.py` consumes the producer-owned real 3.8 API seed after the +[accepted handoff](https://github.com/durable-workflow/server/pull/1210#issuecomment-6860138558). +It does not call the synthetic transform or seed a replacement baseline. +The accepted fixture contract is Server commit +`a09f156399416216d2f097c257a2164f83c7eda7`. The original seed actually executed +commit `09a4ccda3da11853606af031a8970bbde4b2d6ec`, driver SHA256 +`01c0fa1251ac3c809723e58d71817881fe2c5c9498579bc4c2e6097324a3618b`. +Keep these two provenance meanings distinct. + +Use another fresh private directory with the common staged inputs above and: + +| Input | Source/identity | +| --- | --- | +| `CLI_216_PHYSICAL_BASELINE` | Absolute original stopped-backup path supplied by its owner; SHA256 `7d1826f7681342bb3a5974509648e2e8d75e7a21d262f65f3ec1f58727ccbc0c` | +| `CLI_216_PHYSICAL_PACKET` | Absolute original seed JSON path supplied by its owner; SHA256 `8c728ce827df3a11621dfb0e82cdd404bbab48784182a5e92c8d632d49047fc5` | +| `physical-contract.json` | The legacy manifest at accepted Server `a09f1563`; SHA256 `ceb4695643e48c7bfe3f81eea380c6ab9aa0b973da830cb552d070344047319f` | +| `0013_command_intents.sql` | `rust/migrations/sqlite/0013_command_intents.sql` at published Server source `5614274d2629d03f92932a07dfee0fb10349ddd9`; SHA256 `fc3ad1730682c09704027dcbabc17cd6ded9c642a3ff20d7988d5f4b4a5726fd` | + +The original backup and packet are held read-only and rechecked after use. +Only a newly created owned copy is mounted into the published 3.9.2 image. +The image runs its actual `schema-upgrade` command, requiring SQLite 12 → 13. +The comparison preserves all old business rows and schema objects, allowing +only tagged DDL plus explicitly validated schema-version/journal metadata. +The intent table must be empty. The new journal checksum, description, +version, success, timestamp interval and execution-time bound are checked. + +```bash +env -i PATH=/usr/bin:/bin \ + CLI_216_WORKDIR="$experiment" \ + CLI_216_PROXY_SOURCE="$experiment/committed_response_loss.py" \ + CLI_216_PHYSICAL_BASELINE="$original_backup" \ + CLI_216_PHYSICAL_PACKET="$original_packet" \ + timeout 180s python3 -I -B scripts/acceptance/command-keys/physical_cli.py +``` + +Created names are `cli-216-392-upgrade` and `cli-216-392-physical`; cleanup +authority remains their captured IDs. The upgrade container is retained until +its actual exit/log result is read, then removed. The runtime exercises the +installed CLI's original keyed-update readbacks and name/run fences for all +three old-run states. Original NULL-key signals are checked only for preserved +identity/history, never manufactured retained-key or intent equality. +Full request-window snapshots must agree. A later post-stop maintenance +snapshot is a different observation and must not be relabeled as that window. + +The comparison's offline controls require the staged helper and tagged SQL: + +```bash +CLI_216_TEST_HELPER="$experiment/sqlite_legacy.py" \ + CLI_216_TEST_DELTA="$experiment/0013_command_intents.sql" \ + python3 -I -B scripts/acceptance/command-keys/test_physical_cli.py -v +``` + +These controls are not a physical upgrade result. Source review, current +normal gates, fresh actual execution and cleanup still qualify that cell. diff --git a/scripts/acceptance/command-keys/physical_cli.py b/scripts/acceptance/command-keys/physical_cli.py new file mode 100644 index 0000000..7216854 --- /dev/null +++ b/scripts/acceptance/command-keys/physical_cli.py @@ -0,0 +1,201 @@ +"""Installed CLI readback on a copy of the accepted real 3.8 API seed.""" +from contextlib import closing +from copy import deepcopy +import concurrent.futures +import datetime +import hashlib +import json +import os +from pathlib import Path +import sqlite3 +import stat +import time +import types + +SOURCE = Path(__file__).resolve().parent +LEGACY_SHA = "b6d2bee1b689745b626ce2d7b754f0179bdb1d22e53ef33dd1501e6dec520c76" +BASELINE_SHA = "7d1826f7681342bb3a5974509648e2e8d75e7a21d262f65f3ec1f58727ccbc0c" +PACKET_SHA = "8c728ce827df3a11621dfb0e82cdd404bbab48784182a5e92c8d632d49047fc5" +CONTRACT_SHA = "ceb4695643e48c7bfe3f81eea380c6ab9aa0b973da830cb552d070344047319f" +DELTA_SHA = "fc3ad1730682c09704027dcbabc17cd6ded9c642a3ff20d7988d5f4b4a5726fd" +JOURNAL_CHECKSUM = "53b752529a1f7addaefe62ab61160dd42b3990f805cfdc4ecd1b48927a6c2f7ce42526fc04522ac887a37208f343fc73" + + +def load_legacy(): + path = SOURCE / "legacy_cli.py" + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC) + try: + observed = os.fstat(descriptor) + if (not stat.S_ISREG(observed.st_mode) or observed.st_uid != os.getuid() + or observed.st_nlink != 1 or observed.st_mode & 0o022 or observed.st_size > 131072): + raise ValueError("legacy adapter owner/type/mode differs") + data = os.pread(descriptor, 131073, 0) + if hashlib.sha256(data).hexdigest() != LEGACY_SHA: + raise ValueError("legacy adapter source differs") + module = types.ModuleType("cli216_physical_common") + module.__file__ = str(path) + exec(compile(data, str(path), "exec"), module.__dict__) + held = module.d.held(path, LEGACY_SHA) + if held.identity != held._identity(observed): + raise ValueError("legacy adapter changed during held import") + return module + finally: + os.close(descriptor) + + +def compare_upgrade(before, after, delta, started, ended, helper): + """Allow only the tagged DDL and narrowly validated migration metadata.""" + if before["tables"]["dw_server_schema"]["rows"] != [[["text", "rust-development"], ["integer", 12]]]: + raise ValueError("baseline is not native SQLite schema 12") + expected = deepcopy(before) + with closing(sqlite3.connect(":memory:")) as db: + db.execute("CREATE TABLE dw_server_schema(engine TEXT, version INTEGER)") + db.executescript(delta.decode()) + additions = helper.snapshot(db) + expected["schema"].extend(row for row in additions["schema"] if row[2] == "dw_command_intents") + expected["schema"].sort(key=lambda row: (row[0], row[1])) + expected["tables"]["dw_command_intents"] = additions["tables"]["dw_command_intents"] + expected["tables"]["dw_server_schema"]["rows"] = [[["text", "rust-development"], ["integer", 13]]] + journal = after["tables"]["_sqlx_migrations"] + columns = ["version", "description", "installed_on", "success", "checksum", "execution_time"] + if journal["columns"] != columns or len(journal["rows"]) != 1: + raise ValueError("upgraded migration journal shape differs") + row = journal["rows"][0] + old = before["tables"]["_sqlx_migrations"]["rows"] + if len(old) != 1 or old[0][0] != ["integer", 12]: + raise ValueError("baseline migration journal shape differs") + if (row[0] != ["integer", 13] or row[1] != ["text", old[0][1][1] + " and keyed command intents"] + or row[3] != ["integer", 1] or row[4] != ["blob", JOURNAL_CHECKSUM] + or row[5][0] != "integer" or not 0 <= row[5][1] <= 120_000_000_000 + or row[2][0] != "text"): + raise ValueError("migration journal content differs from tagged schema 13") + installed = datetime.datetime.fromisoformat(row[2][1]).replace(tzinfo=datetime.timezone.utc) + if not started - datetime.timedelta(seconds=1) <= installed <= ended + datetime.timedelta(seconds=1): + raise ValueError("migration journal time is outside the actual upgrade window") + expected["tables"]["_sqlx_migrations"]["rows"] = deepcopy(journal["rows"]) + if expected != after: + raise ValueError("upgrade changed an original business row/schema or unexpected metadata") + + +def physical_cases(packet, common): + d = common.d + for original in packet["cases"]: + kind, state = original["kind"], original["state"] + receipt = original["receipt"] + fixture = {"workflow": original["workflow_id"], "run": receipt["run_id"], "namespace": "default"} + d.require(d.history(fixture) == original["history"]["events"], kind + ": original " + state + " API history preserved") + if kind == "signal": + # Real 3.8 ordinary signals stored no caller key. Never issue a + # manufactured retained retry or infer intent equality for them. + with closing(sqlite3.connect(d.DB.as_uri() + "?mode=ro", uri=True)) as db: + key = db.execute("SELECT request_id FROM workflow_commands WHERE id=?", (receipt["command_id"],)).fetchone() + d.require(key == (None,), "physical signal: original NULL key is compatibility only") + continue + record = {"request_id": original["original_body"]["request_id"], + "receipt": {key: receipt[key] for key in ("command_id", "run_id", "command_sequence", "update_id", "accepted_at")}} + case = {"kind": kind, "fixture": fixture, "record": record, + "state": {"open": "open_after_restart", "completed": "completed_after_restart", "continued": "continued_after_restart"}[state]} + with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool: + replies = list(pool.map(lambda _: common.readback(case), range(4))) + d.require(len({reply["command_id"] for reply in replies}) == 1, "physical update: concurrent " + state + " retries retain original command") + common.readback(case, run=fixture["run"]) + for options in [{"name": "other"}, {"run": "00000000000000000000000000"}]: + answer = d.message("update", fixture["workflow"], record["request_id"], value=999, success=False, **options) + refusal = common.typed_error(answer, "request_id_conflict") + d.require(refusal.get("retained_command_id") == receipt["command_id"] and + refusal.get("conflict") == ["name" if "name" in options else "target"], + "physical update: correct original identity/name/run fence") + + +if __name__ == "__main__": + common = load_legacy() + d = common.d + entry = d.held(Path(__file__).absolute(), None, 131072) + baseline = d.held(Path(os.environ["CLI_216_PHYSICAL_BASELINE"]), BASELINE_SHA, 64 * 1024 * 1024) + packet_input = d.held(Path(os.environ["CLI_216_PHYSICAL_PACKET"]), PACKET_SHA) + contract_input = d.held(d.ROOT / "physical-contract.json", CONTRACT_SHA) + delta_input = d.held(d.ROOT / "0013_command_intents.sql", DELTA_SHA) + packet = json.loads(packet_input.data) + if packet["schema"] != "durable-workflow.command-key-physical-seed.v1" or packet["manifest_sha256"] != CONTRACT_SHA: + raise ValueError("physical seed interface differs") + contract = json.loads(contract_input.data) + if packet["expected_external_baseline"] != contract["physical_upgrade"]["baseline"] or len(packet["cases"]) != 6: + raise ValueError("physical baseline/case identities differ") + d.NAME = "cli-216-392-physical" + common.runtime = d.boundary.OwnedContainer(d.docker, d.NAME) + upgrade = d.boundary.OwnedContainer(d.docker, "cli-216-392-upgrade") + outcome, error, phase = "runner-blocked", None, "preflight" + started = datetime.datetime.now(datetime.timezone.utc) + before, after = None, None + try: + observed = json.loads(d.docker("image", "inspect", d.IMAGE, "--format", "{{json .}}")) + d.require(observed["Architecture"] == "amd64" and observed["Os"] == "linux" and + observed["Config"]["Labels"]["org.opencontainers.image.revision"] == + "5614274d2629d03f92932a07dfee0fb10349ddd9", "physical candidate published platform/source identity") + descriptor = os.open(d.DB, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + with os.fdopen(descriptor, "wb") as copied: + copied.write(baseline.data) + before = common.full_snapshot() + phase = "explicit physical schema upgrade" + upgrade_started = datetime.datetime.now(datetime.timezone.utc) + upgrade.run("-d", "--name", upgrade.name, "--platform=linux/amd64", "--init", "--cpus=1", "--memory=1g", + "--pids-limit=256", "--network=none", "--read-only", "--cap-drop=ALL", "--security-opt=no-new-privileges", + "--env", "DB_CONNECTION=sqlite", "--env", "DB_DATABASE=/app/database/database.sqlite", + "--mount", "type=bind,source=" + str(d.ROOT / "data") + ",target=/app/database", + "--tmpfs", "/tmp:rw,size=16m", d.IMAGE, "schema-upgrade") + upgrade.inspect() + d.require(d.docker("wait", upgrade.id) == "0", "tagged image schema-upgrade exits zero") + state = upgrade.inspect()["State"] + d.require(state["ExitCode"] == 0 and not state["OOMKilled"], "upgrade process exits cleanly without OOM") + upgrade.inspect() + result = json.loads(d.docker("logs", upgrade.id)) + d.require(result == {"backend": "sqlite", "from_schema_version": 12, "schema_version": 13, "status": "upgraded"}, + "actual schema transition is 12 to 13") + upgrade_ended = datetime.datetime.now(datetime.timezone.utc) + after = common.full_snapshot() + compare_upgrade(before, after, delta_input.data, upgrade_started, upgrade_ended, common.legacy) + d.require(True, "every original business row/schema preserved; only tagged DDL and validated version/journal metadata changed") + upgrade.remove() + phase = "installed CLI physical readback" + common.start() + window = common.full_snapshot() + physical_cases(packet, common) + common.unchanged(window, "physical upgrade installed request window") + outcome = "pass" + except Exception as failure: + error = (type(failure).__name__ + ": " + str(failure)).replace(d.TOKEN, "[redacted]") + finally: + if d.PROXY: + d.PROXY.shutdown() + d.PROXY.server_close() + cleanup_error = None + try: + for owned in (upgrade, common.runtime): + owned.remove() + for _ in range(50): + if not owned.exists(): + break + time.sleep(.1) + if owned.exists(): + raise RuntimeError("created physical container removal did not finish") + for item in d.HELD: + item.recheck() + except Exception as failure: + outcome = "runner-blocked" + cleanup_error = (type(failure).__name__ + ": " + str(failure)).replace(d.TOKEN, "[redacted]") + (d.ROOT / "runtime.env").unlink(missing_ok=True) + report = {"cell": "physical-3.8-upgrade", "outcome": outcome, "error": error, "cleanup_error": cleanup_error, + "phase": phase, "started_at": started.isoformat(), "finished_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "checks": len(d.CHECKS), "driver_sha256": entry.digest, + "accepted_contract_source": "a09f156399416216d2f097c257a2164f83c7eda7", + "actual_seed_source": "09a4ccda3da11853606af031a8970bbde4b2d6ec", + "actual_seed_driver_sha256": "01c0fa1251ac3c809723e58d71817881fe2c5c9498579bc4c2e6097324a3618b", + "baseline_sha256": baseline.digest, "packet_sha256": packet_input.digest, "contract_sha256": contract_input.digest, + "before_sha256": common.legacy.fingerprint(before) if before else None, + "after_upgrade_sha256": common.legacy.fingerprint(after) if after else None, + "request_windows": common.windows, "server_index": d.IMAGE, + "created_container_ids": upgrade.created_ids + common.runtime.created_ids, + "cleanup": "created IDs/volumes/env removed; original held inputs unchanged" if cleanup_error is None else "requires investigation"} + (d.ROOT / "physical-result.json").write_text(json.dumps(report, indent=2)) + print(json.dumps(report), flush=True) + raise SystemExit(0 if outcome == "pass" else 1) diff --git a/scripts/acceptance/command-keys/test_physical_cli.py b/scripts/acceptance/command-keys/test_physical_cli.py new file mode 100644 index 0000000..f98a18b --- /dev/null +++ b/scripts/acceptance/command-keys/test_physical_cli.py @@ -0,0 +1,91 @@ +"""Offline checks for the physical-upgrade comparison's narrow exceptions.""" +import datetime +import importlib.util +import os +from pathlib import Path +import sqlite3 +import unittest + +SOURCE = Path(__file__).resolve().parent + + +def local_module(name, filename): + spec = importlib.util.spec_from_file_location(name, SOURCE / filename) + value = importlib.util.module_from_spec(spec) + spec.loader.exec_module(value) + return value + + +physical = local_module("physical_comparison", "physical_cli.py") +boundary = local_module("physical_boundaries", "boundaries.py") + + +class UpgradeComparisonTests(unittest.TestCase): + def setUp(self): + helper_input = boundary.HeldInput(Path(os.environ["CLI_216_TEST_HELPER"]), + "6fc6054ce087c136a86d7b6b222daeefaa87c58fa0b613b155a5df39e6a26fad") + self.addCleanup(helper_input.close) + self.helper = helper_input.module("physical_snapshot_helper") + delta_input = boundary.HeldInput(Path(os.environ["CLI_216_TEST_DELTA"]), physical.DELTA_SHA) + self.addCleanup(delta_input.close) + self.delta = delta_input.data + self.db = sqlite3.connect(":memory:") + self.addCleanup(self.db.close) + self.db.executescript(""" + CREATE TABLE dw_server_schema(engine TEXT, version INTEGER); + INSERT INTO dw_server_schema VALUES ('rust-development', 12); + CREATE TABLE _sqlx_migrations(version INTEGER, description TEXT, installed_on TEXT, + success INTEGER, checksum BLOB, execution_time INTEGER); + CREATE TABLE business(value INTEGER); + INSERT INTO business VALUES (1); + """) + self.db.execute("INSERT INTO _sqlx_migrations VALUES (12,'baseline','2026-01-01 00:00:00',1,?,1)", (bytes(48),)) + self.before = self.helper.snapshot(self.db) + self.started = datetime.datetime.now(datetime.timezone.utc) + self.db.executescript(self.delta.decode()) + self.db.execute("UPDATE _sqlx_migrations SET version=13,description='baseline and keyed command intents'," + "installed_on=?,checksum=?", (self.started.strftime("%Y-%m-%d %H:%M:%S"), bytes.fromhex(physical.JOURNAL_CHECKSUM))) + self.ended = datetime.datetime.now(datetime.timezone.utc) + + def compare(self): + return physical.compare_upgrade(self.before, self.helper.snapshot(self.db), self.delta, + self.started, self.ended, self.helper) + + def refuses(self, statement): + self.db.execute(statement) + with self.assertRaises(ValueError): + self.compare() + + def test_exact_tagged_upgrade_and_bounded_metadata_pass(self): + self.compare() + + def test_business_row_change_is_not_a_metadata_exception(self): + self.refuses("UPDATE business SET value=2") + + def test_unexpected_schema_object_is_refused(self): + self.refuses("CREATE INDEX unrelated ON business(value)") + + def test_nonempty_new_intent_table_is_refused(self): + self.refuses("INSERT INTO dw_command_intents VALUES ('command','default','digest')") + + def test_wrong_journal_checksum_is_refused(self): + self.refuses("UPDATE _sqlx_migrations SET checksum=x'00'") + + def test_wrong_journal_description_is_refused(self): + self.refuses("UPDATE _sqlx_migrations SET description='other'") + + def test_wrong_journal_version_is_refused(self): + self.refuses("UPDATE _sqlx_migrations SET version=14") + + def test_extra_journal_row_is_refused(self): + self.refuses("INSERT INTO _sqlx_migrations SELECT * FROM _sqlx_migrations") + + def test_old_journal_timestamp_is_refused(self): + self.refuses("UPDATE _sqlx_migrations SET installed_on='2020-01-01 00:00:00'") + + def test_unbounded_execution_time_is_refused(self): + self.refuses("UPDATE _sqlx_migrations SET execution_time=999999999999999") + + +if __name__ == "__main__": + unittest.main() From 0c4a012c12cf2bd29660294bfccd66618691befd Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 13 Nov 2026 18:03:20 +0000 Subject: [PATCH 2/7] Verify physical experiment implementation before loading it --- scripts/acceptance/command-keys/README.md | 6 +- .../acceptance/command-keys/physical_cli.py | 213 +++--------------- .../acceptance/command-keys/physical_impl.py | 204 +++++++++++++++++ .../command-keys/test_physical_cli.py | 48 +++- 4 files changed, 290 insertions(+), 181 deletions(-) create mode 100644 scripts/acceptance/command-keys/physical_impl.py diff --git a/scripts/acceptance/command-keys/README.md b/scripts/acceptance/command-keys/README.md index 0975960..4b693e5 100644 --- a/scripts/acceptance/command-keys/README.md +++ b/scripts/acceptance/command-keys/README.md @@ -124,7 +124,11 @@ including catalog SQL, columns, indexes, triggers, views and all table rows. ## Physical 3.8 upgrade -`physical_cli.py` consumes the producer-owned real 3.8 API seed after the +`physical_cli.py` verifies and holds the literal-hash `physical_impl.py` bytes +before executing any implementation code. Its reported runner digest comes +from those checked bytes, with a post-cleanup identity recheck. Direct execution +of the implementation refuses before effectful initialization. It consumes the +producer-owned real 3.8 API seed after the [accepted handoff](https://github.com/durable-workflow/server/pull/1210#issuecomment-6860138558). It does not call the synthetic transform or seed a replacement baseline. The accepted fixture contract is Server commit diff --git a/scripts/acceptance/command-keys/physical_cli.py b/scripts/acceptance/command-keys/physical_cli.py index 7216854..0f93ba6 100644 --- a/scripts/acceptance/command-keys/physical_cli.py +++ b/scripts/acceptance/command-keys/physical_cli.py @@ -1,201 +1,56 @@ -"""Installed CLI readback on a copy of the accepted real 3.8 API seed.""" -from contextlib import closing -from copy import deepcopy -import concurrent.futures -import datetime +"""Verify the physical experiment implementation before executing its bytes.""" import hashlib -import json import os from pathlib import Path -import sqlite3 import stat -import time import types SOURCE = Path(__file__).resolve().parent -LEGACY_SHA = "b6d2bee1b689745b626ce2d7b754f0179bdb1d22e53ef33dd1501e6dec520c76" -BASELINE_SHA = "7d1826f7681342bb3a5974509648e2e8d75e7a21d262f65f3ec1f58727ccbc0c" -PACKET_SHA = "8c728ce827df3a11621dfb0e82cdd404bbab48784182a5e92c8d632d49047fc5" -CONTRACT_SHA = "ceb4695643e48c7bfe3f81eea380c6ab9aa0b973da830cb552d070344047319f" -DELTA_SHA = "fc3ad1730682c09704027dcbabc17cd6ded9c642a3ff20d7988d5f4b4a5726fd" -JOURNAL_CHECKSUM = "53b752529a1f7addaefe62ab61160dd42b3990f805cfdc4ecd1b48927a6c2f7ce42526fc04522ac887a37208f343fc73" +BOUNDARY_SHA = "d377e9191ebf79dc990db159629c303c10531a2f27d3ae20933fabc87e3b3525" +IMPLEMENTATION_SHA = "808cff32ead554ecefc931ead0797097312fec3a5155e49bfdeeffc218016f96" -def load_legacy(): - path = SOURCE / "legacy_cli.py" +def load_boundary(): + path = SOURCE / "boundaries.py" descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC) try: observed = os.fstat(descriptor) if (not stat.S_ISREG(observed.st_mode) or observed.st_uid != os.getuid() - or observed.st_nlink != 1 or observed.st_mode & 0o022 or observed.st_size > 131072): - raise ValueError("legacy adapter owner/type/mode differs") - data = os.pread(descriptor, 131073, 0) - if hashlib.sha256(data).hexdigest() != LEGACY_SHA: - raise ValueError("legacy adapter source differs") - module = types.ModuleType("cli216_physical_common") - module.__file__ = str(path) + or observed.st_nlink != 1 or observed.st_mode & 0o022 or observed.st_size > 65536): + raise ValueError("boundary source owner/type/mode differs") + data = os.pread(descriptor, 65537, 0) + if hashlib.sha256(data).hexdigest() != BOUNDARY_SHA: + raise ValueError("boundary source content differs") + module = types.ModuleType("physical_bootstrap_boundary") exec(compile(data, str(path), "exec"), module.__dict__) - held = module.d.held(path, LEGACY_SHA) + held = module.HeldInput(path, BOUNDARY_SHA) if held.identity != held._identity(observed): - raise ValueError("legacy adapter changed during held import") - return module + held.close() + raise ValueError("boundary source changed during held import") + return module, held finally: os.close(descriptor) -def compare_upgrade(before, after, delta, started, ended, helper): - """Allow only the tagged DDL and narrowly validated migration metadata.""" - if before["tables"]["dw_server_schema"]["rows"] != [[["text", "rust-development"], ["integer", 12]]]: - raise ValueError("baseline is not native SQLite schema 12") - expected = deepcopy(before) - with closing(sqlite3.connect(":memory:")) as db: - db.execute("CREATE TABLE dw_server_schema(engine TEXT, version INTEGER)") - db.executescript(delta.decode()) - additions = helper.snapshot(db) - expected["schema"].extend(row for row in additions["schema"] if row[2] == "dw_command_intents") - expected["schema"].sort(key=lambda row: (row[0], row[1])) - expected["tables"]["dw_command_intents"] = additions["tables"]["dw_command_intents"] - expected["tables"]["dw_server_schema"]["rows"] = [[["text", "rust-development"], ["integer", 13]]] - journal = after["tables"]["_sqlx_migrations"] - columns = ["version", "description", "installed_on", "success", "checksum", "execution_time"] - if journal["columns"] != columns or len(journal["rows"]) != 1: - raise ValueError("upgraded migration journal shape differs") - row = journal["rows"][0] - old = before["tables"]["_sqlx_migrations"]["rows"] - if len(old) != 1 or old[0][0] != ["integer", 12]: - raise ValueError("baseline migration journal shape differs") - if (row[0] != ["integer", 13] or row[1] != ["text", old[0][1][1] + " and keyed command intents"] - or row[3] != ["integer", 1] or row[4] != ["blob", JOURNAL_CHECKSUM] - or row[5][0] != "integer" or not 0 <= row[5][1] <= 120_000_000_000 - or row[2][0] != "text"): - raise ValueError("migration journal content differs from tagged schema 13") - installed = datetime.datetime.fromisoformat(row[2][1]).replace(tzinfo=datetime.timezone.utc) - if not started - datetime.timedelta(seconds=1) <= installed <= ended + datetime.timedelta(seconds=1): - raise ValueError("migration journal time is outside the actual upgrade window") - expected["tables"]["_sqlx_migrations"]["rows"] = deepcopy(journal["rows"]) - if expected != after: - raise ValueError("upgrade changed an original business row/schema or unexpected metadata") - - -def physical_cases(packet, common): - d = common.d - for original in packet["cases"]: - kind, state = original["kind"], original["state"] - receipt = original["receipt"] - fixture = {"workflow": original["workflow_id"], "run": receipt["run_id"], "namespace": "default"} - d.require(d.history(fixture) == original["history"]["events"], kind + ": original " + state + " API history preserved") - if kind == "signal": - # Real 3.8 ordinary signals stored no caller key. Never issue a - # manufactured retained retry or infer intent equality for them. - with closing(sqlite3.connect(d.DB.as_uri() + "?mode=ro", uri=True)) as db: - key = db.execute("SELECT request_id FROM workflow_commands WHERE id=?", (receipt["command_id"],)).fetchone() - d.require(key == (None,), "physical signal: original NULL key is compatibility only") - continue - record = {"request_id": original["original_body"]["request_id"], - "receipt": {key: receipt[key] for key in ("command_id", "run_id", "command_sequence", "update_id", "accepted_at")}} - case = {"kind": kind, "fixture": fixture, "record": record, - "state": {"open": "open_after_restart", "completed": "completed_after_restart", "continued": "continued_after_restart"}[state]} - with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool: - replies = list(pool.map(lambda _: common.readback(case), range(4))) - d.require(len({reply["command_id"] for reply in replies}) == 1, "physical update: concurrent " + state + " retries retain original command") - common.readback(case, run=fixture["run"]) - for options in [{"name": "other"}, {"run": "00000000000000000000000000"}]: - answer = d.message("update", fixture["workflow"], record["request_id"], value=999, success=False, **options) - refusal = common.typed_error(answer, "request_id_conflict") - d.require(refusal.get("retained_command_id") == receipt["command_id"] and - refusal.get("conflict") == ["name" if "name" in options else "target"], - "physical update: correct original identity/name/run fence") - - -if __name__ == "__main__": - common = load_legacy() - d = common.d - entry = d.held(Path(__file__).absolute(), None, 131072) - baseline = d.held(Path(os.environ["CLI_216_PHYSICAL_BASELINE"]), BASELINE_SHA, 64 * 1024 * 1024) - packet_input = d.held(Path(os.environ["CLI_216_PHYSICAL_PACKET"]), PACKET_SHA) - contract_input = d.held(d.ROOT / "physical-contract.json", CONTRACT_SHA) - delta_input = d.held(d.ROOT / "0013_command_intents.sql", DELTA_SHA) - packet = json.loads(packet_input.data) - if packet["schema"] != "durable-workflow.command-key-physical-seed.v1" or packet["manifest_sha256"] != CONTRACT_SHA: - raise ValueError("physical seed interface differs") - contract = json.loads(contract_input.data) - if packet["expected_external_baseline"] != contract["physical_upgrade"]["baseline"] or len(packet["cases"]) != 6: - raise ValueError("physical baseline/case identities differ") - d.NAME = "cli-216-392-physical" - common.runtime = d.boundary.OwnedContainer(d.docker, d.NAME) - upgrade = d.boundary.OwnedContainer(d.docker, "cli-216-392-upgrade") - outcome, error, phase = "runner-blocked", None, "preflight" - started = datetime.datetime.now(datetime.timezone.utc) - before, after = None, None +def run_implementation(path, digest, boundary, boundary_input): + implementation = boundary.HeldInput(path, digest, maximum=131072) try: - observed = json.loads(d.docker("image", "inspect", d.IMAGE, "--format", "{{json .}}")) - d.require(observed["Architecture"] == "amd64" and observed["Os"] == "linux" and - observed["Config"]["Labels"]["org.opencontainers.image.revision"] == - "5614274d2629d03f92932a07dfee0fb10349ddd9", "physical candidate published platform/source identity") - descriptor = os.open(d.DB, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) - with os.fdopen(descriptor, "wb") as copied: - copied.write(baseline.data) - before = common.full_snapshot() - phase = "explicit physical schema upgrade" - upgrade_started = datetime.datetime.now(datetime.timezone.utc) - upgrade.run("-d", "--name", upgrade.name, "--platform=linux/amd64", "--init", "--cpus=1", "--memory=1g", - "--pids-limit=256", "--network=none", "--read-only", "--cap-drop=ALL", "--security-opt=no-new-privileges", - "--env", "DB_CONNECTION=sqlite", "--env", "DB_DATABASE=/app/database/database.sqlite", - "--mount", "type=bind,source=" + str(d.ROOT / "data") + ",target=/app/database", - "--tmpfs", "/tmp:rw,size=16m", d.IMAGE, "schema-upgrade") - upgrade.inspect() - d.require(d.docker("wait", upgrade.id) == "0", "tagged image schema-upgrade exits zero") - state = upgrade.inspect()["State"] - d.require(state["ExitCode"] == 0 and not state["OOMKilled"], "upgrade process exits cleanly without OOM") - upgrade.inspect() - result = json.loads(d.docker("logs", upgrade.id)) - d.require(result == {"backend": "sqlite", "from_schema_version": 12, "schema_version": 13, "status": "upgraded"}, - "actual schema transition is 12 to 13") - upgrade_ended = datetime.datetime.now(datetime.timezone.utc) - after = common.full_snapshot() - compare_upgrade(before, after, delta_input.data, upgrade_started, upgrade_ended, common.legacy) - d.require(True, "every original business row/schema preserved; only tagged DDL and validated version/journal metadata changed") - upgrade.remove() - phase = "installed CLI physical readback" - common.start() - window = common.full_snapshot() - physical_cases(packet, common) - common.unchanged(window, "physical upgrade installed request window") - outcome = "pass" - except Exception as failure: - error = (type(failure).__name__ + ": " + str(failure)).replace(d.TOKEN, "[redacted]") + scope = {"__name__": "__main__", "__file__": str(implementation.path), + "BOOTSTRAP_HELD_IMPLEMENTATION": implementation, + "BOOTSTRAP_HELD_BOUNDARY": boundary_input} + # The literal digest and held bytes precede every implementation + # effect, including imports and source-provenance capture. + exec(compile(implementation.data, str(implementation.path), "exec"), scope) + return scope finally: - if d.PROXY: - d.PROXY.shutdown() - d.PROXY.server_close() - cleanup_error = None try: - for owned in (upgrade, common.runtime): - owned.remove() - for _ in range(50): - if not owned.exists(): - break - time.sleep(.1) - if owned.exists(): - raise RuntimeError("created physical container removal did not finish") - for item in d.HELD: - item.recheck() - except Exception as failure: - outcome = "runner-blocked" - cleanup_error = (type(failure).__name__ + ": " + str(failure)).replace(d.TOKEN, "[redacted]") - (d.ROOT / "runtime.env").unlink(missing_ok=True) - report = {"cell": "physical-3.8-upgrade", "outcome": outcome, "error": error, "cleanup_error": cleanup_error, - "phase": phase, "started_at": started.isoformat(), "finished_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), - "checks": len(d.CHECKS), "driver_sha256": entry.digest, - "accepted_contract_source": "a09f156399416216d2f097c257a2164f83c7eda7", - "actual_seed_source": "09a4ccda3da11853606af031a8970bbde4b2d6ec", - "actual_seed_driver_sha256": "01c0fa1251ac3c809723e58d71817881fe2c5c9498579bc4c2e6097324a3618b", - "baseline_sha256": baseline.digest, "packet_sha256": packet_input.digest, "contract_sha256": contract_input.digest, - "before_sha256": common.legacy.fingerprint(before) if before else None, - "after_upgrade_sha256": common.legacy.fingerprint(after) if after else None, - "request_windows": common.windows, "server_index": d.IMAGE, - "created_container_ids": upgrade.created_ids + common.runtime.created_ids, - "cleanup": "created IDs/volumes/env removed; original held inputs unchanged" if cleanup_error is None else "requires investigation"} - (d.ROOT / "physical-result.json").write_text(json.dumps(report, indent=2)) - print(json.dumps(report), flush=True) - raise SystemExit(0 if outcome == "pass" else 1) + implementation.recheck() + boundary_input.recheck() + finally: + implementation.close() + boundary_input.close() + + +if __name__ == "__main__": + trusted_boundary, held_boundary = load_boundary() + run_implementation(SOURCE / "physical_impl.py", IMPLEMENTATION_SHA, trusted_boundary, held_boundary) diff --git a/scripts/acceptance/command-keys/physical_impl.py b/scripts/acceptance/command-keys/physical_impl.py new file mode 100644 index 0000000..e703103 --- /dev/null +++ b/scripts/acceptance/command-keys/physical_impl.py @@ -0,0 +1,204 @@ +"""Installed CLI readback on a copy of the accepted real 3.8 API seed.""" +from contextlib import closing +from copy import deepcopy +import concurrent.futures +import datetime +import hashlib +import json +import os +from pathlib import Path +import sqlite3 +import stat +import time +import types + +SOURCE = Path(__file__).resolve().parent +LEGACY_SHA = "b6d2bee1b689745b626ce2d7b754f0179bdb1d22e53ef33dd1501e6dec520c76" +BASELINE_SHA = "7d1826f7681342bb3a5974509648e2e8d75e7a21d262f65f3ec1f58727ccbc0c" +PACKET_SHA = "8c728ce827df3a11621dfb0e82cdd404bbab48784182a5e92c8d632d49047fc5" +CONTRACT_SHA = "ceb4695643e48c7bfe3f81eea380c6ab9aa0b973da830cb552d070344047319f" +DELTA_SHA = "fc3ad1730682c09704027dcbabc17cd6ded9c642a3ff20d7988d5f4b4a5726fd" +JOURNAL_CHECKSUM = "53b752529a1f7addaefe62ab61160dd42b3990f805cfdc4ecd1b48927a6c2f7ce42526fc04522ac887a37208f343fc73" + + +def load_legacy(): + path = SOURCE / "legacy_cli.py" + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC) + try: + observed = os.fstat(descriptor) + if (not stat.S_ISREG(observed.st_mode) or observed.st_uid != os.getuid() + or observed.st_nlink != 1 or observed.st_mode & 0o022 or observed.st_size > 131072): + raise ValueError("legacy adapter owner/type/mode differs") + data = os.pread(descriptor, 131073, 0) + if hashlib.sha256(data).hexdigest() != LEGACY_SHA: + raise ValueError("legacy adapter source differs") + module = types.ModuleType("cli216_physical_common") + module.__file__ = str(path) + exec(compile(data, str(path), "exec"), module.__dict__) + held = module.d.held(path, LEGACY_SHA) + if held.identity != held._identity(observed): + raise ValueError("legacy adapter changed during held import") + return module + finally: + os.close(descriptor) + + +def compare_upgrade(before, after, delta, started, ended, helper): + """Allow only the tagged DDL and narrowly validated migration metadata.""" + if before["tables"]["dw_server_schema"]["rows"] != [[["text", "rust-development"], ["integer", 12]]]: + raise ValueError("baseline is not native SQLite schema 12") + expected = deepcopy(before) + with closing(sqlite3.connect(":memory:")) as db: + db.execute("CREATE TABLE dw_server_schema(engine TEXT, version INTEGER)") + db.executescript(delta.decode()) + additions = helper.snapshot(db) + expected["schema"].extend(row for row in additions["schema"] if row[2] == "dw_command_intents") + expected["schema"].sort(key=lambda row: (row[0], row[1])) + expected["tables"]["dw_command_intents"] = additions["tables"]["dw_command_intents"] + expected["tables"]["dw_server_schema"]["rows"] = [[["text", "rust-development"], ["integer", 13]]] + journal = after["tables"]["_sqlx_migrations"] + columns = ["version", "description", "installed_on", "success", "checksum", "execution_time"] + if journal["columns"] != columns or len(journal["rows"]) != 1: + raise ValueError("upgraded migration journal shape differs") + row = journal["rows"][0] + old = before["tables"]["_sqlx_migrations"]["rows"] + if len(old) != 1 or old[0][0] != ["integer", 12]: + raise ValueError("baseline migration journal shape differs") + if (row[0] != ["integer", 13] or row[1] != ["text", old[0][1][1] + " and keyed command intents"] + or row[3] != ["integer", 1] or row[4] != ["blob", JOURNAL_CHECKSUM] + or row[5][0] != "integer" or not 0 <= row[5][1] <= 120_000_000_000 + or row[2][0] != "text"): + raise ValueError("migration journal content differs from tagged schema 13") + installed = datetime.datetime.fromisoformat(row[2][1]).replace(tzinfo=datetime.timezone.utc) + if not started - datetime.timedelta(seconds=1) <= installed <= ended + datetime.timedelta(seconds=1): + raise ValueError("migration journal time is outside the actual upgrade window") + expected["tables"]["_sqlx_migrations"]["rows"] = deepcopy(journal["rows"]) + if expected != after: + raise ValueError("upgrade changed an original business row/schema or unexpected metadata") + + +def physical_cases(packet, common): + d = common.d + for original in packet["cases"]: + kind, state = original["kind"], original["state"] + receipt = original["receipt"] + fixture = {"workflow": original["workflow_id"], "run": receipt["run_id"], "namespace": "default"} + d.require(d.history(fixture) == original["history"]["events"], kind + ": original " + state + " API history preserved") + if kind == "signal": + # Real 3.8 ordinary signals stored no caller key. Never issue a + # manufactured retained retry or infer intent equality for them. + with closing(sqlite3.connect(d.DB.as_uri() + "?mode=ro", uri=True)) as db: + key = db.execute("SELECT request_id FROM workflow_commands WHERE id=?", (receipt["command_id"],)).fetchone() + d.require(key == (None,), "physical signal: original NULL key is compatibility only") + continue + record = {"request_id": original["original_body"]["request_id"], + "receipt": {key: receipt[key] for key in ("command_id", "run_id", "command_sequence", "update_id", "accepted_at")}} + case = {"kind": kind, "fixture": fixture, "record": record, + "state": {"open": "open_after_restart", "completed": "completed_after_restart", "continued": "continued_after_restart"}[state]} + with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool: + replies = list(pool.map(lambda _: common.readback(case), range(4))) + d.require(len({reply["command_id"] for reply in replies}) == 1, "physical update: concurrent " + state + " retries retain original command") + common.readback(case, run=fixture["run"]) + for options in [{"name": "other"}, {"run": "00000000000000000000000000"}]: + answer = d.message("update", fixture["workflow"], record["request_id"], value=999, success=False, **options) + refusal = common.typed_error(answer, "request_id_conflict") + d.require(refusal.get("retained_command_id") == receipt["command_id"] and + refusal.get("conflict") == ["name" if "name" in options else "target"], + "physical update: correct original identity/name/run fence") + + +if __name__ == "__main__": + if "BOOTSTRAP_HELD_IMPLEMENTATION" not in globals() or "BOOTSTRAP_HELD_BOUNDARY" not in globals(): + raise SystemExit("Use physical_cli.py to verify the implementation before execution") + common = load_legacy() + d = common.d + entry = BOOTSTRAP_HELD_IMPLEMENTATION + d.HELD.extend([BOOTSTRAP_HELD_BOUNDARY, entry]) + baseline = d.held(Path(os.environ["CLI_216_PHYSICAL_BASELINE"]), BASELINE_SHA, 64 * 1024 * 1024) + packet_input = d.held(Path(os.environ["CLI_216_PHYSICAL_PACKET"]), PACKET_SHA) + contract_input = d.held(d.ROOT / "physical-contract.json", CONTRACT_SHA) + delta_input = d.held(d.ROOT / "0013_command_intents.sql", DELTA_SHA) + packet = json.loads(packet_input.data) + if packet["schema"] != "durable-workflow.command-key-physical-seed.v1" or packet["manifest_sha256"] != CONTRACT_SHA: + raise ValueError("physical seed interface differs") + contract = json.loads(contract_input.data) + if packet["expected_external_baseline"] != contract["physical_upgrade"]["baseline"] or len(packet["cases"]) != 6: + raise ValueError("physical baseline/case identities differ") + d.NAME = "cli-216-392-physical" + common.runtime = d.boundary.OwnedContainer(d.docker, d.NAME) + upgrade = d.boundary.OwnedContainer(d.docker, "cli-216-392-upgrade") + outcome, error, phase = "runner-blocked", None, "preflight" + started = datetime.datetime.now(datetime.timezone.utc) + before, after = None, None + try: + observed = json.loads(d.docker("image", "inspect", d.IMAGE, "--format", "{{json .}}")) + d.require(observed["Architecture"] == "amd64" and observed["Os"] == "linux" and + observed["Config"]["Labels"]["org.opencontainers.image.revision"] == + "5614274d2629d03f92932a07dfee0fb10349ddd9", "physical candidate published platform/source identity") + descriptor = os.open(d.DB, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + with os.fdopen(descriptor, "wb") as copied: + copied.write(baseline.data) + before = common.full_snapshot() + phase = "explicit physical schema upgrade" + upgrade_started = datetime.datetime.now(datetime.timezone.utc) + upgrade.run("-d", "--name", upgrade.name, "--platform=linux/amd64", "--init", "--cpus=1", "--memory=1g", + "--pids-limit=256", "--network=none", "--read-only", "--cap-drop=ALL", "--security-opt=no-new-privileges", + "--env", "DB_CONNECTION=sqlite", "--env", "DB_DATABASE=/app/database/database.sqlite", + "--mount", "type=bind,source=" + str(d.ROOT / "data") + ",target=/app/database", + "--tmpfs", "/tmp:rw,size=16m", d.IMAGE, "schema-upgrade") + upgrade.inspect() + d.require(d.docker("wait", upgrade.id) == "0", "tagged image schema-upgrade exits zero") + state = upgrade.inspect()["State"] + d.require(state["ExitCode"] == 0 and not state["OOMKilled"], "upgrade process exits cleanly without OOM") + upgrade.inspect() + result = json.loads(d.docker("logs", upgrade.id)) + d.require(result == {"backend": "sqlite", "from_schema_version": 12, "schema_version": 13, "status": "upgraded"}, + "actual schema transition is 12 to 13") + upgrade_ended = datetime.datetime.now(datetime.timezone.utc) + after = common.full_snapshot() + compare_upgrade(before, after, delta_input.data, upgrade_started, upgrade_ended, common.legacy) + d.require(True, "every original business row/schema preserved; only tagged DDL and validated version/journal metadata changed") + upgrade.remove() + phase = "installed CLI physical readback" + common.start() + window = common.full_snapshot() + physical_cases(packet, common) + common.unchanged(window, "physical upgrade installed request window") + outcome = "pass" + except Exception as failure: + error = (type(failure).__name__ + ": " + str(failure)).replace(d.TOKEN, "[redacted]") + finally: + if d.PROXY: + d.PROXY.shutdown() + d.PROXY.server_close() + cleanup_error = None + try: + for owned in (upgrade, common.runtime): + owned.remove() + for _ in range(50): + if not owned.exists(): + break + time.sleep(.1) + if owned.exists(): + raise RuntimeError("created physical container removal did not finish") + for item in d.HELD: + item.recheck() + except Exception as failure: + outcome = "runner-blocked" + cleanup_error = (type(failure).__name__ + ": " + str(failure)).replace(d.TOKEN, "[redacted]") + (d.ROOT / "runtime.env").unlink(missing_ok=True) + report = {"cell": "physical-3.8-upgrade", "outcome": outcome, "error": error, "cleanup_error": cleanup_error, + "phase": phase, "started_at": started.isoformat(), "finished_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "checks": len(d.CHECKS), "driver_sha256": entry.digest, + "accepted_contract_source": "a09f156399416216d2f097c257a2164f83c7eda7", + "actual_seed_source": "09a4ccda3da11853606af031a8970bbde4b2d6ec", + "actual_seed_driver_sha256": "01c0fa1251ac3c809723e58d71817881fe2c5c9498579bc4c2e6097324a3618b", + "baseline_sha256": baseline.digest, "packet_sha256": packet_input.digest, "contract_sha256": contract_input.digest, + "before_sha256": common.legacy.fingerprint(before) if before else None, + "after_upgrade_sha256": common.legacy.fingerprint(after) if after else None, + "request_windows": common.windows, "server_index": d.IMAGE, + "created_container_ids": upgrade.created_ids + common.runtime.created_ids, + "cleanup": "created IDs/volumes/env removed; original held inputs unchanged" if cleanup_error is None else "requires investigation"} + (d.ROOT / "physical-result.json").write_text(json.dumps(report, indent=2)) + print(json.dumps(report), flush=True) + raise SystemExit(0 if outcome == "pass" else 1) diff --git a/scripts/acceptance/command-keys/test_physical_cli.py b/scripts/acceptance/command-keys/test_physical_cli.py index f98a18b..76d234f 100644 --- a/scripts/acceptance/command-keys/test_physical_cli.py +++ b/scripts/acceptance/command-keys/test_physical_cli.py @@ -1,9 +1,13 @@ """Offline checks for the physical-upgrade comparison's narrow exceptions.""" import datetime +import hashlib import importlib.util import os from pathlib import Path import sqlite3 +import subprocess +import sys +import tempfile import unittest SOURCE = Path(__file__).resolve().parent @@ -16,8 +20,9 @@ def local_module(name, filename): return value -physical = local_module("physical_comparison", "physical_cli.py") +physical = local_module("physical_comparison", "physical_impl.py") boundary = local_module("physical_boundaries", "boundaries.py") +bootstrap = local_module("physical_bootstrap", "physical_cli.py") class UpgradeComparisonTests(unittest.TestCase): @@ -87,5 +92,46 @@ def test_unbounded_execution_time_is_refused(self): self.refuses("UPDATE _sqlx_migrations SET execution_time=999999999999999") +class PreCaptureTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name).resolve() + self.boundary_input = boundary.HeldInput(SOURCE / "boundaries.py", bootstrap.BOUNDARY_SHA) + self.addCleanup(self.boundary_input.close) + + def test_loaded_predecessor_can_relabel_while_candidate_refuses_before_execution(self): + source = self.root / "implementation.py" + marker = self.root / "effect" + original = ("captured = hashlib.sha256(Path(__file__).read_bytes()).hexdigest()\n" + "Path(" + repr(str(marker)) + ").touch()\n").encode() + source.write_bytes(original) + loaded = compile(original, str(source), "exec") + source.rename(self.root / "original") + source.write_bytes(b"replacement = True\n") + scope = {"__file__": str(source), "hashlib": hashlib, "Path": Path} + exec(loaded, scope) + self.assertEqual(scope["captured"], hashlib.sha256(source.read_bytes()).hexdigest()) + self.assertNotEqual(scope["captured"], hashlib.sha256(original).hexdigest()) + self.assertTrue(marker.exists()) + marker.unlink() + with self.assertRaises(ValueError): + bootstrap.run_implementation(source, hashlib.sha256(original).hexdigest(), boundary, self.boundary_input) + self.assertFalse(marker.exists()) + + def test_direct_implementation_refuses_before_effectful_initialization(self): + result = subprocess.run([sys.executable, "-I", "-B", str(SOURCE / "physical_impl.py")], + env={"PATH": os.defpath}, capture_output=True, text=True, timeout=5) + self.assertNotEqual(result.returncode, 0) + self.assertIn("Use physical_cli.py", result.stderr) + + def test_literal_bound_implementation_executes_checked_bytes(self): + source = self.root / "implementation.py" + data = b"value = 'reviewed'\n" + source.write_bytes(data) + result = bootstrap.run_implementation(source, hashlib.sha256(data).hexdigest(), boundary, self.boundary_input) + self.assertEqual(result["value"], "reviewed") + + if __name__ == "__main__": unittest.main() From 23768faf57d8c4a19f303332309af71b52c35e80 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 13 Nov 2026 18:13:19 +0000 Subject: [PATCH 3/7] Bind every command-key runner before execution and require zero journal time --- scripts/acceptance/command-keys/README.md | 11 ++++---- scripts/acceptance/command-keys/cli_216.py | 9 ++++++- scripts/acceptance/command-keys/legacy_cli.py | 11 ++++++-- .../acceptance/command-keys/physical_impl.py | 6 ++--- .../{physical_cli.py => run_cli_216.py} | 14 +++++++--- .../command-keys/test_physical_cli.py | 26 ++++++++++++++++--- 6 files changed, 60 insertions(+), 17 deletions(-) rename scripts/acceptance/command-keys/{physical_cli.py => run_cli_216.py} (74%) diff --git a/scripts/acceptance/command-keys/README.md b/scripts/acceptance/command-keys/README.md index 4b693e5..c6ab017 100644 --- a/scripts/acceptance/command-keys/README.md +++ b/scripts/acceptance/command-keys/README.md @@ -81,7 +81,7 @@ or registry credentials. Runtime authentication is synthetic and private. env -i PATH=/usr/bin:/bin \ CLI_216_WORKDIR="$experiment" \ CLI_216_PROXY_SOURCE="$experiment/committed_response_loss.py" \ - timeout 180s python3 -I -B scripts/acceptance/command-keys/cli_216.py + timeout 180s python3 -I -B scripts/acceptance/command-keys/run_cli_216.py ``` For the restart phase, preserve the first phase's `admissions.json` as @@ -91,7 +91,7 @@ records separately too. `CLI_216_PHASE=fresh-selected` adds direct fresh explicit-run literal-key markers (the loss cells intentionally withhold those initial response bodies). `CLI_216_PHASE=typed` runs the separate typed-intent cases. Use a **new** private -directory for `legacy_cli.py`, with the same staged inputs and environment. +directory with `CLI_216_RUNNER=legacy`, with the same staged inputs and environment. The legacy adapter generates its own admissions and stopped-copy database. An outer experiment owner must also enforce the timeout and remove only the @@ -124,7 +124,7 @@ including catalog SQL, columns, indexes, triggers, views and all table rows. ## Physical 3.8 upgrade -`physical_cli.py` verifies and holds the literal-hash `physical_impl.py` bytes +`run_cli_216.py` verifies and holds literal-hash modern, legacy or physical implementation bytes before executing any implementation code. Its reported runner digest comes from those checked bytes, with a post-cleanup identity recheck. Direct execution of the implementation refuses before effectful initialization. It consumes the @@ -152,15 +152,16 @@ The image runs its actual `schema-upgrade` command, requiring SQLite 12 → 13. The comparison preserves all old business rows and schema objects, allowing only tagged DDL plus explicitly validated schema-version/journal metadata. The intent table must be empty. The new journal checksum, description, -version, success, timestamp interval and execution-time bound are checked. +version, success, timestamp interval and exact integer-zero execution time are checked. ```bash env -i PATH=/usr/bin:/bin \ CLI_216_WORKDIR="$experiment" \ CLI_216_PROXY_SOURCE="$experiment/committed_response_loss.py" \ + CLI_216_RUNNER=physical \ CLI_216_PHYSICAL_BASELINE="$original_backup" \ CLI_216_PHYSICAL_PACKET="$original_packet" \ - timeout 180s python3 -I -B scripts/acceptance/command-keys/physical_cli.py + timeout 180s python3 -I -B scripts/acceptance/command-keys/run_cli_216.py ``` Created names are `cli-216-392-upgrade` and `cli-216-392-physical`; cleanup diff --git a/scripts/acceptance/command-keys/cli_216.py b/scripts/acceptance/command-keys/cli_216.py index 9e1b052..413ed35 100644 --- a/scripts/acceptance/command-keys/cli_216.py +++ b/scripts/acceptance/command-keys/cli_216.py @@ -14,6 +14,9 @@ import urllib.error import urllib.request +if __name__ == "__main__" and "BOOTSTRAP_HELD_IMPLEMENTATION" not in globals(): + raise SystemExit("Use run_cli_216.py to verify the implementation before execution") + SOURCE = Path(__file__).resolve().parent BOUNDARY_SHA = 'd377e9191ebf79dc990db159629c303c10531a2f27d3ae20933fabc87e3b3525' # Bootstrap the local reviewed boundary module from the same bytes that are @@ -48,7 +51,11 @@ def held(path, digest, maximum=16 * 1024 * 1024): HELD.append(value) return value -ENTRY_INPUT = held(Path(__file__).absolute(), None, 131072) +if "BOOTSTRAP_HELD_IMPLEMENTATION" in globals(): + ENTRY_INPUT = BOOTSTRAP_HELD_IMPLEMENTATION + HELD.extend([BOOTSTRAP_HELD_BOUNDARY, ENTRY_INPUT]) +else: + ENTRY_INPUT = held(Path(__file__).absolute(), None, 131072) PROXY_INPUT = held(PROXY_SOURCE, PROXY_SHA256) shared_proxy = PROXY_INPUT.module('cli216_shared_proxy') CommittedResponseLoss = shared_proxy.CommittedResponseLoss diff --git a/scripts/acceptance/command-keys/legacy_cli.py b/scripts/acceptance/command-keys/legacy_cli.py index d568929..af20ac6 100644 --- a/scripts/acceptance/command-keys/legacy_cli.py +++ b/scripts/acceptance/command-keys/legacy_cli.py @@ -13,6 +13,9 @@ import threading import time +if __name__ == "__main__" and "BOOTSTRAP_HELD_IMPLEMENTATION" not in globals(): + raise SystemExit("Use run_cli_216.py to verify the implementation before execution") + SOURCE = Path(__file__).resolve().parent ROOT = Path(os.environ["CLI_216_WORKDIR"]) @@ -39,9 +42,13 @@ def bound_driver(): os.close(descriptor) -BASE_SHA = "c6b7897a6b5c5208a236d296438fc56fa97d4b15ef28329ac719fade852dc562" +BASE_SHA = "e26565125bb122e6d84cd6a6e230c7d47a56ca043eda7bffa7025b9c19fdbbbe" d = bound_driver() -ENTRY_INPUT = d.held(Path(__file__).absolute(), None, 131072) +if "BOOTSTRAP_HELD_IMPLEMENTATION" in globals(): + ENTRY_INPUT = BOOTSTRAP_HELD_IMPLEMENTATION + d.HELD.extend([BOOTSTRAP_HELD_BOUNDARY, ENTRY_INPUT]) +else: + ENTRY_INPUT = d.held(Path(__file__).absolute(), None, 131072) legacy = d.legacy_helper d.NAME = "cli-216-392-legacy" runtime = d.boundary.OwnedContainer(d.docker, d.NAME) diff --git a/scripts/acceptance/command-keys/physical_impl.py b/scripts/acceptance/command-keys/physical_impl.py index e703103..7ea5f73 100644 --- a/scripts/acceptance/command-keys/physical_impl.py +++ b/scripts/acceptance/command-keys/physical_impl.py @@ -13,7 +13,7 @@ import types SOURCE = Path(__file__).resolve().parent -LEGACY_SHA = "b6d2bee1b689745b626ce2d7b754f0179bdb1d22e53ef33dd1501e6dec520c76" +LEGACY_SHA = "0f0ceb09676d38109bbd57ed9c4d22e0c448a78c65999fe82a5feb3b3ce1062b" BASELINE_SHA = "7d1826f7681342bb3a5974509648e2e8d75e7a21d262f65f3ec1f58727ccbc0c" PACKET_SHA = "8c728ce827df3a11621dfb0e82cdd404bbab48784182a5e92c8d632d49047fc5" CONTRACT_SHA = "ceb4695643e48c7bfe3f81eea380c6ab9aa0b973da830cb552d070344047319f" @@ -66,7 +66,7 @@ def compare_upgrade(before, after, delta, started, ended, helper): raise ValueError("baseline migration journal shape differs") if (row[0] != ["integer", 13] or row[1] != ["text", old[0][1][1] + " and keyed command intents"] or row[3] != ["integer", 1] or row[4] != ["blob", JOURNAL_CHECKSUM] - or row[5][0] != "integer" or not 0 <= row[5][1] <= 120_000_000_000 + or row[5] != ["integer", 0] or row[2][0] != "text"): raise ValueError("migration journal content differs from tagged schema 13") installed = datetime.datetime.fromisoformat(row[2][1]).replace(tzinfo=datetime.timezone.utc) @@ -109,7 +109,7 @@ def physical_cases(packet, common): if __name__ == "__main__": if "BOOTSTRAP_HELD_IMPLEMENTATION" not in globals() or "BOOTSTRAP_HELD_BOUNDARY" not in globals(): - raise SystemExit("Use physical_cli.py to verify the implementation before execution") + raise SystemExit("Use run_cli_216.py to verify the implementation before execution") common = load_legacy() d = common.d entry = BOOTSTRAP_HELD_IMPLEMENTATION diff --git a/scripts/acceptance/command-keys/physical_cli.py b/scripts/acceptance/command-keys/run_cli_216.py similarity index 74% rename from scripts/acceptance/command-keys/physical_cli.py rename to scripts/acceptance/command-keys/run_cli_216.py index 0f93ba6..fd3519c 100644 --- a/scripts/acceptance/command-keys/physical_cli.py +++ b/scripts/acceptance/command-keys/run_cli_216.py @@ -1,4 +1,4 @@ -"""Verify the physical experiment implementation before executing its bytes.""" +"""Verify the selected command-key implementation before executing its bytes.""" import hashlib import os from pathlib import Path @@ -7,7 +7,11 @@ SOURCE = Path(__file__).resolve().parent BOUNDARY_SHA = "d377e9191ebf79dc990db159629c303c10531a2f27d3ae20933fabc87e3b3525" -IMPLEMENTATION_SHA = "808cff32ead554ecefc931ead0797097312fec3a5155e49bfdeeffc218016f96" +IMPLEMENTATIONS = { + 'modern': ('cli_216.py', 'e26565125bb122e6d84cd6a6e230c7d47a56ca043eda7bffa7025b9c19fdbbbe'), + 'legacy': ('legacy_cli.py', '0f0ceb09676d38109bbd57ed9c4d22e0c448a78c65999fe82a5feb3b3ce1062b'), + 'physical': ('physical_impl.py', '81ff88b11a7c402d2206c2856b08080eb149528b31cd0f616ccd676e459fcd3f'), +} def load_boundary(): @@ -52,5 +56,9 @@ def run_implementation(path, digest, boundary, boundary_input): if __name__ == "__main__": + choice = os.environ.get("CLI_216_RUNNER", "modern") + if choice not in IMPLEMENTATIONS: + raise SystemExit("CLI_216_RUNNER must be modern, legacy or physical") + filename, expected = IMPLEMENTATIONS[choice] trusted_boundary, held_boundary = load_boundary() - run_implementation(SOURCE / "physical_impl.py", IMPLEMENTATION_SHA, trusted_boundary, held_boundary) + run_implementation(SOURCE / filename, expected, trusted_boundary, held_boundary) diff --git a/scripts/acceptance/command-keys/test_physical_cli.py b/scripts/acceptance/command-keys/test_physical_cli.py index 76d234f..85366e5 100644 --- a/scripts/acceptance/command-keys/test_physical_cli.py +++ b/scripts/acceptance/command-keys/test_physical_cli.py @@ -22,7 +22,7 @@ def local_module(name, filename): physical = local_module("physical_comparison", "physical_impl.py") boundary = local_module("physical_boundaries", "boundaries.py") -bootstrap = local_module("physical_bootstrap", "physical_cli.py") +bootstrap = local_module("physical_bootstrap", "run_cli_216.py") class UpgradeComparisonTests(unittest.TestCase): @@ -49,7 +49,7 @@ def setUp(self): self.started = datetime.datetime.now(datetime.timezone.utc) self.db.executescript(self.delta.decode()) self.db.execute("UPDATE _sqlx_migrations SET version=13,description='baseline and keyed command intents'," - "installed_on=?,checksum=?", (self.started.strftime("%Y-%m-%d %H:%M:%S"), bytes.fromhex(physical.JOURNAL_CHECKSUM))) + "installed_on=?,checksum=?,execution_time=0", (self.started.strftime("%Y-%m-%d %H:%M:%S"), bytes.fromhex(physical.JOURNAL_CHECKSUM))) self.ended = datetime.datetime.now(datetime.timezone.utc) def compare(self): @@ -88,6 +88,9 @@ def test_extra_journal_row_is_refused(self): def test_old_journal_timestamp_is_refused(self): self.refuses("UPDATE _sqlx_migrations SET installed_on='2020-01-01 00:00:00'") + def test_nonzero_execution_time_is_refused(self): + self.refuses("UPDATE _sqlx_migrations SET execution_time=1") + def test_unbounded_execution_time_is_refused(self): self.refuses("UPDATE _sqlx_migrations SET execution_time=999999999999999") @@ -123,7 +126,24 @@ def test_direct_implementation_refuses_before_effectful_initialization(self): result = subprocess.run([sys.executable, "-I", "-B", str(SOURCE / "physical_impl.py")], env={"PATH": os.defpath}, capture_output=True, text=True, timeout=5) self.assertNotEqual(result.returncode, 0) - self.assertIn("Use physical_cli.py", result.stderr) + self.assertIn("Use run_cli_216.py", result.stderr) + + def test_direct_common_entrypoints_refuse_before_initialization(self): + for filename in ("cli_216.py", "legacy_cli.py"): + with self.subTest(filename=filename): + result = subprocess.run([sys.executable, "-I", "-B", str(SOURCE / filename)], + env={"PATH": os.defpath}, capture_output=True, text=True, timeout=5) + self.assertNotEqual(result.returncode, 0) + self.assertIn("Use run_cli_216.py", result.stderr) + + def test_unknown_loader_choice_refuses(self): + for choice in ("unknown", "../physical_impl.py"): + with self.subTest(choice=choice): + result = subprocess.run([sys.executable, "-I", "-B", str(SOURCE / "run_cli_216.py")], + env={"PATH": os.defpath, "CLI_216_RUNNER": choice}, + capture_output=True, text=True, timeout=5) + self.assertNotEqual(result.returncode, 0) + self.assertIn("CLI_216_RUNNER must be", result.stderr) def test_literal_bound_implementation_executes_checked_bytes(self): source = self.root / "implementation.py" From a38f93ddb4488fd3bc8a39a6ab98143716e05658 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 13 Nov 2026 18:59:44 +0000 Subject: [PATCH 4/7] Hold workspace database and report identities across acceptance phases --- scripts/acceptance/command-keys/README.md | 23 ++ scripts/acceptance/command-keys/boundaries.py | 208 +++++++++++++++++- scripts/acceptance/command-keys/cli_216.py | 33 +-- scripts/acceptance/command-keys/legacy_cli.py | 53 +++-- .../acceptance/command-keys/physical_impl.py | 25 ++- .../acceptance/command-keys/run_cli_216.py | 8 +- .../command-keys/test_boundaries.py | 157 ++++++++++++- 7 files changed, 463 insertions(+), 44 deletions(-) diff --git a/scripts/acceptance/command-keys/README.md b/scripts/acceptance/command-keys/README.md index c6ab017..61e3cd1 100644 --- a/scripts/acceptance/command-keys/README.md +++ b/scripts/acceptance/command-keys/README.md @@ -183,3 +183,26 @@ CLI_216_TEST_HELPER="$experiment/sqlite_legacy.py" \ These controls are not a physical upgrade result. Source review, current normal gates, fresh actual execution and cleanup still qualify that cell. + +The manual experiment holds owned mode-0700 root/data directories and a +single-link mode-0600 database descriptor throughout each phase. SQLite opens +through the held directory so WAL/SHM siblings keep their normal semantics. +Docker binds both the held directory and the held main database file via local +`/proc//fd` paths; the inspected mounts must match both descriptors. +This requires the local Linux Docker daemon, not a remote Docker context. +Identity checks immediately surround snapshots, container creation/inspection, +API and installed CLI calls, cleanup, and report emission. Reports are written +through held no-follow descriptors, then their path, parent, and contents are +rechecked before printing a terminal result. A swap fails the run; exact-ID +container cleanup remains available even after a workspace identity failure. + +The synthetic transform uses a separate stopped sibling copy, verifies its +complete result against the accepted helper's fingerprints, and copies those +bytes into the existing held runtime database inode. It does not rename a new +file over the database that later containers exercise. The physical upgrade +also mutates only that held copied database. Reports include root/data/database +identities so the parent can compare them across modern process restarts. +Offline workspace controls demonstrate predecessor pathname substitution and +require refusal for root/data/database swaps before upgrade or runtime start, +report path/parent swaps during emission, in-place report edits, symlinks and +extra hard links. A positive WAL control preserves actual mutable SQLite use. diff --git a/scripts/acceptance/command-keys/boundaries.py b/scripts/acceptance/command-keys/boundaries.py index 2680401..f1f663e 100644 --- a/scripts/acceptance/command-keys/boundaries.py +++ b/scripts/acceptance/command-keys/boundaries.py @@ -87,12 +87,207 @@ def private_directory(path): return path +class HeldDirectory: + """Keep an owned directory inode, including its no-follow parent binding.""" + def __init__(self, path, parent=None): + self.path, self.parent = Path(path), parent + if not self.path.is_absolute() or self.path.resolve() != self.path: + raise ValueError("directory must be canonical without symlinks") + if parent is not None: + parent.recheck() + if self.path.parent != parent.path: + raise ValueError("held directory must be a direct child") + self.fd = os.open(self.path if parent is None else self.path.name, + os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC, + dir_fd=None if parent is None else parent.fd) + self.identity = self.key(os.fstat(self.fd)) + try: + self.recheck() + except BaseException: + self.close() + raise + + @staticmethod + def key(value): + return (value.st_dev, value.st_ino, value.st_mode, value.st_uid, value.st_gid, value.st_nlink) + + def recheck(self): + if self.parent is not None: + self.parent.recheck() + value = os.fstat(self.fd) + named = os.stat(self.path if self.parent is None else self.path.name, + dir_fd=None if self.parent is None else self.parent.fd, follow_symlinks=False) + if (not stat.S_ISDIR(value.st_mode) or value.st_uid != os.getuid() + or stat.S_IMODE(value.st_mode) != 0o700 or value.st_nlink < 2 + or self.key(value) != self.identity or self.key(named) != self.identity + or self.path.resolve() != self.path or self.key(self.path.lstat()) != self.identity): + raise ValueError("held directory identity changed") + + @property + def anchor(self): + return "/proc/" + str(os.getpid()) + "/fd/" + str(self.fd) + + def close(self): + os.close(self.fd) + + +class HeldWritableFile: + """Mutable SQLite data or a frozen report, always tied to one parent/inode.""" + def __init__(self, parent, name, *, create=False, maximum=64 * 1024 * 1024): + if Path(name).name != name or name in (".", ".."): + raise ValueError("held file requires one filename") + self.parent, self.name, self.maximum, self.digest = parent, name, maximum, None + parent.recheck() + flags = os.O_RDWR | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC + if create: + flags |= os.O_CREAT | os.O_EXCL + self.fd = os.open(name, flags, 0o600, dir_fd=parent.fd) + self.identity = HeldDirectory.key(os.fstat(self.fd)) + try: + self.recheck() + except BaseException: + self.close() + raise + + def recheck(self): + self.parent.recheck() + value = os.fstat(self.fd) + named = os.stat(self.name, dir_fd=self.parent.fd, follow_symlinks=False) + if (not stat.S_ISREG(value.st_mode) or value.st_uid != os.getuid() or value.st_nlink != 1 + or stat.S_IMODE(value.st_mode) != 0o600 or not 0 <= value.st_size <= self.maximum + or HeldDirectory.key(value) != self.identity or HeldDirectory.key(named) != self.identity): + raise ValueError("held writable file identity changed") + if self.digest is not None and hashlib.sha256(os.pread(self.fd, self.maximum + 1, 0)).hexdigest() != self.digest: + raise ValueError("held report content changed") + + @property + def anchor(self): + return "/proc/" + str(os.getpid()) + "/fd/" + str(self.fd) + + def write(self, data, *, freeze=False): + self.recheck() + if len(data) > self.maximum: + raise ValueError("held file byte bound exceeded") + self.digest = None + os.ftruncate(self.fd, 0) + offset = 0 + while offset < len(data): + count = os.pwrite(self.fd, data[offset:], offset) + if count <= 0: + raise OSError("held file write made no progress") + offset += count + os.fsync(self.fd) + self.digest = hashlib.sha256(data).hexdigest() if freeze else None + self.recheck() + + def close(self): + os.close(self.fd) + + +class Workspace: + """Bind comparison, Docker data, and terminal evidence to held objects.""" + def __init__(self, root): + self.root = HeldDirectory(root) + self.data = HeldDirectory(Path(root) / "data", self.root) + self.files = {} + try: + # The first phase creates the file; later phases hold the existing + # inode. Server mutates it across restarts and schema upgrades. + exists = os.path.lexists(Path(root) / "data/database.sqlite") + self.database = HeldWritableFile(self.data, "database.sqlite", create=not exists) + self.recheck() + except BaseException: + self.data.close() + self.root.close() + raise + + def recheck(self): + self.root.recheck() + self.data.recheck() + self.database.recheck() + for value in self.files.values(): + value.recheck() + + def evidence(self): + self.recheck() + return {"root": list(self.root.identity), "data": list(self.data.identity), + "database": list(self.database.identity)} + + def connection(self, *, readonly=True): + from contextlib import contextmanager + import sqlite3 + + @contextmanager + def opened(): + self.recheck() + # The directory descriptor preserves SQLite's WAL/SHM sibling + # semantics. Check SQLite's opened main file against our held DB. + uri = "file:" + self.data.anchor + "/database.sqlite?mode=" + ("ro" if readonly else "rw") + db = sqlite3.connect(uri, uri=True, timeout=2) + try: + named = db.execute("PRAGMA database_list").fetchone()[2] + if HeldDirectory.key(os.stat(named)) != self.database.identity: + raise ValueError("SQLite opened a different database") + self.recheck() + yield db + finally: + db.close() + self.recheck() + return opened() + + def mount_arguments(self): + self.recheck() + # Docker resolves these local /proc descriptors while the parent holds + # them open. Both directory and main-file mounts keep the exact inode, + # even if a same-UID process renames a workspace pathname transiently. + return ("--mount", "type=bind,source=" + self.data.anchor + ",target=/app/database", + "--mount", "type=bind,source=" + self.database.anchor + ",target=/app/database/database.sqlite") + + def inspect_mounts(self, value): + self.recheck() + expected = {"/app/database": self.data.anchor, "/app/database/database.sqlite": self.database.anchor} + for destination, source in expected.items(): + matches = [item for item in value["Mounts"] if item["Destination"] == destination] + if (len(matches) != 1 or matches[0]["Type"] != "bind" or matches[0]["Source"] != source + or matches[0]["RW"] is not True): + raise ValueError("Docker did not bind the held workspace objects") + self.recheck() + + def write(self, name, data): + self.recheck() + if name not in self.files: + exists = os.path.lexists(self.root.path / name) + self.files[name] = HeldWritableFile(self.root, name, create=not exists, maximum=32 * 1024 * 1024) + self.files[name].write(data, freeze=True) + self.recheck() + + def unlink(self, name): + self.recheck() + item = self.files.get(name) + if item is not None: + item.recheck() + os.unlink(name, dir_fd=self.root.fd) + item.close() + del self.files[name] + self.recheck() + + def close(self): + for value in self.files.values(): + value.close() + self.database.close() + self.data.close() + self.root.close() + + class OwnedContainer: - def __init__(self, docker, name): + def __init__(self, docker, name, workspace=None): self.docker, self.name, self.id = docker, name, None + self.workspace = workspace self.created_ids = [] def run(self, *arguments): + if self.workspace is not None: + self.workspace.recheck() if self.exists() or self.docker("ps", "-aq", "--filter", "name=^" + self.name + "$"): raise ValueError("container name already exists; no cleanup authority acquired") created = self.docker("run", *arguments) @@ -105,19 +300,26 @@ def run(self, *arguments): def exists(self): return self.id is not None and self.docker("ps", "-aq", "--no-trunc", "--filter", "id=" + self.id) == self.id - def inspect(self): + def inspect(self, *, check_workspace=True): + if check_workspace and self.workspace is not None: + self.workspace.recheck() if not self.exists(): raise ValueError("created container no longer exists") value = json.loads(self.docker("inspect", self.id, "--format", "{{json .}}")) if value["Id"] != self.id or value["Name"] != "/" + self.name: raise ValueError("created container ID/name changed") + if check_workspace and self.workspace is not None: + self.workspace.inspect_mounts(value) return value def stop(self): self.inspect() self.docker("stop", "--time=10", self.id) + if self.workspace is not None: + self.workspace.recheck() def remove(self): if self.exists(): - self.inspect() + # A failed workspace check must not prevent exact-ID cleanup. + self.inspect(check_workspace=False) self.docker("rm", "-fv", self.id) diff --git a/scripts/acceptance/command-keys/cli_216.py b/scripts/acceptance/command-keys/cli_216.py index 413ed35..7b2087c 100644 --- a/scripts/acceptance/command-keys/cli_216.py +++ b/scripts/acceptance/command-keys/cli_216.py @@ -18,7 +18,7 @@ raise SystemExit("Use run_cli_216.py to verify the implementation before execution") SOURCE = Path(__file__).resolve().parent -BOUNDARY_SHA = 'd377e9191ebf79dc990db159629c303c10531a2f27d3ae20933fabc87e3b3525' +BOUNDARY_SHA = '8e1374d497741cf1668c43bd884af69f72282448bc7151d00f6ff4718c4546b8' # Bootstrap the local reviewed boundary module from the same bytes that are # checked. Subsequent inputs use its held descriptors and sealed memory files. _boundary_path = SOURCE / 'boundaries.py' @@ -79,6 +79,7 @@ def held(path, digest, maximum=16 * 1024 * 1024): 'DURABLE_WORKFLOW_AUTH_TOKEN': TOKEN} for directory in ('data', 'home', 'config', 'tmp', 'docker-empty'): boundary.private_directory(ROOT / directory) +WORKSPACE = boundary.Workspace(ROOT) HTTP = urllib.request.build_opener(urllib.request.ProxyHandler({})) @@ -98,6 +99,7 @@ def docker(*args): def request(method, path, body=None, namespace='default'): + WORKSPACE.recheck() headers = {'Authorization': 'Bearer ' + TOKEN, 'X-Namespace': namespace, 'Content-Type': 'application/json'} worker = path.startswith('/api/worker/') headers['X-Durable-Workflow-Protocol-Version' if worker else 'X-Durable-Workflow-Control-Plane-Version'] = '1.20' if worker else '2' @@ -108,13 +110,17 @@ def request(method, path, body=None, namespace='default'): return response.status, json.load(response) except urllib.error.HTTPError as e: return e.code, json.load(e) + finally: + WORKSPACE.recheck() def cli(*args, namespace='default', success=True): + WORKSPACE.recheck() CLI_INPUT.recheck() p = subprocess.run(['dw', *args, '--server=' + PROXY.url, '--namespace=' + namespace, '--json'], executable=CLI_INPUT.executable, env=ENV, capture_output=True, text=True, timeout=45, pass_fds=(CLI_INPUT.sealed_fd,)) CLI_INPUT.recheck() + WORKSPACE.recheck() try: body = json.loads(p.stdout) except ValueError: @@ -191,8 +197,7 @@ def history(f): def snapshot(): - from contextlib import closing - with closing(sqlite3.connect(DB.as_uri() + '?mode=ro', uri=True, timeout=2)) as db: + with WORKSPACE.connection() as db: return legacy_helper.snapshot(db) @@ -406,17 +411,16 @@ def run_typed(): if __name__ == '__main__': outcome, error = 'runner-blocked', None - runtime = boundary.OwnedContainer(docker, NAME) + runtime = boundary.OwnedContainer(docker, NAME, WORKSPACE) try: observed = json.loads(docker('image', 'inspect', IMAGE, '--format', '{{json .}}')) require(observed['Architecture'] == 'amd64' and observed['Os'] == 'linux', 'observed native platform is linux/amd64') require(observed['Config']['Labels']['org.opencontainers.image.revision'] == '5614274d2629d03f92932a07dfee0fb10349ddd9', 'observed image source matches qualified producer') envfile = ROOT / 'runtime.env' - envfile.write_text('DW_AUTH_TOKEN=' + TOKEN + '\nDW_RUST_MAINTENANCE_LOOP=0\n') - envfile.chmod(0o600) + WORKSPACE.write('runtime.env', ('DW_AUTH_TOKEN=' + TOKEN + '\nDW_RUST_MAINTENANCE_LOOP=0\n').encode()) runtime.run('--rm', '-d', '--name', NAME, '--platform=linux/amd64', '--init', '--cpus=1', '--memory=1g', '--pids-limit=256', '--read-only', '--cap-drop=ALL', '--security-opt=no-new-privileges', '--env-file', str(envfile), - '--mount', 'type=bind,source=' + str(ROOT / 'data') + ',target=/app/database', + *WORKSPACE.mount_arguments(), '--tmpfs', '/app/storage:rw,uid=1000,gid=1000,mode=750,size=64m', '--tmpfs', '/tmp:rw,size=16m', '-p', '127.0.0.1::8080', IMAGE) binding = runtime.inspect()['NetworkSettings']['Ports']['8080/tcp'][0] @@ -455,6 +459,7 @@ def run_typed(): time.sleep(.1) if runtime.exists(): raise RuntimeError('created container removal did not complete') + WORKSPACE.recheck() for item in HELD: item.recheck() cleanup = 'created container and anonymous volumes removed; held inputs unchanged' @@ -462,15 +467,17 @@ def run_typed(): cleanup = 'cleanup or input identity requires investigation' outcome = 'runner-blocked' error = (type(failure).__name__ + ': ' + str(failure)).replace(TOKEN, '[redacted]') - (ROOT / 'runtime.env').unlink(missing_ok=True) + WORKSPACE.unlink('runtime.env') report = {'started_at': STARTED, 'finished_at': datetime.datetime.now(datetime.timezone.utc).isoformat(), - 'outcome': outcome, 'created_container_ids': runtime.created_ids, 'checks': CHECKS, 'error': error, 'cleanup': cleanup, + 'outcome': outcome, 'workspace_identity': WORKSPACE.evidence(), 'created_container_ids': runtime.created_ids, 'checks': CHECKS, 'error': error, 'cleanup': cleanup, 'runner_sha256': ENTRY_INPUT.digest, 'proxy_sha256': PROXY_INPUT.digest, 'server_index': IMAGE, 'server_source': '5614274d2629d03f92932a07dfee0fb10349ddd9', 'phase': os.environ.get('CLI_216_PHASE', 'modern')} - (ROOT / 'result.json').write_text(json.dumps(report, indent=2)) - (ROOT / 'invocations.json').write_text(json.dumps(INVOCATIONS, indent=2).replace(TOKEN, '[redacted]')) - (ROOT / 'admissions.json').write_text(json.dumps(ADMISSIONS, indent=2)) + WORKSPACE.write('result.json', json.dumps(report, indent=2).encode()) + WORKSPACE.write('invocations.json', json.dumps(INVOCATIONS, indent=2).replace(TOKEN, '[redacted]').encode()) + WORKSPACE.write('admissions.json', json.dumps(ADMISSIONS, indent=2).encode()) if PROXY: - (ROOT / 'wire.json').write_text(json.dumps(PROXY.records, indent=2).replace(TOKEN, '[redacted]')) + WORKSPACE.write('wire.json', json.dumps(PROXY.records, indent=2).replace(TOKEN, '[redacted]').encode()) + WORKSPACE.recheck() print(json.dumps(report), flush=True) + WORKSPACE.close() raise SystemExit(0 if outcome == 'pass' else 1) diff --git a/scripts/acceptance/command-keys/legacy_cli.py b/scripts/acceptance/command-keys/legacy_cli.py index af20ac6..af51a79 100644 --- a/scripts/acceptance/command-keys/legacy_cli.py +++ b/scripts/acceptance/command-keys/legacy_cli.py @@ -42,7 +42,7 @@ def bound_driver(): os.close(descriptor) -BASE_SHA = "e26565125bb122e6d84cd6a6e230c7d47a56ca043eda7bffa7025b9c19fdbbbe" +BASE_SHA = "cd8638f7594b8fd72ac0e72dd44d38f6be27167430518b4bebd6c5c244c7c0cd" d = bound_driver() if "BOOTSTRAP_HELD_IMPLEMENTATION" in globals(): ENTRY_INPUT = BOOTSTRAP_HELD_IMPLEMENTATION @@ -51,25 +51,24 @@ def bound_driver(): ENTRY_INPUT = d.held(Path(__file__).absolute(), None, 131072) legacy = d.legacy_helper d.NAME = "cli-216-392-legacy" -runtime = d.boundary.OwnedContainer(d.docker, d.NAME) +runtime = d.boundary.OwnedContainer(d.docker, d.NAME, d.WORKSPACE) manifest = legacy.manifest() cases, selections, windows = [], [], [] started = datetime.datetime.now(datetime.timezone.utc).isoformat() def full_snapshot(): - with closing(sqlite3.connect(d.DB.as_uri() + "?mode=ro", uri=True, timeout=2)) as db: + with d.WORKSPACE.connection() as db: return legacy.snapshot(db) def start(): envfile = ROOT / "runtime.env" - envfile.write_text("DW_AUTH_TOKEN=" + d.TOKEN + "\nDW_RUST_MAINTENANCE_LOOP=0\n") - envfile.chmod(0o600) + d.WORKSPACE.write("runtime.env", ("DW_AUTH_TOKEN=" + d.TOKEN + "\nDW_RUST_MAINTENANCE_LOOP=0\n").encode()) runtime.run("--rm", "-d", "--name", d.NAME, "--platform=linux/amd64", "--init", "--cpus=1", "--memory=1g", "--pids-limit=256", "--read-only", "--cap-drop=ALL", "--security-opt=no-new-privileges", "--env-file", str(envfile), - "--mount", "type=bind,source=" + str(ROOT / "data") + ",target=/app/database", + *d.WORKSPACE.mount_arguments(), "--tmpfs", "/app/storage:rw,uid=1000,gid=1000,mode=750,size=64m", "--tmpfs", "/tmp:rw,size=16m", "-p", "127.0.0.1::8080", d.IMAGE) binding = runtime.inspect()["NetworkSettings"]["Ports"]["8080/tcp"][0] @@ -230,11 +229,36 @@ def exercise(): phase = "offline accepted transform" # This is a current-producer fixture. Checkpoint only after observed # process absence, then freeze the helper's untouched source inode. - with closing(sqlite3.connect(d.DB, timeout=2)) as db: + with d.WORKSPACE.connection(readonly=False) as db: db.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone() + # Preserve the held runtime DB inode. Transform a distinct stopped + # sibling copy, verify its complete result, then copy through the held + # DB descriptor before starting the next runtime on that same inode. + d.WORKSPACE.recheck() original = d.DB.with_name("seed.sqlite") - d.DB.rename(original) - transform = legacy.transform_copy(original, d.DB, selections, stopped=True) + descriptor = os.open(original.name, os.O_RDWR | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, + 0o600, dir_fd=d.WORKSPACE.data.fd) + try: + with os.fdopen(os.dup(descriptor), "wb") as copied: + copied.write(os.pread(d.WORKSPACE.database.fd, 64 * 1024 * 1024 + 1, 0)) + finally: + os.close(descriptor) + seed_input = d.held(original, None, 64 * 1024 * 1024) + d.WORKSPACE.recheck() + transformed = d.DB.with_name("transformed.sqlite") + before = full_snapshot() + transform = legacy.transform_copy(original, transformed, selections, stopped=True) + copied_input = d.held(transformed, None, 64 * 1024 * 1024) + d.WORKSPACE.recheck() + seed_input.recheck() + with closing(sqlite3.connect(Path(copied_input.executable).as_uri() + "?mode=ro&immutable=1", uri=True)) as db: + transformed_snapshot = legacy.snapshot(db) + if (transform["before_sha256"] != legacy.fingerprint(before) + or transform["after_sha256"] != legacy.fingerprint(transformed_snapshot)): + raise ValueError("held transform source/result differs") + d.WORKSPACE.database.write(copied_input.data) + if full_snapshot() != transformed_snapshot: + raise ValueError("held runtime DB differs from accepted transformed copy") d.require(transform["transformed"] == 12, "exact twelve approved legacy transforms; all other schema/rows preserved") start() phase = "installed legacy readback" @@ -257,14 +281,15 @@ def exercise(): time.sleep(.1) if runtime.exists(): raise RuntimeError("created container removal did not complete") + d.WORKSPACE.recheck() for item in d.HELD: item.recheck() except Exception as failure: cleanup_error = (type(failure).__name__ + ": " + str(failure)).replace(d.TOKEN, "[redacted]") outcome = "runner-blocked" - (ROOT / "runtime.env").unlink(missing_ok=True) + d.WORKSPACE.unlink("runtime.env") absent = not runtime.exists() - report = {"cell": "synthetic-legacy-shape", "outcome": outcome, "created_container_ids": runtime.created_ids, "error": error, + report = {"cell": "synthetic-legacy-shape", "outcome": outcome, "workspace_identity": d.WORKSPACE.evidence(), "created_container_ids": runtime.created_ids, "error": error, "cleanup_error": cleanup_error, "phase": phase, "checks": len(d.CHECKS), "started_at": started, "finished_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), "driver_sha256": ENTRY_INPUT.digest, @@ -273,7 +298,9 @@ def exercise(): "manifest_sha256": legacy.MANIFEST_SHA256, "server_index": d.IMAGE, "transform": transform, "windows": windows, "cleanup": "created container/volumes/env removed; held inputs unchanged" if absent and cleanup_error is None else "cleanup/input identity failed"} - (ROOT / "legacy-result.json").write_text(json.dumps(report, indent=2)) - (ROOT / "legacy-wire.json").write_text(json.dumps({"seed": seed_wire, "readback": wire}, indent=2)) + d.WORKSPACE.write("legacy-result.json", json.dumps(report, indent=2).encode()) + d.WORKSPACE.write("legacy-wire.json", json.dumps({"seed": seed_wire, "readback": wire}, indent=2).encode()) + d.WORKSPACE.recheck() print(json.dumps(report), flush=True) + d.WORKSPACE.close() raise SystemExit(0 if outcome == "pass" and absent else 1) diff --git a/scripts/acceptance/command-keys/physical_impl.py b/scripts/acceptance/command-keys/physical_impl.py index 7ea5f73..25adb52 100644 --- a/scripts/acceptance/command-keys/physical_impl.py +++ b/scripts/acceptance/command-keys/physical_impl.py @@ -13,7 +13,7 @@ import types SOURCE = Path(__file__).resolve().parent -LEGACY_SHA = "0f0ceb09676d38109bbd57ed9c4d22e0c448a78c65999fe82a5feb3b3ce1062b" +LEGACY_SHA = "7324676373db9d44c63423fc50cb995a9040a791fa16a26a04690f00d5b0259e" BASELINE_SHA = "7d1826f7681342bb3a5974509648e2e8d75e7a21d262f65f3ec1f58727ccbc0c" PACKET_SHA = "8c728ce827df3a11621dfb0e82cdd404bbab48784182a5e92c8d632d49047fc5" CONTRACT_SHA = "ceb4695643e48c7bfe3f81eea380c6ab9aa0b973da830cb552d070344047319f" @@ -125,8 +125,8 @@ def physical_cases(packet, common): if packet["expected_external_baseline"] != contract["physical_upgrade"]["baseline"] or len(packet["cases"]) != 6: raise ValueError("physical baseline/case identities differ") d.NAME = "cli-216-392-physical" - common.runtime = d.boundary.OwnedContainer(d.docker, d.NAME) - upgrade = d.boundary.OwnedContainer(d.docker, "cli-216-392-upgrade") + common.runtime = d.boundary.OwnedContainer(d.docker, d.NAME, d.WORKSPACE) + upgrade = d.boundary.OwnedContainer(d.docker, "cli-216-392-upgrade", d.WORKSPACE) outcome, error, phase = "runner-blocked", None, "preflight" started = datetime.datetime.now(datetime.timezone.utc) before, after = None, None @@ -135,16 +135,18 @@ def physical_cases(packet, common): d.require(observed["Architecture"] == "amd64" and observed["Os"] == "linux" and observed["Config"]["Labels"]["org.opencontainers.image.revision"] == "5614274d2629d03f92932a07dfee0fb10349ddd9", "physical candidate published platform/source identity") - descriptor = os.open(d.DB, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) - with os.fdopen(descriptor, "wb") as copied: - copied.write(baseline.data) + d.WORKSPACE.recheck() + if os.fstat(d.WORKSPACE.database.fd).st_size != 0: + raise ValueError("physical upgrade needs a newly created empty DB inode") + d.WORKSPACE.database.write(baseline.data) + d.WORKSPACE.recheck() before = common.full_snapshot() phase = "explicit physical schema upgrade" upgrade_started = datetime.datetime.now(datetime.timezone.utc) upgrade.run("-d", "--name", upgrade.name, "--platform=linux/amd64", "--init", "--cpus=1", "--memory=1g", "--pids-limit=256", "--network=none", "--read-only", "--cap-drop=ALL", "--security-opt=no-new-privileges", "--env", "DB_CONNECTION=sqlite", "--env", "DB_DATABASE=/app/database/database.sqlite", - "--mount", "type=bind,source=" + str(d.ROOT / "data") + ",target=/app/database", + *d.WORKSPACE.mount_arguments(), "--tmpfs", "/tmp:rw,size=16m", d.IMAGE, "schema-upgrade") upgrade.inspect() d.require(d.docker("wait", upgrade.id) == "0", "tagged image schema-upgrade exits zero") @@ -181,13 +183,14 @@ def physical_cases(packet, common): time.sleep(.1) if owned.exists(): raise RuntimeError("created physical container removal did not finish") + d.WORKSPACE.recheck() for item in d.HELD: item.recheck() except Exception as failure: outcome = "runner-blocked" cleanup_error = (type(failure).__name__ + ": " + str(failure)).replace(d.TOKEN, "[redacted]") - (d.ROOT / "runtime.env").unlink(missing_ok=True) - report = {"cell": "physical-3.8-upgrade", "outcome": outcome, "error": error, "cleanup_error": cleanup_error, + d.WORKSPACE.unlink("runtime.env") + report = {"cell": "physical-3.8-upgrade", "outcome": outcome, "workspace_identity": d.WORKSPACE.evidence(), "error": error, "cleanup_error": cleanup_error, "phase": phase, "started_at": started.isoformat(), "finished_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), "checks": len(d.CHECKS), "driver_sha256": entry.digest, "accepted_contract_source": "a09f156399416216d2f097c257a2164f83c7eda7", @@ -199,6 +202,8 @@ def physical_cases(packet, common): "request_windows": common.windows, "server_index": d.IMAGE, "created_container_ids": upgrade.created_ids + common.runtime.created_ids, "cleanup": "created IDs/volumes/env removed; original held inputs unchanged" if cleanup_error is None else "requires investigation"} - (d.ROOT / "physical-result.json").write_text(json.dumps(report, indent=2)) + d.WORKSPACE.write("physical-result.json", json.dumps(report, indent=2).encode()) + d.WORKSPACE.recheck() print(json.dumps(report), flush=True) + d.WORKSPACE.close() raise SystemExit(0 if outcome == "pass" else 1) diff --git a/scripts/acceptance/command-keys/run_cli_216.py b/scripts/acceptance/command-keys/run_cli_216.py index fd3519c..4dd64bc 100644 --- a/scripts/acceptance/command-keys/run_cli_216.py +++ b/scripts/acceptance/command-keys/run_cli_216.py @@ -6,11 +6,11 @@ import types SOURCE = Path(__file__).resolve().parent -BOUNDARY_SHA = "d377e9191ebf79dc990db159629c303c10531a2f27d3ae20933fabc87e3b3525" +BOUNDARY_SHA = "8e1374d497741cf1668c43bd884af69f72282448bc7151d00f6ff4718c4546b8" IMPLEMENTATIONS = { - 'modern': ('cli_216.py', 'e26565125bb122e6d84cd6a6e230c7d47a56ca043eda7bffa7025b9c19fdbbbe'), - 'legacy': ('legacy_cli.py', '0f0ceb09676d38109bbd57ed9c4d22e0c448a78c65999fe82a5feb3b3ce1062b'), - 'physical': ('physical_impl.py', '81ff88b11a7c402d2206c2856b08080eb149528b31cd0f616ccd676e459fcd3f'), + 'modern': ('cli_216.py', 'cd8638f7594b8fd72ac0e72dd44d38f6be27167430518b4bebd6c5c244c7c0cd'), + 'legacy': ('legacy_cli.py', '7324676373db9d44c63423fc50cb995a9040a791fa16a26a04690f00d5b0259e'), + 'physical': ('physical_impl.py', 'f1babc7103039e18e3d88a9d780926e5b73a0447ba17b56173d9dffd06c1c353'), } diff --git a/scripts/acceptance/command-keys/test_boundaries.py b/scripts/acceptance/command-keys/test_boundaries.py index d84ae68..54ee9d3 100644 --- a/scripts/acceptance/command-keys/test_boundaries.py +++ b/scripts/acceptance/command-keys/test_boundaries.py @@ -223,7 +223,13 @@ def test_schema_only_changes_do_not_pass_zero_write_comparison(self): database = Path(temporary.name).resolve() / "snapshot.sqlite" tree = ast.parse((SOURCE / "cli_216.py").read_text()) function = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "snapshot") - scope = {"DB": database, "sqlite3": sqlite3, "legacy_helper": helper} + data = database.parent / "data" + data.mkdir(mode=0o700) + database = data / "database.sqlite" + database.parent.parent.chmod(0o700) + workspace = boundary.Workspace(database.parent.parent) + self.addCleanup(workspace.close) + scope = {"WORKSPACE": workspace, "legacy_helper": helper} exec(compile(ast.Module(body=[function], type_ignores=[]), "snapshot-check", "exec"), scope) with closing(sqlite3.connect(database)) as db: db.execute("CREATE TABLE fixture(value INTEGER)") @@ -247,5 +253,154 @@ def predecessor(): self.assertNotEqual(current, scope["snapshot"]()) +class WorkspaceTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.parent = Path(self.temporary.name).resolve() + self.root = self.parent / "work" + self.root.mkdir(mode=0o700) + (self.root / "data").mkdir(mode=0o700) + self.workspace = boundary.Workspace(self.root) + self.addCleanup(self.workspace.close) + self.database = self.root / "data/database.sqlite" + with self.workspace.connection(readonly=False) as db: + db.execute("CREATE TABLE fixture(value INTEGER)") + db.execute("INSERT INTO fixture VALUES (1)") + db.commit() + + def replace(self, target): + path = {"root": self.root, "data": self.root / "data", "database": self.database}[target] + renamed = path.with_name(path.name + "-original") + path.rename(renamed) + if target == "database": + path.write_bytes(renamed.read_bytes()) + path.chmod(0o600) + else: + import shutil + shutil.copytree(renamed, path) + # The predecessor could reopen the new pathname and observe the same + # schema/rows, even though Docker would exercise a different object. + with closing(sqlite3.connect(self.database)) as db: + self.assertEqual(db.execute("SELECT * FROM fixture").fetchall(), [(1,)]) + + def test_parent_or_db_swap_after_copy_refuses_before_upgrade(self): + # Each subcase needs an independent held workspace. + for target in ("root", "data", "database"): + with self.subTest(target=target): + if target != "root": + self.setUp() + self.workspace.database.recheck() + self.replace(target) + docker = FakeDocker() + owned = boundary.OwnedContainer(docker, "cli-test", self.workspace) + with self.assertRaises(ValueError): + owned.run("--name", "cli-test") + self.assertFalse(docker.calls) + + def test_parent_or_db_swap_after_comparison_refuses_runtime_start(self): + for target in ("data", "database"): + with self.subTest(target=target): + if target == "database": + self.setUp() + with self.workspace.connection() as db: + self.assertEqual(db.execute("SELECT * FROM fixture").fetchall(), [(1,)]) + self.replace(target) + with self.assertRaises(ValueError): + self.workspace.mount_arguments() + with self.assertRaises(ValueError): + with self.workspace.connection(): + self.fail("replaced database was opened") + + def test_root_swap_refuses_report_emission(self): + self.replace("root") + with self.assertRaises(ValueError): + self.workspace.write("physical-result.json", b'{"outcome":"pass"}') + self.assertFalse((self.root / "physical-result.json").exists()) + + def test_report_replacement_is_refused_before_and_during_write(self): + self.workspace.write("physical-result.json", b"original") + path = self.root / "physical-result.json" + path.rename(self.root / "original-report") + path.write_bytes(b"replacement") + path.chmod(0o600) + with self.assertRaises(ValueError): + self.workspace.write("physical-result.json", b"pass") + self.assertEqual(path.read_bytes(), b"replacement") + + def test_report_path_swap_during_emission_cannot_publish_pass(self): + original = os.pwrite + path = self.root / "physical-result.json" + def swapped(fd, data, offset): + result = original(fd, data, offset) + path.rename(self.root / "held-report") + path.write_bytes(b"forged") + path.chmod(0o600) + return result + with patch.object(boundary.os, "pwrite", swapped): + with self.assertRaises(ValueError): + self.workspace.write("physical-result.json", b'{"outcome":"pass"}') + self.assertEqual(path.read_bytes(), b"forged") + + def test_report_parent_swap_during_emission_is_refused(self): + original = os.pwrite + def swapped(fd, data, offset): + result = original(fd, data, offset) + self.replace("root") + return result + with patch.object(boundary.os, "pwrite", swapped): + with self.assertRaises(ValueError): + self.workspace.write("physical-result.json", b'{"outcome":"pass"}') + + def test_report_in_place_edit_is_refused(self): + self.workspace.write("physical-result.json", b"reviewed") + (self.root / "physical-result.json").write_bytes(b"tampered") + with self.assertRaises(ValueError): + self.workspace.recheck() + + def test_db_symlink_and_extra_link_are_refused(self): + original = self.database.with_name("original.sqlite") + self.database.rename(original) + self.database.symlink_to(original) + with self.assertRaises(ValueError): + self.workspace.recheck() + self.database.unlink() + original.rename(self.database) + os.link(self.database, self.database.with_name("second.sqlite")) + with self.assertRaises(ValueError): + self.workspace.recheck() + + def test_mutable_database_keeps_inode_and_wal_semantics(self): + identity = self.workspace.database.identity + with self.workspace.connection(readonly=False) as db: + db.execute("PRAGMA journal_mode=WAL") + db.execute("INSERT INTO fixture VALUES (2)") + db.commit() + with self.workspace.connection() as reader: + self.assertEqual(reader.execute("SELECT * FROM fixture").fetchall(), [(1,), (2,)]) + self.workspace.recheck() + self.assertEqual(self.workspace.database.identity, identity) + + def test_docker_mounts_are_exact_held_directory_and_file_descriptors(self): + arguments = self.workspace.mount_arguments() + mounts = [{"Destination": target, "Source": source, "Type": "bind", "RW": True} + for target, source in (("/app/database", self.workspace.data.anchor), + ("/app/database/database.sqlite", self.workspace.database.anchor))] + self.assertIn("source=" + self.workspace.database.anchor, arguments[-1]) + self.workspace.inspect_mounts({"Mounts": mounts}) + mounts[1]["Source"] = str(self.database) + with self.assertRaises(ValueError): + self.workspace.inspect_mounts({"Mounts": mounts}) + + def test_failed_workspace_check_does_not_block_exact_id_cleanup(self): + docker = FakeDocker() + owned = boundary.OwnedContainer(docker, "cli-test") + owned.run("--name", "cli-test") + owned.workspace = self.workspace + self.replace("data") + owned.remove() + self.assertIn(("rm", "-fv", "a" * 64), docker.calls) + + if __name__ == "__main__": unittest.main() From d2bdd0a0789abfd8c83e580e4c76085d71204137 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 13 Nov 2026 19:14:51 +0000 Subject: [PATCH 5/7] Bind the outer bootstrap and verify native mount identity before exec --- scripts/acceptance/command-keys/README.md | 48 ++++++- scripts/acceptance/command-keys/boundaries.py | 69 ++++++++-- scripts/acceptance/command-keys/cli_216.py | 20 +-- .../acceptance/command-keys/git_bootstrap.py | 22 +++ scripts/acceptance/command-keys/legacy_cli.py | 9 +- .../acceptance/command-keys/physical_impl.py | 9 +- .../acceptance/command-keys/run_cli_216.py | 15 +- .../command-keys/test_boundaries.py | 37 ++++- .../command-keys/test_physical_cli.py | 43 +++++- .../command-keys/test_workspace_container.py | 128 ++++++++++++++++++ 10 files changed, 348 insertions(+), 52 deletions(-) create mode 100644 scripts/acceptance/command-keys/git_bootstrap.py create mode 100644 scripts/acceptance/command-keys/test_workspace_container.py diff --git a/scripts/acceptance/command-keys/README.md b/scripts/acceptance/command-keys/README.md index 61e3cd1..dce2f17 100644 --- a/scripts/acceptance/command-keys/README.md +++ b/scripts/acceptance/command-keys/README.md @@ -73,7 +73,22 @@ The runner holds these exact bytes and compares the actual retained Avro result with this known fixture. It never executes an external codec program or accepts its self-reported decoded result. -Run source with isolated host Python and the private directory in +Obtain the outer bootstrap from the frozen reviewed Git commit, independently +verify its SHA256 from the owning review receipt, then pass those literal bytes +to isolated Python with `-c`. Do not execute any dispatcher or bootstrap by a +host pathname. The bootstrap obtains the dispatcher from the same frozen Git +object and checks its separately fixed SHA256 before compile/exec. Host-path +substitution of the outer dispatcher cannot replace those Git bytes. + +```bash +reviewed_source_commit=FULL_REVIEWED_COMMIT_FROM_THE_PR +bootstrap="$(git --no-replace-objects show "$reviewed_source_commit:scripts/acceptance/command-keys/git_bootstrap.py")"$'\n' +expected_bootstrap_sha256=368d78748c12bb51b60a01336702f942a9fe11f9dff918edacdb710a8950ed44 +actual_bootstrap_sha256="$(printf '%s' "$bootstrap" | sha256sum | awk '{print $1}')" +test "$actual_bootstrap_sha256" = "$expected_bootstrap_sha256" +``` + +Run from that repository with the private directory in `CLI_216_WORKDIR`. Provide a clean environment without cloud, GitHub, Composer or registry credentials. Runtime authentication is synthetic and private. @@ -81,7 +96,7 @@ or registry credentials. Runtime authentication is synthetic and private. env -i PATH=/usr/bin:/bin \ CLI_216_WORKDIR="$experiment" \ CLI_216_PROXY_SOURCE="$experiment/committed_response_loss.py" \ - timeout 180s python3 -I -B scripts/acceptance/command-keys/run_cli_216.py + timeout 180s python3 -I -B -c "$bootstrap" "$reviewed_source_commit" ``` For the restart phase, preserve the first phase's `admissions.json` as @@ -161,7 +176,7 @@ env -i PATH=/usr/bin:/bin \ CLI_216_RUNNER=physical \ CLI_216_PHYSICAL_BASELINE="$original_backup" \ CLI_216_PHYSICAL_PACKET="$original_packet" \ - timeout 180s python3 -I -B scripts/acceptance/command-keys/run_cli_216.py + timeout 180s python3 -I -B -c "$bootstrap" "$reviewed_source_commit" ``` Created names are `cli-216-392-upgrade` and `cli-216-392-physical`; cleanup @@ -187,9 +202,14 @@ normal gates, fresh actual execution and cleanup still qualify that cell. The manual experiment holds owned mode-0700 root/data directories and a single-link mode-0600 database descriptor throughout each phase. SQLite opens through the held directory so WAL/SHM siblings keep their normal semantics. -Docker binds both the held directory and the held main database file via local -`/proc//fd` paths; the inspected mounts must match both descriptors. -This requires the local Linux Docker daemon, not a remote Docker context. +Docker acquires the guarded canonical data bind. A literal in-container shell +preflight checks the actual mounted directory and database device/inode, +UID/GID, mode and link count before executing the native binary. Host checks +immediately before/after acquisition and inspection retain their held identities. +The env-file and Docker config client use inherited `/proc/self/fd` descriptors; +HOME/config/tmp/bin consumer directories are separately held and inherited too. +Exact-ID cleanup uses that held Docker configuration even after a workspace +pathname failure. This requires the local Linux Docker daemon. Identity checks immediately surround snapshots, container creation/inspection, API and installed CLI calls, cleanup, and report emission. Reports are written through held no-follow descriptors, then their path, parent, and contents are @@ -206,3 +226,19 @@ Offline workspace controls demonstrate predecessor pathname substitution and require refusal for root/data/database swaps before upgrade or runtime start, report path/parent swaps during emission, in-place report edits, symlinks and extra hard links. A positive WAL control preserves actual mutable SQLite use. + +Opt-in actual-container boundary controls (separate from installed qualification): + +```bash +CLI_216_CONTAINER_CONTROLS=1 \ + python3 -I -B scripts/acceptance/command-keys/test_workspace_container.py -v +``` + +These use five sequential `cli-152-boundary-*` containers with the same pinned +native image, one CPU, 512 MiB, 128 PIDs, no network or ports. Directory/file +substitution must exit 78 before native execution. The positive mount control +executes the native `schema-check` command and expects its SQLite-unsupported +diagnostic, proving the preflight reached the native binary without starting a +runtime or upgrading a database. Env/config controls use the real Docker reader +and retain exact-ID cleanup after substitution. Check host capacity first and +set TMPDIR to owned disk-backed scratch. No image build or credentials are used. diff --git a/scripts/acceptance/command-keys/boundaries.py b/scripts/acceptance/command-keys/boundaries.py index f1f663e..72a3ede 100644 --- a/scripts/acceptance/command-keys/boundaries.py +++ b/scripts/acceptance/command-keys/boundaries.py @@ -87,6 +87,18 @@ def private_directory(path): return path +# Adapted from sdk-python 377a3eee workspace_guards.py. The qualified +# image checks the actual acquired bind before executing its native binary. +NATIVE_PREFLIGHT = """set -eu +test -d "$1" && test ! -L "$1" || exit 78 +test -f "$1/$2" && test ! -L "$1/$2" || exit 78 +test "$(stat -c '%d:%i:%u:%g:%a:%h' -- "$1")" = "$3" || exit 78 +test "$(stat -c '%d:%i:%u:%g:%a:%h' -- "$1/$2")" = "$4" || exit 78 +shift 4 +exec /usr/local/bin/durable-workflow-server "$@" +""" + + class HeldDirectory: """Keep an owned directory inode, including its no-follow parent binding.""" def __init__(self, path, parent=None): @@ -187,9 +199,13 @@ def close(self): class Workspace: """Bind comparison, Docker data, and terminal evidence to held objects.""" def __init__(self, root): + for name in ("home", "config", "tmp", "docker-empty", "bin"): + private_directory(Path(root) / name) self.root = HeldDirectory(root) self.data = HeldDirectory(Path(root) / "data", self.root) self.files = {} + self.children = {name: HeldDirectory(Path(root) / name, self.root) + for name in ("home", "config", "tmp", "docker-empty", "bin")} try: # The first phase creates the file; later phases hold the existing # inode. Server mutates it across restarts and schema upgrades. @@ -205,6 +221,8 @@ def recheck(self): self.root.recheck() self.data.recheck() self.database.recheck() + for value in self.children.values(): + value.recheck() for value in self.files.values(): value.recheck() @@ -237,21 +255,40 @@ def opened(): def mount_arguments(self): self.recheck() - # Docker resolves these local /proc descriptors while the parent holds - # them open. Both directory and main-file mounts keep the exact inode, - # even if a same-UID process renames a workspace pathname transiently. - return ("--mount", "type=bind,source=" + self.data.anchor + ",target=/app/database", - "--mount", "type=bind,source=" + self.database.anchor + ",target=/app/database/database.sqlite") + return ("--mount", "type=bind,source=" + str(self.data.path) + ",target=/app/database") + + def native_arguments(self, *arguments): + self.recheck() + def identity(descriptor): + value = os.fstat(descriptor) + return f"{value.st_dev}:{value.st_ino}:{value.st_uid}:{value.st_gid}:{stat.S_IMODE(value.st_mode):o}:{value.st_nlink}" + return ("-c", NATIVE_PREFLIGHT, "checked-native", "/app/database", "database.sqlite", + identity(self.data.fd), identity(self.database.fd), *arguments) def inspect_mounts(self, value): self.recheck() - expected = {"/app/database": self.data.anchor, "/app/database/database.sqlite": self.database.anchor} - for destination, source in expected.items(): - matches = [item for item in value["Mounts"] if item["Destination"] == destination] - if (len(matches) != 1 or matches[0]["Type"] != "bind" or matches[0]["Source"] != source - or matches[0]["RW"] is not True): - raise ValueError("Docker did not bind the held workspace objects") + matches = [item for item in value["Mounts"] if item["Destination"] == "/app/database"] + if (len(matches) != 1 or matches[0]["Type"] != "bind" or matches[0]["Source"] != str(self.data.path) + or matches[0]["RW"] is not True): + raise ValueError("Docker did not acquire the guarded canonical data bind") + self.recheck() + + def child_anchor(self, name): + return self.children[name].anchor + + def inherited(self): + return tuple(item.fd for item in [*self.children.values(), *self.files.values()]) + + def read(self, name): + self.recheck() + if name not in self.files: + self.files[name] = HeldWritableFile(self.root, name, maximum=32 * 1024 * 1024) + item = self.files[name] + item.recheck() + data = os.pread(item.fd, item.maximum + 1, 0) + item.digest = hashlib.sha256(data).hexdigest() self.recheck() + return data def write(self, name, data): self.recheck() @@ -262,19 +299,23 @@ def write(self, name, data): self.recheck() def unlink(self, name): - self.recheck() + # Unlink only the captured entry through its held parent; root-path + # refusal must not prevent credential cleanup or touch a replacement. item = self.files.get(name) if item is not None: - item.recheck() + value = os.stat(item.name, dir_fd=item.parent.fd, follow_symlinks=False) + if HeldDirectory.key(value) != item.identity or HeldDirectory.key(os.fstat(item.fd)) != item.identity: + raise ValueError("cleanup entry was replaced; preserving it") os.unlink(name, dir_fd=self.root.fd) item.close() del self.files[name] - self.recheck() def close(self): for value in self.files.values(): value.close() self.database.close() + for value in self.children.values(): + value.close() self.data.close() self.root.close() diff --git a/scripts/acceptance/command-keys/cli_216.py b/scripts/acceptance/command-keys/cli_216.py index 7b2087c..b18cf58 100644 --- a/scripts/acceptance/command-keys/cli_216.py +++ b/scripts/acceptance/command-keys/cli_216.py @@ -18,7 +18,7 @@ raise SystemExit("Use run_cli_216.py to verify the implementation before execution") SOURCE = Path(__file__).resolve().parent -BOUNDARY_SHA = '8e1374d497741cf1668c43bd884af69f72282448bc7151d00f6ff4718c4546b8' +BOUNDARY_SHA = '5de532a128931b56bd7ca2232d60e07a2fa65d550cfbf56d9cd78d16ed3a5095' # Bootstrap the local reviewed boundary module from the same bytes that are # checked. Subsequent inputs use its held descriptors and sealed memory files. _boundary_path = SOURCE / 'boundaries.py' @@ -80,6 +80,9 @@ def held(path, digest, maximum=16 * 1024 * 1024): for directory in ('data', 'home', 'config', 'tmp', 'docker-empty'): boundary.private_directory(ROOT / directory) WORKSPACE = boundary.Workspace(ROOT) +ENV.update({'PATH': WORKSPACE.child_anchor('bin') + ':/usr/bin:/bin', + 'HOME': WORKSPACE.child_anchor('home'), 'DW_CONFIG_HOME': WORKSPACE.child_anchor('config'), + 'TMPDIR': WORKSPACE.child_anchor('tmp')}) HTTP = urllib.request.build_opener(urllib.request.ProxyHandler({})) @@ -92,7 +95,7 @@ def require(condition, label, detail=None): def docker(*args): - p = subprocess.run(['docker', '--config', str(ROOT / 'docker-empty'), *args], capture_output=True, text=True, timeout=50, env={'PATH': os.defpath, 'HOME': str(ROOT / 'home')}) + p = subprocess.run(['docker', '--config', '/proc/self/fd/' + str(WORKSPACE.children['docker-empty'].fd), *args], capture_output=True, text=True, timeout=50, env={'PATH': os.defpath, 'HOME': WORKSPACE.child_anchor('home')}, pass_fds=WORKSPACE.inherited()) if p.returncode: raise RuntimeError('docker operation failed: ' + p.stderr.replace(TOKEN, '[redacted]')[:2000]) return p.stdout.strip() @@ -118,7 +121,7 @@ def cli(*args, namespace='default', success=True): WORKSPACE.recheck() CLI_INPUT.recheck() p = subprocess.run(['dw', *args, '--server=' + PROXY.url, '--namespace=' + namespace, '--json'], - executable=CLI_INPUT.executable, env=ENV, capture_output=True, text=True, timeout=45, pass_fds=(CLI_INPUT.sealed_fd,)) + executable=CLI_INPUT.executable, env=ENV, capture_output=True, text=True, timeout=45, pass_fds=(CLI_INPUT.sealed_fd, *WORKSPACE.inherited())) CLI_INPUT.recheck() WORKSPACE.recheck() try: @@ -270,7 +273,7 @@ def result_is_one(answer): def run_modern(): - require(shutil.which('dw', path=ENV['PATH']) == str(ROOT / 'bin/dw'), 'ordinary scoped PATH selects installed release') + require(shutil.which('dw', path=ENV['PATH']) == WORKSPACE.child_anchor('bin') + '/dw', 'ordinary scoped PATH selects installed release') require(hashlib.sha256((ROOT / 'bin/dw').read_bytes()).hexdigest() == 'da366c79c90978a3c0a92d723c66a96a2d0963f9dccbd41c8207f2c41848f814', 'published CLI binary SHA matches approved release') status, info = request('GET', '/api/cluster/info') require(status == 200 and info['version'] == '3.9.2' and info['capabilities']['command_request_keys'] is True, 'published Server 3.9.2 advertises command_request_keys', info.get('version')) @@ -344,7 +347,7 @@ def run_modern(): def run_selected_and_restart(): status, info = request('GET', '/api/cluster/info') require(status == 200 and info['version'] == '3.9.2', 'same published Server version after process restart') - previous = json.loads((ROOT / 'modern-admissions.json').read_text()) + previous = json.loads(WORKSPACE.read('modern-admissions.json')) recovered = 0 for record in previous: if not record['dropped']: @@ -419,10 +422,10 @@ def run_typed(): envfile = ROOT / 'runtime.env' WORKSPACE.write('runtime.env', ('DW_AUTH_TOKEN=' + TOKEN + '\nDW_RUST_MAINTENANCE_LOOP=0\n').encode()) runtime.run('--rm', '-d', '--name', NAME, '--platform=linux/amd64', '--init', '--cpus=1', '--memory=1g', '--pids-limit=256', - '--read-only', '--cap-drop=ALL', '--security-opt=no-new-privileges', '--env-file', str(envfile), + '--read-only', '--cap-drop=ALL', '--security-opt=no-new-privileges', '--env-file', '/proc/self/fd/' + str(WORKSPACE.files['runtime.env'].fd), '--entrypoint=/bin/sh', *WORKSPACE.mount_arguments(), '--tmpfs', '/app/storage:rw,uid=1000,gid=1000,mode=750,size=64m', - '--tmpfs', '/tmp:rw,size=16m', '-p', '127.0.0.1::8080', IMAGE) + '--tmpfs', '/tmp:rw,size=16m', '-p', '127.0.0.1::8080', IMAGE, *WORKSPACE.native_arguments()) binding = runtime.inspect()['NetworkSettings']['Ports']['8080/tcp'][0] BASE = 'http://127.0.0.1:' + binding['HostPort'] for _ in range(100): @@ -470,7 +473,8 @@ def run_typed(): WORKSPACE.unlink('runtime.env') report = {'started_at': STARTED, 'finished_at': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'outcome': outcome, 'workspace_identity': WORKSPACE.evidence(), 'created_container_ids': runtime.created_ids, 'checks': CHECKS, 'error': error, 'cleanup': cleanup, - 'runner_sha256': ENTRY_INPUT.digest, + 'runner_sha256': ENTRY_INPUT.digest, 'launcher_sha256': BOOTSTRAP_LAUNCHER_SHA256, + 'runner_source_commit': BOOTSTRAP_SOURCE_COMMIT, 'proxy_sha256': PROXY_INPUT.digest, 'server_index': IMAGE, 'server_source': '5614274d2629d03f92932a07dfee0fb10349ddd9', 'phase': os.environ.get('CLI_216_PHASE', 'modern')} WORKSPACE.write('result.json', json.dumps(report, indent=2).encode()) WORKSPACE.write('invocations.json', json.dumps(INVOCATIONS, indent=2).replace(TOKEN, '[redacted]').encode()) diff --git a/scripts/acceptance/command-keys/git_bootstrap.py b/scripts/acceptance/command-keys/git_bootstrap.py new file mode 100644 index 0000000..2eceb46 --- /dev/null +++ b/scripts/acceptance/command-keys/git_bootstrap.py @@ -0,0 +1,22 @@ +"""Literal -c bootstrap: obtain reviewed dispatcher bytes from a frozen Git object.""" +import hashlib +import os +from pathlib import Path +import re +import subprocess +import sys + +if sys.argv[0] != "-c": + raise SystemExit("Use the reviewed literal -c Git bootstrap, not a host pathname") +commit = sys.argv.pop(1) +if not re.fullmatch(r"[0-9a-f]{40}", commit, re.ASCII): + raise ValueError("bootstrap requires one reviewed full Git commit") +relative = "scripts/acceptance/command-keys/run_cli_216.py" +expected = "5762dd388597e1a5fcb7e088a6abb54aee8ad6976a240aebd88be1f615314150" +data = subprocess.check_output(["/usr/bin/git", "--no-replace-objects", "show", commit + ":" + relative], + env={"PATH": os.defpath}, timeout=10) +if hashlib.sha256(data).hexdigest() != expected: + raise ValueError("reviewed dispatcher Git bytes differ") +context = {"__name__": "__main__", "__file__": str(Path.cwd() / relative), + "BOOTSTRAP_LAUNCHER_SHA256": expected, "BOOTSTRAP_SOURCE_COMMIT": commit} +exec(compile(data, context["__file__"], "exec"), context) diff --git a/scripts/acceptance/command-keys/legacy_cli.py b/scripts/acceptance/command-keys/legacy_cli.py index af51a79..b8915f0 100644 --- a/scripts/acceptance/command-keys/legacy_cli.py +++ b/scripts/acceptance/command-keys/legacy_cli.py @@ -42,7 +42,7 @@ def bound_driver(): os.close(descriptor) -BASE_SHA = "cd8638f7594b8fd72ac0e72dd44d38f6be27167430518b4bebd6c5c244c7c0cd" +BASE_SHA = "f8a51ec2119cd24acf6b861b0a3c09e5ab7f651539136db506d6d8ef4b8e448b" d = bound_driver() if "BOOTSTRAP_HELD_IMPLEMENTATION" in globals(): ENTRY_INPUT = BOOTSTRAP_HELD_IMPLEMENTATION @@ -67,10 +67,10 @@ def start(): d.WORKSPACE.write("runtime.env", ("DW_AUTH_TOKEN=" + d.TOKEN + "\nDW_RUST_MAINTENANCE_LOOP=0\n").encode()) runtime.run("--rm", "-d", "--name", d.NAME, "--platform=linux/amd64", "--init", "--cpus=1", "--memory=1g", "--pids-limit=256", "--read-only", "--cap-drop=ALL", - "--security-opt=no-new-privileges", "--env-file", str(envfile), + "--security-opt=no-new-privileges", "--env-file", "/proc/self/fd/" + str(d.WORKSPACE.files["runtime.env"].fd), "--entrypoint=/bin/sh", *d.WORKSPACE.mount_arguments(), "--tmpfs", "/app/storage:rw,uid=1000,gid=1000,mode=750,size=64m", - "--tmpfs", "/tmp:rw,size=16m", "-p", "127.0.0.1::8080", d.IMAGE) + "--tmpfs", "/tmp:rw,size=16m", "-p", "127.0.0.1::8080", d.IMAGE, *d.WORKSPACE.native_arguments()) binding = runtime.inspect()["NetworkSettings"]["Ports"]["8080/tcp"][0] d.BASE = "http://127.0.0.1:" + binding["HostPort"] for _ in range(100): @@ -292,7 +292,8 @@ def exercise(): report = {"cell": "synthetic-legacy-shape", "outcome": outcome, "workspace_identity": d.WORKSPACE.evidence(), "created_container_ids": runtime.created_ids, "error": error, "cleanup_error": cleanup_error, "phase": phase, "checks": len(d.CHECKS), "started_at": started, "finished_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), - "driver_sha256": ENTRY_INPUT.digest, + "driver_sha256": ENTRY_INPUT.digest, "launcher_sha256": BOOTSTRAP_LAUNCHER_SHA256, + "runner_source_commit": BOOTSTRAP_SOURCE_COMMIT, "base_driver_sha256": d.ENTRY_INPUT.digest, "helper_sha256": d.HELPER_INPUT.digest, "manifest_sha256": legacy.MANIFEST_SHA256, "server_index": d.IMAGE, diff --git a/scripts/acceptance/command-keys/physical_impl.py b/scripts/acceptance/command-keys/physical_impl.py index 25adb52..be161ad 100644 --- a/scripts/acceptance/command-keys/physical_impl.py +++ b/scripts/acceptance/command-keys/physical_impl.py @@ -13,7 +13,7 @@ import types SOURCE = Path(__file__).resolve().parent -LEGACY_SHA = "7324676373db9d44c63423fc50cb995a9040a791fa16a26a04690f00d5b0259e" +LEGACY_SHA = "c2d19bd944bfb1cca7d370de923be2ccbd788f51b7ed4131622de691d58a08eb" BASELINE_SHA = "7d1826f7681342bb3a5974509648e2e8d75e7a21d262f65f3ec1f58727ccbc0c" PACKET_SHA = "8c728ce827df3a11621dfb0e82cdd404bbab48784182a5e92c8d632d49047fc5" CONTRACT_SHA = "ceb4695643e48c7bfe3f81eea380c6ab9aa0b973da830cb552d070344047319f" @@ -145,9 +145,9 @@ def physical_cases(packet, common): upgrade_started = datetime.datetime.now(datetime.timezone.utc) upgrade.run("-d", "--name", upgrade.name, "--platform=linux/amd64", "--init", "--cpus=1", "--memory=1g", "--pids-limit=256", "--network=none", "--read-only", "--cap-drop=ALL", "--security-opt=no-new-privileges", - "--env", "DB_CONNECTION=sqlite", "--env", "DB_DATABASE=/app/database/database.sqlite", + "--entrypoint=/bin/sh", "--env", "DB_CONNECTION=sqlite", "--env", "DB_DATABASE=/app/database/database.sqlite", *d.WORKSPACE.mount_arguments(), - "--tmpfs", "/tmp:rw,size=16m", d.IMAGE, "schema-upgrade") + "--tmpfs", "/tmp:rw,size=16m", d.IMAGE, *d.WORKSPACE.native_arguments("schema-upgrade")) upgrade.inspect() d.require(d.docker("wait", upgrade.id) == "0", "tagged image schema-upgrade exits zero") state = upgrade.inspect()["State"] @@ -192,7 +192,8 @@ def physical_cases(packet, common): d.WORKSPACE.unlink("runtime.env") report = {"cell": "physical-3.8-upgrade", "outcome": outcome, "workspace_identity": d.WORKSPACE.evidence(), "error": error, "cleanup_error": cleanup_error, "phase": phase, "started_at": started.isoformat(), "finished_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), - "checks": len(d.CHECKS), "driver_sha256": entry.digest, + "checks": len(d.CHECKS), "driver_sha256": entry.digest, "launcher_sha256": BOOTSTRAP_LAUNCHER_SHA256, + "runner_source_commit": BOOTSTRAP_SOURCE_COMMIT, "accepted_contract_source": "a09f156399416216d2f097c257a2164f83c7eda7", "actual_seed_source": "09a4ccda3da11853606af031a8970bbde4b2d6ec", "actual_seed_driver_sha256": "01c0fa1251ac3c809723e58d71817881fe2c5c9498579bc4c2e6097324a3618b", diff --git a/scripts/acceptance/command-keys/run_cli_216.py b/scripts/acceptance/command-keys/run_cli_216.py index 4dd64bc..e78cf8f 100644 --- a/scripts/acceptance/command-keys/run_cli_216.py +++ b/scripts/acceptance/command-keys/run_cli_216.py @@ -5,12 +5,15 @@ import stat import types +if __name__ == "__main__" and "BOOTSTRAP_LAUNCHER_SHA256" not in globals(): + raise SystemExit("Use the reviewed literal -c Git bootstrap before the dispatcher") + SOURCE = Path(__file__).resolve().parent -BOUNDARY_SHA = "8e1374d497741cf1668c43bd884af69f72282448bc7151d00f6ff4718c4546b8" +BOUNDARY_SHA = "5de532a128931b56bd7ca2232d60e07a2fa65d550cfbf56d9cd78d16ed3a5095" IMPLEMENTATIONS = { - 'modern': ('cli_216.py', 'cd8638f7594b8fd72ac0e72dd44d38f6be27167430518b4bebd6c5c244c7c0cd'), - 'legacy': ('legacy_cli.py', '7324676373db9d44c63423fc50cb995a9040a791fa16a26a04690f00d5b0259e'), - 'physical': ('physical_impl.py', 'f1babc7103039e18e3d88a9d780926e5b73a0447ba17b56173d9dffd06c1c353'), + 'modern': ('cli_216.py', 'f8a51ec2119cd24acf6b861b0a3c09e5ab7f651539136db506d6d8ef4b8e448b'), + 'legacy': ('legacy_cli.py', 'c2d19bd944bfb1cca7d370de923be2ccbd788f51b7ed4131622de691d58a08eb'), + 'physical': ('physical_impl.py', '446676dbfb755cc2f3aebeffdcf56977d3d064e2796bf4f86379bda0cf341ddc'), } @@ -41,7 +44,9 @@ def run_implementation(path, digest, boundary, boundary_input): try: scope = {"__name__": "__main__", "__file__": str(implementation.path), "BOOTSTRAP_HELD_IMPLEMENTATION": implementation, - "BOOTSTRAP_HELD_BOUNDARY": boundary_input} + "BOOTSTRAP_HELD_BOUNDARY": boundary_input, + "BOOTSTRAP_LAUNCHER_SHA256": globals().get("BOOTSTRAP_LAUNCHER_SHA256"), + "BOOTSTRAP_SOURCE_COMMIT": globals().get("BOOTSTRAP_SOURCE_COMMIT")} # The literal digest and held bytes precede every implementation # effect, including imports and source-provenance capture. exec(compile(implementation.data, str(implementation.path), "exec"), scope) diff --git a/scripts/acceptance/command-keys/test_boundaries.py b/scripts/acceptance/command-keys/test_boundaries.py index 54ee9d3..ac23fd3 100644 --- a/scripts/acceptance/command-keys/test_boundaries.py +++ b/scripts/acceptance/command-keys/test_boundaries.py @@ -381,17 +381,42 @@ def test_mutable_database_keeps_inode_and_wal_semantics(self): self.workspace.recheck() self.assertEqual(self.workspace.database.identity, identity) - def test_docker_mounts_are_exact_held_directory_and_file_descriptors(self): + def test_canonical_docker_mount_requires_literal_native_identity_preflight(self): arguments = self.workspace.mount_arguments() - mounts = [{"Destination": target, "Source": source, "Type": "bind", "RW": True} - for target, source in (("/app/database", self.workspace.data.anchor), - ("/app/database/database.sqlite", self.workspace.database.anchor))] - self.assertIn("source=" + self.workspace.database.anchor, arguments[-1]) + mounts = [{"Destination": "/app/database", "Source": str(self.root / "data"), "Type": "bind", "RW": True}] + self.assertEqual(arguments, ("--mount", "type=bind,source=" + str(self.root / "data") + ",target=/app/database")) + native = self.workspace.native_arguments("schema-check") + self.assertEqual(native[1], boundary.NATIVE_PREFLIGHT) + self.assertEqual(native[-1], "schema-check") + self.assertEqual(native[-2], ":".join(str(value) for value in [self.database.stat().st_dev, + self.database.stat().st_ino, os.getuid(), os.getgid(), 600, 1])) self.workspace.inspect_mounts({"Mounts": mounts}) - mounts[1]["Source"] = str(self.database) + mounts[0]["Source"] = str(self.root) with self.assertRaises(ValueError): self.workspace.inspect_mounts({"Mounts": mounts}) + def test_consumer_state_directory_swaps_are_refused(self): + for name in ("home", "config", "tmp", "docker-empty", "bin"): + with self.subTest(name=name): + path = self.root / name + path.rename(self.root / (name + "-held")) + path.mkdir(mode=0o700) + with self.assertRaises(ValueError): + self.workspace.recheck() + path.rmdir() + (self.root / (name + "-held")).rename(path) + self.workspace.recheck() + + def test_env_cleanup_uses_held_parent_after_root_rename(self): + self.workspace.write("runtime.env", b"SYNTHETIC=value") + original = self.root.with_name("held-root") + self.root.rename(original) + self.root.mkdir(mode=0o700) + (self.root / "runtime.env").write_bytes(b"foreign") + self.workspace.unlink("runtime.env") + self.assertFalse((original / "runtime.env").exists()) + self.assertEqual((self.root / "runtime.env").read_bytes(), b"foreign") + def test_failed_workspace_check_does_not_block_exact_id_cleanup(self): docker = FakeDocker() owned = boundary.OwnedContainer(docker, "cli-test") diff --git a/scripts/acceptance/command-keys/test_physical_cli.py b/scripts/acceptance/command-keys/test_physical_cli.py index 85366e5..156ef31 100644 --- a/scripts/acceptance/command-keys/test_physical_cli.py +++ b/scripts/acceptance/command-keys/test_physical_cli.py @@ -5,10 +5,12 @@ import os from pathlib import Path import sqlite3 +import shutil import subprocess import sys import tempfile import unittest +import unittest.mock SOURCE = Path(__file__).resolve().parent @@ -139,11 +141,42 @@ def test_direct_common_entrypoints_refuse_before_initialization(self): def test_unknown_loader_choice_refuses(self): for choice in ("unknown", "../physical_impl.py"): with self.subTest(choice=choice): - result = subprocess.run([sys.executable, "-I", "-B", str(SOURCE / "run_cli_216.py")], - env={"PATH": os.defpath, "CLI_216_RUNNER": choice}, - capture_output=True, text=True, timeout=5) - self.assertNotEqual(result.returncode, 0) - self.assertIn("CLI_216_RUNNER must be", result.stderr) + scope = {"__name__": "__main__", "__file__": str(SOURCE / "run_cli_216.py"), + "BOOTSTRAP_LAUNCHER_SHA256": "trusted-test"} + with unittest.mock.patch.dict(os.environ, {"CLI_216_RUNNER": choice}): + with self.assertRaisesRegex(SystemExit, "CLI_216_RUNNER must be"): + exec(compile((SOURCE / "run_cli_216.py").read_bytes(), scope["__file__"], "exec"), scope) + + def test_direct_outer_dispatcher_refuses_pathname_authority(self): + result = subprocess.run([sys.executable, "-I", "-B", str(SOURCE / "run_cli_216.py")], + env={"PATH": os.defpath}, capture_output=True, text=True, timeout=5) + self.assertNotEqual(result.returncode, 0) + self.assertIn("literal -c Git bootstrap", result.stderr) + + def test_frozen_git_bootstrap_ignores_preopen_outer_path_substitution(self): + root = self.root / "repo" + root.mkdir() + folder = root / "scripts/acceptance/command-keys" + folder.mkdir(parents=True) + launcher = folder / "run_cli_216.py" + launcher.write_bytes((SOURCE / "run_cli_216.py").read_bytes()) + env = {"PATH": os.defpath, "HOME": str(self.root), "GIT_CONFIG_NOSYSTEM": "1"} + def git(*args): + return subprocess.check_output(["git", "-C", str(root), *args], env=env, stderr=subprocess.DEVNULL) + git("init"); git("add", ".") + git("-c", "user.name=Fixture", "-c", "user.email=fixture@example.invalid", "commit", "-m", "Frozen dispatcher") + commit = git("rev-parse", "HEAD").decode().strip() + marker = self.root / "substitute-executed" + launcher.write_text("from pathlib import Path\nPath(" + repr(str(marker)) + ").touch()\n") + subprocess.run([sys.executable, "-I", "-B", str(launcher)], env=env, check=True) + self.assertTrue(marker.exists()) + marker.unlink() + code = (SOURCE / "git_bootstrap.py").read_text() + result = subprocess.run([sys.executable, "-I", "-B", "-c", code, commit], cwd=root, + env={**env, "CLI_216_RUNNER": "unknown"}, capture_output=True, text=True, timeout=10) + self.assertNotEqual(result.returncode, 0) + self.assertIn("CLI_216_RUNNER must be", result.stderr) + self.assertFalse(marker.exists()) def test_literal_bound_implementation_executes_checked_bytes(self): source = self.root / "implementation.py" diff --git a/scripts/acceptance/command-keys/test_workspace_container.py b/scripts/acceptance/command-keys/test_workspace_container.py new file mode 100644 index 0000000..8121d8a --- /dev/null +++ b/scripts/acceptance/command-keys/test_workspace_container.py @@ -0,0 +1,128 @@ +"""Opt-in local Docker boundary controls, not installed CLI conformance.""" +import importlib.util +import json +import os +from pathlib import Path +import secrets +import shutil +import subprocess +import tempfile +import unittest + +SOURCE = Path(__file__).resolve().parent +spec = importlib.util.spec_from_file_location("container_boundaries", SOURCE / "boundaries.py") +boundary = importlib.util.module_from_spec(spec) +spec.loader.exec_module(boundary) +IMAGE = "ghcr.io/durable-workflow/server@sha256:8ffcdb2a36af3129a1987849aafe8715fe0d3bf9a9648467f0b5cb8eeef43ba9" + + +@unittest.skipUnless(os.environ.get("CLI_216_CONTAINER_CONTROLS") == "1", "explicit local Docker control required") +class ContainerWorkspaceTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name).resolve() / "fixture" + self.root.mkdir(mode=0o700) + (self.root / "data").mkdir(mode=0o700) + self.workspace = boundary.Workspace(self.root) + self.addCleanup(self.workspace.close) + self.created = [] + self.addCleanup(self.cleanup) + + def docker(self, *arguments): + result = subprocess.run(["docker", "--config", "/proc/self/fd/" + str(self.workspace.children["docker-empty"].fd), *arguments], + env={"PATH": os.defpath, "HOME": self.workspace.child_anchor("home")}, + pass_fds=self.workspace.inherited(), capture_output=True, text=True, timeout=40) + if result.returncode: + raise RuntimeError("local Docker control failed: " + result.stderr[:1000]) + return (result.stdout + (result.stderr if arguments[0] == "logs" else "")).strip() + + def cleanup(self): + for owned in self.created: + owned.remove() + self.assertFalse(owned.exists(), "exact created control ID remains") + print(json.dumps({"control": self.id(), "created_id": owned.id, "absent": True}), flush=True) + + def run_container(self, *extra, command): + name = "cli-152-boundary-" + secrets.token_hex(5) + owned = boundary.OwnedContainer(self.docker, name) + self.created.append(owned) + owned.run("-d", "--name", name, "--platform=linux/amd64", "--init", "--cpus=1", "--memory=512m", + "--pids-limit=128", "--read-only", "--network=none", "--cap-drop=ALL", + "--security-opt=no-new-privileges", "--entrypoint=/bin/sh", "--env", "DB_CONNECTION=sqlite", *extra, IMAGE, *command) + code = self.docker("wait", owned.id) + state = owned.inspect()["State"] + self.assertFalse(state["OOMKilled"]) + return code, self.docker("logs", owned.id) + + def test_native_exec_on_actual_guarded_canonical_mount(self): + mounts = self.workspace.mount_arguments() + native = self.workspace.native_arguments("schema-check") + code, output = self.run_container(*mounts, "--tmpfs", "/tmp:rw,size=16m", command=native) + # This finite native command intentionally refuses SQLite. Its exact + # native diagnostic proves the mount preflight reached exec, without + # starting a runtime or performing an installed upgrade cohort. + self.assertEqual(code, "1", output) + self.assertIn("these development schema commands require DB_CONNECTION=pgsql, mysql or mariadb", output) + self.workspace.recheck() + self.workspace.inspect_mounts(self.created[-1].inspect()) + self.assertEqual(os.fstat(self.workspace.database.fd).st_size, 0) + + def replacement_refused(self, target): + mounts = self.workspace.mount_arguments() + native = self.workspace.native_arguments("schema-check") + path = self.root / "data" if target == "data" else self.root / "data/database.sqlite" + original = path.with_name(path.name + "-original") + path.rename(original) + if target == "data": + shutil.copytree(original, path) + else: + path.write_bytes(b"") + path.chmod(0o600) + code, output = self.run_container(*mounts, command=native) + self.assertEqual(code, "78") + self.assertEqual(output, "") + self.assertEqual((self.root / "data/database.sqlite").stat().st_size, 0, + "native bootstrap executed against replacement") + with self.assertRaises(ValueError): + self.workspace.recheck() + + def test_actual_preflight_refuses_directory_replacement_before_native_exec(self): + self.replacement_refused("data") + + def test_actual_preflight_refuses_db_replacement_before_native_exec(self): + self.replacement_refused("database") + + def test_actual_docker_env_reader_uses_held_descriptor(self): + self.workspace.write("runtime.env", b"CLI_FIXTURE_VALUE=held\n") + descriptor = self.workspace.files["runtime.env"].fd + path = self.root / "runtime.env" + path.rename(self.root / "original-env") + path.write_text("CLI_FIXTURE_VALUE=forged\n") + path.chmod(0o600) + # A pathname reader would consume the forged value. The actual Docker + # client inherits the original FD, even after that pathname changes. + self.assertIn("forged", path.read_text()) + code, output = self.run_container("--env-file", "/proc/self/fd/" + str(descriptor), + command=("-c", 'printf %s "$CLI_FIXTURE_VALUE"')) + self.assertEqual((code, output), ("0", "held")) + with self.assertRaises(ValueError): + self.workspace.recheck() + + def test_actual_docker_config_reader_and_cleanup_ignore_replaced_path(self): + path = self.root / "docker-empty" + path.rename(self.root / "original-docker") + path.mkdir(mode=0o700) + (path / "config.json").write_text('{"currentContext":"not-an-owned-context"}') + old = subprocess.run(["docker", "--config", str(path), "info", "--format", "{{.Name}}"], + env={"PATH": os.defpath}, capture_output=True, text=True, timeout=10) + self.assertNotEqual(old.returncode, 0) + code, output = self.run_container(command=("-c", "printf held-config")) + self.assertEqual((code, output), ("0", "held-config")) + with self.assertRaises(ValueError): + self.workspace.recheck() + # addCleanup still uses the held config FD and captured container ID. + + +if __name__ == "__main__": + unittest.main() From 8579c734e5c1b75b687e6d153d57043fd3abfcf1 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 13 Nov 2026 19:23:09 +0000 Subject: [PATCH 6/7] Read the held transformed SQLite copy from its disk descriptor path --- scripts/acceptance/command-keys/README.md | 2 +- .../acceptance/command-keys/git_bootstrap.py | 2 +- scripts/acceptance/command-keys/legacy_cli.py | 11 ++++++++- .../acceptance/command-keys/physical_impl.py | 2 +- .../acceptance/command-keys/run_cli_216.py | 4 ++-- .../command-keys/test_boundaries.py | 24 +++++++++++++++++++ 6 files changed, 39 insertions(+), 6 deletions(-) diff --git a/scripts/acceptance/command-keys/README.md b/scripts/acceptance/command-keys/README.md index dce2f17..fd51107 100644 --- a/scripts/acceptance/command-keys/README.md +++ b/scripts/acceptance/command-keys/README.md @@ -83,7 +83,7 @@ substitution of the outer dispatcher cannot replace those Git bytes. ```bash reviewed_source_commit=FULL_REVIEWED_COMMIT_FROM_THE_PR bootstrap="$(git --no-replace-objects show "$reviewed_source_commit:scripts/acceptance/command-keys/git_bootstrap.py")"$'\n' -expected_bootstrap_sha256=368d78748c12bb51b60a01336702f942a9fe11f9dff918edacdb710a8950ed44 +expected_bootstrap_sha256=80d8c383351eb18cd110da078029ad5aaa8b1b3518e63bf62b74304aec62a124 actual_bootstrap_sha256="$(printf '%s' "$bootstrap" | sha256sum | awk '{print $1}')" test "$actual_bootstrap_sha256" = "$expected_bootstrap_sha256" ``` diff --git a/scripts/acceptance/command-keys/git_bootstrap.py b/scripts/acceptance/command-keys/git_bootstrap.py index 2eceb46..b2513a2 100644 --- a/scripts/acceptance/command-keys/git_bootstrap.py +++ b/scripts/acceptance/command-keys/git_bootstrap.py @@ -12,7 +12,7 @@ if not re.fullmatch(r"[0-9a-f]{40}", commit, re.ASCII): raise ValueError("bootstrap requires one reviewed full Git commit") relative = "scripts/acceptance/command-keys/run_cli_216.py" -expected = "5762dd388597e1a5fcb7e088a6abb54aee8ad6976a240aebd88be1f615314150" +expected = "443fb73678de46ae6d52e2b388f23d1e6b0adf5aa2bc3fc85720c427718c9c7a" data = subprocess.check_output(["/usr/bin/git", "--no-replace-objects", "show", commit + ":" + relative], env={"PATH": os.defpath}, timeout=10) if hashlib.sha256(data).hexdigest() != expected: diff --git a/scripts/acceptance/command-keys/legacy_cli.py b/scripts/acceptance/command-keys/legacy_cli.py index b8915f0..0428f07 100644 --- a/scripts/acceptance/command-keys/legacy_cli.py +++ b/scripts/acceptance/command-keys/legacy_cli.py @@ -251,8 +251,17 @@ def exercise(): copied_input = d.held(transformed, None, 64 * 1024 * 1024) d.WORKSPACE.recheck() seed_input.recheck() - with closing(sqlite3.connect(Path(copied_input.executable).as_uri() + "?mode=ro&immutable=1", uri=True)) as db: + # SQLite cannot open this sealed memfd as a database. Use its held + # disk sibling through the held data directory, check the actual + # opened main-file identity, then compare against the helper's full + # logical result before writing the already-captured bytes to DB. + copied_input.recheck() + with closing(sqlite3.connect("file:" + d.WORKSPACE.data.anchor + "/transformed.sqlite?mode=ro&immutable=1", uri=True)) as db: + opened = db.execute("PRAGMA database_list").fetchone()[2] + if d.boundary.HeldInput._identity(os.stat(opened)) != copied_input.identity: + raise ValueError("SQLite opened a different transformed copy") transformed_snapshot = legacy.snapshot(db) + copied_input.recheck() if (transform["before_sha256"] != legacy.fingerprint(before) or transform["after_sha256"] != legacy.fingerprint(transformed_snapshot)): raise ValueError("held transform source/result differs") diff --git a/scripts/acceptance/command-keys/physical_impl.py b/scripts/acceptance/command-keys/physical_impl.py index be161ad..8d23c56 100644 --- a/scripts/acceptance/command-keys/physical_impl.py +++ b/scripts/acceptance/command-keys/physical_impl.py @@ -13,7 +13,7 @@ import types SOURCE = Path(__file__).resolve().parent -LEGACY_SHA = "c2d19bd944bfb1cca7d370de923be2ccbd788f51b7ed4131622de691d58a08eb" +LEGACY_SHA = "cb3afd7348d1e556e57e820897ab5ca46fbaf944dcc02a5b04758a19065ed5cc" BASELINE_SHA = "7d1826f7681342bb3a5974509648e2e8d75e7a21d262f65f3ec1f58727ccbc0c" PACKET_SHA = "8c728ce827df3a11621dfb0e82cdd404bbab48784182a5e92c8d632d49047fc5" CONTRACT_SHA = "ceb4695643e48c7bfe3f81eea380c6ab9aa0b973da830cb552d070344047319f" diff --git a/scripts/acceptance/command-keys/run_cli_216.py b/scripts/acceptance/command-keys/run_cli_216.py index e78cf8f..6cb38bd 100644 --- a/scripts/acceptance/command-keys/run_cli_216.py +++ b/scripts/acceptance/command-keys/run_cli_216.py @@ -12,8 +12,8 @@ BOUNDARY_SHA = "5de532a128931b56bd7ca2232d60e07a2fa65d550cfbf56d9cd78d16ed3a5095" IMPLEMENTATIONS = { 'modern': ('cli_216.py', 'f8a51ec2119cd24acf6b861b0a3c09e5ab7f651539136db506d6d8ef4b8e448b'), - 'legacy': ('legacy_cli.py', 'c2d19bd944bfb1cca7d370de923be2ccbd788f51b7ed4131622de691d58a08eb'), - 'physical': ('physical_impl.py', '446676dbfb755cc2f3aebeffdcf56977d3d064e2796bf4f86379bda0cf341ddc'), + 'legacy': ('legacy_cli.py', 'cb3afd7348d1e556e57e820897ab5ca46fbaf944dcc02a5b04758a19065ed5cc'), + 'physical': ('physical_impl.py', 'b3e42c81cc8855e11aad1eba68ef5651cd81460c65fd97b2708f5b08f576b8ca'), } diff --git a/scripts/acceptance/command-keys/test_boundaries.py b/scripts/acceptance/command-keys/test_boundaries.py index ac23fd3..54860f5 100644 --- a/scripts/acceptance/command-keys/test_boundaries.py +++ b/scripts/acceptance/command-keys/test_boundaries.py @@ -370,6 +370,30 @@ def test_db_symlink_and_extra_link_are_refused(self): with self.assertRaises(ValueError): self.workspace.recheck() + def test_transformed_copy_sqlite_read_uses_held_disk_identity_not_memfd(self): + path = self.root / "data/transformed.sqlite" + with closing(sqlite3.connect(path)) as db: + db.execute("CREATE TABLE transformed(value INTEGER)") + db.execute("INSERT INTO transformed VALUES (7)") + db.commit() + path.chmod(0o600) + held = boundary.HeldInput(path, hashlib.sha256(path.read_bytes()).hexdigest()) + self.addCleanup(held.close) + with self.assertRaises(sqlite3.OperationalError): + with closing(sqlite3.connect(Path(held.executable).as_uri() + "?mode=ro&immutable=1", uri=True)) as db: + db.execute("SELECT * FROM transformed").fetchall() + held.recheck() + with closing(sqlite3.connect("file:" + self.workspace.data.anchor + "/transformed.sqlite?mode=ro&immutable=1", uri=True)) as db: + opened = db.execute("PRAGMA database_list").fetchone()[2] + self.assertEqual(boundary.HeldInput._identity(os.stat(opened)), held.identity) + self.assertEqual(db.execute("SELECT * FROM transformed").fetchall(), [(7,)]) + held.recheck() + path.rename(path.with_name("original-transformed.sqlite")) + path.write_bytes(held.data) + path.chmod(0o600) + with self.assertRaises(ValueError): + held.recheck() + def test_mutable_database_keeps_inode_and_wal_semantics(self): identity = self.workspace.database.identity with self.workspace.connection(readonly=False) as db: From fcfcdce7bac92f5aa71a3c01cb32587a796c6b1a Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 13 Nov 2026 19:29:28 +0000 Subject: [PATCH 7/7] Open SQLite through held real disk file descriptors --- scripts/acceptance/command-keys/README.md | 5 +-- scripts/acceptance/command-keys/boundaries.py | 11 +++---- scripts/acceptance/command-keys/cli_216.py | 2 +- .../acceptance/command-keys/git_bootstrap.py | 2 +- scripts/acceptance/command-keys/legacy_cli.py | 14 +++----- .../acceptance/command-keys/physical_impl.py | 2 +- .../acceptance/command-keys/run_cli_216.py | 8 ++--- .../command-keys/test_boundaries.py | 33 +++++++++++++++++-- 8 files changed, 50 insertions(+), 27 deletions(-) diff --git a/scripts/acceptance/command-keys/README.md b/scripts/acceptance/command-keys/README.md index fd51107..2ac8c03 100644 --- a/scripts/acceptance/command-keys/README.md +++ b/scripts/acceptance/command-keys/README.md @@ -83,7 +83,7 @@ substitution of the outer dispatcher cannot replace those Git bytes. ```bash reviewed_source_commit=FULL_REVIEWED_COMMIT_FROM_THE_PR bootstrap="$(git --no-replace-objects show "$reviewed_source_commit:scripts/acceptance/command-keys/git_bootstrap.py")"$'\n' -expected_bootstrap_sha256=80d8c383351eb18cd110da078029ad5aaa8b1b3518e63bf62b74304aec62a124 +expected_bootstrap_sha256=4b34ceb7b8c1366bc46405310bc4779df6a3ee63eb274af41868265028a5bc12 actual_bootstrap_sha256="$(printf '%s' "$bootstrap" | sha256sum | awk '{print $1}')" test "$actual_bootstrap_sha256" = "$expected_bootstrap_sha256" ``` @@ -201,7 +201,8 @@ normal gates, fresh actual execution and cleanup still qualify that cell. The manual experiment holds owned mode-0700 root/data directories and a single-link mode-0600 database descriptor throughout each phase. SQLite opens -through the held directory so WAL/SHM siblings keep their normal semantics. +through the held real disk-file descriptor, preserving normal WAL/SHM behavior +without reopening the mutable database name. Docker acquires the guarded canonical data bind. A literal in-container shell preflight checks the actual mounted directory and database device/inode, UID/GID, mode and link count before executing the native binary. Host checks diff --git a/scripts/acceptance/command-keys/boundaries.py b/scripts/acceptance/command-keys/boundaries.py index 72a3ede..33edff1 100644 --- a/scripts/acceptance/command-keys/boundaries.py +++ b/scripts/acceptance/command-keys/boundaries.py @@ -238,14 +238,13 @@ def connection(self, *, readonly=True): @contextmanager def opened(): self.recheck() - # The directory descriptor preserves SQLite's WAL/SHM sibling - # semantics. Check SQLite's opened main file against our held DB. - uri = "file:" + self.data.anchor + "/database.sqlite?mode=" + ("ro" if readonly else "rw") + # The real disk descriptor binds SQLite's opened main inode. + # PRAGMA database_list followed by stat would reopen a name and + # could accept swap-away/open-substitute/swap-back. SQLite retains + # normal disk-backed WAL/SHM behavior through this descriptor. + uri = "file:" + self.database.anchor + "?mode=" + ("ro" if readonly else "rw") db = sqlite3.connect(uri, uri=True, timeout=2) try: - named = db.execute("PRAGMA database_list").fetchone()[2] - if HeldDirectory.key(os.stat(named)) != self.database.identity: - raise ValueError("SQLite opened a different database") self.recheck() yield db finally: diff --git a/scripts/acceptance/command-keys/cli_216.py b/scripts/acceptance/command-keys/cli_216.py index b18cf58..bc2c1ba 100644 --- a/scripts/acceptance/command-keys/cli_216.py +++ b/scripts/acceptance/command-keys/cli_216.py @@ -18,7 +18,7 @@ raise SystemExit("Use run_cli_216.py to verify the implementation before execution") SOURCE = Path(__file__).resolve().parent -BOUNDARY_SHA = '5de532a128931b56bd7ca2232d60e07a2fa65d550cfbf56d9cd78d16ed3a5095' +BOUNDARY_SHA = '49f44b870296b27955f116295f4e177a1fd285cc976809c13e5f57dc3cb1ebbf' # Bootstrap the local reviewed boundary module from the same bytes that are # checked. Subsequent inputs use its held descriptors and sealed memory files. _boundary_path = SOURCE / 'boundaries.py' diff --git a/scripts/acceptance/command-keys/git_bootstrap.py b/scripts/acceptance/command-keys/git_bootstrap.py index b2513a2..114977e 100644 --- a/scripts/acceptance/command-keys/git_bootstrap.py +++ b/scripts/acceptance/command-keys/git_bootstrap.py @@ -12,7 +12,7 @@ if not re.fullmatch(r"[0-9a-f]{40}", commit, re.ASCII): raise ValueError("bootstrap requires one reviewed full Git commit") relative = "scripts/acceptance/command-keys/run_cli_216.py" -expected = "443fb73678de46ae6d52e2b388f23d1e6b0adf5aa2bc3fc85720c427718c9c7a" +expected = "b19a7a7be644f708f9855cbc5e9cc07e8bd67c60d17996ab69948f15d6db28ae" data = subprocess.check_output(["/usr/bin/git", "--no-replace-objects", "show", commit + ":" + relative], env={"PATH": os.defpath}, timeout=10) if hashlib.sha256(data).hexdigest() != expected: diff --git a/scripts/acceptance/command-keys/legacy_cli.py b/scripts/acceptance/command-keys/legacy_cli.py index 0428f07..d467de3 100644 --- a/scripts/acceptance/command-keys/legacy_cli.py +++ b/scripts/acceptance/command-keys/legacy_cli.py @@ -42,7 +42,7 @@ def bound_driver(): os.close(descriptor) -BASE_SHA = "f8a51ec2119cd24acf6b861b0a3c09e5ab7f651539136db506d6d8ef4b8e448b" +BASE_SHA = "507ff78a1a25bca43f4b5ce84abd395d59d24641bbac6db8c6ee86a665113417" d = bound_driver() if "BOOTSTRAP_HELD_IMPLEMENTATION" in globals(): ENTRY_INPUT = BOOTSTRAP_HELD_IMPLEMENTATION @@ -251,15 +251,11 @@ def exercise(): copied_input = d.held(transformed, None, 64 * 1024 * 1024) d.WORKSPACE.recheck() seed_input.recheck() - # SQLite cannot open this sealed memfd as a database. Use its held - # disk sibling through the held data directory, check the actual - # opened main-file identity, then compare against the helper's full - # logical result before writing the already-captured bytes to DB. + # A real disk descriptor works with SQLite and binds the main inode; + # the sealed executable memfd does not. A post-open pathname stat + # cannot establish the connection's identity after a swap-back. copied_input.recheck() - with closing(sqlite3.connect("file:" + d.WORKSPACE.data.anchor + "/transformed.sqlite?mode=ro&immutable=1", uri=True)) as db: - opened = db.execute("PRAGMA database_list").fetchone()[2] - if d.boundary.HeldInput._identity(os.stat(opened)) != copied_input.identity: - raise ValueError("SQLite opened a different transformed copy") + with closing(sqlite3.connect("file:/proc/self/fd/" + str(copied_input.fd) + "?mode=ro&immutable=1", uri=True)) as db: transformed_snapshot = legacy.snapshot(db) copied_input.recheck() if (transform["before_sha256"] != legacy.fingerprint(before) diff --git a/scripts/acceptance/command-keys/physical_impl.py b/scripts/acceptance/command-keys/physical_impl.py index 8d23c56..6da4e6a 100644 --- a/scripts/acceptance/command-keys/physical_impl.py +++ b/scripts/acceptance/command-keys/physical_impl.py @@ -13,7 +13,7 @@ import types SOURCE = Path(__file__).resolve().parent -LEGACY_SHA = "cb3afd7348d1e556e57e820897ab5ca46fbaf944dcc02a5b04758a19065ed5cc" +LEGACY_SHA = "cc4018a807ff01e557676fa7ff4cb73f30872b004d04b42338fe96f65e18aec2" BASELINE_SHA = "7d1826f7681342bb3a5974509648e2e8d75e7a21d262f65f3ec1f58727ccbc0c" PACKET_SHA = "8c728ce827df3a11621dfb0e82cdd404bbab48784182a5e92c8d632d49047fc5" CONTRACT_SHA = "ceb4695643e48c7bfe3f81eea380c6ab9aa0b973da830cb552d070344047319f" diff --git a/scripts/acceptance/command-keys/run_cli_216.py b/scripts/acceptance/command-keys/run_cli_216.py index 6cb38bd..8c8bae3 100644 --- a/scripts/acceptance/command-keys/run_cli_216.py +++ b/scripts/acceptance/command-keys/run_cli_216.py @@ -9,11 +9,11 @@ raise SystemExit("Use the reviewed literal -c Git bootstrap before the dispatcher") SOURCE = Path(__file__).resolve().parent -BOUNDARY_SHA = "5de532a128931b56bd7ca2232d60e07a2fa65d550cfbf56d9cd78d16ed3a5095" +BOUNDARY_SHA = "49f44b870296b27955f116295f4e177a1fd285cc976809c13e5f57dc3cb1ebbf" IMPLEMENTATIONS = { - 'modern': ('cli_216.py', 'f8a51ec2119cd24acf6b861b0a3c09e5ab7f651539136db506d6d8ef4b8e448b'), - 'legacy': ('legacy_cli.py', 'cb3afd7348d1e556e57e820897ab5ca46fbaf944dcc02a5b04758a19065ed5cc'), - 'physical': ('physical_impl.py', 'b3e42c81cc8855e11aad1eba68ef5651cd81460c65fd97b2708f5b08f576b8ca'), + 'modern': ('cli_216.py', '507ff78a1a25bca43f4b5ce84abd395d59d24641bbac6db8c6ee86a665113417'), + 'legacy': ('legacy_cli.py', 'cc4018a807ff01e557676fa7ff4cb73f30872b004d04b42338fe96f65e18aec2'), + 'physical': ('physical_impl.py', '640a3a8b9e3c802e965971ca1e8980200493513a81249a9265817b9a9bce3ba4'), } diff --git a/scripts/acceptance/command-keys/test_boundaries.py b/scripts/acceptance/command-keys/test_boundaries.py index 54860f5..2f05a5e 100644 --- a/scripts/acceptance/command-keys/test_boundaries.py +++ b/scripts/acceptance/command-keys/test_boundaries.py @@ -383,9 +383,7 @@ def test_transformed_copy_sqlite_read_uses_held_disk_identity_not_memfd(self): with closing(sqlite3.connect(Path(held.executable).as_uri() + "?mode=ro&immutable=1", uri=True)) as db: db.execute("SELECT * FROM transformed").fetchall() held.recheck() - with closing(sqlite3.connect("file:" + self.workspace.data.anchor + "/transformed.sqlite?mode=ro&immutable=1", uri=True)) as db: - opened = db.execute("PRAGMA database_list").fetchone()[2] - self.assertEqual(boundary.HeldInput._identity(os.stat(opened)), held.identity) + with closing(sqlite3.connect("file:/proc/self/fd/" + str(held.fd) + "?mode=ro&immutable=1", uri=True)) as db: self.assertEqual(db.execute("SELECT * FROM transformed").fetchall(), [(7,)]) held.recheck() path.rename(path.with_name("original-transformed.sqlite")) @@ -394,6 +392,35 @@ def test_transformed_copy_sqlite_read_uses_held_disk_identity_not_memfd(self): with self.assertRaises(ValueError): held.recheck() + def test_sqlite_swap_open_swap_back_fools_path_stat_but_not_held_fd(self): + connect = sqlite3.connect + def raced(*arguments, **keywords): + original = self.database.with_name("opened-original.sqlite") + self.database.rename(original) + self.database.write_bytes(original.read_bytes()) + self.database.chmod(0o600) + with closing(connect(self.database)) as replacement: + replacement.execute("UPDATE fixture SET value=99") + replacement.commit() + try: + opened = connect(*arguments, **keywords) + finally: + self.database.unlink() + original.rename(self.database) + # The predecessor's post-open name points back at the original, + # even if the returned connection already opened the substitute. + if "/database.sqlite?" in str(arguments[0]): + named = opened.execute("PRAGMA database_list").fetchone()[2] + self.assertEqual(boundary.HeldDirectory.key(os.stat(named)), self.workspace.database.identity) + self.workspace.recheck() + return opened + with patch.object(sqlite3, "connect", raced): + with closing(sqlite3.connect("file:" + self.workspace.data.anchor + "/database.sqlite?mode=ro", uri=True)) as old: + self.assertEqual(old.execute("SELECT * FROM fixture").fetchall(), [(99,)]) + with self.workspace.connection() as current: + self.assertEqual(current.execute("SELECT * FROM fixture").fetchall(), [(1,)]) + self.workspace.recheck() + def test_mutable_database_keeps_inode_and_wal_semantics(self): identity = self.workspace.database.identity with self.workspace.connection(readonly=False) as db: