From cc1d50942732d606fb3b2228595884561d1f3ecf Mon Sep 17 00:00:00 2001 From: Kc Date: Sat, 24 Oct 2026 10:05:13 +0900 Subject: [PATCH 1/2] chore: add a Ruff config and a report-only format check for the Python tools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ruff.toml formats the maintained Python tools (core/tools, tools): Python 3.11 (tomllib), width 120. Docs, evidence and core/vendor are outside its include list. It pins Ruff 0.17.2 and gives the command. .github/workflows/format.yml runs `ruff format --diff` when those files or the config change: one job of about fifteen seconds. Like ci.yml it reports and never gates: it is not a required check and nothing runs before a push. Lint findings are not checked. AGENTS.md §4 names the config and the lane. Production LOC: 102160 -> 102160 (delta 0) core 96788 -> 96788, api 5372 -> 5372; reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). Method: python3 -B tools/production_loc.py (parent's copy) --root on git archive of the parent (4497b0cb) and the commit tree. --- .github/workflows/format.yml | 37 ++++++++++++++++++++++++++++++++++++ AGENTS.md | 10 +++++++--- ruff.toml | 9 +++++++++ 3 files changed, 53 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/format.yml create mode 100644 ruff.toml diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml new file mode 100644 index 000000000..fc9016873 --- /dev/null +++ b/.github/workflows/format.yml @@ -0,0 +1,37 @@ +# Format lane: checks that the Python tools are formatted (ruff.toml). Reports +# only, like ci.yml: never a required check and never blocks a push or merge +# (AGENTS.md §4). Lint findings are not checked. The Ruff version matches the +# command in ruff.toml. +name: format + +on: + push: + branches: [main] + paths: + - "core/tools/**/*.py" + - "tools/**/*.py" + - "ruff.toml" + - ".github/workflows/format.yml" + pull_request: + paths: + - "core/tools/**/*.py" + - "tools/**/*.py" + - "ruff.toml" + - ".github/workflows/format.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: format-${{ github.ref }} + cancel-in-progress: true + +jobs: + ruff: + name: ruff (Python tools) + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - uses: actions/checkout@v7.0.1 + - run: pipx run --spec ruff==0.17.2 ruff format --diff diff --git a/AGENTS.md b/AGENTS.md index f44463f9d..beafed0c8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -308,9 +308,10 @@ remain binding; moving them out of this file does not weaken them. - **CI reports; it never gates (owner, 2026-10-23).** GitHub Actions runs a fast lane after each push or pull request (`.github/workflows/ci.yml`: fmt, guards, Clippy and the fast tests) and a nightly lane (`nightly.yml`: slow and - FUSE mount tests). [core/tools/ci_tests.py](core/tools/ci_tests.py) decides - which tests go in which lane. Neither is a required check, and nothing runs - before a push. Do not wait for CI, and do not treat a green run as + FUSE mount tests); `format.yml` checks the Ruff formatting below. + [core/tools/ci_tests.py](core/tools/ci_tests.py) decides which tests go in + which lane. None is a required check, and nothing runs before a push. Do + not wait for CI, and do not treat a green run as verification of your change: verify the changed scope yourself with [core checks](core/AGENTS.md#checks-and-completion) and report checks/gaps. There is still no pre-push gate: `tools/preflight.sh` stays retired; do not @@ -347,6 +348,9 @@ remain binding; moving them out of this file does not weaken them. - New core/application production files obey the 999-line ceiling and 200-line declaration/delegation ceiling for `lib.rs`/`mod.rs`. Product-only source, external tests and guard coverage are detailed in the core guide. +- Format changed Python tools with Ruff; `ruff.toml` names the pinned version, + scope and command. `format.yml` reports formatting drift after a push; + nothing runs before one. - Update affected source architecture/API documentation with implementation changes. Keep proposal, research, implemented and measured claims distinct. diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 000000000..72d8b0ed2 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,9 @@ +# Ruff formats the maintained Python tools. .github/workflows/format.yml reports +# drift after a push; nothing runs before one. From the repository root, with +# the pinned version: +# pipx run --spec ruff==0.17.2 ruff format +# pipx run --spec ruff==0.17.2 ruff check +# Docs, evidence and core/vendor are outside `include` and stay as recorded. +include = ["core/tools/**/*.py", "tools/**/*.py"] +target-version = "py311" # tomllib +line-length = 120 From 69d9cdc7a54ab6f54365af670c9b2dfc049e75b7 Mon Sep 17 00:00:00 2001 From: Kc Date: Sat, 24 Oct 2026 10:05:24 +0900 Subject: [PATCH 2/2] style: format the Python tools with Ruff One mechanical pass with the previous commit's config, no hand edits: `ruff format` at the root. 31 files in core/tools and tools change; a second run changes nothing. Lint findings are not fixed here. Checked: `ruff format --check` is clean, and each changed file parses to the same AST as before (docstring indentation aside). Tests were not run. Production LOC: 102160 -> 102160 (delta 0) core 96788 -> 96788, api 5372 -> 5372; reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). Development tools are outside every counted scope. Method: as in the previous commit. --- core/tools/check_fuser_integrity.py | 120 +++-- core/tools/check_product_boundary.py | 156 +++++-- core/tools/ci_tests.py | 8 +- core/tools/phase7_services.py | 365 ++++++++++----- core/tools/production_loc.py | 6 +- core/tools/sdk_telemetry_report.py | 75 ++-- core/tools/substitution-proof/run.py | 75 +++- core/tools/test_check_fuser_integrity.py | 128 +++--- core/tools/test_check_product_boundary.py | 260 +++++++---- core/tools/test_ci_tests.py | 20 +- core/tools/test_phase7_services.py | 119 +++-- core/tools/test_sdk_telemetry_report.py | 20 +- tools/cleanup.py | 303 +++++++++---- tools/cleanup_archive.py | 73 +-- tools/cleanup_attrs.py | 16 +- tools/cleanup_discover.py | 88 +++- tools/cleanup_docker.py | 37 +- tools/cleanup_retirement.py | 6 +- tools/cleanup_state.py | 52 ++- tools/cleanup_volume_cache.py | 122 +++--- tools/decode_observations.py | 512 +++++++++++++++++----- tools/production_loc.py | 24 +- tools/stage5_component_comparison.py | 74 ++-- tools/test_cleanup.py | 352 ++++++++++----- tools/test_decode_observations.py | 38 +- tools/test_production_loc.py | 83 ++-- tools/test_worktree.py | 121 ++++- tools/worktree.py | 4 +- tools/worktree_build.py | 153 +++++-- tools/worktree_resources.py | 108 +++-- tools/worktree_run.py | 127 ++++-- 31 files changed, 2538 insertions(+), 1107 deletions(-) diff --git a/core/tools/check_fuser_integrity.py b/core/tools/check_fuser_integrity.py index 9fc96b40c..8be08d70a 100644 --- a/core/tools/check_fuser_integrity.py +++ b/core/tools/check_fuser_integrity.py @@ -7,6 +7,7 @@ receive-buffer patches have separate pinned identities; all other fuser overrides remain rejected. """ + from __future__ import annotations import argparse @@ -37,9 +38,7 @@ def is_fuser(name, value): - return name.split(":", 1)[0] == "fuser" or ( - isinstance(value, dict) and value.get("package") == "fuser" - ) + return name.split(":", 1)[0] == "fuser" or (isinstance(value, dict) and value.get("package") == "fuser") def manifest_errors(document, approved=APPROVED_REVISIONS, patch_path=None): @@ -51,9 +50,13 @@ def manifest_errors(document, approved=APPROVED_REVISIONS, patch_path=None): if isinstance(group, dict): for name, value in group.items(): if is_fuser(name, value): - permitted = (section == "patch" and registry == "crates-io" - and name == "fuser" and patch_path is not None - and value == {"version": "=0.18.0", "path": patch_path}) + permitted = ( + section == "patch" + and registry == "crates-io" + and name == "fuser" + and patch_path is not None + and value == {"version": "=0.18.0", "path": patch_path} + ) if not permitted: errors.append(f"{section} override of fuser is not the authorized timestamp patch") @@ -86,7 +89,8 @@ def visit(table): visit(document) if patch_path is not None and document.get("patch", {}).get("crates-io", {}).get("fuser") != { - "version": "=0.18.0", "path": patch_path + "version": "=0.18.0", + "path": patch_path, }: errors.append("owning manifest is missing its exact fuser timestamp patch") return errors @@ -105,7 +109,8 @@ def lock_errors(document, approved=APPROVED_REVISIONS, patched=False): errors.append("registry fuser requires its locked checksum") elif source.startswith("git+"): valid = any( - source in ( + source + in ( f"git+{OFFICIAL_GIT}?rev={revision}#{revision}", f"git+{OFFICIAL_GIT.removesuffix('.git')}?rev={revision}#{revision}", ) @@ -113,8 +118,7 @@ def lock_errors(document, approved=APPROVED_REVISIONS, patched=False): ) if not valid: errors.append("locked fuser Git source/revision is not approved") - elif (patched and not source and package.get("version") == "0.18.0" - and "checksum" not in package): + elif patched and not source and package.get("version") == "0.18.0" and "checksum" not in package: pass # Local patched bytes have a separate checked provenance record. else: errors.append("locked fuser has a local/replacement source") @@ -134,9 +138,10 @@ def patched_package_errors(root: Path): return ["patched fuser package directory is redirected"] if metadata["archive_sha256"] != BASE_CHECKSUM: return ["patched fuser has the wrong published archive identity"] - if metadata["patch_sha256"] != PATCH_CHECKSUM or hashlib.sha256( - (record / "signed-timestamps.patch").read_bytes() - ).hexdigest() != PATCH_CHECKSUM: + if ( + metadata["patch_sha256"] != PATCH_CHECKSUM + or hashlib.sha256((record / "signed-timestamps.patch").read_bytes()).hexdigest() != PATCH_CHECKSUM + ): return ["fuser timestamp diff changed"] if metadata["modified_files"] != {"src/time.rs": TIME_CHECKSUM}: return ["fuser patch changes files beyond the authorized timestamp correction"] @@ -147,13 +152,16 @@ def patched_package_errors(root: Path): if hashlib.sha256(lifecycle_raw).hexdigest() != LIFECYCLE_PROVENANCE_CHECKSUM: return ["fuser lifecycle provenance changed"] lifecycle = json.loads(lifecycle_raw) - if (lifecycle["timestamp_provenance_sha256"] != PROVENANCE_CHECKSUM - or lifecycle["archive_sha256"] != BASE_CHECKSUM - or set(lifecycle["files"]) != LIFECYCLE_FILES): + if ( + lifecycle["timestamp_provenance_sha256"] != PROVENANCE_CHECKSUM + or lifecycle["archive_sha256"] != BASE_CHECKSUM + or set(lifecycle["files"]) != LIFECYCLE_FILES + ): return ["fuser lifecycle delta has an unauthorized base or file scope"] - if (lifecycle["patch_sha256"] != LIFECYCLE_PATCH_CHECKSUM - or hashlib.sha256((record / "session-lifecycle.patch").read_bytes()).hexdigest() - != LIFECYCLE_PATCH_CHECKSUM): + if ( + lifecycle["patch_sha256"] != LIFECYCLE_PATCH_CHECKSUM + or hashlib.sha256((record / "session-lifecycle.patch").read_bytes()).hexdigest() != LIFECYCLE_PATCH_CHECKSUM + ): return ["fuser lifecycle diff changed"] expected |= lifecycle["files"] if len(expected) != 86: @@ -162,14 +170,17 @@ def patched_package_errors(root: Path): if hashlib.sha256(buffer_raw).hexdigest() != BUFFER_PROVENANCE_CHECKSUM: return ["fuser receive-buffer provenance changed"] buffer = json.loads(buffer_raw) - if (buffer["timestamp_provenance_sha256"] != PROVENANCE_CHECKSUM - or buffer["lifecycle_provenance_sha256"] != LIFECYCLE_PROVENANCE_CHECKSUM - or buffer["archive_sha256"] != BASE_CHECKSUM - or set(buffer["files"]) != BUFFER_FILES): + if ( + buffer["timestamp_provenance_sha256"] != PROVENANCE_CHECKSUM + or buffer["lifecycle_provenance_sha256"] != LIFECYCLE_PROVENANCE_CHECKSUM + or buffer["archive_sha256"] != BASE_CHECKSUM + or set(buffer["files"]) != BUFFER_FILES + ): return ["fuser receive-buffer delta has an unauthorized base or file scope"] - if (buffer["patch_sha256"] != BUFFER_PATCH_CHECKSUM - or hashlib.sha256((record / "receive-buffer.patch").read_bytes()).hexdigest() - != BUFFER_PATCH_CHECKSUM): + if ( + buffer["patch_sha256"] != BUFFER_PATCH_CHECKSUM + or hashlib.sha256((record / "receive-buffer.patch").read_bytes()).hexdigest() != BUFFER_PATCH_CHECKSUM + ): return ["fuser receive-buffer diff changed"] expected |= buffer["files"] if len(expected) != 86: @@ -194,8 +205,7 @@ def config_errors(document): errors = [] if document.get("paths"): errors.append("Cargo path overrides can replace fuser and are forbidden") - if any(isinstance(source, dict) and "directory" in source - for source in document.get("source", {}).values()): + if any(isinstance(source, dict) and "directory" in source for source in document.get("source", {}).values()): errors.append("Cargo directory source replacement can vendor fuser") return errors @@ -213,7 +223,7 @@ def package_errors(archive: Path, source: Path, checksum: str): if not entry.isfile() or not entry.name.startswith(prefix): errors.append(f"unexpected archive entry: {entry.name}") continue - relative = Path(entry.name[len(prefix):]) + relative = Path(entry.name[len(prefix) :]) if relative.is_absolute() or ".." in relative.parts: errors.append("invalid archive path") continue @@ -234,11 +244,29 @@ def package_errors(archive: Path, source: Path, checksum: str): def repository_errors(root: Path): - paths = subprocess.check_output( - ["git", "-C", str(root), "ls-files", "-z", "--cached", "--others", - "--exclude-standard", "--", "Cargo.toml", "Cargo.lock", "**/Cargo.toml", - "**/Cargo.lock", ".cargo/config.toml", ".cargo/config"] - ).decode().split("\0") + paths = ( + subprocess.check_output( + [ + "git", + "-C", + str(root), + "ls-files", + "-z", + "--cached", + "--others", + "--exclude-standard", + "--", + "Cargo.toml", + "Cargo.lock", + "**/Cargo.toml", + "**/Cargo.lock", + ".cargo/config.toml", + ".cargo/config", + ] + ) + .decode() + .split("\0") + ) errors = [] checked = 0 for name in sorted(set(paths) - {""}): @@ -248,9 +276,13 @@ def repository_errors(root: Path): checked += 1 try: document = tomllib.loads(path.read_text()) - findings = (manifest_errors(document, patch_path=PATCH_MANIFESTS.get(name)) if path.name == "Cargo.toml" - else lock_errors(document, patched=name in PATCH_LOCKS) if path.name == "Cargo.lock" - else config_errors(document)) + findings = ( + manifest_errors(document, patch_path=PATCH_MANIFESTS.get(name)) + if path.name == "Cargo.toml" + else lock_errors(document, patched=name in PATCH_LOCKS) + if path.name == "Cargo.lock" + else config_errors(document) + ) errors.extend(f"{name}: {error}" for error in findings) except (OSError, ValueError) as error: errors.append(f"{name}: cannot verify: {error}") @@ -270,9 +302,17 @@ def main(): checked, errors = repository_errors(args.root) if all(supplied): errors.extend(package_errors(*supplied)) - print(json.dumps({"checked_manifests_locks_configs": checked, "errors": errors, - "authorized_patch_verified": not errors, - "registry_package_verified": bool(all(supplied)) and not errors}, indent=2)) + print( + json.dumps( + { + "checked_manifests_locks_configs": checked, + "errors": errors, + "authorized_patch_verified": not errors, + "registry_package_verified": bool(all(supplied)) and not errors, + }, + indent=2, + ) + ) return 1 if errors else 0 diff --git a/core/tools/check_product_boundary.py b/core/tools/check_product_boundary.py index e13c2e6e5..30c246e39 100644 --- a/core/tools/check_product_boundary.py +++ b/core/tools/check_product_boundary.py @@ -10,7 +10,9 @@ TEST_ATTR = re.compile(r"^(?:\w+\s*::\s*)*(?:test|bench)\b") TEST_CFG = re.compile(r'\btest\b|"[^"\n]*(?:test|bench|mock|fixture|fault)[^"\n]*"') CFG_MACRO = re.compile(r"\bcfg\s*!\s*\(([^;]*)\)", re.DOTALL) -IMPL = re.compile(r"\b(?:struct|enum|trait|impl|union|static|const|let|if|else|match|loop|while|for)\b|\bmacro_rules\s*!") +IMPL = re.compile( + r"\b(?:struct|enum|trait|impl|union|static|const|let|if|else|match|loop|while|for)\b|\bmacro_rules\s*!" +) DOC_CODE = re.compile(r"^\s*(?:///|//!)\s*```", re.MULTILINE) UNSAFE = re.compile(r"\bunsafe\b") @@ -23,7 +25,19 @@ "layerfs-storage": "src/encoding/codec.rs", "layerfs-persistence": "src/backend/sqlite/file_control.rs", } -UNSAFE_FREE_CRATES = ("layerfs-content", "layerfs-telemetry", "layerfs-project", "layerfs-overlay", "layerfs-workspace", "layerfs-store-service", "layerfs-fuse", "layerfs-daemon", "layerfs-api", "layerfs-bridge", "layerfs-sandbox") +UNSAFE_FREE_CRATES = ( + "layerfs-content", + "layerfs-telemetry", + "layerfs-project", + "layerfs-overlay", + "layerfs-workspace", + "layerfs-store-service", + "layerfs-fuse", + "layerfs-daemon", + "layerfs-api", + "layerfs-bridge", + "layerfs-sandbox", +) UNSAFE_ROOT_ATTR = { "layerfs-storage": "#![deny(unsafe_code)]", "layerfs-content": "#![forbid(unsafe_code)]", @@ -55,27 +69,72 @@ "layerfs-fuse": {"layerfs-content", "layerfs-overlay", "layerfs-workspace"}, # The Store engine (#352 F-020): the SDK's Store calls in-process and the # Store/overlay ports daemon Workspace Commits use; it opens the Store. - "layerfs-store-service": {"layerfs-overlay", "layerfs-workspace", "layerfs-bridge", "layerfs-content", "layerfs-storage", "layerfs-history", "layerfs-persistence", "layerfs-telemetry"}, + "layerfs-store-service": { + "layerfs-overlay", + "layerfs-workspace", + "layerfs-bridge", + "layerfs-content", + "layerfs-storage", + "layerfs-history", + "layerfs-persistence", + "layerfs-telemetry", + }, # Concrete Store open/install belongs to application composition. The # provider-independent daemon store/ subtree has a narrower source boundary. - "layerfs-daemon": {"layerfs-overlay", "layerfs-workspace", "layerfs-store-service", "layerfs-fuse", "layerfs-bridge", "layerfs-content", "layerfs-storage", "layerfs-history", "layerfs-persistence", "layerfs-telemetry"}, - "layerfs-api": {"layerfs-content", "layerfs-storage", "layerfs-history", "layerfs-persistence", "layerfs-telemetry", "layerfs-bridge", "layerfs-project", "layerfs-sandbox", "layerfs-store-service"}, + "layerfs-daemon": { + "layerfs-overlay", + "layerfs-workspace", + "layerfs-store-service", + "layerfs-fuse", + "layerfs-bridge", + "layerfs-content", + "layerfs-storage", + "layerfs-history", + "layerfs-persistence", + "layerfs-telemetry", + }, + "layerfs-api": { + "layerfs-content", + "layerfs-storage", + "layerfs-history", + "layerfs-persistence", + "layerfs-telemetry", + "layerfs-bridge", + "layerfs-project", + "layerfs-sandbox", + "layerfs-store-service", + }, "layerfs-bridge": {"layerfs-content", "layerfs-history"}, "layerfs-sandbox": set(), } DOMAIN_CRATES = {"layerfs-content", "layerfs-storage", "layerfs-history", "layerfs-project"} ENGINES_AND_CLUSTER2 = { - "layerfs-s3", "layerfs-metadata", "layerfs-persistence", "layerfs-overlay", "layerfs-workspace", - "layerfs-store-service", "layerfs-fuse", "layerfs-daemon", "layerfs-bridge", "layerfs-sandbox", - "layerfs-server", "layerfs-sdk", "layerfs-api", "layerfs-api-core", + "layerfs-s3", + "layerfs-metadata", + "layerfs-persistence", + "layerfs-overlay", + "layerfs-workspace", + "layerfs-store-service", + "layerfs-fuse", + "layerfs-daemon", + "layerfs-bridge", + "layerfs-sandbox", + "layerfs-server", + "layerfs-sdk", + "layerfs-api", + "layerfs-api-core", } # Package directories under core/crates that R7-retire removed. Their source # stays in Git history; a directory or manifest that returns fails the scan. RETIRED_PACKAGES = ( - "layerfs-server", "layerfs-fuse-legacy", "layerfs-sandbox-legacy", - "layerfs-sdk-legacy", "layerfs-daemon-legacy", "layerfs-bridge-legacy", + "layerfs-server", + "layerfs-fuse-legacy", + "layerfs-sandbox-legacy", + "layerfs-sdk-legacy", + "layerfs-daemon-legacy", + "layerfs-bridge-legacy", "layerfs-workspace-legacy", ) RETIRED_PACKAGE = "retired package; its directory was removed and is not restored" @@ -96,7 +155,13 @@ def dependency_violations(source): for section in sections: for alias, specification in section.items(): dependency = specification.get("package", alias) if isinstance(specification, dict) else alias - if name in DOMAIN_CRATES and dependency in {"rusqlite", "postgres", "tokio-postgres", "native-tls", "reqwest"}: + if name in DOMAIN_CRATES and dependency in { + "rusqlite", + "postgres", + "tokio-postgres", + "native-tls", + "reqwest", + }: guidance = "" if name == "layerfs-project" and dependency == "rusqlite": guidance = ( @@ -104,7 +169,7 @@ def dependency_violations(source): "an owning backend-neutral port; SQLite-backed Project " "acquisition is allowed through that boundary" ) - found.append((1,f"engine dependency in domain {name} -> {dependency}{guidance}")) + found.append((1, f"engine dependency in domain {name} -> {dependency}{guidance}")) if dependency.startswith("layerfs-") and dependency not in ALLOWED_DEPENDENCIES[name]: found.append((1, f"forbidden production dependency {name} -> {dependency}")) return found @@ -129,19 +194,28 @@ def component_violations(path, source): if name == "layerfs-daemon" and "/src/store/" in path.as_posix(): concrete = r"\b(?:layerfs_persistence|layerfs_api|layerfs_bridge|rusqlite|sqlite)\s*::|\b(?:Path|PathBuf)\b" for match in re.finditer(concrete, source): - found.append((source.count("\n", 0, match.start()) + 1, - "concrete provider/transport/path in daemon store/; use opened Storage/History ports")) + found.append( + ( + source.count("\n", 0, match.start()) + 1, + "concrete provider/transport/path in daemon store/; use opened Storage/History ports", + ) + ) if name in DOMAIN_CRATES: for dependency in ENGINES_AND_CLUSTER2: spelling = dependency.replace("-", "[-_]") for match in re.finditer(r"\b" + spelling + r"\b", source): - found.append((source.count("\n", 0, match.start()) + 1, - f"domain source names forbidden component {dependency}")) + found.append( + (source.count("\n", 0, match.start()) + 1, f"domain source names forbidden component {dependency}") + ) for match in re.finditer(r"\b(layerfs_[a-z0-9_]+)\s*::", source): dependency = match[1].replace("_", "-") if dependency != name and dependency not in ALLOWED_DEPENDENCIES[name]: - found.append((source.count("\n", 0, match.start()) + 1, - f"forbidden first-party source import {name} -> {dependency}")) + found.append( + ( + source.count("\n", 0, match.start()) + 1, + f"forbidden first-party source import {name} -> {dependency}", + ) + ) return found @@ -162,7 +236,7 @@ def unsafe_violations(path, source): crate = crate_name(path) if crate is None or crate not in UNSAFE_ROOT_ATTR or path.suffix != ".rs": return found - relative = Path(*path.parts[path.parts.index("src"):]) + relative = Path(*path.parts[path.parts.index("src") :]) code = "\n".join(line.split("//", 1)[0] for line in source.splitlines()) if UNSAFE.search(code): if crate in UNSAFE_FREE_CRATES or str(relative) != UNSAFE_AUDITED_MODULE[crate]: @@ -186,9 +260,7 @@ def violations(path, source): return found for match in ATTR.finditer(source): body = match[1].strip() - if TEST_ATTR.match(body) or ( - re.match(r"cfg(?:_attr)?\b", body) and TEST_CFG.search(body) - ): + if TEST_ATTR.match(body) or (re.match(r"cfg(?:_attr)?\b", body) and TEST_CFG.search(body)): found.append((source.count("\n", 0, match.start()) + 1, "test-only attribute in product source")) for match in CFG_MACRO.finditer(source): if TEST_CFG.search(match[1]): @@ -202,7 +274,9 @@ def violations(path, source): found.append((number, "implementation construct in declaration/delegation entry file")) if crate_name(path) == "layerfs-storage" and any(part in ("encoding", "pack") for part in path.parts): for match in re.finditer(r"\b(?:rusqlite|sqlite)\s*::", source): - found.append((source.count("\n", 0, match.start()) + 1, "engine access under encoding/ or pack/; use source seam")) + found.append( + (source.count("\n", 0, match.start()) + 1, "engine access under encoding/ or pack/; use source seam") + ) found.extend(unsafe_violations(path, source)) found.extend(component_violations(path, source)) return found @@ -212,16 +286,22 @@ def overlay_terminology_violations(path, source): """Guard the renamed SQL/API concepts; generic temporary buffers remain valid.""" if crate_name(path) not in {"layerfs-overlay", "layerfs-workspace", "layerfs-daemon"}: return [] - retired = (r"\b(?:Dentry|Dentries|DENTRY\w*|dentry\w*|ScratchRecord|SCRATCH_BYTES|" - r"OverlayScratch|IndexedScratch\w*|indexed_scratch\w*|owned_scratch\w*|" - r"scratch_(?:rows|bytes|page|contains|get|apply|keys\w*)|put_scratch|" - r"IndexedScope|IndexedKey|IndexedChange|IndexedApply|ExpectedValue)\b|" - r"\b(?:StatementKind|ServiceClass)::Scratch\b|" - r"\b(?:TABLE|FROM|INTO|UPDATE|ON)\s+scratch\b|" - r"\bpub\s+(?:struct|enum|trait)\s+Scratch\w*\b") - found = [(source.count("\n", 0, match.start()) + 1, - "retired overlay concept name; use directory_entry or operation_record terminology") - for match in re.finditer(retired, source)] + retired = ( + r"\b(?:Dentry|Dentries|DENTRY\w*|dentry\w*|ScratchRecord|SCRATCH_BYTES|" + r"OverlayScratch|IndexedScratch\w*|indexed_scratch\w*|owned_scratch\w*|" + r"scratch_(?:rows|bytes|page|contains|get|apply|keys\w*)|put_scratch|" + r"IndexedScope|IndexedKey|IndexedChange|IndexedApply|ExpectedValue)\b|" + r"\b(?:StatementKind|ServiceClass)::Scratch\b|" + r"\b(?:TABLE|FROM|INTO|UPDATE|ON)\s+scratch\b|" + r"\bpub\s+(?:struct|enum|trait)\s+Scratch\w*\b" + ) + found = [ + ( + source.count("\n", 0, match.start()) + 1, + "retired overlay concept name; use directory_entry or operation_record terminology", + ) + for match in re.finditer(retired, source) + ] if "scratch" in path.name or "dentry" in path.name: found.append((1, "retired overlay concept filename")) return found @@ -233,14 +313,18 @@ def production_files(core, include_sdk=True): for package in (core / "crates").glob("*"): if not package.is_dir(): continue - files.update(path for path in (package / "src").rglob("*") - if path.is_file() and path.suffix in (".rs", ".sql")) + files.update(path for path in (package / "src").rglob("*") if path.is_file() and path.suffix in (".rs", ".sql")) files.update(path for path in (package / "sql").rglob("*.sql") if path.is_file()) if not include_sdk: return sorted(files) api = core.parent / "api" files.update((api / "binding" / "src").rglob("*.rs")) - for directory in (api / "agent" / "sdk" / "src", api / "agent" / "cli" / "src", api / "backend" / "src", api / "runtime" / "src"): + for directory in ( + api / "agent" / "sdk" / "src", + api / "agent" / "cli" / "src", + api / "backend" / "src", + api / "runtime" / "src", + ): files.update(directory.rglob("*.js")) return sorted(files) diff --git a/core/tools/ci_tests.py b/core/tools/ci_tests.py index c7e6d2768..e981a9254 100644 --- a/core/tools/ci_tests.py +++ b/core/tools/ci_tests.py @@ -26,6 +26,7 @@ exists in layerfs-overlay (fast) and layerfs-workspace (heavy), so the push job also builds the heavy one; the filter keeps it from running. """ + import json import subprocess import sys @@ -75,9 +76,10 @@ def metadata(): out = subprocess.run( - ["cargo", "+1.85.1", "metadata", "--manifest-path", str(MANIFEST), - "--no-deps", "--format-version", "1"], - check=True, capture_output=True, text=True, + ["cargo", "+1.85.1", "metadata", "--manifest-path", str(MANIFEST), "--no-deps", "--format-version", "1"], + check=True, + capture_output=True, + text=True, ).stdout return json.loads(out) diff --git a/core/tools/phase7_services.py b/core/tools/phase7_services.py index 53384f2dc..6dff054f8 100644 --- a/core/tools/phase7_services.py +++ b/core/tools/phase7_services.py @@ -16,97 +16,126 @@ from urllib.parse import urlparse ROOT = Path(__file__).resolve().parents[2] -STATE = ROOT / 'target' / 'phase7-services' / 'settings.json' -POSTGRES_IMAGE = 'postgres@sha256:639ab7ceb90e13123085b741fb31ef493fba25463002f6da665352e7b534b652' -MINIO_IMAGE = 'cgr.dev/chainguard/minio@sha256:4692462f35d97d7e82c30371d82f057703c5d9489bcae726010594c812f2d285' -OWNER_LABEL = 'dev.layerfs.phase7.worktree' -NAMES = {'postgres': 'layerfs-postgres', 'minio': 'layerfs-minio'} -NETWORK = 'layerfs-phase7-cluster1' -VOLUMES = {'postgres': 'layerfs-phase7-postgres', 'minio': 'layerfs-phase7-minio'} +STATE = ROOT / "target" / "phase7-services" / "settings.json" +POSTGRES_IMAGE = "postgres@sha256:639ab7ceb90e13123085b741fb31ef493fba25463002f6da665352e7b534b652" +MINIO_IMAGE = "cgr.dev/chainguard/minio@sha256:4692462f35d97d7e82c30371d82f057703c5d9489bcae726010594c812f2d285" +OWNER_LABEL = "dev.layerfs.phase7.worktree" +NAMES = {"postgres": "layerfs-postgres", "minio": "layerfs-minio"} +NETWORK = "layerfs-phase7-cluster1" +VOLUMES = {"postgres": "layerfs-phase7-postgres", "minio": "layerfs-phase7-minio"} def docker(*args, input_text=None, check=True): - result = subprocess.run(['docker', *args], input=input_text, text=True, - capture_output=True, timeout=60) + result = subprocess.run(["docker", *args], input=input_text, text=True, capture_output=True, timeout=60) if check and result.returncode: - raise RuntimeError(f'docker {args[0]} failed: {result.stderr.strip()}') + raise RuntimeError(f"docker {args[0]} failed: {result.stderr.strip()}") return result def inspect(kind, name): - result = docker(kind, 'inspect', name, check=False) if kind in ('volume', 'network') else docker('inspect', name, check=False) + result = ( + docker(kind, "inspect", name, check=False) + if kind in ("volume", "network") + else docker("inspect", name, check=False) + ) if result.returncode: - missing = ('No such object:', 'No such container:', 'no such volume', 'No such network:') - if result.returncode == 1 and (any(marker.casefold() in result.stderr.casefold() for marker in missing) or (kind == 'network' and f'network {name} not found' in result.stderr.casefold())): + missing = ("No such object:", "No such container:", "no such volume", "No such network:") + if result.returncode == 1 and ( + any(marker.casefold() in result.stderr.casefold() for marker in missing) + or (kind == "network" and f"network {name} not found" in result.stderr.casefold()) + ): return None - raise RuntimeError(f'cannot inspect {kind} {name}: {result.stderr.strip()}') + raise RuntimeError(f"cannot inspect {kind} {name}: {result.stderr.strip()}") return json.loads(result.stdout)[0] def owned(kind, name): item = inspect(kind, name) if item is not None: - labels = item.get('Config', {}).get('Labels', {}) if kind == 'container' else item.get('Labels', {}) + labels = item.get("Config", {}).get("Labels", {}) if kind == "container" else item.get("Labels", {}) if (labels or {}).get(OWNER_LABEL) != str(ROOT): - raise RuntimeError(f'refusing foreign {kind}: {name}') + raise RuntimeError(f"refusing foreign {kind}: {name}") return item def ownership_inventory(): """Validate all resources before removing or creating any of them.""" - return {kind: {name: owned(kind, name) for name in names} - for kind, names in [('container', NAMES.values()), ('volume', VOLUMES.values()), ('network', [NETWORK])]} + return { + kind: {name: owned(kind, name) for name in names} + for kind, names in [("container", NAMES.values()), ("volume", VOLUMES.values()), ("network", [NETWORK])] + } def save(settings): STATE.parent.mkdir(parents=True, exist_ok=True) - STATE.write_text(json.dumps(settings, indent=2) + '\n') + STATE.write_text(json.dumps(settings, indent=2) + "\n") STATE.chmod(0o600) - exports = STATE.with_name('services.env') - exports.write_text(''.join(f'export {key}={shlex.quote(value)}\n' for key, value in settings['environment'].items())) + exports = STATE.with_name("services.env") + exports.write_text( + "".join(f"export {key}={shlex.quote(value)}\n" for key, value in settings["environment"].items()) + ) exports.chmod(0o600) def load(): if not STATE.exists(): - raise RuntimeError('no owned settings; run up first') + raise RuntimeError("no owned settings; run up first") settings = json.loads(STATE.read_text()) - if settings['owner'] != str(ROOT): - raise RuntimeError('settings belong to another worktree') + if settings["owner"] != str(ROOT): + raise RuntimeError("settings belong to another worktree") return settings def fresh_settings(): - return {'owner': str(ROOT), 'complete': False, 'epoch': secrets.token_hex(12), 'environment': { - 'LAYERFS_PG_HOST': '127.0.0.1', 'LAYERFS_PG_PORT': '5432', - 'LAYERFS_PG_DATABASE': 'layerfs', 'LAYERFS_PG_SCHEMA': 'layerfs', - 'LAYERFS_PG_USER': 'layerfs', 'LAYERFS_PG_PASSWORD': secrets.token_hex(24), - 'LAYERFS_PG_TLS': 'disabled', - 'LAYERFS_S3_ENDPOINT': 'http://127.0.0.1:9000', 'LAYERFS_S3_BUCKET': 'layerfs', - 'LAYERFS_S3_PREFIX': 'cluster1', 'LAYERFS_S3_REGION': 'us-east-1', - 'LAYERFS_S3_ACCESS_KEY': 'layerfs' + secrets.token_hex(8), - 'LAYERFS_S3_SECRET_KEY': secrets.token_hex(24), - }, 'images': {'postgres': POSTGRES_IMAGE, 'minio': MINIO_IMAGE}} + return { + "owner": str(ROOT), + "complete": False, + "epoch": secrets.token_hex(12), + "environment": { + "LAYERFS_PG_HOST": "127.0.0.1", + "LAYERFS_PG_PORT": "5432", + "LAYERFS_PG_DATABASE": "layerfs", + "LAYERFS_PG_SCHEMA": "layerfs", + "LAYERFS_PG_USER": "layerfs", + "LAYERFS_PG_PASSWORD": secrets.token_hex(24), + "LAYERFS_PG_TLS": "disabled", + "LAYERFS_S3_ENDPOINT": "http://127.0.0.1:9000", + "LAYERFS_S3_BUCKET": "layerfs", + "LAYERFS_S3_PREFIX": "cluster1", + "LAYERFS_S3_REGION": "us-east-1", + "LAYERFS_S3_ACCESS_KEY": "layerfs" + secrets.token_hex(8), + "LAYERFS_S3_SECRET_KEY": secrets.token_hex(24), + }, + "images": {"postgres": POSTGRES_IMAGE, "minio": MINIO_IMAGE}, + } def signed_bucket_request(environment, now=None): """Only the development bucket bootstrap; product requests live in layerfs-s3.""" now = now or datetime.now(timezone.utc) - date = now.strftime('%Y%m%d'); stamp = now.strftime('%Y%m%dT%H%M%SZ') - endpoint = environment['LAYERFS_S3_ENDPOINT']; host = urlparse(endpoint).netloc - path = '/' + environment['LAYERFS_S3_BUCKET']; body_hash = hashlib.sha256(b'').hexdigest() - headers = f'host:{host}\nx-amz-content-sha256:{body_hash}\nx-amz-date:{stamp}\n' - names = 'host;x-amz-content-sha256;x-amz-date' - canonical = f'PUT\n{path}\n\n{headers}\n{names}\n{body_hash}' - region = environment['LAYERFS_S3_REGION']; scope = f'{date}/{region}/s3/aws4_request' - to_sign = f'AWS4-HMAC-SHA256\n{stamp}\n{scope}\n{hashlib.sha256(canonical.encode()).hexdigest()}' - key = ('AWS4' + environment['LAYERFS_S3_SECRET_KEY']).encode() - for value in (date, region, 's3', 'aws4_request'): + date = now.strftime("%Y%m%d") + stamp = now.strftime("%Y%m%dT%H%M%SZ") + endpoint = environment["LAYERFS_S3_ENDPOINT"] + host = urlparse(endpoint).netloc + path = "/" + environment["LAYERFS_S3_BUCKET"] + body_hash = hashlib.sha256(b"").hexdigest() + headers = f"host:{host}\nx-amz-content-sha256:{body_hash}\nx-amz-date:{stamp}\n" + names = "host;x-amz-content-sha256;x-amz-date" + canonical = f"PUT\n{path}\n\n{headers}\n{names}\n{body_hash}" + region = environment["LAYERFS_S3_REGION"] + scope = f"{date}/{region}/s3/aws4_request" + to_sign = f"AWS4-HMAC-SHA256\n{stamp}\n{scope}\n{hashlib.sha256(canonical.encode()).hexdigest()}" + key = ("AWS4" + environment["LAYERFS_S3_SECRET_KEY"]).encode() + for value in (date, region, "s3", "aws4_request"): key = hmac.new(key, value.encode(), hashlib.sha256).digest() signature = hmac.new(key, to_sign.encode(), hashlib.sha256).hexdigest() - authorization = f'AWS4-HMAC-SHA256 Credential={environment["LAYERFS_S3_ACCESS_KEY"]}/{scope}, SignedHeaders={names}, Signature={signature}' - return urllib.request.Request(endpoint + path, data=b'', method='PUT', headers={ - 'Host': host, 'x-amz-content-sha256': body_hash, 'x-amz-date': stamp, 'Authorization': authorization}) + authorization = f"AWS4-HMAC-SHA256 Credential={environment['LAYERFS_S3_ACCESS_KEY']}/{scope}, SignedHeaders={names}, Signature={signature}" + return urllib.request.Request( + endpoint + path, + data=b"", + method="PUT", + headers={"Host": host, "x-amz-content-sha256": body_hash, "x-amz-date": stamp, "Authorization": authorization}, + ) def wait_ready(settings): @@ -115,134 +144,230 @@ def wait_ready(settings): pg = minio = False while time.monotonic() < deadline: if not pg: - pg = docker('exec', NAMES['postgres'], 'pg_isready', '-h', '127.0.0.1', '-U', 'layerfs', '-d', 'layerfs', check=False).returncode == 0 + pg = ( + docker( + "exec", + NAMES["postgres"], + "pg_isready", + "-h", + "127.0.0.1", + "-U", + "layerfs", + "-d", + "layerfs", + check=False, + ).returncode + == 0 + ) if not minio: try: - with urllib.request.urlopen('http://127.0.0.1:9000/minio/health/live', timeout=1) as response: + with urllib.request.urlopen("http://127.0.0.1:9000/minio/health/live", timeout=1) as response: minio = response.status == 200 except (urllib.error.URLError, TimeoutError, ConnectionError): pass if pg and minio: return for name in NAMES.values(): - item = owned('container', name) - if item is None or not item['State']['Running']: - raise RuntimeError(f'service exited during readiness: {name}') + item = owned("container", name) + if item is None or not item["State"]["Running"]: + raise RuntimeError(f"service exited during readiness: {name}") time.sleep(0.1) - raise RuntimeError('service readiness failed within declared setup budget') + raise RuntimeError("service readiness failed within declared setup budget") def postgres_sql(sql): - return docker('exec', '-i', NAMES['postgres'], 'psql', '-X', '-v', 'ON_ERROR_STOP=1', - '-U', 'layerfs', '-d', 'layerfs', '-At', input_text=sql).stdout + return docker( + "exec", + "-i", + NAMES["postgres"], + "psql", + "-X", + "-v", + "ON_ERROR_STOP=1", + "-U", + "layerfs", + "-d", + "layerfs", + "-At", + input_text=sql, + ).stdout def bootstrap(settings): - postgres_sql('CREATE SCHEMA layerfs;\nCHECKPOINT;\n') - with urllib.request.urlopen(signed_bucket_request(settings['environment']), timeout=5) as response: + postgres_sql("CREATE SCHEMA layerfs;\nCHECKPOINT;\n") + with urllib.request.urlopen(signed_bucket_request(settings["environment"]), timeout=5) as response: if response.status != 200: - raise RuntimeError('bucket create was not acknowledged') + raise RuntimeError("bucket create was not acknowledged") # Runtime tables are frozen in steps 6/7 and bootstrapped explicitly there. - settings['schema_bootstrap'] = 'namespace-only; C2/C5 runtime tables enter in steps 6/7' + settings["schema_bootstrap"] = "namespace-only; C2/C5 runtime tables enter in steps 6/7" def up(): inventory = ownership_inventory() - existing = [item for item in inventory['container'].values() if item is not None] + existing = [item for item in inventory["container"].values() if item is not None] if existing: settings = load() - if len(existing) != 2 or not settings['complete']: - raise RuntimeError('partial owned setup; inspect evidence and use explicit reset') + if len(existing) != 2 or not settings["complete"]: + raise RuntimeError("partial owned setup; inspect evidence and use explicit reset") validate(settings) - return status(settings, dependency_reuse='owned-services-and-volumes') - if any(inventory['volume'].values()) or any(inventory['network'].values()): - raise RuntimeError('orphaned owned setup; explicit reset required') - settings = fresh_settings(); save(settings) - for key, image in settings['images'].items(): - image_info = json.loads(docker('image', 'inspect', image).stdout)[0] - settings.setdefault('image_ids', {})[key] = image_info['Id'] + return status(settings, dependency_reuse="owned-services-and-volumes") + if any(inventory["volume"].values()) or any(inventory["network"].values()): + raise RuntimeError("orphaned owned setup; explicit reset required") + settings = fresh_settings() save(settings) - label = f'{OWNER_LABEL}={ROOT}' - docker('network', 'create', '--label', label, NETWORK) + for key, image in settings["images"].items(): + image_info = json.loads(docker("image", "inspect", image).stdout)[0] + settings.setdefault("image_ids", {})[key] = image_info["Id"] + save(settings) + label = f"{OWNER_LABEL}={ROOT}" + docker("network", "create", "--label", label, NETWORK) for volume in VOLUMES.values(): - docker('volume', 'create', '--label', label, volume) - e = settings['environment'] - common = ['--detach', '--label', label, '--network', NETWORK, '--cpus', '2', - '--memory', '512m', '--memory-swap', '512m', '--pids-limit', '256'] + docker("volume", "create", "--label", label, volume) + e = settings["environment"] + common = [ + "--detach", + "--label", + label, + "--network", + NETWORK, + "--cpus", + "2", + "--memory", + "512m", + "--memory-swap", + "512m", + "--pids-limit", + "256", + ] # Pass secrets in private environment files, never in printed argv/evidence. - pg_env = STATE.with_name('postgres.env'); minio_env = STATE.with_name('minio.env') - pg_env.write_text(f'POSTGRES_USER=layerfs\nPOSTGRES_DB=layerfs\nPOSTGRES_PASSWORD={e["LAYERFS_PG_PASSWORD"]}\nTZ=UTC\n') - minio_env.write_text(f'MINIO_ROOT_USER={e["LAYERFS_S3_ACCESS_KEY"]}\nMINIO_ROOT_PASSWORD={e["LAYERFS_S3_SECRET_KEY"]}\nMINIO_COMPRESS_ENABLE=off\nMINIO_KMS_AUTO_ENCRYPTION=off\nMINIO_BROWSER=off\n') - pg_env.chmod(0o600); minio_env.chmod(0o600) - docker('run', *common, '--name', NAMES['postgres'], '--publish', '127.0.0.1:5432:5432', - '--env-file', str(pg_env), '--volume', VOLUMES['postgres'] + ':/var/lib/postgresql/data', POSTGRES_IMAGE) + pg_env = STATE.with_name("postgres.env") + minio_env = STATE.with_name("minio.env") + pg_env.write_text( + f"POSTGRES_USER=layerfs\nPOSTGRES_DB=layerfs\nPOSTGRES_PASSWORD={e['LAYERFS_PG_PASSWORD']}\nTZ=UTC\n" + ) + minio_env.write_text( + f"MINIO_ROOT_USER={e['LAYERFS_S3_ACCESS_KEY']}\nMINIO_ROOT_PASSWORD={e['LAYERFS_S3_SECRET_KEY']}\nMINIO_COMPRESS_ENABLE=off\nMINIO_KMS_AUTO_ENCRYPTION=off\nMINIO_BROWSER=off\n" + ) + pg_env.chmod(0o600) + minio_env.chmod(0o600) + docker( + "run", + *common, + "--name", + NAMES["postgres"], + "--publish", + "127.0.0.1:5432:5432", + "--env-file", + str(pg_env), + "--volume", + VOLUMES["postgres"] + ":/var/lib/postgresql/data", + POSTGRES_IMAGE, + ) # The minimal image's named data volume is root-owned; this profile states UID 0. - docker('run', *common, '--name', NAMES['minio'], '--user', '0:0', '--publish', '127.0.0.1:9000:9000', - '--env-file', str(minio_env), '--volume', VOLUMES['minio'] + ':/data', MINIO_IMAGE, 'server', '/data') - wait_ready(settings); bootstrap(settings) - settings['complete'] = True; save(settings) + docker( + "run", + *common, + "--name", + NAMES["minio"], + "--user", + "0:0", + "--publish", + "127.0.0.1:9000:9000", + "--env-file", + str(minio_env), + "--volume", + VOLUMES["minio"] + ":/data", + MINIO_IMAGE, + "server", + "/data", + ) + wait_ready(settings) + bootstrap(settings) + settings["complete"] = True + save(settings) validate(settings) - return status(settings, dependency_reuse='pinned-local-images') + return status(settings, dependency_reuse="pinned-local-images") def validate(settings): for service, name in NAMES.items(): - item = owned('container', name) - if item is None or not item['State']['Running'] or item['Image'] != settings['image_ids'][service]: - raise RuntimeError(f'wrong image or unavailable owned service: {name}') - port = '5432/tcp' if service == 'postgres' else '9000/tcp' - binding = item['NetworkSettings']['Ports'].get(port) - if binding != [{'HostIp': '127.0.0.1', 'HostPort': port.split('/')[0]}]: - raise RuntimeError(f'wrong published endpoint: {name}') - limits = item['HostConfig'] - if limits['Memory'] != 512 * 1024 * 1024 or limits['MemorySwap'] != limits['Memory'] or limits['NanoCpus'] != 2_000_000_000: - raise RuntimeError(f'wrong declared resource profile: {name}') + item = owned("container", name) + if item is None or not item["State"]["Running"] or item["Image"] != settings["image_ids"][service]: + raise RuntimeError(f"wrong image or unavailable owned service: {name}") + port = "5432/tcp" if service == "postgres" else "9000/tcp" + binding = item["NetworkSettings"]["Ports"].get(port) + if binding != [{"HostIp": "127.0.0.1", "HostPort": port.split("/")[0]}]: + raise RuntimeError(f"wrong published endpoint: {name}") + limits = item["HostConfig"] + if ( + limits["Memory"] != 512 * 1024 * 1024 + or limits["MemorySwap"] != limits["Memory"] + or limits["NanoCpus"] != 2_000_000_000 + ): + raise RuntimeError(f"wrong declared resource profile: {name}") def status(settings=None, dependency_reuse=None): - settings = settings or load(); validate(settings) - pg_settings = postgres_sql("SELECT name || '=' || setting FROM pg_settings WHERE name IN ('server_version','server_encoding','fsync','synchronous_commit','full_page_writes','wal_level','shared_buffers','default_transaction_isolation','statement_timeout','TimeZone') ORDER BY name;\n") - return {'status': 'PASS', 'owner': str(ROOT), 'epoch': settings['epoch'], - 'images': settings['images'], 'image_ids': settings['image_ids'], - 'postgres_settings': pg_settings.splitlines(), - 'postgres_settings_sha256': hashlib.sha256(pg_settings.encode()).hexdigest(), - 'minio_version': docker('exec', NAMES['minio'], '/usr/bin/minio', '--version').stdout.splitlines()[0], - 'schema_bootstrap': settings.get('schema_bootstrap'), 'dependency_reuse': dependency_reuse, - 'environment_file': str(STATE.with_name('services.env')), - 'resource_profile': {'cpus_per_service': 2, 'memory_bytes_per_service': 512 * 1024 * 1024, 'swap': 'disabled', 'minio_uid': 0}} + settings = settings or load() + validate(settings) + pg_settings = postgres_sql( + "SELECT name || '=' || setting FROM pg_settings WHERE name IN ('server_version','server_encoding','fsync','synchronous_commit','full_page_writes','wal_level','shared_buffers','default_transaction_isolation','statement_timeout','TimeZone') ORDER BY name;\n" + ) + return { + "status": "PASS", + "owner": str(ROOT), + "epoch": settings["epoch"], + "images": settings["images"], + "image_ids": settings["image_ids"], + "postgres_settings": pg_settings.splitlines(), + "postgres_settings_sha256": hashlib.sha256(pg_settings.encode()).hexdigest(), + "minio_version": docker("exec", NAMES["minio"], "/usr/bin/minio", "--version").stdout.splitlines()[0], + "schema_bootstrap": settings.get("schema_bootstrap"), + "dependency_reuse": dependency_reuse, + "environment_file": str(STATE.with_name("services.env")), + "resource_profile": { + "cpus_per_service": 2, + "memory_bytes_per_service": 512 * 1024 * 1024, + "swap": "disabled", + "minio_uid": 0, + }, + } def down(): inventory = ownership_inventory() - for name, item in inventory['container'].items(): + for name, item in inventory["container"].items(): if item is not None: - docker('rm', '--force', name) - for name, item in inventory['volume'].items(): + docker("rm", "--force", name) + for name, item in inventory["volume"].items(): if item is not None: - docker('volume', 'rm', name) - for name, item in inventory['network'].items(): + docker("volume", "rm", name) + for name, item in inventory["network"].items(): if item is not None: - docker('network', 'rm', name) + docker("network", "rm", name) if STATE.exists(): - settings = load(); settings['complete'] = False; save(settings) - return {'status': 'PASS', 'cleanup': 'owned-containers-volumes-network-removed'} + settings = load() + settings["complete"] = False + save(settings) + return {"status": "PASS", "cleanup": "owned-containers-volumes-network-removed"} def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('command', choices=['up', 'reset', 'down', 'status']) + parser.add_argument("command", choices=["up", "reset", "down", "status"]) args = parser.parse_args() try: - if args.command == 'reset': - down(); result = up() + if args.command == "reset": + down() + result = up() else: - result = {'up': up, 'down': down, 'status': status}[args.command]() + result = {"up": up, "down": down, "status": status}[args.command]() print(json.dumps(result, indent=2)) return 0 except (RuntimeError, subprocess.TimeoutExpired, urllib.error.URLError, OSError) as error: - print(json.dumps({'status': 'FAIL', 'reason': str(error)})) + print(json.dumps({"status": "FAIL", "reason": str(error)})) return 1 -if __name__ == '__main__': +if __name__ == "__main__": raise SystemExit(main()) diff --git a/core/tools/production_loc.py b/core/tools/production_loc.py index 3aeb5c8fb..431e1c3e6 100644 --- a/core/tools/production_loc.py +++ b/core/tools/production_loc.py @@ -13,6 +13,7 @@ Usage: python3 core/tools/production_loc.py [--json] [--root DIR] """ + import argparse import json from pathlib import Path @@ -46,7 +47,7 @@ def strip_rust(text): else: index += 1 continue - if text.startswith('#[cfg(test)]', index) or text.startswith('#[cfg(any(test', index): + if text.startswith("#[cfg(test)]", index) or text.startswith("#[cfg(any(test", index): pending_test_attribute = True index = text.find("\n", index) index = length if index < 0 else index + 1 @@ -144,8 +145,7 @@ def main(): if arguments.json: print(json.dumps(totals, sort_keys=True)) else: - print(f"core {totals['core']} legacy {totals['legacy']} combined {totals['combined']} " - f"files {totals['files']}") + print(f"core {totals['core']} legacy {totals['legacy']} combined {totals['combined']} files {totals['files']}") if __name__ == "__main__": diff --git a/core/tools/sdk_telemetry_report.py b/core/tools/sdk_telemetry_report.py index 850c08bcb..75c5c8d7a 100644 --- a/core/tools/sdk_telemetry_report.py +++ b/core/tools/sdk_telemetry_report.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Summarize one #236 route from retained LayerFS LFT1 telemetry records.""" + import argparse import hashlib import json @@ -22,9 +23,14 @@ def records(path): def one(events, key=None, name=None): - matches = [event for event in events if event.get("kind") == "operation" - and (key is None or event.get("key") == key) - and event.get("timing") and event["timing"].get("name") == name] + matches = [ + event + for event in events + if event.get("kind") == "operation" + and (key is None or event.get("key") == key) + and event.get("timing") + and event["timing"].get("name") == name + ] if len(matches) != 1: raise ValueError(f"expected one {name} telemetry event, got {len(matches)}") return matches[0] @@ -53,11 +59,13 @@ def metrics(event): def summarize(root): identity = json.loads((root / "identities.json").read_text()) run = f"{identity['telemetry_run']:032x}" - host = [event for event in records(root / "raw-host.log") - if event.get("run") == run and event.get("role") == 1] + host = [event for event in records(root / "raw-host.log") if event.get("run") == run and event.get("role") == 1] sandbox = (root / "primary-sandbox-id.txt").read_text().strip() - daemon = [event for event in records(root / f"layerfs-{sandbox}.stderr") - if event.get("run") == run and event.get("role") == 2] + daemon = [ + event + for event in records(root / f"layerfs-{sandbox}.stderr") + if event.get("run") == run and event.get("role") == 2 + ] route = metrics(one(host, 1000, "sdk.route")) lookup_recorded = any(event.get("key") in (2001, 2002) for event in host) phases = [] @@ -66,20 +74,24 @@ def summarize(root): phase = {"name": name, "sdk_host_process": metrics(sdk)} if lookup_recorded and name != "create": start = sdk["opened_ns"] - within = [event for event in host - if start <= event.get("opened_ns", -1) < start + sdk["timing"]["elapsed_ns"]] + within = [ + event for event in host if start <= event.get("opened_ns", -1) < start + sdk["timing"]["elapsed_ns"] + ] # A retained control session reuses one checked Hello across rapid # calls, so a window may legitimately carry no Hello of its own. # Report the observed count rather than assuming one per call. - hellos = [event for event in within - if event.get("kind") == "operation" and event.get("key") == 2002 - and event.get("timing", {}).get("name") == "owner.hello"] + hellos = [ + event + for event in within + if event.get("kind") == "operation" + and event.get("key") == 2002 + and event.get("timing", {}).get("name") == "owner.hello" + ] phase["owner_lookup"] = { "hello_count": len(hellos), "hello": metrics(hellos[0]) if len(hellos) == 1 else None, } - ports = [event for event in within - if event.get("kind") == "operation" and event.get("key") == 2001] + ports = [event for event in within if event.get("kind") == "operation" and event.get("key") == 2001] if ports: phase["owner_lookup"]["docker_port"] = metrics(ports[0]) if daemon_name: @@ -87,9 +99,11 @@ def summarize(root): # not comparable with the host's. The route runs more than one call # for some names (the test issues several Execs), and the measured # one is the first: it is the call the SDK phase window covers. - matches = [event for event in daemon - if event.get("kind") == "operation" - and event.get("timing", {}).get("name") == daemon_name] + matches = [ + event + for event in daemon + if event.get("kind") == "operation" and event.get("timing", {}).get("name") == daemon_name + ] if not matches: raise ValueError(f"no {daemon_name} daemon event") phase["daemon_process"] = metrics(matches[0]) @@ -97,9 +111,10 @@ def summarize(root): else: phase["daemon_process"] = None phases.append(phase) - if not route["success"] or not all(p["sdk_host_process"]["success"] - and (p["daemon_process"] is None or p["daemon_process"]["success"]) - for p in phases): + if not route["success"] or not all( + p["sdk_host_process"]["success"] and (p["daemon_process"] is None or p["daemon_process"]["success"]) + for p in phases + ): raise ValueError("one operation failed") call_sum = sum(p["sdk_host_process"]["elapsed_ns"] for p in phases) if route["elapsed_ns"] < call_sum: @@ -125,12 +140,22 @@ def main(): root = args.output report = summarize(root) (root / "report.json").open("x").write(json.dumps(report, indent=2, sort_keys=True) + "\n") - manifest = {path.name: hashlib.sha256(path.read_bytes()).hexdigest() - for path in sorted(root.iterdir()) if path.is_file() and path.name != "manifest.json"} + manifest = { + path.name: hashlib.sha256(path.read_bytes()).hexdigest() + for path in sorted(root.iterdir()) + if path.is_file() and path.name != "manifest.json" + } (root / "manifest.json").open("x").write(json.dumps(manifest, indent=2, sort_keys=True) + "\n") - print(json.dumps({"status": report["functional_status"], "admission": report["performance_admission"], - "route_elapsed_ns": report["route_host_process"]["elapsed_ns"], - "files": len(manifest)})) + print( + json.dumps( + { + "status": report["functional_status"], + "admission": report["performance_admission"], + "route_elapsed_ns": report["route_host_process"]["elapsed_ns"], + "files": len(manifest), + } + ) + ) if __name__ == "__main__": diff --git a/core/tools/substitution-proof/run.py b/core/tools/substitution-proof/run.py index b62408eae..55f5d29d5 100755 --- a/core/tools/substitution-proof/run.py +++ b/core/tools/substitution-proof/run.py @@ -44,7 +44,7 @@ "c2-only": ("baseline", "candidate"), "combined": ("candidate", "candidate"), } -MANIFEST = '''# Substitution arm: {arm} (generated by run.py; the record is matrix.json) +MANIFEST = """# Substitution arm: {arm} (generated by run.py; the record is matrix.json) # # The consumer source is frozen and shared (`../../consumer/`); the only difference # between the four arms is which revision each product dependency resolves to. @@ -76,7 +76,7 @@ [[bin]] name = "cross_revision" path = "../../consumer/cross_revision.rs" -''' +""" def sh(*command: str, **kwargs) -> subprocess.CompletedProcess: @@ -116,8 +116,7 @@ def materialize(root: Path, arm: str, revisions: dict[str, Path]) -> Path: # so that a mixed arm still has exactly one copy in its graph. for crate, side in zip(CRATES, ("head", *ARMS[arm])): source = (REPO if side == "head" else revisions[side]) / "core" / "crates" / crate - shutil.copytree(source, tree / "crates" / crate, - ignore=shutil.ignore_patterns("target", "*.pyc")) + shutil.copytree(source, tree / "crates" / crate, ignore=shutil.ignore_patterns("target", "*.pyc")) return tree @@ -151,7 +150,8 @@ def main() -> int: arm_record: dict[str, object] = { "c1": commits[c1_side] if c1_side != "head" else commits["candidate"], "c2": commits[c2_side] if c2_side != "head" else commits["candidate"], - "c1_source": c1_side, "c2_source": c2_side, + "c1_source": c1_side, + "c2_source": c2_side, "manifest_sha256": digest(manifest), "tree": str(tree), } @@ -160,9 +160,10 @@ def main() -> int: source = (REPO if side == "head" else revisions[side]) / "core" / "crates" / crate live = tree / "crates" / crate differing = [ - str(p.relative_to(source)) for p in source.rglob("*.rs") - if p.is_file() and (not (live / p.relative_to(source)).exists() - or digest(p) != digest(live / p.relative_to(source))) + str(p.relative_to(source)) + for p in source.rglob("*.rs") + if p.is_file() + and (not (live / p.relative_to(source)).exists() or digest(p) != digest(live / p.relative_to(source))) ] arm_record[f"{crate}_differs_from_revision"] = differing environment = {"CARGO_TARGET_DIR": str(root / "target")} @@ -172,9 +173,24 @@ def main() -> int: "passed": "test result: ok" in test.stdout, "tail": (test.stdout + test.stderr).strip().splitlines()[-3:], } - example = sh("cargo", "+1.85.1", "run", "--quiet", "--manifest-path", str(manifest), - "--example", "filesystem_primitives_candidate", "--", - "--files", "2000", "--changes", "200", "--samples", "1", env={**os.environ, **environment}) + example = sh( + "cargo", + "+1.85.1", + "run", + "--quiet", + "--manifest-path", + str(manifest), + "--example", + "filesystem_primitives_candidate", + "--", + "--files", + "2000", + "--changes", + "200", + "--samples", + "1", + env={**os.environ, **environment}, + ) stdout = example.stdout.strip().splitlines() # The example prints both identity and one timing field; only identity is # compared across arms, so the elapsed field is stripped from the record. @@ -185,8 +201,11 @@ def main() -> int: "identity_lines": identity, } matrix["arms"][arm] = arm_record - print(f"{arm:9s} test={'ok' if arm_record['test']['passed'] else 'FAILED'} " - f"example={'ok' if example.returncode == 0 else 'FAILED'}", flush=True) + print( + f"{arm:9s} test={'ok' if arm_record['test']['passed'] else 'FAILED'} " + f"example={'ok' if example.returncode == 0 else 'FAILED'}", + flush=True, + ) # Cross-revision persisted-data checks: create the Store under one arm and read # it under another. A refusal is recorded with its typed error and exit code. @@ -194,9 +213,19 @@ def main() -> int: workdir.mkdir(parents=True, exist_ok=True) def arm_binary(arm: str, *arguments: str) -> subprocess.CompletedProcess: - return sh("cargo", "+1.85.1", "run", "--quiet", "--manifest-path", - str(HERE / "arms" / arm / "Cargo.toml"), "--bin", "cross_revision", - "--", *arguments, env={**os.environ, "CARGO_TARGET_DIR": str(root / "target")}) + return sh( + "cargo", + "+1.85.1", + "run", + "--quiet", + "--manifest-path", + str(HERE / "arms" / arm / "Cargo.toml"), + "--bin", + "cross_revision", + "--", + *arguments, + env={**os.environ, "CARGO_TARGET_DIR": str(root / "target")}, + ) checks = ( ("baseline-create->baseline-read", "baseline", "baseline"), @@ -223,22 +252,22 @@ def arm_binary(arm: str, *arguments: str) -> subprocess.CompletedProcess: "read_stdout": "" if opened is None else opened.stdout.strip(), "read_refused": bool(opened is not None and opened.returncode == 3), } - print(f"{name:34s} create={created.returncode} read={cross[name]['read_exit_code']} " - f"{cross[name]['read_stdout'][:60]}", flush=True) + print( + f"{name:34s} create={created.returncode} read={cross[name]['read_exit_code']} " + f"{cross[name]['read_stdout'][:60]}", + flush=True, + ) matrix["cross_revision"] = cross baseline_identity = matrix["arms"]["baseline"]["example"]["identity_lines"] matrix["canonical_identity"] = { "lines_compared": len(baseline_identity), "identical_in_every_arm": all( - record["example"]["identity_lines"] == baseline_identity - for record in matrix["arms"].values() + record["example"]["identity_lines"] == baseline_identity for record in matrix["arms"].values() ), "baseline": baseline_identity, } - matrix["test_passed_in_every_arm"] = all( - record["test"]["passed"] for record in matrix["arms"].values() - ) + matrix["test_passed_in_every_arm"] = all(record["test"]["passed"] for record in matrix["arms"].values()) Path(arguments.out).write_text(json.dumps(matrix, indent=2, sort_keys=True) + "\n") print(f"matrix -> {arguments.out}") return 0 diff --git a/core/tools/test_check_fuser_integrity.py b/core/tools/test_check_fuser_integrity.py index 298f9fe59..3ba015b80 100644 --- a/core/tools/test_check_fuser_integrity.py +++ b/core/tools/test_check_fuser_integrity.py @@ -7,9 +7,17 @@ import unittest import tomllib -from check_fuser_integrity import (OFFICIAL_GIT, config_errors, lock_errors, - manifest_errors, package_errors, patched_package_errors, - PATCH_PACKAGE, PATCH_RECORD, PATCH_MANIFESTS) +from check_fuser_integrity import ( + OFFICIAL_GIT, + config_errors, + lock_errors, + manifest_errors, + package_errors, + patched_package_errors, + PATCH_PACKAGE, + PATCH_RECORD, + PATCH_MANIFESTS, +) REV = "e48279fab0ddbe4a6e22aefad3cbfdf796d0b0b7" @@ -20,55 +28,59 @@ def test_official_registry_dependency_is_allowed(self): def test_direct_and_renamed_patch_are_rejected(self): for dependency in ('fuser={path="../fuser"}', 'alias={package="fuser",git="https://example.com/fuser"}'): - self.assertTrue(manifest_errors(tomllib.loads('[patch.crates-io]\n' + dependency))) + self.assertTrue(manifest_errors(tomllib.loads("[patch.crates-io]\n" + dependency))) def test_replace_is_rejected(self): self.assertTrue(manifest_errors(tomllib.loads('[replace]\n"fuser:0.18.0"={path="../fuser"}'))) def test_nested_target_path_override_is_rejected(self): - self.assertTrue(manifest_errors(tomllib.loads('[target."cfg(unix)".dependencies]\nalias={package="fuser",path="../fuser"}'))) + self.assertTrue( + manifest_errors(tomllib.loads('[target."cfg(unix)".dependencies]\nalias={package="fuser",path="../fuser"}')) + ) def test_git_requires_explicit_approved_immutable_official_revision(self): doc = tomllib.loads(f'[dependencies]\nfuser={{git="{OFFICIAL_GIT}",rev="{REV}"}}') self.assertTrue(manifest_errors(doc)) self.assertFalse(manifest_errors(doc, (REV,))) - for git, rev in [('https://github.com/other/fuser.git', REV), (OFFICIAL_GIT, 'main')]: - self.assertTrue(manifest_errors({'dependencies': {'fuser': {'git': git, 'rev': rev}}}, (REV,))) + for git, rev in [("https://github.com/other/fuser.git", REV), (OFFICIAL_GIT, "main")]: + self.assertTrue(manifest_errors({"dependencies": {"fuser": {"git": git, "rev": rev}}}, (REV,))) def test_branch_and_custom_registry_are_rejected(self): - self.assertTrue(manifest_errors({'dependencies': {'fuser': {'git': OFFICIAL_GIT, 'rev': REV, 'branch': 'master'}}}, (REV,))) - self.assertTrue(manifest_errors({'dependencies': {'fuser': {'registry': 'custom', 'version': '0.18'}}})) + self.assertTrue( + manifest_errors({"dependencies": {"fuser": {"git": OFFICIAL_GIT, "rev": REV, "branch": "master"}}}, (REV,)) + ) + self.assertTrue(manifest_errors({"dependencies": {"fuser": {"registry": "custom", "version": "0.18"}}})) def test_local_lock_and_unapproved_git_lock_are_rejected(self): - self.assertTrue(lock_errors({'package': [{'name': 'fuser', 'version': '0.18.0'}]})) - source = f'git+{OFFICIAL_GIT}?rev={REV}#{REV}' - self.assertTrue(lock_errors({'package': [{'name': 'fuser', 'source': source}]})) - self.assertFalse(lock_errors({'package': [{'name': 'fuser', 'source': source}]}, (REV,))) + self.assertTrue(lock_errors({"package": [{"name": "fuser", "version": "0.18.0"}]})) + source = f"git+{OFFICIAL_GIT}?rev={REV}#{REV}" + self.assertTrue(lock_errors({"package": [{"name": "fuser", "source": source}]})) + self.assertFalse(lock_errors({"package": [{"name": "fuser", "source": source}]}, (REV,))) def test_registry_lock_requires_checksum(self): - item = {'name': 'fuser', 'source': 'registry+https://github.com/rust-lang/crates.io-index'} - self.assertTrue(lock_errors({'package': [item]})) - item['checksum'] = '1' * 64 - self.assertFalse(lock_errors({'package': [item]})) + item = {"name": "fuser", "source": "registry+https://github.com/rust-lang/crates.io-index"} + self.assertTrue(lock_errors({"package": [item]})) + item["checksum"] = "1" * 64 + self.assertFalse(lock_errors({"package": [item]})) def test_only_exact_owner_authorized_manifest_patch_is_allowed(self): - path = PATCH_MANIFESTS['core/Cargo.toml'] - doc = {'patch': {'crates-io': {'fuser': {'version': '=0.18.0', 'path': path}}}} + path = PATCH_MANIFESTS["core/Cargo.toml"] + doc = {"patch": {"crates-io": {"fuser": {"version": "=0.18.0", "path": path}}}} self.assertFalse(manifest_errors(doc, patch_path=path)) self.assertTrue(manifest_errors(doc)) - for key, value in [('path', '../elsewhere'), ('version', '0.18')]: - changed = {'patch': {'crates-io': {'fuser': dict(doc['patch']['crates-io']['fuser'])}}} - changed['patch']['crates-io']['fuser'][key] = value + for key, value in [("path", "../elsewhere"), ("version", "0.18")]: + changed = {"patch": {"crates-io": {"fuser": dict(doc["patch"]["crates-io"]["fuser"])}}} + changed["patch"]["crates-io"]["fuser"][key] = value self.assertTrue(manifest_errors(changed, patch_path=path)) self.assertTrue(manifest_errors({}, patch_path=path)) def test_patch_lock_requires_owning_context_exact_version_and_no_false_checksum(self): - item = {'name': 'fuser', 'version': '0.18.0'} - self.assertFalse(lock_errors({'package': [item]}, patched=True)) - for key, value in [('version', '0.18.1'), ('checksum', '1' * 64)]: - self.assertTrue(lock_errors({'package': [item | {key: value}]}, patched=True)) - registry = item | {'source': 'registry+https://github.com/rust-lang/crates.io-index', 'checksum': '1' * 64} - self.assertTrue(lock_errors({'package': [registry]}, patched=True)) + item = {"name": "fuser", "version": "0.18.0"} + self.assertFalse(lock_errors({"package": [item]}, patched=True)) + for key, value in [("version", "0.18.1"), ("checksum", "1" * 64)]: + self.assertTrue(lock_errors({"package": [item | {key: value}]}, patched=True)) + registry = item | {"source": "registry+https://github.com/rust-lang/crates.io-index", "checksum": "1" * 64} + self.assertTrue(lock_errors({"package": [registry]}, patched=True)) def test_authorized_package_rejects_unrelated_edits_inventory_changes_and_redirects(self): repository = Path(__file__).resolve().parents[2] @@ -77,23 +89,29 @@ def test_authorized_package_rejects_unrelated_edits_inventory_changes_and_redire shutil.copytree(repository / PATCH_RECORD, root / PATCH_RECORD) shutil.copytree(repository / PATCH_PACKAGE, root / PATCH_PACKAGE) self.assertFalse(patched_package_errors(root)) - for name in ['src/time.rs', 'src/session.rs', 'src/session/lifecycle.rs', 'src/lib.rs', 'CHANGELOG.md', - 'src/read_buf.rs']: + for name in [ + "src/time.rs", + "src/session.rs", + "src/session/lifecycle.rs", + "src/lib.rs", + "CHANGELOG.md", + "src/read_buf.rs", + ]: path = root / PATCH_PACKAGE / name original = path.read_bytes() - path.write_bytes(original + b'// unauthorized edit\n') + path.write_bytes(original + b"// unauthorized edit\n") self.assertTrue(patched_package_errors(root)) path.write_bytes(original) - path = root / PATCH_RECORD / 'provenance.json' + path = root / PATCH_RECORD / "provenance.json" original = path.read_bytes() - path.write_bytes(original + b'\n') + path.write_bytes(original + b"\n") self.assertTrue(patched_package_errors(root)) path.write_bytes(original) - injected = root / PATCH_PACKAGE / 'src/extra.rs' - injected.write_text('extra') + injected = root / PATCH_PACKAGE / "src/extra.rs" + injected.write_text("extra") self.assertTrue(patched_package_errors(root)) injected.unlink() - injected.symlink_to(root / PATCH_PACKAGE / 'src/time.rs') + injected.symlink_to(root / PATCH_PACKAGE / "src/time.rs") self.assertTrue(patched_package_errors(root)) def test_lifecycle_provenance_and_diff_are_both_required_and_exact(self): @@ -103,11 +121,15 @@ def test_lifecycle_provenance_and_diff_are_both_required_and_exact(self): shutil.copytree(repository / PATCH_RECORD, root / PATCH_RECORD) shutil.copytree(repository / PATCH_PACKAGE, root / PATCH_PACKAGE) self.assertFalse(patched_package_errors(root)) - for name in ['lifecycle-provenance.json', 'session-lifecycle.patch', - 'receive-buffer-provenance.json', 'receive-buffer.patch']: + for name in [ + "lifecycle-provenance.json", + "session-lifecycle.patch", + "receive-buffer-provenance.json", + "receive-buffer.patch", + ]: path = root / PATCH_RECORD / name original = path.read_bytes() - path.write_bytes(original + b'\n') + path.write_bytes(original + b"\n") self.assertTrue(patched_package_errors(root)) path.unlink() self.assertTrue(patched_package_errors(root)) @@ -115,27 +137,27 @@ def test_lifecycle_provenance_and_diff_are_both_required_and_exact(self): self.assertFalse(patched_package_errors(root)) def test_cargo_directory_and_path_overrides_are_rejected(self): - self.assertTrue(config_errors({'paths': ['../fuser']})) - self.assertTrue(config_errors({'source': {'vendor': {'directory': '../vendor'}}})) - self.assertFalse(config_errors({'target': {'aarch64': {'rustflags': ['--cfg', 'aes_armv8']}}})) + self.assertTrue(config_errors({"paths": ["../fuser"]})) + self.assertTrue(config_errors({"source": {"vendor": {"directory": "../vendor"}}})) + self.assertFalse(config_errors({"target": {"aarch64": {"rustflags": ["--cfg", "aes_armv8"]}}})) def test_archive_verification_catches_modified_and_extra_source(self): with TemporaryDirectory() as root: root = Path(root) - source = root / 'fuser-0.18.0' - (source / 'src').mkdir(parents=True) - path = source / 'src/time.rs' - path.write_bytes(b'original source\n') - archive = root / 'fuser.crate' - with tarfile.open(archive, 'w:gz') as file: - info = tarfile.TarInfo('fuser-0.18.0/src/time.rs') + source = root / "fuser-0.18.0" + (source / "src").mkdir(parents=True) + path = source / "src/time.rs" + path.write_bytes(b"original source\n") + archive = root / "fuser.crate" + with tarfile.open(archive, "w:gz") as file: + info = tarfile.TarInfo("fuser-0.18.0/src/time.rs") info.size = len(path.read_bytes()) file.addfile(info, io.BytesIO(path.read_bytes())) checksum = hashlib.sha256(archive.read_bytes()).hexdigest() self.assertFalse(package_errors(archive, source, checksum)) - path.write_bytes(b'modified\n') + path.write_bytes(b"modified\n") self.assertTrue(package_errors(archive, source, checksum)) - path.write_bytes(b'original source\n') - (source / 'src/injected.rs').write_text('extra') + path.write_bytes(b"original source\n") + (source / "src/injected.rs").write_text("extra") self.assertTrue(package_errors(archive, source, checksum)) - self.assertTrue(package_errors(archive, source, '0' * 64)) + self.assertTrue(package_errors(archive, source, "0" * 64)) diff --git a/core/tools/test_check_product_boundary.py b/core/tools/test_check_product_boundary.py index aae012550..7130c7ed3 100644 --- a/core/tools/test_check_product_boundary.py +++ b/core/tools/test_check_product_boundary.py @@ -2,18 +2,29 @@ from tempfile import TemporaryDirectory import unittest -from check_product_boundary import (RETIRED_PACKAGES, dependency_violations, production_files, - unsafe_violations, violations) +from check_product_boundary import ( + RETIRED_PACKAGES, + dependency_violations, + production_files, + unsafe_violations, + violations, +) class ProductBoundaryTests(unittest.TestCase): def test_native_fuse_owns_requests_without_reverse_daemon_dependency(self): path = Path("core/crates/layerfs-fuse/src/ports.rs") - for source in ["use layerfs_workspace::SourceView;", "use layerfs_overlay::NativeMount;", - "use layerfs_content::filesystem::PathName;"]: + for source in [ + "use layerfs_workspace::SourceView;", + "use layerfs_overlay::NativeMount;", + "use layerfs_content::filesystem::PathName;", + ]: self.assertFalse(violations(path, source)) - for source in ["use layerfs_daemon::Pending;", "use layerfs_persistence::Handles;", - "use layerfs_api::WorkspaceApi;"]: + for source in [ + "use layerfs_daemon::Pending;", + "use layerfs_persistence::Handles;", + "use layerfs_api::WorkspaceApi;", + ]: self.assertTrue(violations(path, source)) self.assertTrue(unsafe_violations(path, "unsafe fn syscall() {}")) root = Path("core/crates/layerfs-fuse/src/lib.rs") @@ -21,12 +32,20 @@ def test_native_fuse_owns_requests_without_reverse_daemon_dependency(self): self.assertTrue(unsafe_violations(root, "mod mount;")) prefix = '[package]\nname="layerfs-fuse"\n[dependencies]\n' self.assertTrue(dependency_violations(prefix + 'layerfs-daemon={path="../layerfs-daemon"}')) - self.assertFalse(dependency_violations('[package]\nname="layerfs-daemon"\n[dependencies]\nlayerfs-fuse={path="../layerfs-fuse"}')) + self.assertFalse( + dependency_violations( + '[package]\nname="layerfs-daemon"\n[dependencies]\nlayerfs-fuse={path="../layerfs-fuse"}' + ) + ) def test_overlay_terminology_distinguishes_records_from_temporary_buffers(self): path = Path("core/crates/layerfs-overlay/src/contract/types.rs") - for source in ["pub struct Dentry {}", "pub struct ScratchRecord {}", - "pub struct IndexedScope {}", "fn f() { StatementKind::Scratch; }"]: + for source in [ + "pub struct Dentry {}", + "pub struct ScratchRecord {}", + "pub struct IndexedScope {}", + "fn f() { StatementKind::Scratch; }", + ]: self.assertTrue(violations(path, source)) self.assertFalse(violations(path, "pub struct DirectoryEntry {}\npub struct OperationRecord {}")) self.assertFalse(violations(path, "fn encode() { let scratch = Vec::new(); } // temporary buffer")) @@ -40,8 +59,12 @@ def test_daemon_store_is_provider_independent(self): path = Path("core/crates/layerfs-daemon/src/store/open.rs") for source in ["use layerfs_storage::Storage;", "use layerfs_history::HistoryCatalog;"]: self.assertFalse(violations(path, source)) - for source in ["use layerfs_persistence::Handles;", "use layerfs_sdk::RemoteObjects;", - "use rusqlite::Connection;", "use std::path::{Path, PathBuf};"]: + for source in [ + "use layerfs_persistence::Handles;", + "use layerfs_sdk::RemoteObjects;", + "use rusqlite::Connection;", + "use std::path::{Path, PathBuf};", + ]: self.assertTrue(violations(path, source)) bootstrap = Path("core/crates/layerfs-daemon/src/bootstrap.rs") self.assertFalse(violations(bootstrap, "use layerfs_persistence::Handles;")) @@ -56,19 +79,31 @@ def test_persistence_seal_ffi_is_limited_to_the_authorized_module(self): self.assertTrue(unsafe_violations(base / "lib.rs", "pub mod store;")) def test_retired_transport_paths_cannot_return(self): - for relative in ("layerfs-api/src/client/call.rs", "layerfs-api/src/runtime/owner.rs", - "layerfs-daemon/src/upstream/owner.rs", "layerfs-bridge/src/codec/frame.rs", - "layerfs-bridge/src/contract/frame.rs", "layerfs-bridge/src/native/framing.rs", - "layerfs-api/core/src/lib.rs"): + for relative in ( + "layerfs-api/src/client/call.rs", + "layerfs-api/src/runtime/owner.rs", + "layerfs-daemon/src/upstream/owner.rs", + "layerfs-bridge/src/codec/frame.rs", + "layerfs-bridge/src/contract/frame.rs", + "layerfs-bridge/src/native/framing.rs", + "layerfs-api/core/src/lib.rs", + ): findings = violations(Path("core/crates") / relative, "pub use std::io::Read;") self.assertTrue(any("retired host-mediated" in message for _, message in findings), relative) def test_retired_packages_cannot_return(self): - self.assertEqual(RETIRED_PACKAGES, ( - "layerfs-server", "layerfs-fuse-legacy", "layerfs-sandbox-legacy", - "layerfs-sdk-legacy", "layerfs-daemon-legacy", "layerfs-bridge-legacy", - "layerfs-workspace-legacy", - )) + self.assertEqual( + RETIRED_PACKAGES, + ( + "layerfs-server", + "layerfs-fuse-legacy", + "layerfs-sandbox-legacy", + "layerfs-sdk-legacy", + "layerfs-daemon-legacy", + "layerfs-bridge-legacy", + "layerfs-workspace-legacy", + ), + ) for name in RETIRED_PACKAGES: findings = violations(Path("core/crates") / name / "src/lib.rs", "pub use std::io::Read;") self.assertTrue(any("retired package" in message for _, message in findings), name) @@ -84,13 +119,23 @@ def test_sdk_source_edges_and_unsafe_boundary(self): self.assertFalse(unsafe_violations(root, "#![forbid(unsafe_code)]\nmod init;")) def test_sandbox_active_edges_and_unsafe_boundary(self): - path=Path("core/crates/layerfs-sandbox/src/backend/docker/http.rs") - self.assertTrue(unsafe_violations(path,"unsafe fn signal_pid() {}")) - self.assertFalse(unsafe_violations(Path("core/crates/layerfs-sandbox/src/lib.rs"),"#![forbid(unsafe_code)]")) - prefix='[package]\nname="layerfs-sandbox"\n[dependencies]\n' - for dependency in ("layerfs-api-core","layerfs-server","layerfs-daemon","layerfs-persistence","layerfs-api"): - self.assertTrue(dependency_violations(prefix+f'{dependency}={{path="../{dependency}"}}')) - self.assertTrue(dependency_violations('[package]\nname="layerfs-sandbox-legacy"\n[dependencies]\nlayerfs-api-core={path="../layerfs-api/core"}')) + path = Path("core/crates/layerfs-sandbox/src/backend/docker/http.rs") + self.assertTrue(unsafe_violations(path, "unsafe fn signal_pid() {}")) + self.assertFalse(unsafe_violations(Path("core/crates/layerfs-sandbox/src/lib.rs"), "#![forbid(unsafe_code)]")) + prefix = '[package]\nname="layerfs-sandbox"\n[dependencies]\n' + for dependency in ( + "layerfs-api-core", + "layerfs-server", + "layerfs-daemon", + "layerfs-persistence", + "layerfs-api", + ): + self.assertTrue(dependency_violations(prefix + f'{dependency}={{path="../{dependency}"}}')) + self.assertTrue( + dependency_violations( + '[package]\nname="layerfs-sandbox-legacy"\n[dependencies]\nlayerfs-api-core={path="../layerfs-api/core"}' + ) + ) def test_bridge_control_reuses_domain_records_without_engine_edges(self): prefix = '[package]\nname="layerfs-bridge"\n[dependencies]\n' @@ -109,18 +154,22 @@ def test_sdk_dependency_edges(self): def test_daemon_direct_store_and_control_edges_without_host_runtime(self): prefix = '[package]\nname="layerfs-daemon"\n[dependencies]\n' path = Path("core/crates/layerfs-daemon/src/bootstrap.rs") - for dependency in ("layerfs-bridge", "layerfs-content", - "layerfs-storage", "layerfs-history", "layerfs-persistence"): + for dependency in ( + "layerfs-bridge", + "layerfs-content", + "layerfs-storage", + "layerfs-history", + "layerfs-persistence", + ): with self.subTest(dependency=dependency): self.assertFalse(dependency_violations(prefix + f'{dependency}={{path="../{dependency}"}}')) - self.assertFalse(violations(path, f'use {dependency.replace("-", "_")}::PublicType;')) + self.assertFalse(violations(path, f"use {dependency.replace('-', '_')}::PublicType;")) self.assertTrue(dependency_violations(prefix + 'layerfs-server={path="../layerfs-server"}')) self.assertTrue(dependency_violations(prefix + 'layerfs-api={path="../layerfs-api"}')) - self.assertTrue(violations(path, 'use layerfs_api::Runtime;')) + self.assertTrue(violations(path, "use layerfs_api::Runtime;")) sdk = '[package]\nname="layerfs-api"\n[dependencies]\n' self.assertTrue(dependency_violations(sdk + 'layerfs-daemon={path="../../layerfs-daemon"}')) - self.assertTrue(violations(Path("core/crates/layerfs-api/src/project/init.rs"), - 'use layerfs_daemon::Owner;')) + self.assertTrue(violations(Path("core/crates/layerfs-api/src/project/init.rs"), "use layerfs_daemon::Owner;")) def test_store_service_engine_edges(self): prefix = '[package]\nname="layerfs-store-service"\n[dependencies]\n' @@ -131,23 +180,39 @@ def test_store_service_engine_edges(self): for package in ("layerfs-api", "layerfs-daemon"): source = f'[package]\nname="{package}"\n[dependencies]\nlayerfs-store-service={{path="../layerfs-store-service"}}\n' self.assertFalse(dependency_violations(source)) - self.assertTrue(violations(Path("core/crates/layerfs-history/src/lib.rs"), - "use layerfs_store_service::history::MAIN;")) + self.assertTrue( + violations(Path("core/crates/layerfs-history/src/lib.rs"), "use layerfs_store_service::history::MAIN;") + ) root = Path("core/crates/layerfs-store-service/src/lib.rs") self.assertTrue(unsafe_violations(root, "pub mod store;")) self.assertFalse(unsafe_violations(root, "#![forbid(unsafe_code)]\npub mod store;")) - self.assertFalse(dependency_violations( - '[package]\nname="layerfs-persistence"\n[dependencies]\nlayerfs-telemetry={path="../layerfs-telemetry"}\n')) + self.assertFalse( + dependency_violations( + '[package]\nname="layerfs-persistence"\n[dependencies]\nlayerfs-telemetry={path="../layerfs-telemetry"}\n' + ) + ) def test_workspace_composition_edges_preserve_domain_and_reverse_boundaries(self): for package in ("layerfs-api", "layerfs-daemon"): - source=f'[package]\nname="{package}"\n[dependencies]\nlayerfs-workspace={{path="../layerfs-workspace"}}\n' + source = f'[package]\nname="{package}"\n[dependencies]\nlayerfs-workspace={{path="../layerfs-workspace"}}\n' self.assertEqual(bool(dependency_violations(source)), package == "layerfs-api") - path=Path("core/crates/layerfs-api/src/project/init.rs") if package=="layerfs-api" else Path("core/crates/layerfs-daemon/src/commands.rs") - self.assertEqual(bool(violations(path,"use layerfs_workspace::Workspace;")), package == "layerfs-api") - self.assertTrue(violations(path,"use layerfs_server::Server;")) - self.assertTrue(dependency_violations('[package]\nname="layerfs-workspace"\n[dependencies]\nlayerfs-daemon={path="../layerfs-daemon"}\n')) - self.assertTrue(dependency_violations('[package]\nname="layerfs-content"\n[dependencies]\nlayerfs-workspace={path="../layerfs-workspace"}\n')) + path = ( + Path("core/crates/layerfs-api/src/project/init.rs") + if package == "layerfs-api" + else Path("core/crates/layerfs-daemon/src/commands.rs") + ) + self.assertEqual(bool(violations(path, "use layerfs_workspace::Workspace;")), package == "layerfs-api") + self.assertTrue(violations(path, "use layerfs_server::Server;")) + self.assertTrue( + dependency_violations( + '[package]\nname="layerfs-workspace"\n[dependencies]\nlayerfs-daemon={path="../layerfs-daemon"}\n' + ) + ) + self.assertTrue( + dependency_violations( + '[package]\nname="layerfs-content"\n[dependencies]\nlayerfs-workspace={path="../layerfs-workspace"}\n' + ) + ) def test_cluster1_dependency_edges(self): for package in ("layerfs-storage", "layerfs-history", "layerfs-project", "layerfs-content"): @@ -157,29 +222,35 @@ def test_cluster1_dependency_edges(self): self.assertTrue(dependency_violations(source)) source = f'[package]\nname="{package}"\n[dev-dependencies]\n"{dependency}"={{path="../{dependency}"}}\n' self.assertFalse(dependency_violations(source)) - self.assertFalse(dependency_violations('[package]\nname="layerfs-persistence"\n[dependencies]\nlayerfs-history={path="../layerfs-history"}\n')) - self.assertTrue(dependency_violations('[package]\nname="layerfs-persistence"\n[dependencies]\nlayerfs-workspace={path="../layerfs-workspace"}\n')) + self.assertFalse( + dependency_violations( + '[package]\nname="layerfs-persistence"\n[dependencies]\nlayerfs-history={path="../layerfs-history"}\n' + ) + ) + self.assertTrue( + dependency_violations( + '[package]\nname="layerfs-persistence"\n[dependencies]\nlayerfs-workspace={path="../layerfs-workspace"}\n' + ) + ) def test_project_sqlite_backing_uses_the_owning_provider_boundary(self): project = '[package]\nname="layerfs-project"\n[dependencies]\n' - self.assertFalse(dependency_violations( - project + 'layerfs-storage={path="../layerfs-storage"}\n')) + self.assertFalse(dependency_violations(project + 'layerfs-storage={path="../layerfs-storage"}\n')) provider = '[package]\nname="layerfs-persistence"\n[dependencies]\n' - self.assertFalse(dependency_violations( - provider + 'layerfs-storage={path="../layerfs-storage"}\n' - + 'rusqlite={version="=0.40.2"}\n')) - for table in ("dependencies", "build-dependencies", - "target.'cfg(target_os = \"linux\")'.dependencies"): + self.assertFalse( + dependency_violations( + provider + 'layerfs-storage={path="../layerfs-storage"}\n' + 'rusqlite={version="=0.40.2"}\n' + ) + ) + for table in ("dependencies", "build-dependencies", "target.'cfg(target_os = \"linux\")'.dependencies"): with self.subTest(table=table): - source = ('[package]\nname="layerfs-project"\n' - f'[{table}]\n' - 'database={package="rusqlite",version="=0.40.2"}\n') + source = ( + f'[package]\nname="layerfs-project"\n[{table}]\ndatabase={{package="rusqlite",version="=0.40.2"}}\n' + ) findings = dependency_violations(source) self.assertEqual(len(findings), 1) - self.assertIn("engine dependency in domain layerfs-project -> rusqlite", - findings[0][1]) - self.assertIn("SQLite-backed Project acquisition is allowed", - findings[0][1]) + self.assertIn("engine dependency in domain layerfs-project -> rusqlite", findings[0][1]) + self.assertIn("SQLite-backed Project acquisition is allowed", findings[0][1]) self.assertIn("layerfs-persistence", findings[0][1]) def test_domain_source_component_names(self): @@ -198,7 +269,9 @@ def test_storage_encoding_engine_boundary(self): for source in ("use rusqlite::Connection;", "crate::sqlite::lookup::location();"): self.assertTrue(violations(path, source)) self.assertFalse(violations(path, "use crate::source::Source;")) - self.assertFalse(violations(Path("core/crates/layerfs-storage/src/sqlite/source.rs"), "use rusqlite::Connection;")) + self.assertFalse( + violations(Path("core/crates/layerfs-storage/src/sqlite/source.rs"), "use rusqlite::Connection;") + ) def test_product_attributes(self): for source in ( @@ -207,25 +280,25 @@ def test_product_attributes(self): "#[cfg(any(unix,\n test))]\nmod checks;", '#[cfg(feature = "test-instrumentation")]\nfn inject() {}', '#[cfg_attr(feature = "checks", test)]\nfn case() {}', - 'fn f() { if cfg!(any(all(unix), test)) {} }', - 'fn f() { if cfg!(any(\n all(unix),\n test\n)) {} }', + "fn f() { if cfg!(any(all(unix), test)) {} }", + "fn f() { if cfg!(any(\n all(unix),\n test\n)) {} }", ): with self.subTest(source=source): self.assertTrue(violations(Path("scope.rs"), source)) - self.assertFalse(violations(Path("scope.rs"), '#[cfg(unix)]\nfn f() { assert!(true); }')) + self.assertFalse(violations(Path("scope.rs"), "#[cfg(unix)]\nfn f() { assert!(true); }")) def test_entry_files(self): for name in ("lib.rs", "mod.rs"): path = Path(name) self.assertFalse(violations(path, "// documentation\n" * 200)) self.assertTrue(violations(path, "// documentation\n" * 201)) - self.assertFalse(violations(path, 'mod scope;\npub use scope::Timing;\npub fn run() { scope::run() }')) - for source in ('pub struct State;', 'impl State {}', 'fn f() { let value = 1; }', 'fn f() { if true {} }'): + self.assertFalse(violations(path, "mod scope;\npub use scope::Timing;\npub fn run() { scope::run() }")) + for source in ("pub struct State;", "impl State {}", "fn f() { let value = 1; }", "fn f() { if true {} }"): self.assertTrue(violations(path, source)) self.assertFalse(violations(Path("recording.rs"), "pub struct State;\nimpl State {}")) def test_executable_rustdoc(self): - self.assertTrue(violations(Path("report.rs"), '/// ```rust\n/// let x = 1;\n/// ```')) + self.assertTrue(violations(Path("report.rs"), "/// ```rust\n/// let x = 1;\n/// ```")) def test_production_file_size_boundary(self): for name in ("save.rs", "schema.sql"): @@ -242,7 +315,9 @@ def test_unsafe_boundary(self): telemetry = Path("core/crates/layerfs-telemetry/src/timer/recording.rs") # The audited module and the lint attribute names never trip the scan. self.assertFalse(unsafe_violations(audited, "fn f() { unsafe { call(); } }")) - self.assertFalse(unsafe_violations(elsewhere, "#![deny(unsafe_code)]\n// the word `unsafe` in a comment\nfn f() {}")) + self.assertFalse( + unsafe_violations(elsewhere, "#![deny(unsafe_code)]\n// the word `unsafe` in a comment\nfn f() {}") + ) self.assertFalse(unsafe_violations(audited, "unsafe fn parse() {}\nfn code() { let _ = unsafe {}; }")) # Unsafe code outside the audited module is rejected, comment or not. for path in (elsewhere, content, telemetry): @@ -273,15 +348,23 @@ def test_unsafe_boundary(self): def test_shipped_sql_and_external_tests_scope(self): with TemporaryDirectory() as directory: core = Path(directory) - names = ("src/lib.rs", "src/sql/query.sql", "sql/schema.sql", "tests/case.rs", - "tests/fixture.sql", "examples/demo.rs", "benches/save.rs") + names = ( + "src/lib.rs", + "src/sql/query.sql", + "sql/schema.sql", + "tests/case.rs", + "tests/fixture.sql", + "examples/demo.rs", + "benches/save.rs", + ) for name in names: path = core / "crates" / "store" / name path.parent.mkdir(parents=True, exist_ok=True) path.write_text("") - self.assertEqual([str(p.relative_to(core / "crates" / "store")) - for p in production_files(core)], - ["sql/schema.sql", "src/lib.rs", "src/sql/query.sql"]) + self.assertEqual( + [str(p.relative_to(core / "crates" / "store")) for p in production_files(core)], + ["sql/schema.sql", "src/lib.rs", "src/sql/query.sql"], + ) def test_flat_api_product_scope(self): with TemporaryDirectory() as directory: @@ -290,33 +373,44 @@ def test_flat_api_product_scope(self): path = core / "crates/layerfs-api" / name path.parent.mkdir(parents=True, exist_ok=True) path.write_text("") - self.assertEqual([str(p.relative_to(core / "crates/layerfs-api")) - for p in production_files(core)], - ["src/fs/read.rs", "src/lib.rs"]) + self.assertEqual( + [str(p.relative_to(core / "crates/layerfs-api")) for p in production_files(core)], + ["src/fs/read.rs", "src/lib.rs"], + ) def test_skip_sdk_leaves_only_the_sdk_out(self): with TemporaryDirectory() as directory: root = Path(directory) core = root / "core" - for path in (core / "crates/store/src/lib.rs", root / "api/binding/src/lib.rs", - root / "api/agent/sdk/src/index.js"): + for path in ( + core / "crates/store/src/lib.rs", + root / "api/binding/src/lib.rs", + root / "api/agent/sdk/src/index.js", + ): path.parent.mkdir(parents=True, exist_ok=True) path.write_text("") everything = [str(p.relative_to(root)) for p in production_files(core)] self.assertIn("api/binding/src/lib.rs", everything) self.assertIn("api/agent/sdk/src/index.js", everything) - self.assertEqual([str(p.relative_to(root)) - for p in production_files(core, include_sdk=False)], - ["core/crates/store/src/lib.rs"]) + self.assertEqual( + [str(p.relative_to(root)) for p in production_files(core, include_sdk=False)], + ["core/crates/store/src/lib.rs"], + ) if __name__ == "__main__": unittest.main() + class DurablePersistenceEdges(unittest.TestCase): def test_domain_engine_dependencies_are_refused(self): - for name in ("layerfs-storage","layerfs-history","layerfs-project"): - for engine in ("rusqlite","postgres","tokio-postgres"): + for name in ("layerfs-storage", "layerfs-history", "layerfs-project"): + for engine in ("rusqlite", "postgres", "tokio-postgres"): self.assertTrue(dependency_violations(f'[package]\nname="{name}"\n[dependencies]\n{engine}="1"\n')) + def test_persistence_adapter_accepts_its_domain_edges(self): - self.assertFalse(dependency_violations('[package]\nname="layerfs-persistence"\n[dependencies]\nlayerfs-storage={path="../layerfs-storage"}\nlayerfs-history={path="../layerfs-history"}\n')) + self.assertFalse( + dependency_violations( + '[package]\nname="layerfs-persistence"\n[dependencies]\nlayerfs-storage={path="../layerfs-storage"}\nlayerfs-history={path="../layerfs-history"}\n' + ) + ) diff --git a/core/tools/test_ci_tests.py b/core/tools/test_ci_tests.py index a1ef43852..dffad6749 100644 --- a/core/tools/test_ci_tests.py +++ b/core/tools/test_ci_tests.py @@ -48,16 +48,17 @@ def test_fast_filter_excludes_the_nightly_groups_and_nightly_selects_them(self): class ExecutablesFromCargoJson(unittest.TestCase): def test_keeps_only_mount_test_binaries(self): - meta = {"packages": [{"name": "layerfs-daemon", - "manifest_path": "/w/core/crates/layerfs-daemon/Cargo.toml"}]} + meta = {"packages": [{"name": "layerfs-daemon", "manifest_path": "/w/core/crates/layerfs-daemon/Cargo.toml"}]} def artifact(name, kind, exe): - return json.dumps({ - "reason": "compiler-artifact", - "manifest_path": "/w/core/crates/layerfs-daemon/Cargo.toml", - "target": {"name": name, "kind": [kind]}, - "executable": exe, - }) + return json.dumps( + { + "reason": "compiler-artifact", + "manifest_path": "/w/core/crates/layerfs-daemon/Cargo.toml", + "target": {"name": name, "kind": [kind]}, + "executable": exe, + } + ) lines = [ "not json", @@ -65,8 +66,7 @@ def artifact(name, kind, exe): artifact("lane_merge", "test", "/t/lane_merge-1"), artifact("layerfs-daemon", "bin", "/t/layerfs-daemon"), ] - found = list(ci_tests.executables( - lines, [("layerfs-daemon", "mounted_commit")], meta)) + found = list(ci_tests.executables(lines, [("layerfs-daemon", "mounted_commit")], meta)) self.assertEqual(found, [(Path("/w/core/crates/layerfs-daemon"), "/t/mounted_commit-1")]) diff --git a/core/tools/test_phase7_services.py b/core/tools/test_phase7_services.py index bad026ec1..e39ba3a26 100644 --- a/core/tools/test_phase7_services.py +++ b/core/tools/test_phase7_services.py @@ -16,81 +16,110 @@ class Phase7ServicesTests(unittest.TestCase): def test_both_images_are_immutable_pins(self): for image in (services.POSTGRES_IMAGE, services.MINIO_IMAGE): - self.assertRegex(image, r'^[^@]+@sha256:[0-9a-f]{64}$') + self.assertRegex(image, r"^[^@]+@sha256:[0-9a-f]{64}$") def test_absent_object_is_distinct_from_daemon_failure(self): - missing = subprocess.CompletedProcess([], 1, '', 'error: no such object: layerfs-postgres') - with patch.object(services, 'docker', return_value=missing): - self.assertIsNone(services.inspect('container', 'layerfs-postgres')) - network = subprocess.CompletedProcess([], 1, '', 'Error response from daemon: network layerfs-phase7-cluster1 not found') - with patch.object(services, 'docker', return_value=network): - self.assertIsNone(services.inspect('network', 'layerfs-phase7-cluster1')) - unavailable = subprocess.CompletedProcess([], 1, '', 'Cannot connect to the Docker daemon') - with patch.object(services, 'docker', return_value=unavailable): - with self.assertRaisesRegex(RuntimeError, 'cannot inspect'): - services.inspect('container', 'layerfs-postgres') + missing = subprocess.CompletedProcess([], 1, "", "error: no such object: layerfs-postgres") + with patch.object(services, "docker", return_value=missing): + self.assertIsNone(services.inspect("container", "layerfs-postgres")) + network = subprocess.CompletedProcess( + [], 1, "", "Error response from daemon: network layerfs-phase7-cluster1 not found" + ) + with patch.object(services, "docker", return_value=network): + self.assertIsNone(services.inspect("network", "layerfs-phase7-cluster1")) + unavailable = subprocess.CompletedProcess([], 1, "", "Cannot connect to the Docker daemon") + with patch.object(services, "docker", return_value=unavailable): + with self.assertRaisesRegex(RuntimeError, "cannot inspect"): + services.inspect("container", "layerfs-postgres") def test_foreign_resource_is_refused_before_any_removal(self): - foreign = {'Config': {'Labels': {services.OWNER_LABEL: '/another/worktree'}}} - with patch.object(services, 'inspect', return_value=foreign), patch.object(services, 'docker') as docker: - with self.assertRaisesRegex(RuntimeError, 'foreign container'): + foreign = {"Config": {"Labels": {services.OWNER_LABEL: "/another/worktree"}}} + with patch.object(services, "inspect", return_value=foreign), patch.object(services, "docker") as docker: + with self.assertRaisesRegex(RuntimeError, "foreign container"): services.down() docker.assert_not_called() def test_partial_owned_setup_is_never_resumed_silently(self): - inventory = {'container': {'pg': {'State': {'Running': True}}, 'minio': None}, - 'volume': {}, 'network': {}} - with patch.object(services, 'ownership_inventory', return_value=inventory), patch.object(services, 'load', return_value={'complete': False}), patch.object(services, 'docker') as docker: - with self.assertRaisesRegex(RuntimeError, 'partial owned setup'): + inventory = {"container": {"pg": {"State": {"Running": True}}, "minio": None}, "volume": {}, "network": {}} + with ( + patch.object(services, "ownership_inventory", return_value=inventory), + patch.object(services, "load", return_value={"complete": False}), + patch.object(services, "docker") as docker, + ): + with self.assertRaisesRegex(RuntimeError, "partial owned setup"): services.up() docker.assert_not_called() def test_credentials_stay_in_private_files(self): settings = services.fresh_settings() with TemporaryDirectory() as tmp: - path = Path(tmp) / 'settings.json' - with patch.object(services, 'STATE', path): + path = Path(tmp) / "settings.json" + with patch.object(services, "STATE", path): services.save(settings) self.assertEqual(path.stat().st_mode & 0o777, 0o600) - self.assertEqual(path.with_name('services.env').stat().st_mode & 0o777, 0o600) - self.assertEqual(services.load()['environment'], settings['environment']) - self.assertNotEqual(settings['environment']['LAYERFS_PG_PASSWORD'], settings['environment']['LAYERFS_S3_SECRET_KEY']) + self.assertEqual(path.with_name("services.env").stat().st_mode & 0o777, 0o600) + self.assertEqual(services.load()["environment"], settings["environment"]) + self.assertNotEqual( + settings["environment"]["LAYERFS_PG_PASSWORD"], settings["environment"]["LAYERFS_S3_SECRET_KEY"] + ) def test_bucket_signature_has_the_declared_body_and_date(self): - environment = services.fresh_settings()['environment'] + environment = services.fresh_settings()["environment"] request = services.signed_bucket_request(environment, datetime(2026, 10, 3, 12, 0, tzinfo=timezone.utc)) headers = {key.lower(): value for key, value in request.header_items()} - self.assertEqual(request.method, 'PUT') - self.assertEqual(request.full_url, 'http://127.0.0.1:9000/layerfs') - self.assertEqual(request.data, b'') - self.assertEqual(headers['x-amz-date'], '20261003T120000Z') - self.assertRegex(headers['authorization'], r'/20261003/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=[0-9a-f]{64}$') - self.assertNotIn(environment['LAYERFS_S3_SECRET_KEY'], headers['authorization']) + self.assertEqual(request.method, "PUT") + self.assertEqual(request.full_url, "http://127.0.0.1:9000/layerfs") + self.assertEqual(request.data, b"") + self.assertEqual(headers["x-amz-date"], "20261003T120000Z") + self.assertRegex( + headers["authorization"], + r"/20261003/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=[0-9a-f]{64}$", + ) + self.assertNotIn(environment["LAYERFS_S3_SECRET_KEY"], headers["authorization"]) def test_readiness_handles_startup_close_and_checks_tcp(self): - response = MagicMock(); response.__enter__.return_value.status = 200 - ready = subprocess.CompletedProcess([], 0, '', '') - with patch.object(services, 'docker', return_value=ready) as docker, patch.object(services.urllib.request, 'urlopen', side_effect=[RemoteDisconnected('startup'), response]), patch.object(services, 'owned', return_value={'State': {'Running': True}}), patch.object(services.time, 'sleep'): + response = MagicMock() + response.__enter__.return_value.status = 200 + ready = subprocess.CompletedProcess([], 0, "", "") + with ( + patch.object(services, "docker", return_value=ready) as docker, + patch.object(services.urllib.request, "urlopen", side_effect=[RemoteDisconnected("startup"), response]), + patch.object(services, "owned", return_value={"State": {"Running": True}}), + patch.object(services.time, "sleep"), + ): services.wait_ready({}) - self.assertIn('-h', docker.call_args.args) - self.assertIn('127.0.0.1', docker.call_args.args) + self.assertIn("-h", docker.call_args.args) + self.assertIn("127.0.0.1", docker.call_args.args) def test_private_credentials_are_omitted_from_status(self): - settings = services.fresh_settings(); settings['image_ids'] = {'postgres': 'id-pg', 'minio': 'id-minio'} - version = subprocess.CompletedProcess([], 0, 'minio version RELEASE.test\n', '') - with patch.object(services, 'validate'), patch.object(services, 'postgres_sql', return_value='fsync=on\nsynchronous_commit=on\n'), patch.object(services, 'docker', return_value=version): + settings = services.fresh_settings() + settings["image_ids"] = {"postgres": "id-pg", "minio": "id-minio"} + version = subprocess.CompletedProcess([], 0, "minio version RELEASE.test\n", "") + with ( + patch.object(services, "validate"), + patch.object(services, "postgres_sql", return_value="fsync=on\nsynchronous_commit=on\n"), + patch.object(services, "docker", return_value=version), + ): result = services.status(settings) text = json.dumps(result) - for key in ('LAYERFS_PG_PASSWORD', 'LAYERFS_S3_SECRET_KEY'): - self.assertNotIn(settings['environment'][key], text) - self.assertEqual(result['resource_profile']['memory_bytes_per_service'], 512 * 1024 * 1024) + for key in ("LAYERFS_PG_PASSWORD", "LAYERFS_S3_SECRET_KEY"): + self.assertNotIn(settings["environment"][key], text) + self.assertEqual(result["resource_profile"]["memory_bytes_per_service"], 512 * 1024 * 1024) def test_cleanup_removes_only_verified_owned_names(self): - inventory = {'container': {'pg': {}}, 'volume': {'pg-data': {}}, 'network': {'private-net': {}}} - with TemporaryDirectory() as tmp, patch.object(services, 'STATE', Path(tmp) / 'absent'), patch.object(services, 'ownership_inventory', return_value=inventory), patch.object(services, 'docker') as docker: + inventory = {"container": {"pg": {}}, "volume": {"pg-data": {}}, "network": {"private-net": {}}} + with ( + TemporaryDirectory() as tmp, + patch.object(services, "STATE", Path(tmp) / "absent"), + patch.object(services, "ownership_inventory", return_value=inventory), + patch.object(services, "docker") as docker, + ): services.down() - self.assertEqual([call.args for call in docker.call_args_list], [('rm', '--force', 'pg'), ('volume', 'rm', 'pg-data'), ('network', 'rm', 'private-net')]) + self.assertEqual( + [call.args for call in docker.call_args_list], + [("rm", "--force", "pg"), ("volume", "rm", "pg-data"), ("network", "rm", "private-net")], + ) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/core/tools/test_sdk_telemetry_report.py b/core/tools/test_sdk_telemetry_report.py index 63bba2ec4..8382c6872 100644 --- a/core/tools/test_sdk_telemetry_report.py +++ b/core/tools/test_sdk_telemetry_report.py @@ -42,8 +42,7 @@ def test_owner_lookup_events_are_assigned_to_the_public_call_window(self): host.append(event) if daemon_name: daemon.append(self.event(2, 1, daemon_name, 10)) - for offset, lookup_key, label in ((10, 2001, "owner.docker_port"), - (40, 2002, "owner.hello")): + for offset, lookup_key, label in ((10, 2001, "owner.docker_port"), (40, 2002, "owner.hello")): step = self.event(1, lookup_key, label, 20) step["opened_ns"] = event["opened_ns"] + offset host.append(step) @@ -65,10 +64,19 @@ def test_owner_lookup_events_are_assigned_to_the_public_call_window(self): @staticmethod def event(role, key, name, elapsed): - return {"kind": "operation", "run": f"{1:032x}", "role": role, "key": key, - "success": True, "resource_status": "sampled", "samples": 2, "gaps": 0, - "cpu_shared_ns": [10, 5], "sampled_max_rss": 1024, - "timing": {"name": name, "elapsed_ns": elapsed}} + return { + "kind": "operation", + "run": f"{1:032x}", + "role": role, + "key": key, + "success": True, + "resource_status": "sampled", + "samples": 2, + "gaps": 0, + "cpu_shared_ns": [10, 5], + "sampled_max_rss": 1024, + "timing": {"name": name, "elapsed_ns": elapsed}, + } if __name__ == "__main__": diff --git a/tools/cleanup.py b/tools/cleanup.py index 2712d5d78..b2fbd1d69 100644 --- a/tools/cleanup.py +++ b/tools/cleanup.py @@ -17,14 +17,41 @@ from cleanup_archive import archive from cleanup_discover import discover from cleanup_volume_cache import prepare as prepare_volume_cache, clear as clear_volume_cache -from cleanup_docker import guard as guard_container, identity as container_identity, inspect as inspect_container, retire +from cleanup_docker import ( + guard as guard_container, + identity as container_identity, + inspect as inspect_container, + retire, +) from cleanup_retirement import retirement_open_files -from cleanup_state import (Refused, absolute, allocated, docker_state, ensure_idle, - ensure_volume_idle, git, inventory, open_files, overlaps, - run, same_volume, volume_identity, worktrees) +from cleanup_state import ( + Refused, + absolute, + allocated, + docker_state, + ensure_idle, + ensure_volume_idle, + git, + inventory, + open_files, + overlaps, + run, + same_volume, + volume_identity, + worktrees, +) CACHE_DIRS = ("incremental", "deps", "build", ".fingerprint", "examples") -KINDS = {"cargo-cache", "archive-tree", "archive-file", "worktree", "docker-volume", "docker-container", "discard-tree", "docker-cargo-cache"} +KINDS = { + "cargo-cache", + "archive-tree", + "archive-file", + "worktree", + "docker-volume", + "docker-container", + "discard-tree", + "docker-cargo-cache", +} EVENT_LOCK = threading.Lock() @@ -47,8 +74,13 @@ def event(output, **record): def free_space(repo): value = shutil.disk_usage(repo) - return {"at_utc": now(), "filesystem_path": str(repo), "free_bytes": value.free, - "free_GB": value.free / 1e9, "free_GiB": value.free / 1024**3} + return { + "at_utc": now(), + "filesystem_path": str(repo), + "free_bytes": value.free, + "free_GB": value.free / 1e9, + "free_GiB": value.free / 1024**3, + } def byte_target(gb): @@ -59,12 +91,17 @@ def byte_target(gb): def check_space(args): state = free_space(Path(args.repo).resolve()) - state.update(min_free_bytes=byte_target(args.min_free_gb), - reclaim_target_bytes=byte_target(args.reclaim_gb), - units="GB = 1,000,000,000 bytes; GiB = 1,073,741,824 bytes") + state.update( + min_free_bytes=byte_target(args.min_free_gb), + reclaim_target_bytes=byte_target(args.reclaim_gb), + units="GB = 1,000,000,000 bytes; GiB = 1,073,741,824 bytes", + ) state["cleanup_required"] = state["free_bytes"] < state["min_free_bytes"] - state["action"] = ("Plan with --discover --authorization or an exact scope, then apply. Preserve active work and pins." - if state["cleanup_required"] else "Headroom threshold satisfied") + state["action"] = ( + "Plan with --discover --authorization or an exact scope, then apply. Preserve active work and pins." + if state["cleanup_required"] + else "Headroom threshold satisfied" + ) print(json.dumps(state, indent=2)) return 2 if state["cleanup_required"] else 0 @@ -132,13 +169,18 @@ def observations(repo, roots, sizes=False): try: info = path.lstat() if info.st_size >= 5 * 1024**2: - large.append({"path": str(path), "bytes": info.st_size, - "allocated_bytes": info.st_blocks * 512}) + large.append( + {"path": str(path), "bytes": info.st_size, "allocated_bytes": info.st_blocks * 512} + ) except OSError as error: errors.append(str(error)) large.sort(key=lambda row: row["allocated_bytes"], reverse=True) - return {"worktrees": result, "large_files": large, "scan_errors": errors, - "note": "Allocation estimates may overlap or share APFS blocks. Age/size is not release authorization."} + return { + "worktrees": result, + "large_files": large, + "scan_errors": errors, + "note": "Allocation estimates may overlap or share APFS blocks. Age/size is not release authorization.", + } def make_plan(args): @@ -150,13 +192,22 @@ def make_plan(args): scope = json.loads(Path(args.scope).read_text()) if args.scope else {"version": 1, "actions": []} if scope.get("version") != 1: raise Refused("Unsupported scope version") - scope["protected_paths"] = list(dict.fromkeys([ - *scope.get("protected_paths", []), *getattr(args, "protect_path", [])])) - scope["protected_containers"] = list(dict.fromkeys([ - *scope.get("protected_containers", []), *getattr(args, "protect_container", [])])) - plan = {"version": 1, "created_utc": now(), "repository": str(repo), - "pass_directory": str(output), "scope": scope, "actions": [], "blocked": [], - "free_space": free_space(repo)} + scope["protected_paths"] = list( + dict.fromkeys([*scope.get("protected_paths", []), *getattr(args, "protect_path", [])]) + ) + scope["protected_containers"] = list( + dict.fromkeys([*scope.get("protected_containers", []), *getattr(args, "protect_container", [])]) + ) + plan = { + "version": 1, + "created_utc": now(), + "repository": str(repo), + "pass_directory": str(output), + "scope": scope, + "actions": [], + "blocked": [], + "free_space": free_space(repo), + } plan["reclaim_target_bytes"] = byte_target(args.reclaim_gb) registered = {r["worktree"]: r for r in worktrees(repo)} try: @@ -166,17 +217,24 @@ def make_plan(args): containers, opened = None, None plan["runtime_error"] = str(error) cache_worktrees = [str(absolute(p)) for p in getattr(args, "cache_worktree", [])] - if cache_worktrees and (not getattr(args, "discover_caches", False) - or any(p not in registered for p in cache_worktrees)): + if cache_worktrees and ( + not getattr(args, "discover_caches", False) or any(p not in registered for p in cache_worktrees) + ): raise Refused("--cache-worktree requires --discover-caches and registered checkout paths") if getattr(args, "discover", False) or getattr(args, "discover_caches", False): if containers is None: raise Refused(plan["runtime_error"]) scope, plan["discovery_skipped"] = discover( - repo, scope, registered, containers, opened, args.authorization or "", + repo, + scope, + registered, + containers, + opened, + args.authorization or "", caches_only=getattr(args, "discover_caches", False), retire_idle_sessions=getattr(args, "retire_idle_sessions", False), - cache_worktrees=[str(absolute(p)) for p in getattr(args, "cache_worktree", [])]) + cache_worktrees=[str(absolute(p)) for p in getattr(args, "cache_worktree", [])], + ) plan["scope"] = scope write(output / "scope.json", scope) # A planned container removal may release its bind mount before a later @@ -210,9 +268,14 @@ def make_plan(args): if action["kind"] == "cargo-cache": # Select only generated children. Direct executables/helpers remain in place. target = absolute(action["target_root"]) - if (not target.is_dir() or target.is_symlink() or path.parent != target - or not path.is_dir() or path.name not in {"debug", "release"} - or not (target / ".rustc_info.json").is_file()): + if ( + not target.is_dir() + or target.is_symlink() + or path.parent != target + or not path.is_dir() + or path.name not in {"debug", "release"} + or not (target / ".rustc_info.json").is_file() + ): raise Refused("cargo-cache needs a direct debug/release profile under a real Cargo target_root") paths = [path / child for child in CACHE_DIRS if (path / child).exists()] if containers is None: @@ -229,8 +292,14 @@ def make_plan(args): # Cargo hard-links object files between incremental and deps. # Check all siblings before removing any: unlinking one changes # another's ctime even though no external writer touched it. - chosen.append({**action, "path": str(path), "guard_path": str(path), - "cache_snapshots": {str(item): inventory(item)[1] for item in paths}}) + chosen.append( + { + **action, + "path": str(path), + "guard_path": str(path), + "cache_snapshots": {str(item): inventory(item)[1] for item in paths}, + } + ) continue for item in paths: if action["kind"] == "worktree": @@ -238,19 +307,20 @@ def make_plan(args): snapshot = {"device": info.st_dev, "inode": info.st_ino} else: snapshot = inventory(item)[1] - selection = {**action, "path": str(item), "snapshot": snapshot, - "guard_path": str(path)} + selection = {**action, "path": str(item), "snapshot": snapshot, "guard_path": str(path)} if action["kind"] == "worktree": selection["git_head"] = git(item, "rev-parse", "HEAD").decode().strip() - selection["staged_sha256"] = hashlib.sha256(git( - item, "diff", "--cached", "--binary", "--full-index", "--no-ext-diff", "--no-textconv")).hexdigest() + selection["staged_sha256"] = hashlib.sha256( + git(item, "diff", "--cached", "--binary", "--full-index", "--no-ext-diff", "--no-textconv") + ).hexdigest() chosen.append(selection) except (Refused, OSError, KeyError, TimeoutError) as error: plan["blocked"].append({"selection": action, "reason": str(error)}) # Refuse overlapping selections rather than changing ordering semantics. for i, item in enumerate(chosen): - if "path" in item and any(overlaps(item["path"], other["path"]) - for other in chosen[i + 1:] if "path" in other): + if "path" in item and any( + overlaps(item["path"], other["path"]) for other in chosen[i + 1 :] if "path" in other + ): raise Refused("Overlapping path selections; use one action per resource") names = [x["name"] for x in chosen if x["kind"] in {"docker-volume", "docker-cargo-cache"}] if len(names) != len(set(names)): @@ -259,10 +329,20 @@ def make_plan(args): plan["protected_presence"] = {str(p): p.exists() for p in pins(scope)} write(output / "inventory.json", observations(repo, scope.get("scan_roots", []), getattr(args, "sizes", False))) write(output / "plan.json", plan) - print(json.dumps({"plan": str(output / "plan.json"), "selected": len(chosen), - "blocked": plan["blocked"], "mode": "inventory only; nothing deleted", - "next": "Review plan.json, then apply --plan " + str(output / "plan.json") if chosen - else "No removals selected. Use --discover --authorization for authorized cleanup."}, indent=2)) + print( + json.dumps( + { + "plan": str(output / "plan.json"), + "selected": len(chosen), + "blocked": plan["blocked"], + "mode": "inventory only; nothing deleted", + "next": "Review plan.json, then apply --plan " + str(output / "plan.json") + if chosen + else "No removals selected. Use --discover --authorization for authorized cleanup.", + }, + indent=2, + ) + ) return 0 @@ -290,8 +370,12 @@ def apply_action(repo, output, plan, action): raise Refused("Discard tree must be a directory") registered = {r["worktree"]: r for r in worktrees(repo)} validate_path(path, action["kind"], repo, output, registered) - ensure_idle([path], pins(scope), containers, - retirement_open_files(path, open_files(), action.get("retire_idle_sessions", False))) + ensure_idle( + [path], + pins(scope), + containers, + retirement_open_files(path, open_files(), action.get("retire_idle_sessions", False)), + ) if action["kind"] == "cargo-cache": guard = absolute(action["guard_path"]) if "cache_snapshots" in action: @@ -308,8 +392,12 @@ def apply_action(repo, output, plan, action): raise Refused("Cache changed since plan; preserve the whole profile") # One final activity inspection covers the whole profile. Its # generated siblings are one planned job, not a replay or refresh. - ensure_idle([guard], pins(scope), docker_state(), retirement_open_files( - guard, open_files(), action.get("retire_idle_sessions", False))) + ensure_idle( + [guard], + pins(scope), + docker_state(), + retirement_open_files(guard, open_files(), action.get("retire_idle_sessions", False)), + ) event(output, state="removing", action=action) for child in children: shutil.rmtree(child) @@ -321,8 +409,12 @@ def apply_action(repo, output, plan, action): target = absolute(action["target_root"]) if guard.parent != target or guard.name not in {"debug", "release"}: raise Refused("Invalid Cargo target profile") - ensure_idle([guard], [], containers, retirement_open_files( - guard, open_files(), action.get("retire_idle_sessions", False))) + ensure_idle( + [guard], + [], + containers, + retirement_open_files(guard, open_files(), action.get("retire_idle_sessions", False)), + ) checkout = action["kind"] == "worktree" if checkout and "device" in action["snapshot"]: info = path.stat() @@ -364,8 +456,12 @@ def apply_action(repo, output, plan, action): git(repo, "update-ref", recovery["head_ref"], head) event(output, state="archived", action=action, recovery=recovery) # Archival may take time. Refresh every ownership guard before unlinking. - ensure_idle([path], pins(scope), docker_state(), - retirement_open_files(path, open_files(), action.get("retire_idle_sessions", False))) + ensure_idle( + [path], + pins(scope), + docker_state(), + retirement_open_files(path, open_files(), action.get("retire_idle_sessions", False)), + ) if inventory(path, checkout)[1] != recovery["snapshot"]: raise Refused("Path changed after archival; preserve it") if checkout: @@ -398,14 +494,21 @@ def apply_plan(args): jobs = getattr(args, "jobs", 1) if not 1 <= jobs <= 4: raise Refused("Cleanup jobs must be between 1 and 4") - if jobs > 1 and any(a["kind"] not in {"worktree", "docker-container", "archive-tree", "archive-file"} for a in plan["actions"]): + if jobs > 1 and any( + a["kind"] not in {"worktree", "docker-container", "archive-tree", "archive-file"} for a in plan["actions"] + ): raise Refused("Parallel cleanup supports disjoint archives, worktrees or containers only") # A pass can be attempted exactly once, including failed or interrupted passes. with (output / "apply.started").open("x") as stream: stream.write(now() + "\n") - plan["pass_id"] = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + "-" + hashlib.sha256(str(output).encode()).hexdigest()[:12] + plan["pass_id"] = ( + datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + + "-" + + hashlib.sha256(str(output).encode()).hexdigest()[:12] + ) before = free_space(repo) results = [] + def attempt(action): event(output, state="started", action=action) try: @@ -420,7 +523,7 @@ def attempt(action): try: with ThreadPoolExecutor(max_workers=jobs) as pool: for offset in range(0, len(plan["actions"]), jobs): - batch = plan["actions"][offset:offset + jobs] + batch = plan["actions"][offset : offset + jobs] completed = list(pool.map(attempt, batch)) results.extend(completed) # Settle already-running independent jobs before reporting a @@ -440,32 +543,55 @@ def attempt(action): touched.add(key) if row["state"] in {"removed", "failed_or_preserved"}: terminal[key] = row - results = [terminal[json.dumps(a, sort_keys=True)] for a in plan["actions"] - if json.dumps(a, sort_keys=True) in terminal] + results = [ + terminal[json.dumps(a, sort_keys=True)] + for a in plan["actions"] + if json.dumps(a, sort_keys=True) in terminal + ] unattempted = [a for a in plan["actions"] if json.dumps(a, sort_keys=True) not in touched] - uncertain = [a for a in plan["actions"] if json.dumps(a, sort_keys=True) in touched - and json.dumps(a, sort_keys=True) not in terminal] + uncertain = [ + a + for a in plan["actions"] + if json.dumps(a, sort_keys=True) in touched and json.dumps(a, sort_keys=True) not in terminal + ] after = free_space(repo) present = {raw: Path(raw).exists() for raw in plan["protected_presence"]} lost_pins = [raw for raw, existed in plan["protected_presence"].items() if existed and not present[raw]] - report = {"finished_utc": now(), "before": before, "after": after, - "outcome": "no_actions_selected" if not plan["actions"] else "attempted", - "net_free_bytes_change": after["free_bytes"] - before["free_bytes"], - "results": results, "protected_presence": present, "lost_pins": lost_pins, - "not_attempted": unattempted, "uncertain_actions": uncertain, - "interrupted": isinstance(sys.exc_info()[1], KeyboardInterrupt), - "blocked_at_plan": plan["blocked"], - "note": "Net change includes concurrent I/O; allocation estimates are not physical recovery. " - "Keep recovery refs/objects and manifests. No product build, test, Store call, " - "global prune or measurement scheduling is performed. Only explicitly selected " - "retired daemons may be stopped; container actions retain their mounted data."} + report = { + "finished_utc": now(), + "before": before, + "after": after, + "outcome": "no_actions_selected" if not plan["actions"] else "attempted", + "net_free_bytes_change": after["free_bytes"] - before["free_bytes"], + "results": results, + "protected_presence": present, + "lost_pins": lost_pins, + "not_attempted": unattempted, + "uncertain_actions": uncertain, + "interrupted": isinstance(sys.exc_info()[1], KeyboardInterrupt), + "blocked_at_plan": plan["blocked"], + "note": "Net change includes concurrent I/O; allocation estimates are not physical recovery. " + "Keep recovery refs/objects and manifests. No product build, test, Store call, " + "global prune or measurement scheduling is performed. Only explicitly selected " + "retired daemons may be stopped; container actions retain their mounted data.", + } report["reclaim_target_bytes"] = plan.get("reclaim_target_bytes", 0) - report["reclaim_target_met"] = (report["reclaim_target_bytes"] == 0 - or report["net_free_bytes_change"] >= report["reclaim_target_bytes"]) + report["reclaim_target_met"] = ( + report["reclaim_target_bytes"] == 0 or report["net_free_bytes_change"] >= report["reclaim_target_bytes"] + ) write(output / "report.json", report) - print(json.dumps({"report": str(output / "report.json"), "removed": sum(r["state"] == "removed" for r in results), - "net_free_bytes_change": report["net_free_bytes_change"], "lost_pins": lost_pins, - "reclaim_target_met": report["reclaim_target_met"]}, indent=2)) + print( + json.dumps( + { + "report": str(output / "report.json"), + "removed": sum(r["state"] == "removed" for r in results), + "net_free_bytes_change": report["net_free_bytes_change"], + "lost_pins": lost_pins, + "reclaim_target_met": report["reclaim_target_met"], + }, + indent=2, + ) + ) if lost_pins or any(r["state"] != "removed" for r in results): return 1 return 0 if report["reclaim_target_met"] else 2 @@ -481,19 +607,38 @@ def main(): plan = commands.add_parser("plan", help="Inventory only; no deletion or recovery-ref writes") plan.add_argument("--repo", default=str(Path(__file__).resolve().parents[1])) plan.add_argument("--scope", help="Explicit release selections and protected resources (JSON)") - plan.add_argument("--discover", action="store_true", help="Select idle linked worktrees and stopped LayerFS containers") - plan.add_argument("--discover-caches", action="store_true", help="Select idle generated Cargo caches in linked worktrees, retaining binaries and helpers") - plan.add_argument("--cache-worktree", action="append", default=[], help="Restrict --discover-caches to this registered checkout; repeat as needed; explicitly allows the primary checkout") - plan.add_argument("--retire-idle-sessions", action="store_true", help="Allow retired checkouts held only by idle agent/shell cwd handles; active files/commands stay protected") + plan.add_argument( + "--discover", action="store_true", help="Select idle linked worktrees and stopped LayerFS containers" + ) + plan.add_argument( + "--discover-caches", + action="store_true", + help="Select idle generated Cargo caches in linked worktrees, retaining binaries and helpers", + ) + plan.add_argument( + "--cache-worktree", + action="append", + default=[], + help="Restrict --discover-caches to this registered checkout; repeat as needed; explicitly allows the primary checkout", + ) + plan.add_argument( + "--retire-idle-sessions", + action="store_true", + help="Allow retired checkouts held only by idle agent/shell cwd handles; active files/commands stay protected", + ) plan.add_argument("--authorization", help="User instruction authorizing discovery; no separate release file needed") - plan.add_argument("--protect-path", action="append", default=[], help="Preserve this path and overlapping resources") + plan.add_argument( + "--protect-path", action="append", default=[], help="Preserve this path and overlapping resources" + ) plan.add_argument("--protect-container", action="append", default=[], help="Preserve exact container ID or name") plan.add_argument("--sizes", action="store_true", help="Also run potentially expensive du scans of all worktrees") plan.add_argument("--out", required=True, help="New pass directory outside the repository") plan.add_argument("--reclaim-gb", type=float, default=0, help="Net host-space recovery target, decimal GB") apply = commands.add_parser("apply", help="Attempt a saved plan once; archive selected data before removal") apply.add_argument("--plan", required=True) - apply.add_argument("--jobs", type=int, default=1, help="Up to 4 independent archive/worktree/container jobs; default 1") + apply.add_argument( + "--jobs", type=int, default=1, help="Up to 4 independent archive/worktree/container jobs; default 1" + ) args = parser.parse_args() try: if args.command == "check": diff --git a/tools/cleanup_archive.py b/tools/cleanup_archive.py index 4de22f512..e899e3a5e 100644 --- a/tools/cleanup_archive.py +++ b/tools/cleanup_archive.py @@ -21,8 +21,9 @@ def file_hashes(repo, files, write=False): def flush(): flags = ["-w"] if write else [] data = "".join(str(path) + "\n" for path in batch).encode() - result.extend(git(repo, "hash-object", "--no-filters", *flags, - "--stdin-paths", data=data).decode().splitlines()) + result.extend( + git(repo, "hash-object", "--no-filters", *flags, "--stdin-paths", data=data).decode().splitlines() + ) for path in files: size = path.stat().st_size @@ -47,20 +48,28 @@ def archive(repo, path, output, pass_id, checkout=False): # re-encoding tens of thousands of immutable tracked files per checkout. head = git(path, "rev-parse", "HEAD").decode().strip() git(repo, "cat-file", "-e", head + "^{tree}") - selected = set(os.fsdecode(value).rstrip("/") for value in ( - git(path, "ls-files", "--others", "-z") + - git(path, "diff", "HEAD", "--name-only", "-z", "--no-ext-diff", "--no-textconv") - ).split(b"\0") if value) + selected = set( + os.fsdecode(value).rstrip("/") + for value in ( + git(path, "ls-files", "--others", "-z") + + git(path, "diff", "HEAD", "--name-only", "-z", "--no-ext-diff", "--no-textconv") + ).split(b"\0") + if value + ) kept = [] for row in file_rows: relative = Path(row["path"]) - if (".git" in relative.parts or row["path"] in selected - or any(str(parent) in selected for parent in relative.parents)): + if ( + ".git" in relative.parts + or row["path"] in selected + or any(str(parent) in selected for parent in relative.parents) + ): kept.append(row) else: row["from_head"] = True file_rows = kept regular = [r for r in file_rows if r["target"] is None] + # For a single file, wrap it under its original leaf name in the recovery tree. def name(row): raw = path.name if row["path"] == "." else row["path"] @@ -86,11 +95,11 @@ def source(row): row["oid"] = oid for row in file_rows: if row["target"] is not None: - row["oid"] = git(repo, "hash-object", "-w", "--stdin", - data=os.fsencode(row["target"])).decode().strip() + row["oid"] = git(repo, "hash-object", "-w", "--stdin", data=os.fsencode(row["target"])).decode().strip() sizes = {r["oid"]: r["size"] for r in file_rows} - checks = git(repo, "cat-file", "--batch-check", - data="".join(oid + "\n" for oid in sizes).encode()).decode().splitlines() + checks = ( + git(repo, "cat-file", "--batch-check", data="".join(oid + "\n" for oid in sizes).encode()).decode().splitlines() + ) if len(checks) != len(sizes): raise Refused("Archive object count mismatch") for line in checks: @@ -104,8 +113,7 @@ def source(row): git(repo, "read-tree", "--empty", env=env) entries = [] for row in file_rows: - mode = "120000" if row["target"] is not None else ( - "100755" if row["mode"] & stat.S_IXUSR else "100644") + mode = "120000" if row["target"] is not None else ("100755" if row["mode"] & stat.S_IXUSR else "100644") entries.append(f"{mode} {row['oid']}\t{name(row)}\0") git(repo, "update-index", "-z", "--index-info", data="".join(entries).encode(), env=env) tree = git(repo, "write-tree", env=env).decode().strip() @@ -136,16 +144,22 @@ def source(row): row["xattrs_base64"] = attributes(source(row)) if inventory(path, checkout)[1] != snapshot: raise Refused("Source metadata changed during attribute capture") - manifest = {"version": 2 if checkout else 1, "original_path": str(path), "repository": str(repo), - "recovery_ref": reference, "tree": tree, "rows": rows, - "tracked_head": head, - "snapshot": snapshot, - "limits": "Raw bytes, symlinks, modes, ownership IDs and times recorded. " - "Git archive does not restore empty dirs, metadata or ACLs automatically. " - "Restore xattrs_base64 with base64 decoding and os.setxattr. " - "For worktrees, restore tracked_head and staged/unstaged patches first, " - "then overlay raw archived edits/untracked/ignored files. Rows marked " - "from_head are backed by the pinned commit, not duplicated in this tree."} + manifest = { + "version": 2 if checkout else 1, + "original_path": str(path), + "repository": str(repo), + "recovery_ref": reference, + "tree": tree, + "rows": rows, + "tracked_head": head, + "snapshot": snapshot, + "limits": "Raw bytes, symlinks, modes, ownership IDs and times recorded. " + "Git archive does not restore empty dirs, metadata or ACLs automatically. " + "Restore xattrs_base64 with base64 decoding and os.setxattr. " + "For worktrees, restore tracked_head and staged/unstaged patches first, " + "then overlay raw archived edits/untracked/ignored files. Rows marked " + "from_head are backed by the pinned commit, not duplicated in this tree.", + } raw = json.dumps(manifest, sort_keys=True).encode() dest = output / (key + ".json.gz") with gzip.open(dest, "wb", compresslevel=1) as stream: @@ -153,6 +167,11 @@ def source(row): with gzip.open(dest, "rb") as stream: if stream.read() != raw: raise Refused("Archive manifest readback mismatch") - return {"manifest": str(dest), "manifest_sha256": hashlib.sha256(raw).hexdigest(), - "recovery_ref": reference, "tree": tree, "snapshot": snapshot, - "tracked_head": head} + return { + "manifest": str(dest), + "manifest_sha256": hashlib.sha256(raw).hexdigest(), + "recovery_ref": reference, + "tree": tree, + "snapshot": snapshot, + "tracked_head": head, + } diff --git a/tools/cleanup_attrs.py b/tools/cleanup_attrs.py index 2c24ed51f..b25cbf06d 100644 --- a/tools/cleanup_attrs.py +++ b/tools/cleanup_attrs.py @@ -15,16 +15,24 @@ def darwin_library(): library = ctypes.CDLL(None, use_errno=True) library.listxattr.argtypes = [ctypes.c_char_p, ctypes.c_void_p, ctypes.c_size_t, ctypes.c_int] library.listxattr.restype = ctypes.c_ssize_t - library.getxattr.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_void_p, - ctypes.c_size_t, ctypes.c_uint32, ctypes.c_int] + library.getxattr.argtypes = [ + ctypes.c_char_p, + ctypes.c_char_p, + ctypes.c_void_p, + ctypes.c_size_t, + ctypes.c_uint32, + ctypes.c_int, + ] library.getxattr.restype = ctypes.c_ssize_t return library def attributes(path): if hasattr(os, "listxattr"): - return {key: base64.b64encode(os.getxattr(path, key, follow_symlinks=False)).decode() - for key in os.listxattr(path, follow_symlinks=False)} + return { + key: base64.b64encode(os.getxattr(path, key, follow_symlinks=False)).decode() + for key in os.listxattr(path, follow_symlinks=False) + } if sys.platform != "darwin": raise Refused("Extended attribute inspection is unsupported on this platform") library = darwin_library() diff --git a/tools/cleanup_discover.py b/tools/cleanup_discover.py index 8f3ad9287..580492e29 100644 --- a/tools/cleanup_discover.py +++ b/tools/cleanup_discover.py @@ -12,8 +12,7 @@ def repository_container(container, repo, registered): return True roots = [repo, *(Path(p) for p in registered)] for mount in container["mounts"]: - if mount.get("Type") == "bind" and any( - overlaps(host_bind_path(mount["Source"]), root) for root in roots): + if mount.get("Type") == "bind" and any(overlaps(host_bind_path(mount["Source"]), root) for root in roots): return True if mount.get("Destination") == "/layerfs-store": return True @@ -27,8 +26,11 @@ def cargo_profiles(worktree): continue for base, dirs, files in os.walk(root): target = Path(base) - dirs[:] = [d for d in dirs if d not in {"debug", "release", "registry", "git", "node_modules"} - and not (target / d).is_symlink()] + dirs[:] = [ + d + for d in dirs + if d not in {"debug", "release", "registry", "git", "node_modules"} and not (target / d).is_symlink() + ] if len(target.relative_to(root).parts) >= 3: dirs[:] = [] if ".rustc_info.json" in files: @@ -37,7 +39,17 @@ def cargo_profiles(worktree): yield target, target / profile -def discover(repo, scope, registered, containers, opened, authorization, caches_only=False, retire_idle_sessions=False, cache_worktrees=None): +def discover( + repo, + scope, + registered, + containers, + opened, + authorization, + caches_only=False, + retire_idle_sessions=False, + cache_worktrees=None, +): """User authorization replaces a separate release file, never an activity guard.""" if not authorization.strip(): raise Refused("Discovery needs --authorization recording the user's cleanup instruction") @@ -48,12 +60,19 @@ def discover(repo, scope, registered, containers, opened, authorization, caches_ skipped = [] selected = {a.get("path") or a.get("name") for a in scope["actions"]} retiring = {a.get("name") for a in scope["actions"] if a["kind"] == "docker-container"} - path_containers = [c for c in containers if c["running"] and c["id"] not in retiring - or not repository_container(c, repo, registered) - or c["id"] in protected_containers - or c["name"].lstrip("/") in protected_containers - or any(m.get("Type") == "bind" and any(overlaps( - host_bind_path(m["Source"]), p) for p in protected) for m in c["mounts"])] + path_containers = [ + c + for c in containers + if c["running"] + and c["id"] not in retiring + or not repository_container(c, repo, registered) + or c["id"] in protected_containers + or c["name"].lstrip("/") in protected_containers + or any( + m.get("Type") == "bind" and any(overlaps(host_bind_path(m["Source"]), p) for p in protected) + for m in c["mounts"] + ) + ] for raw, record in registered.items(): path = Path(raw) explicit_cache = caches_only and cache_worktrees and raw in cache_worktrees @@ -68,19 +87,32 @@ def discover(repo, scope, registered, containers, opened, authorization, caches_ except Refused as error: skipped.append({"kind": "cargo-cache", "path": str(profile), "reason": str(error)}) continue - scope["actions"].append({"kind": "cargo-cache", "path": str(profile), - "target_root": str(target), "released": True, - "owner": "user-authorized cleanup", "reason": authorization}) + scope["actions"].append( + { + "kind": "cargo-cache", + "path": str(profile), + "target_root": str(target), + "released": True, + "owner": "user-authorized cleanup", + "reason": authorization, + } + ) continue try: - ensure_idle([path], protected, path_containers, - retirement_open_files(path, opened, retire_idle_sessions)) + ensure_idle([path], protected, path_containers, retirement_open_files(path, opened, retire_idle_sessions)) except Refused as error: skipped.append({"kind": "worktree", "path": raw, "reason": str(error)}) continue - scope["actions"].append({"kind": "worktree", "path": raw, "released": True, - "retire_idle_sessions": retire_idle_sessions, - "owner": "user-authorized cleanup", "reason": authorization}) + scope["actions"].append( + { + "kind": "worktree", + "path": raw, + "released": True, + "retire_idle_sessions": retire_idle_sessions, + "owner": "user-authorized cleanup", + "reason": authorization, + } + ) for container in containers: if caches_only: break @@ -90,13 +122,21 @@ def discover(repo, scope, registered, containers, opened, authorization, caches_ continue if container["id"] in protected_containers or container["name"].lstrip("/") in protected_containers: continue - if any(m.get("Type") == "bind" and any(overlaps(host_bind_path(m["Source"]), p) - for p in protected) for m in container["mounts"]): + if any( + m.get("Type") == "bind" and any(overlaps(host_bind_path(m["Source"]), p) for p in protected) + for m in container["mounts"] + ): continue if container["id"] not in selected: - scope["actions"].append({"kind": "docker-container", "name": container["id"], - "released": True, "owner": "user-authorized cleanup", - "reason": authorization}) + scope["actions"].append( + { + "kind": "docker-container", + "name": container["id"], + "released": True, + "owner": "user-authorized cleanup", + "reason": authorization, + } + ) # Containers first: removing stopped containers releases their bind mounts. scope["actions"].sort(key=lambda action: action["kind"] != "docker-container") return scope, skipped diff --git a/tools/cleanup_docker.py b/tools/cleanup_docker.py index c1e2bc6d0..629e470bc 100644 --- a/tools/cleanup_docker.py +++ b/tools/cleanup_docker.py @@ -17,7 +17,8 @@ def inspect(name): def identity(info): return {key: info[key] for key in ("Id", "Created", "Image", "Config", "Mounts")} | { - "state": {key: info["State"][key] for key in ("Status", "Running", "StartedAt", "FinishedAt")}} + "state": {key: info["State"][key] for key in ("Status", "Running", "StartedAt", "FinishedAt")} + } def guard(info, action, scope): @@ -27,8 +28,9 @@ def guard(info, action, scope): for mount in info.get("Mounts", []): if mount.get("Name") in scope.get("protected_volumes", []): raise Refused("Container uses a protected volume: " + info["Name"]) - if mount.get("Type") == "bind" and any(overlaps( - host_bind_path(mount["Source"]), Path(p).resolve()) for p in scope.get("protected_paths", [])): + if mount.get("Type") == "bind" and any( + overlaps(host_bind_path(mount["Source"]), Path(p).resolve()) for p in scope.get("protected_paths", []) + ): raise Refused("Container binds a protected path: " + info["Name"]) if info["State"]["Running"]: if action.get("retire_running") is not True: @@ -36,8 +38,7 @@ def guard(info, action, scope): # Explicit retirement supports abandoned filesystem daemons only. Ordinary # commands, Exec descendants and other services remain protected. top = run(["docker", "top", info["Id"], "-eo", "pid,args"]).stdout.decode().splitlines()[1:] - if (len(top) != 1 or len(top[0].split()) < 2 - or Path(top[0].split()[1]).name != "layerfs-daemon"): + if len(top) != 1 or len(top[0].split()) < 2 or Path(top[0].split()[1]).name != "layerfs-daemon": raise Refused("Container has active commands or is not a lone LayerFS daemon") elif info["State"]["Status"] not in {"exited", "created"}: raise Refused("Container is not quiescent") @@ -71,19 +72,25 @@ def retire(action, scope, output, pass_id, record_event, repo=None): rootfs = archive(repo, selected, dest, pass_id) rootfs["paths"] = action.get("preserve_paths", []) shutil.rmtree(selected) - rootfs["limits"] = "Other ephemeral rootfs data explicitly discarded; original image and all mounted data retained" + rootfs["limits"] = ( + "Other ephemeral rootfs data explicitly discarded; original image and all mounted data retained" + ) else: tag = "layerfs-cleanup-recovery:" + pass_id.lower() + "-" + info["Id"][:12] - image = run(["docker", "commit", "--message", "Cleanup recovery; retain mounted volumes", - info["Id"], tag]).stdout.decode().strip() + image = ( + run(["docker", "commit", "--message", "Cleanup recovery; retain mounted volumes", info["Id"], tag]) + .stdout.decode() + .strip() + ) saved = json.loads(run(["docker", "image", "inspect", image]).stdout)[0] if saved["Id"] != image or saved.get("Parent") != info["Image"]: raise Refused("Recovery image identity/parent mismatch") rootfs = {"image": image, "image_tag": tag} log = dest / "logs.txt" with log.open("xb") as stream: - run_logs = subprocess.run(["docker", "logs", "--timestamps", info["Id"]], - stdout=stream, stderr=subprocess.STDOUT, timeout=115) + run_logs = subprocess.run( + ["docker", "logs", "--timestamps", info["Id"]], stdout=stream, stderr=subprocess.STDOUT, timeout=115 + ) if run_logs.returncode: raise Refused("Container log capture failed; container and recovery image retained") original_hash = hashlib.sha256() @@ -98,9 +105,13 @@ def retire(action, scope, output, pass_id, record_event, repo=None): if verified.digest() != original_hash.digest(): raise Refused("Archived log verification failed") log.unlink() - recovery = {"directory": str(dest), "rootfs": rootfs, - "logs_sha256": original_hash.hexdigest(), "volumes": "retained", - "limits": "See rootfs recovery limits; mounted data remains at its original mounts"} + recovery = { + "directory": str(dest), + "rootfs": rootfs, + "logs_sha256": original_hash.hexdigest(), + "volumes": "retained", + "limits": "See rootfs recovery limits; mounted data remains at its original mounts", + } (dest / "recovery.json").write_text(json.dumps(recovery, indent=2) + "\n") record_event(output, state="archived", action=action, recovery=recovery) current = inspect(info["Id"]) diff --git a/tools/cleanup_retirement.py b/tools/cleanup_retirement.py index 74ca2527d..854b45e51 100644 --- a/tools/cleanup_retirement.py +++ b/tools/cleanup_retirement.py @@ -34,8 +34,7 @@ def retirement_open_files(path, opened, enabled): ignored = set() root = str(path) docker_disks = Path.home() / "Library/Containers/com.docker.docker/Data/vms" - docker_vms = {pid for pid, name in opened if Path(name).name == "Docker.raw" - and docker_disks in Path(name).parents} + docker_vms = {pid for pid, name in opened if Path(name).name == "Docker.raw" and docker_disks in Path(name).parents} relevant = {} for pid, name in opened: if name == root or name.startswith(root + os.sep): @@ -50,8 +49,7 @@ def retirement_open_files(path, opened, enabled): # A shell with no children, or an idle agent, may retain only the root # directory. Open files below it remain protected. This never kills an # agent and is re-evaluated before unlinking the backed-up checkout. - if all(name == root for name in names) and ( - pid in idle or executable == "zsh" and not children.get(pid)): + if all(name == root for name in names) and (pid in idle or executable == "zsh" and not children.get(pid)): ignored.add(pid) # Docker Desktop may retain cached host file descriptors after a bind # is gone. The caller independently checks every current container bind. diff --git a/tools/cleanup_state.py b/tools/cleanup_state.py index b3d017a33..3920abf4e 100644 --- a/tools/cleanup_state.py +++ b/tools/cleanup_state.py @@ -20,8 +20,9 @@ def run(args, data=None, env=None, allowed=(0,), timeout=115): except subprocess.TimeoutExpired as error: raise Refused(f"Command exceeded {timeout} seconds; no retry: {args!r}") from error if result.returncode not in allowed: - raise Refused(f"Command failed ({result.returncode}): {args!r}: " - + result.stderr.decode(errors="replace")[-4000:]) + raise Refused( + f"Command failed ({result.returncode}): {args!r}: " + result.stderr.decode(errors="replace")[-4000:] + ) return result @@ -47,7 +48,7 @@ def host_bind_path(raw): if sys.platform == "darwin": for prefix in ("/host_mnt", "/run/desktop/mnt/host"): if raw.startswith(prefix + "/"): - raw = raw[len(prefix):] + raw = raw[len(prefix) :] break return Path(raw).resolve() @@ -63,17 +64,25 @@ def visit(item): relative = str(item.relative_to(path)) if "\n" in str(item) or "\r" in str(item): raise Refused(f"Unsupported newline in path: {item!r}") - if not (stat.S_ISREG(info.st_mode) or stat.S_ISDIR(info.st_mode) - or stat.S_ISLNK(info.st_mode)): + if not (stat.S_ISREG(info.st_mode) or stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode)): raise Refused(f"Special file is not disposable: {item}") if info.st_dev != root_device: raise Refused(f"Nested filesystem: {item}") - rows.append({"path": relative, "mode": info.st_mode, "size": info.st_size, - "mtime_ns": info.st_mtime_ns, "ctime_ns": info.st_ctime_ns, - "inode": info.st_ino, "device": info.st_dev, - "uid": info.st_uid, "gid": info.st_gid, - "allocated": info.st_blocks * 512, - "target": os.readlink(item) if stat.S_ISLNK(info.st_mode) else None}) + rows.append( + { + "path": relative, + "mode": info.st_mode, + "size": info.st_size, + "mtime_ns": info.st_mtime_ns, + "ctime_ns": info.st_ctime_ns, + "inode": info.st_ino, + "device": info.st_dev, + "uid": info.st_uid, + "gid": info.st_gid, + "allocated": info.st_blocks * 512, + "target": os.readlink(item) if stat.S_ISLNK(info.st_mode) else None, + } + ) if stat.S_ISDIR(info.st_mode): for child in sorted(item.iterdir()): if child.name == ".git": @@ -85,8 +94,7 @@ def visit(item): visit(path) digest = hashlib.sha256(json.dumps(rows, sort_keys=True).encode()).hexdigest() - return rows, {"fingerprint": digest, "files": len(rows), - "allocated_bytes": sum(x["allocated"] for x in rows)} + return rows, {"fingerprint": digest, "files": len(rows), "allocated_bytes": sum(x["allocated"] for x in rows)} def worktrees(repo): @@ -109,11 +117,19 @@ def docker_state(): ids = run(["docker", "ps", "-aq"]).stdout.decode().split() containers = [] for offset in range(0, len(ids), 100): - containers.extend(json.loads(run(["docker", "inspect", *ids[offset:offset + 100]]).stdout)) - return [{"id": c["Id"], "name": c["Name"], "running": c["State"]["Running"], - "status": c["State"]["Status"], "image": c["Image"], - "labels": c["Config"].get("Labels") or {}, - "mounts": c.get("Mounts", [])} for c in containers] + containers.extend(json.loads(run(["docker", "inspect", *ids[offset : offset + 100]]).stdout)) + return [ + { + "id": c["Id"], + "name": c["Name"], + "running": c["State"]["Running"], + "status": c["State"]["Status"], + "image": c["Image"], + "labels": c["Config"].get("Labels") or {}, + "mounts": c.get("Mounts", []), + } + for c in containers + ] def open_files(): diff --git a/tools/cleanup_volume_cache.py b/tools/cleanup_volume_cache.py index 15994bd11..f509bc531 100644 --- a/tools/cleanup_volume_cache.py +++ b/tools/cleanup_volume_cache.py @@ -1,82 +1,102 @@ """Clear released Cargo volume caches while retaining their direct binaries.""" + import json import hashlib import re from pathlib import Path import uuid -from cleanup_state import (Refused, run, docker_state, ensure_volume_idle, - volume_identity, same_volume) +from cleanup_state import Refused, run, docker_state, ensure_volume_idle, volume_identity, same_volume + +IMAGE = "node:24.21.0-bookworm" +SCRIPT = Path(__file__).with_suffix(".mjs").resolve() -IMAGE = 'node:24.21.0-bookworm' -SCRIPT = Path(__file__).with_suffix('.mjs').resolve() def helper(name, image, output, mode, cache=None, target_subdir=None): - if target_subdir is not None and (not isinstance(target_subdir, str) - or not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_.-]*', target_subdir)): - raise Refused('target_subdir must be one literal directory name inside the volume') - mount = f'type=volume,src={name},dst=/target' + if target_subdir is not None and ( + not isinstance(target_subdir, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", target_subdir) + ): + raise Refused("target_subdir must be one literal directory name inside the volume") + mount = f"type=volume,src={name},dst=/target" if target_subdir is not None: - mount += ',volume-subpath=' + target_subdir - if mode == 'plan': - mount += ',readonly' + mount += ",volume-subpath=" + target_subdir + if mode == "plan": + mount += ",readonly" token = uuid.uuid4().hex - prefix = output / ('cargo-volume-' + token) - args = ['docker', 'create', '--name', 'layerfs-cleanup-cache-' + token, - '--label', 'layerfs.cleanup.pass=' + output.name, '--pull', 'never', - '--network', 'none', '--read-only', '--mount', - mount, - '--mount', f'type=bind,src={SCRIPT},dst=/cleanup.mjs,readonly'] + prefix = output / ("cargo-volume-" + token) + args = [ + "docker", + "create", + "--name", + "layerfs-cleanup-cache-" + token, + "--label", + "layerfs.cleanup.pass=" + output.name, + "--pull", + "never", + "--network", + "none", + "--read-only", + "--mount", + mount, + "--mount", + f"type=bind,src={SCRIPT},dst=/cleanup.mjs,readonly", + ] if cache is not None: - plan = prefix.with_suffix('.plan.json') - plan.write_text(json.dumps(cache) + '\n') - args += ['--mount', f'type=bind,src={plan},dst=/plan.json,readonly'] - args += ['--entrypoint', 'node', image, '/cleanup.mjs', mode, '/plan.json'] + plan = prefix.with_suffix(".plan.json") + plan.write_text(json.dumps(cache) + "\n") + args += ["--mount", f"type=bind,src={plan},dst=/plan.json,readonly"] + args += ["--entrypoint", "node", image, "/cleanup.mjs", mode, "/plan.json"] ident = run(args).stdout.decode().strip() - prefix.with_suffix('.container.json').write_text(json.dumps({'id': ident, 'command': args}, indent=2)+'\n') + prefix.with_suffix(".container.json").write_text(json.dumps({"id": ident, "command": args}, indent=2) + "\n") completed = False try: # Our created helper is the sole allowed consumer; every other owner, # including stopped containers, still protects this volume. - ensure_volume_idle(name, [], [c for c in docker_state() if c['id'] != ident]) - result = run(['docker', 'start', '--attach', ident]) - prefix.with_suffix('.output.json').write_bytes(result.stdout) - state = json.loads(run(['docker', 'inspect', ident]).stdout)[0]['State'] - if state['Running'] or state['ExitCode'] != 0: - raise Refused('Cargo cache helper did not exit successfully; inspect ' + ident) + ensure_volume_idle(name, [], [c for c in docker_state() if c["id"] != ident]) + result = run(["docker", "start", "--attach", ident]) + prefix.with_suffix(".output.json").write_bytes(result.stdout) + state = json.loads(run(["docker", "inspect", ident]).stdout)[0]["State"] + if state["Running"] or state["ExitCode"] != 0: + raise Refused("Cargo cache helper did not exit successfully; inspect " + ident) value = json.loads(result.stdout) completed = True return value finally: - state = json.loads(run(['docker', 'inspect', ident], timeout=10).stdout)[0]['State'] - if state['Running']: - run(['docker', 'kill', ident], timeout=10) + state = json.loads(run(["docker", "inspect", ident], timeout=10).stdout)[0]["State"] + if state["Running"]: + run(["docker", "kill", ident], timeout=10) if completed: - run(['docker', 'rm', ident], timeout=10) + run(["docker", "rm", ident], timeout=10) # Failed helpers remain stopped for custody; never repeat a removal. def prepare(action, scope, containers, output): - name = action['name'] - ensure_volume_idle(name, scope.get('protected_volumes', []), containers) + name = action["name"] + ensure_volume_idle(name, scope.get("protected_volumes", []), containers) identity = volume_identity(name) - image = json.loads(run(['docker', 'image', 'inspect', IMAGE]).stdout)[0]['Id'] - cache = helper(name, image, output, 'plan', target_subdir=action.get('target_subdir')) + image = json.loads(run(["docker", "image", "inspect", IMAGE]).stdout)[0]["Id"] + cache = helper(name, image, output, "plan", target_subdir=action.get("target_subdir")) if not same_volume(identity, volume_identity(name)): - raise Refused('Cargo volume identity changed during planning') - return {**action, 'identity': identity, 'image_id': image, 'cache': cache, - 'helper_sha256': hashlib.sha256(SCRIPT.read_bytes()).hexdigest()} + raise Refused("Cargo volume identity changed during planning") + return { + **action, + "identity": identity, + "image_id": image, + "cache": cache, + "helper_sha256": hashlib.sha256(SCRIPT.read_bytes()).hexdigest(), + } def clear(action, scope, output, event): - name = action['name'] - if hashlib.sha256(SCRIPT.read_bytes()).hexdigest() != action['helper_sha256']: - raise Refused('Cargo cache helper changed since plan') - ensure_volume_idle(name, scope.get('protected_volumes', []), docker_state()) - if not same_volume(action['identity'], volume_identity(name)): - raise Refused('Cargo volume identity changed since plan') - event(output, state='removing', action=action) - value = helper(name, action['image_id'], output, 'apply', action['cache'], - target_subdir=action.get('target_subdir')) - if not same_volume(action['identity'], volume_identity(name)): - raise Refused('Cargo volume identity changed during cleanup') - return {**value, 'volume': 'retained', 'image_id': action['image_id']} + name = action["name"] + if hashlib.sha256(SCRIPT.read_bytes()).hexdigest() != action["helper_sha256"]: + raise Refused("Cargo cache helper changed since plan") + ensure_volume_idle(name, scope.get("protected_volumes", []), docker_state()) + if not same_volume(action["identity"], volume_identity(name)): + raise Refused("Cargo volume identity changed since plan") + event(output, state="removing", action=action) + value = helper( + name, action["image_id"], output, "apply", action["cache"], target_subdir=action.get("target_subdir") + ) + if not same_volume(action["identity"], volume_identity(name)): + raise Refused("Cargo volume identity changed during cleanup") + return {**value, "volume": "retained", "image_id": action["image_id"]} diff --git a/tools/decode_observations.py b/tools/decode_observations.py index 766eed849..a21104887 100644 --- a/tools/decode_observations.py +++ b/tools/decode_observations.py @@ -20,135 +20,431 @@ no closing section 99 is reported with "complete": false. Values are the daemon's cumulative counters at that call, not per-call deltas. """ + import argparse import json import sys PREFIX = '{"layerfs_observation":' -OPERATIONS = {1: "Mount", 2: "Commit", 3: "Status", 4: "Unmount", 5: "Fork", 6: "History", - 7: "Hello", 8: "EndSession", 9: "Attach", 10: "Locate", 11: "ForceUnmount", - 12: "Observed", 13: "Cleanup", 14: "Branches", 15: "Sdk"} -STATEMENT_KINDS = ["startup", "begin", "commit", "rollback", "inode", "directory_entry", "payload", - "frontier", "capture", "operation_record", "explain", "reclaim"] -OPCODES = ["lookup", "forget", "batch_forget", "getattr", "setattr", "readlink", "mknod", "mkdir", - "unlink", "rmdir", "symlink", "rename", "link", "open", "read", "write", "flush", - "release", "fsync", "opendir", "readdir", "readdirplus", "releasedir", "fsyncdir", - "statfs", "setxattr", "getxattr", "listxattr", "removexattr", "access", "create", - "getlk", "setlk", "bmap", "ioctl", "poll", "fallocate", "lseek", "copy_file_range"] +OPERATIONS = { + 1: "Mount", + 2: "Commit", + 3: "Status", + 4: "Unmount", + 5: "Fork", + 6: "History", + 7: "Hello", + 8: "EndSession", + 9: "Attach", + 10: "Locate", + 11: "ForceUnmount", + 12: "Observed", + 13: "Cleanup", + 14: "Branches", + 15: "Sdk", +} +STATEMENT_KINDS = [ + "startup", + "begin", + "commit", + "rollback", + "inode", + "directory_entry", + "payload", + "frontier", + "capture", + "operation_record", + "explain", + "reclaim", +] +OPCODES = [ + "lookup", + "forget", + "batch_forget", + "getattr", + "setattr", + "readlink", + "mknod", + "mkdir", + "unlink", + "rmdir", + "symlink", + "rename", + "link", + "open", + "read", + "write", + "flush", + "release", + "fsync", + "opendir", + "readdir", + "readdirplus", + "releasedir", + "fsyncdir", + "statfs", + "setxattr", + "getxattr", + "listxattr", + "removexattr", + "access", + "create", + "getlk", + "setlk", + "bmap", + "ioctl", + "poll", + "fallocate", + "lseek", + "copy_file_range", +] SQL_PHASES = ["prepare", "bind", "next", "map", "drop_cursor", "status", "drop_statement"] -STATEMENT = ["attempts", "executions", "rows_returned", "rows_changed", "direct_rows_changed", - "returned_blob_bytes", "returned_value_bytes", "vm_steps", "fullscan_steps", "sorts", - "autoindex_rows", "reprepares", "bound_bytes", "sql_bytes", "statement_memory_samples", - "statement_memory_sample_bytes", "elapsed_ns"] -PAYLOAD = ["write_input_bytes", "write_cells", "partial_write_cells", "cell_copy_bytes", - "cell_zeroed_bytes", "read_window_zeroed_bytes", "read_local_copy_bytes"] -ALLOCATION = ["attempts", "requested_bytes", "admitted_jobs", "refusals", "observations", - "freelist_queries"] -SQL = (["statements", "vm_steps", "fullscan_steps", "sorts", "autoindex_rows", "reprepares", - "returned_rows", "bound_bytes", "statement_ns", "commit_ns", "transactions", - "write_transactions", "write_commits", "commits", "read_snapshots", "rollbacks", - "transaction_ns", "read_snapshot_ns", "sealed_inserts", "sealed_body_bytes", - "blob_open_calls", "blob_reopen_calls", "blob_read_calls", "blob_requested_bytes", - "blob_read_bytes", "blob_read_ns", "blob_close_calls"] - + [f"statement_{p}_{f}" for p in SQL_PHASES for f in ("calls", "wall_ns")] - + [f"commit_{p}_{f}" for p in SQL_PHASES for f in ("calls", "wall_ns")]) -STORAGE = ["policy", "locate", "read_packs", "read_pack_selections", "whole_selected", - "whole_due_density", "whole_due_singleton", "whole_due_small", "whole_due_reuse", - "range_selected", "range_scan_bytes", "range_acquired_bytes", "range_materialized_bytes", - "value_groups", "signatures", "reserve", "initial_reservations", "reservation_refills", - "publish", "payload_reads", "payload_read_bytes", "pack_read_bytes", "pack_write_bytes", - "pooled_packs", "pooled_groups", "reserved_directory_bytes", "ordinal_reservations", - "locator_hits", "locator_misses", "locator_evictions", "locator_eviction_probes", - "locator_second_chances", "locator_bookkeeping_live_peak_bytes", "pack_hits", - "pack_misses", "pack_evictions", "pack_evicted_bytes", "directory_validations", - "directory_unshared_walks", "directory_entry_bounds", "prefetch_group_decodes", - "forced_seals"] -PAGES = ["pages_read", "read_waves", "pages_created", "pages_reused", "change_keys", - "untouched_subtrees", "peak_scratch_bytes"] +STATEMENT = [ + "attempts", + "executions", + "rows_returned", + "rows_changed", + "direct_rows_changed", + "returned_blob_bytes", + "returned_value_bytes", + "vm_steps", + "fullscan_steps", + "sorts", + "autoindex_rows", + "reprepares", + "bound_bytes", + "sql_bytes", + "statement_memory_samples", + "statement_memory_sample_bytes", + "elapsed_ns", +] +PAYLOAD = [ + "write_input_bytes", + "write_cells", + "partial_write_cells", + "cell_copy_bytes", + "cell_zeroed_bytes", + "read_window_zeroed_bytes", + "read_local_copy_bytes", +] +ALLOCATION = ["attempts", "requested_bytes", "admitted_jobs", "refusals", "observations", "freelist_queries"] +SQL = ( + [ + "statements", + "vm_steps", + "fullscan_steps", + "sorts", + "autoindex_rows", + "reprepares", + "returned_rows", + "bound_bytes", + "statement_ns", + "commit_ns", + "transactions", + "write_transactions", + "write_commits", + "commits", + "read_snapshots", + "rollbacks", + "transaction_ns", + "read_snapshot_ns", + "sealed_inserts", + "sealed_body_bytes", + "blob_open_calls", + "blob_reopen_calls", + "blob_read_calls", + "blob_requested_bytes", + "blob_read_bytes", + "blob_read_ns", + "blob_close_calls", + ] + + [f"statement_{p}_{f}" for p in SQL_PHASES for f in ("calls", "wall_ns")] + + [f"commit_{p}_{f}" for p in SQL_PHASES for f in ("calls", "wall_ns")] +) +STORAGE = [ + "policy", + "locate", + "read_packs", + "read_pack_selections", + "whole_selected", + "whole_due_density", + "whole_due_singleton", + "whole_due_small", + "whole_due_reuse", + "range_selected", + "range_scan_bytes", + "range_acquired_bytes", + "range_materialized_bytes", + "value_groups", + "signatures", + "reserve", + "initial_reservations", + "reservation_refills", + "publish", + "payload_reads", + "payload_read_bytes", + "pack_read_bytes", + "pack_write_bytes", + "pooled_packs", + "pooled_groups", + "reserved_directory_bytes", + "ordinal_reservations", + "locator_hits", + "locator_misses", + "locator_evictions", + "locator_eviction_probes", + "locator_second_chances", + "locator_bookkeeping_live_peak_bytes", + "pack_hits", + "pack_misses", + "pack_evictions", + "pack_evicted_bytes", + "directory_validations", + "directory_unshared_walks", + "directory_entry_bounds", + "prefetch_group_decodes", + "forced_seals", +] +PAGES = [ + "pages_read", + "read_waves", + "pages_created", + "pages_reused", + "change_keys", + "untouched_subtrees", + "peak_scratch_bytes", +] # section -> (name, fields, how the index is named) SECTIONS = { 0: ("operation", ["operation", "configured_readers", "cache_bytes"], "namespace"), - 1: ("owner", ["jobs", "maintenance_jobs", "maintenance_rows", "closed_namespaces", - "maintenance_data_bytes", "maintenance_ns", "jobs_general", "jobs_mutation", - "jobs_source", "lock_wait_ns_general", "lock_wait_ns_mutation", - "lock_wait_ns_source", "hold_ns_general", "hold_ns_mutation", "hold_ns_source"], None), + 1: ( + "owner", + [ + "jobs", + "maintenance_jobs", + "maintenance_rows", + "closed_namespaces", + "maintenance_data_bytes", + "maintenance_ns", + "jobs_general", + "jobs_mutation", + "jobs_source", + "lock_wait_ns_general", + "lock_wait_ns_mutation", + "lock_wait_ns_source", + "hold_ns_general", + "hold_ns_mutation", + "hold_ns_source", + ], + None, + ), 2: ("sql_foreground", STATEMENT, "statement"), 3: ("sql_maintenance", STATEMENT, "statement"), 4: ("payload_foreground", PAYLOAD, None), 5: ("payload_maintenance", PAYLOAD, None), 6: ("allocation_foreground", ALLOCATION, None), 7: ("allocation_maintenance", ALLOCATION, None), - 8: ("store", ["object_batches", "object_ids", "length_batches", "length_ids", - "serial_reservations"], None), - 9: ("store_readers", ["stopping", "poisoned", "readers", "quarantined", "opening_failures", - "requests", "lock_wait_ns"], None), - 10: ("immutable_cache", ["cache_hits", "cache_misses", "upstream_batches", "authenticated_bytes", - "evictions", "charged_cache_bytes", "cached_objects", "seeded_objects", - "seeded_bytes", "seeded_lengths"], None), + 8: ("store", ["object_batches", "object_ids", "length_batches", "length_ids", "serial_reservations"], None), + 9: ( + "store_readers", + ["stopping", "poisoned", "readers", "quarantined", "opening_failures", "requests", "lock_wait_ns"], + None, + ), + 10: ( + "immutable_cache", + [ + "cache_hits", + "cache_misses", + "upstream_batches", + "authenticated_bytes", + "evictions", + "charged_cache_bytes", + "cached_objects", + "seeded_objects", + "seeded_bytes", + "seeded_lengths", + ], + None, + ), 12: ("kernel_mount", ["owner_id", "root_serial", "serving", "detached", "abort_bound"], None), 13: ("writer_sql", SQL, None), 14: ("reader_sql", SQL, "reader"), - 15: ("commit_namespace", ["inode_rows", "entry_rows", "inode_pages", "entry_pages", - "change_pages", "largest_page", "parent_points", "base_lookups", - "headers_written", "headers_dropped", "values_written", - "fresh_serials", "tombstones_skipped", "files_constructed", - "files_unchanged", "symlinks", "metadata_built", "metadata_patched", - "record_jobs", "header_opens", "header_rows", "header_points", - "change_opens", "change_rows", "change_points", "value_opens", - "value_rows", "value_points", "fresh_opens", "fresh_rows", - "fresh_points", "window_jobs", "normalize_ns", "values_ns", - "update_ns", "inode_page_ns", "file_ns", "file_bind_ns", - "file_source_ns", "values_emit_ns", "values_record_ns", - "update_record_ns", "update_capture_ns", "update_emit_ns", - "backing_calls", "backing_keys", "backing_ns"], None), + 15: ( + "commit_namespace", + [ + "inode_rows", + "entry_rows", + "inode_pages", + "entry_pages", + "change_pages", + "largest_page", + "parent_points", + "base_lookups", + "headers_written", + "headers_dropped", + "values_written", + "fresh_serials", + "tombstones_skipped", + "files_constructed", + "files_unchanged", + "symlinks", + "metadata_built", + "metadata_patched", + "record_jobs", + "header_opens", + "header_rows", + "header_points", + "change_opens", + "change_rows", + "change_points", + "value_opens", + "value_rows", + "value_points", + "fresh_opens", + "fresh_rows", + "fresh_points", + "window_jobs", + "normalize_ns", + "values_ns", + "update_ns", + "inode_page_ns", + "file_ns", + "file_bind_ns", + "file_source_ns", + "values_emit_ns", + "values_record_ns", + "update_record_ns", + "update_capture_ns", + "update_emit_ns", + "backing_calls", + "backing_keys", + "backing_ns", + ], + None, + ), 16: ("commit_storage", STORAGE, None), - 17: ("content_objects", ["objects_read", "read_waves", "bytes_read", "objects_emitted", - "bytes_emitted"], None), + 17: ("content_objects", ["objects_read", "read_waves", "bytes_read", "objects_emitted", "bytes_emitted"], None), 18: ("content_directories", PAGES, None), 19: ("content_inodes", PAGES, None), - 20: ("content_references", ["rows_touched", "rows_spilled", "base_records_read", "base_waves", - "final_values", "final_removals", "serials_scanned", - "peak_pending", "runs_rows_written", "runs_rows_read", - "runs_created", "runs_merges", "runs_peak_level", - "runs_peak_live_runs", "runs_peak_run_bytes"], None), - 21: ("content_release", ["pages", "entries", "base_records", "released", - "traversed_directories", "peak_depth"], None), - 22: ("content_validation", ["objects_read", "read_waves", "inode_demands", "inode_pages_read", - "directory_pages_read", "entries_examined", - "inode_pages_allocation", "inode_pages_prefetch", - "inode_pages_bindings", "inode_pages_aliases", - "inode_pages_cycles", "inode_pages_reachability", "placements", - "ancestry_steps", "territory_directories", "territory_entries", - "peak_window_rows", "in_place_scans", "in_place_rows", - "in_place_directories"], None), - 23: ("content_bindings", ["bindings_added", "bindings_removed", "directory_updates", - "base_records_read"], None), - 24: ("kernel_loops", ["phase", "revision", "configured", "created", "entered", "exited", - "joined"], None), + 20: ( + "content_references", + [ + "rows_touched", + "rows_spilled", + "base_records_read", + "base_waves", + "final_values", + "final_removals", + "serials_scanned", + "peak_pending", + "runs_rows_written", + "runs_rows_read", + "runs_created", + "runs_merges", + "runs_peak_level", + "runs_peak_live_runs", + "runs_peak_run_bytes", + ], + None, + ), + 21: ( + "content_release", + ["pages", "entries", "base_records", "released", "traversed_directories", "peak_depth"], + None, + ), + 22: ( + "content_validation", + [ + "objects_read", + "read_waves", + "inode_demands", + "inode_pages_read", + "directory_pages_read", + "entries_examined", + "inode_pages_allocation", + "inode_pages_prefetch", + "inode_pages_bindings", + "inode_pages_aliases", + "inode_pages_cycles", + "inode_pages_reachability", + "placements", + "ancestry_steps", + "territory_directories", + "territory_entries", + "peak_window_rows", + "in_place_scans", + "in_place_rows", + "in_place_directories", + ], + None, + ), + 23: ("content_bindings", ["bindings_added", "bindings_removed", "directory_updates", "base_records_read"], None), + 24: ("kernel_loops", ["phase", "revision", "configured", "created", "entered", "exited", "joined"], None), 25: ("kernel_work", ["active", "retained", "requests", "completed", "terminal"], None), - 26: ("kernel_opcodes", OPCODES + ["served", "inline", "refused", "terminal", "forget_units", - "store_units", "lookup_missing", "served_ns"], None), + 26: ( + "kernel_opcodes", + OPCODES + + ["served", "inline", "refused", "terminal", "forget_units", "store_units", "lookup_missing", "served_ns"], + None, + ), 27: ("kernel_abort", ["disposition", "value"], None), - 28: ("workspace_resources", ["namespaces", "inode_rows", "directory_entry_rows", - "payload_cells", "payload_bytes", "shrink_rows", - "operation_record_rows", "operation_record_bytes", "owner_rows", - "source_rows", "owner_details", "reply_tickets", - "maintenance_targets", "ready_targets", "logical_bytes", - "allocated_bytes", "high_water_allocated_bytes", - "reserved_tail_bytes", "cleanup_headroom_bytes"] - + [f"allocation_{f}" for f in ALLOCATION] - + ["database_pages", "free_pages", "debt_upper_bytes"], None), + 28: ( + "workspace_resources", + [ + "namespaces", + "inode_rows", + "directory_entry_rows", + "payload_cells", + "payload_bytes", + "shrink_rows", + "operation_record_rows", + "operation_record_bytes", + "owner_rows", + "source_rows", + "owner_details", + "reply_tickets", + "maintenance_targets", + "ready_targets", + "logical_bytes", + "allocated_bytes", + "high_water_allocated_bytes", + "reserved_tail_bytes", + "cleanup_headroom_bytes", + ] + + [f"allocation_{f}" for f in ALLOCATION] + + ["database_pages", "free_pages", "debt_upper_bytes"], + None, + ), 29: ("reader_storage", STORAGE, "reader"), - 30: ("resource_observer", ["attempted", "completed"] + [f"sql_{f}" for f in STATEMENT] - + [f"payload_{f}" for f in PAYLOAD] - + [f"allocation_{f}" for f in ALLOCATION] - + ["lock_wait_ns", "hold_ns"], None), - 32: ("open_state", ["kernel_entries", "directory_parents", "file_handles", "file_reads", - "lookup_owners", "file_custody", "orphan_files", "generation_waits", - "mounts", "directory_handles", "cookie_pages"], None), + 30: ( + "resource_observer", + ["attempted", "completed"] + + [f"sql_{f}" for f in STATEMENT] + + [f"payload_{f}" for f in PAYLOAD] + + [f"allocation_{f}" for f in ALLOCATION] + + ["lock_wait_ns", "hold_ns"], + None, + ), + 32: ( + "open_state", + [ + "kernel_entries", + "directory_parents", + "file_handles", + "file_reads", + "lookup_owners", + "file_custody", + "orphan_files", + "generation_waits", + "mounts", + "directory_handles", + "cookie_pages", + ], + None, + ), 33: ("template_memory_management", ["enabled"], None), 34: ("resource_jobs", ["jobs"], None), 35: ("owner_lock_holds", ["observations", "reply_attempts", "reply_hold_ns"], None), @@ -173,7 +469,7 @@ def named(section, index, available, values): if not available: return name, None value = {field: values[i] for i, field in enumerate(fields) if i < len(values)} - extra = values[len(fields):] + extra = values[len(fields) :] if extra: value["unnamed"] = extra if section == 0: @@ -194,8 +490,14 @@ def decode(lines, traces): continue key = (words_hex(row["daemon"]), words_hex(row["scope"]), row["call"]) if key not in groups: - groups[key] = {"daemon": key[0], "scope": key[1], "call": key[2], "slot": row["slot"], - "complete": False, "sections": {}} + groups[key] = { + "daemon": key[0], + "scope": key[1], + "call": key[2], + "slot": row["slot"], + "complete": False, + "sections": {}, + } order.append(key) group = groups[key] name, value = named(row["section"], row["index"], row["available"], row["values"]) diff --git a/tools/production_loc.py b/tools/production_loc.py index caf176ba6..43a566e01 100755 --- a/tools/production_loc.py +++ b/tools/production_loc.py @@ -222,9 +222,7 @@ def declared_modules(path: Path, text: str) -> list: file is itself test only, which is what makes the exclusion transitive. """ declarations = [] - pattern = re.compile( - r"(?P(?:#\[[^\]]*\]\s*)*)mod\s+(?P[A-Za-z_][A-Za-z0-9_]*)\s*;" - ) + pattern = re.compile(r"(?P(?:#\[[^\]]*\]\s*)*)mod\s+(?P[A-Za-z_][A-Za-z0-9_]*)\s*;") directory = path.parent if path.stem in {"mod", "lib", "main"} else path.parent / path.stem for match in pattern.finditer(text): attributes = match.group("attributes") @@ -232,19 +230,14 @@ def declared_modules(path: Path, text: str) -> list: implies_test(attribute[attribute.find("(") + 1 : -2]) for attribute in re.findall(r"#\[cfg\([^\]]*\)\]", attributes) ) or any( - "test" in attribute and "cfg(" not in attribute - for attribute in re.findall(r"#\[[^\]]*\]", attributes) + "test" in attribute and "cfg(" not in attribute for attribute in re.findall(r"#\[[^\]]*\]", attributes) ) explicit = re.search(r'#\[path\s*=\s*"([^"]+)"\]', attributes) if explicit: target = path.parent / explicit.group(1) else: candidate = directory / match.group("name") - target = ( - candidate.with_suffix(".rs") - if (candidate.with_suffix(".rs")).exists() - else candidate / "mod.rs" - ) + target = candidate.with_suffix(".rs") if (candidate.with_suffix(".rs")).exists() else candidate / "mod.rs" declarations.append((test_only, target)) return declarations @@ -460,12 +453,10 @@ def scope_files(root: Path, scope: str) -> list: if any(part in SKIP_DIRS for part in parts): continue source = (len(parts) > 2 and parts[1] == "src") or ( - len(parts) > 3 and parts[0] == "layerfs-api" - and parts[1] in ("core", "sdk") and parts[2] == "src" + len(parts) > 3 and parts[0] == "layerfs-api" and parts[1] in ("core", "sdk") and parts[2] == "src" ) sql = (len(parts) > 2 and parts[1] == "sql") or ( - len(parts) > 3 and parts[0] == "layerfs-api" - and parts[1] in ("core", "sdk") and parts[2] == "sql" + len(parts) > 3 and parts[0] == "layerfs-api" and parts[1] in ("core", "sdk") and parts[2] == "sql" ) if source and path.suffix in CODE_SUFFIXES: files.append(path) @@ -534,10 +525,7 @@ def main() -> int: report = per_file(Path(args.root).resolve()) print(f"{'path':<78} {'scope':<9} {'prod':>7} {'physical':>9}") for entry in report["files"]: - print( - f"{entry['path']:<78} {entry['scope']:<9} " - f"{entry['production_loc']:>7} {entry['physical_lines']:>9}" - ) + print(f"{entry['path']:<78} {entry['scope']:<9} {entry['production_loc']:>7} {entry['physical_lines']:>9}") return 0 result = scan(Path(args.root).resolve()) if args.json: diff --git a/tools/stage5_component_comparison.py b/tools/stage5_component_comparison.py index 323963237..8fb9a5c34 100755 --- a/tools/stage5_component_comparison.py +++ b/tools/stage5_component_comparison.py @@ -40,14 +40,10 @@ def run(command, cwd): started = time.monotonic() - result = subprocess.run( - command, cwd=cwd, capture_output=True, text=True, timeout=120 - ) + result = subprocess.run(command, cwd=cwd, capture_output=True, text=True, timeout=120) wall = time.monotonic() - started if result.returncode != 0: - raise SystemExit( - f"command failed ({result.returncode}): {' '.join(command)}\n{result.stderr}" - ) + raise SystemExit(f"command failed ({result.returncode}): {' '.join(command)}\n{result.stderr}") return result.stdout, wall @@ -79,17 +75,14 @@ def main(): arguments = parser.parse_args() if not (ROOT / "Cargo.toml").is_file(): raise SystemExit( - "the reference arm's root workspace is retired (R9c); " - "run this driver at local tag reference-v0.1.6-final" + "the reference arm's root workspace is retired (R9c); run this driver at local tag reference-v0.1.6-final" ) output = pathlib.Path(arguments.output) if output.exists(): raise SystemExit(f"refusing to reuse an existing output directory: {output}") output.mkdir(parents=True) - commit = subprocess.check_output( - ["git", "rev-parse", "HEAD"], cwd=ROOT, text=True - ).strip() + commit = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip() dirty = subprocess.check_output( ["git", "status", "--porcelain", "--", "core/crates", "crates"], cwd=ROOT, @@ -117,26 +110,30 @@ def main(): "filesystem_primitives_candidate", ), ): - command = [ - "cargo", - TOOLCHAIN, - "run", - "--locked", - "--manifest-path", - manifest, - "-p", - package, - "--example", - example, - ] + build_flags + [ - "--", - "--files", - str(files), - "--changes", - str(changes), - "--samples", - "1", - ] + command = ( + [ + "cargo", + TOOLCHAIN, + "run", + "--locked", + "--manifest-path", + manifest, + "-p", + package, + "--example", + example, + ] + + build_flags + + [ + "--", + "--files", + str(files), + "--changes", + str(changes), + "--samples", + "1", + ] + ) stdout, wall = run(command, ROOT) fields = parse(stdout) row[arm] = {"fields": fields, "wall_s": round(wall, 3), "command": command} @@ -150,9 +147,7 @@ def main(): "root", ) mismatch = [ - key - for key in identity_keys - if row["reference"]["fields"].get(key) != row["candidate"]["fields"].get(key) + key for key in identity_keys if row["reference"]["fields"].get(key) != row["candidate"]["fields"].get(key) ] if mismatch: row["identity"] = "MISMATCH" @@ -163,10 +158,7 @@ def main(): reference_ns = row["reference"]["fields"]["elapsed_ns"] candidate_ns = row["candidate"]["fields"]["elapsed_ns"] row["candidate_over_reference"] = round(candidate_ns / reference_ns, 6) - print( - f"case {name} identity MATCH candidate_over_reference " - f"{row['candidate_over_reference']}" - ) + print(f"case {name} identity MATCH candidate_over_reference {row['candidate_over_reference']}") receipts.append(row) sealed = { @@ -174,8 +166,7 @@ def main(): "working_tree_dirty": bool(dirty), "profile": arguments.profile, "toolchain": TOOLCHAIN, - "scope": "component comparison: sorted directory merge, sorted inline-inode " - "merge and root encoding", + "scope": "component comparison: sorted directory merge, sorted inline-inode merge and root encoding", "excluded_on_both_arms": [ "input normalization", "deriving final reference counts", @@ -183,8 +174,7 @@ def main(): "reference ordering preparation", "physical persistence", ], - "cache_state": "warm in-process fixture; the base tree is built in the arm " - "process before the timed region", + "cache_state": "warm in-process fixture; the base tree is built in the arm process before the timed region", "samples": "one per case per arm", "cases": receipts, } diff --git a/tools/test_cleanup.py b/tools/test_cleanup.py index 6d9b34284..0ce61e7da 100644 --- a/tools/test_cleanup.py +++ b/tools/test_cleanup.py @@ -40,18 +40,27 @@ def git(self, *args): return git(self.repo, *args) def action(self, path, kind="archive-tree"): - return {"kind": kind, "path": str(path), "owner": "fixture", - "reason": "completed disposable fixture", "released": True} + return { + "kind": kind, + "path": str(path), + "owner": "fixture", + "reason": "completed disposable fixture", + "released": True, + } def apply(self, action, protected=None): path = Path(action["path"]) action["snapshot"] = inventory(path, action["kind"] == "worktree")[1] if action["kind"] == "worktree": action["git_head"] = git(path, "rev-parse", "HEAD").decode().strip() - action["staged_sha256"] = hashlib.sha256(git( - path, "diff", "--cached", "--binary", "--full-index", "--no-ext-diff", "--no-textconv")).hexdigest() + action["staged_sha256"] = hashlib.sha256( + git(path, "diff", "--cached", "--binary", "--full-index", "--no-ext-diff", "--no-textconv") + ).hexdigest() plan = {"pass_id": "fixture", "scope": {"protected_paths": protected or []}} - with patch.object(cleanup, "docker_state", return_value=[]), patch.object(cleanup, "open_files", return_value=[]): + with ( + patch.object(cleanup, "docker_state", return_value=[]), + patch.object(cleanup, "open_files", return_value=[]), + ): return cleanup.apply_action(self.repo, self.output, plan, action) def test_raw_archive_roundtrip_including_symlink_and_executable(self): @@ -110,7 +119,10 @@ def test_changed_plan_preserves_original(self): action = self.action(source) action["snapshot"] = inventory(source)[1] file.write_text("after") - with patch.object(cleanup, "docker_state", return_value=[]), patch.object(cleanup, "open_files", return_value=[]): + with ( + patch.object(cleanup, "docker_state", return_value=[]), + patch.object(cleanup, "open_files", return_value=[]), + ): with self.assertRaises(Refused): cleanup.apply_action(self.repo, self.output, {"scope": {}, "pass_id": "fixture"}, action) self.assertEqual(file.read_text(), "after") @@ -157,19 +169,23 @@ def test_dirty_worktree_preserves_staged_and_working_versions(self): self.assertEqual(self.git("rev-parse", result["head_ref"]).decode().strip(), result["head"]) def test_volume_consumer_even_stopped_is_protected(self): - stopped = {"name": "stopped", "running": False, - "mounts": [{"Type": "volume", "Name": "cache"}]} + stopped = {"name": "stopped", "running": False, "mounts": [{"Type": "volume", "Name": "cache"}]} with self.assertRaises(Refused): ensure_volume_idle("cache", [], [stopped]) with self.assertRaises(Refused): ensure_volume_idle("cache", ["cache"], []) - self.assertFalse(same_volume({"Name": "cache", "CreatedAt": "old"}, - {"Name": "cache", "CreatedAt": "new"})) + self.assertFalse(same_volume({"Name": "cache", "CreatedAt": "old"}, {"Name": "cache", "CreatedAt": "new"})) def test_apply_is_single_attempt_and_stops_after_failure(self): - plan = {"version": 1, "pass_directory": str(self.output), "repository": str(self.repo), - "scope": {}, "actions": [{"kind": "archive-file"}, {"kind": "archive-tree"}], - "protected_presence": {}, "blocked": []} + plan = { + "version": 1, + "pass_directory": str(self.output), + "repository": str(self.repo), + "scope": {}, + "actions": [{"kind": "archive-file"}, {"kind": "archive-tree"}], + "protected_presence": {}, + "blocked": [], + } path = self.output / "plan.json" path.write_text(json.dumps(plan)) args = type("Args", (), {"plan": str(path)})() @@ -183,10 +199,13 @@ def test_apply_is_single_attempt_and_stops_after_failure(self): cleanup.apply_plan(args) def test_default_plan_has_no_actions(self): - args = type("Args", (), {"repo": str(self.repo), "scope": None, - "reclaim_gb": 0, - "out": str(self.root / "plan-only")})() - with patch.object(cleanup, "docker_state", return_value=[]), patch.object(cleanup, "open_files", return_value=[]): + args = type( + "Args", (), {"repo": str(self.repo), "scope": None, "reclaim_gb": 0, "out": str(self.root / "plan-only")} + )() + with ( + patch.object(cleanup, "docker_state", return_value=[]), + patch.object(cleanup, "open_files", return_value=[]), + ): with contextlib.redirect_stdout(io.StringIO()): self.assertEqual(cleanup.make_plan(args), 0) plan = json.loads((Path(args.out) / "plan.json").read_text()) @@ -202,9 +221,16 @@ def test_low_disk_threshold_uses_decimal_gb(self): self.assertEqual(cleanup.byte_target(300), 300_000_000_000) def test_recovery_shortfall_returns_two_and_reports_actual_delta(self): - plan = {"version": 1, "pass_directory": str(self.output), "repository": str(self.repo), - "scope": {}, "actions": [], "reclaim_target_bytes": 300_000_000_000, - "protected_presence": {}, "blocked": []} + plan = { + "version": 1, + "pass_directory": str(self.output), + "repository": str(self.repo), + "scope": {}, + "actions": [], + "reclaim_target_bytes": 300_000_000_000, + "protected_presence": {}, + "blocked": [], + } path = self.output / "plan.json" path.write_text(json.dumps(plan)) args = type("Args", (), {"plan": str(path)})() @@ -216,18 +242,34 @@ def test_recovery_shortfall_returns_two_and_reports_actual_delta(self): self.assertFalse(report["reclaim_target_met"]) def test_changed_volume_identity_refused_before_remove(self): - action = {"kind": "docker-volume", "name": "exact-fixture", "released": True, - "owner": "fixture", "reason": "closed", "identity": {"CreatedAt": "old"}} - with patch.object(cleanup, "docker_state", return_value=[]), patch.object(cleanup, "volume_identity", return_value={"CreatedAt": "new"}): + action = { + "kind": "docker-volume", + "name": "exact-fixture", + "released": True, + "owner": "fixture", + "reason": "closed", + "identity": {"CreatedAt": "old"}, + } + with ( + patch.object(cleanup, "docker_state", return_value=[]), + patch.object(cleanup, "volume_identity", return_value={"CreatedAt": "new"}), + ): with patch.object(cleanup, "run") as operation: with self.assertRaises(Refused): cleanup.apply_action(self.repo, self.output, {"scope": {}}, action) operation.assert_not_called() def test_no_space_target_does_not_fail_for_concurrent_writes(self): - plan = {"version": 1, "pass_directory": str(self.output), "repository": str(self.repo), - "scope": {}, "actions": [], "reclaim_target_bytes": 0, - "protected_presence": {}, "blocked": []} + plan = { + "version": 1, + "pass_directory": str(self.output), + "repository": str(self.repo), + "scope": {}, + "actions": [], + "reclaim_target_bytes": 0, + "protected_presence": {}, + "blocked": [], + } path = self.output / "plan.json" path.write_text(json.dumps(plan)) args = type("Args", (), {"plan": str(path)})() @@ -242,8 +284,9 @@ def test_authorized_discovery_selects_idle_dirty_worktree_but_protects_active(se self.git("worktree", "add", "--detach", str(active), "HEAD") (idle / "uncommitted").write_text("recover me") registered = {str(p): {"worktree": str(p)} for p in (self.repo, idle, active)} - scope, skipped = discover(self.repo, {}, registered, [], - [(os.getpid() + 10000, str(active / "tracked"))], "user cleanup") + scope, skipped = discover( + self.repo, {}, registered, [], [(os.getpid() + 10000, str(active / "tracked"))], "user cleanup" + ) self.assertEqual([a["path"] for a in scope["actions"]], [str(idle)]) self.assertEqual(skipped[0]["path"], str(active)) with self.assertRaises(Refused): @@ -251,17 +294,25 @@ def test_authorized_discovery_selects_idle_dirty_worktree_but_protects_active(se def test_discovery_keeps_unrelated_and_running_containers(self): def container(name, running, mounts): - return {"id": name, "name": "/" + name, "running": running, - "status": "running" if running else "exited", "mounts": mounts} + return { + "id": name, + "name": "/" + name, + "running": running, + "status": "running" if running else "exited", + "mounts": mounts, + } + store = [{"Type": "volume", "Destination": "/layerfs-store"}] - containers = [container("closed", False, store), container("live", True, store), - container("unrelated", False, [])] + containers = [ + container("closed", False, store), + container("live", True, store), + container("unrelated", False, []), + ] scope, _ = discover(self.repo, {}, {}, containers, [], "authorized") self.assertEqual([a["name"] for a in scope["actions"]], ["closed"]) def test_running_container_retirement_refuses_exec_descendant(self): - info = {"Id": "exact", "Name": "/daemon", "Mounts": [], - "State": {"Running": True}} + info = {"Id": "exact", "Name": "/daemon", "Mounts": [], "State": {"Running": True}} with self.assertRaises(Refused): guard_container(info, {}, {}) with patch("cleanup_docker.run") as operation: @@ -277,8 +328,12 @@ def test_daemon_inspection_includes_docker_required_pid_column(self): operation.assert_called_once_with(["docker", "top", "exact", "-eo", "pid,args"]) def test_container_using_pinned_volume_cannot_be_retired(self): - info = {"Id": "exact", "Name": "/daemon", "Mounts": [{"Name": "pinned"}], - "State": {"Running": False, "Status": "exited"}} + info = { + "Id": "exact", + "Name": "/daemon", + "Mounts": [{"Name": "pinned"}], + "State": {"Running": False, "Status": "exited"}, + } with self.assertRaises(Refused): guard_container(info, {}, {"protected_volumes": ["pinned"]}) @@ -292,9 +347,15 @@ def test_caches_discovery_keeps_helpers_and_active_profile(self): registered = {str(tree): {"worktree": str(tree)}} scope, _ = discover(self.repo, {}, registered, [], [], "cleanup", caches_only=True) self.assertEqual(scope["actions"][0]["path"], str(profile)) - scope, skipped = discover(self.repo, {}, registered, [], - [(os.getpid() + 10000, str(profile / "deps/file"))], - "cleanup", caches_only=True) + scope, skipped = discover( + self.repo, + {}, + registered, + [], + [(os.getpid() + 10000, str(profile / "deps/file"))], + "cleanup", + caches_only=True, + ) self.assertEqual(scope["actions"], []) self.assertEqual(skipped[0]["kind"], "cargo-cache") self.assertEqual((profile / "helper").read_text(), "preserve") @@ -310,9 +371,15 @@ def test_cargo_hardlinked_siblings_are_checked_before_either_is_removed(self): os.link(first / "object.o", second / "object.o") (profile / "helper").write_bytes(b"keep") action = self.action(profile, "cargo-cache") - action.update(target_root=str(target), guard_path=str(profile), - cache_snapshots={str(p): inventory(p)[1] for p in (first, second)}) - with patch.object(cleanup, "docker_state", return_value=[]), patch.object(cleanup, "open_files", return_value=[]): + action.update( + target_root=str(target), + guard_path=str(profile), + cache_snapshots={str(p): inventory(p)[1] for p in (first, second)}, + ) + with ( + patch.object(cleanup, "docker_state", return_value=[]), + patch.object(cleanup, "open_files", return_value=[]), + ): cleanup.apply_action(self.repo, self.output, {"scope": {}}, action) self.assertFalse(first.exists()) self.assertFalse(second.exists()) @@ -327,9 +394,12 @@ def retired_cargo_fixture(self): (profile / "daemon").write_bytes(b"pinned") (target / ".rustc_info.json").write_text("{}") action = self.action(profile, "cargo-cache") - action.update(target_root=str(target), guard_path=str(profile), - retire_idle_sessions=True, - cache_snapshots={str(cache): inventory(cache)[1]}) + action.update( + target_root=str(target), + guard_path=str(profile), + retire_idle_sessions=True, + cache_snapshots={str(cache): inventory(cache)[1]}, + ) return action, profile, cache def test_retired_cargo_profile_filters_cached_vm_handles_at_plan_and_apply(self): @@ -337,14 +407,17 @@ def test_retired_cargo_profile_filters_cached_vm_handles_at_plan_and_apply(self) scope = self.root / "scope.json" scope.write_text(json.dumps({"version": 1, "actions": [action]})) self.output.rmdir() - args = type("Args", (), {"repo": str(self.repo), "scope": str(scope), - "out": str(self.output), "reclaim_gb": 0})() + args = type( + "Args", (), {"repo": str(self.repo), "scope": str(scope), "out": str(self.output), "reclaim_gb": 0} + )() disk = self.root / "Library/Containers/com.docker.docker/Data/vms/0/data/Docker.raw" opened = [(4242, str(cache / "object.o")), (4242, str(disk))] - with patch("cleanup_retirement.Path.home", return_value=self.root), \ - patch("cleanup_retirement.run") as process, \ - patch.object(cleanup, "docker_state", return_value=[]), \ - patch.object(cleanup, "open_files", return_value=opened): + with ( + patch("cleanup_retirement.Path.home", return_value=self.root), + patch("cleanup_retirement.run") as process, + patch.object(cleanup, "docker_state", return_value=[]), + patch.object(cleanup, "open_files", return_value=opened), + ): process.return_value.stdout = b"4242 1 S /System/com.apple.Virtualization.VirtualMachine\n" with contextlib.redirect_stdout(io.StringIO()): cleanup.make_plan(args) @@ -361,16 +434,22 @@ def test_retired_cargo_cache_keeps_live_consumers_and_requires_opt_in(self): vm = [(4242, str(cache / "object.o")), (4242, str(disk))] ordinary = [(4343, str(cache / "object.o"))] bound = {"name": "live", "mounts": [{"Type": "bind", "Source": str(self.repo)}]} - cases = [(False, vm, [[], []]), (True, ordinary, [[], []]), - (True, vm, [[bound], [bound]]), (True, vm, [[], [bound]])] - with patch("cleanup_retirement.Path.home", return_value=self.root), \ - patch("cleanup_retirement.run") as process: - process.return_value.stdout = (b"4242 1 S /System/com.apple.Virtualization.VirtualMachine\n" - b"4343 1 S /usr/bin/active-command\n") + cases = [ + (False, vm, [[], []]), + (True, ordinary, [[], []]), + (True, vm, [[bound], [bound]]), + (True, vm, [[], [bound]]), + ] + with patch("cleanup_retirement.Path.home", return_value=self.root), patch("cleanup_retirement.run") as process: + process.return_value.stdout = ( + b"4242 1 S /System/com.apple.Virtualization.VirtualMachine\n4343 1 S /usr/bin/active-command\n" + ) for enabled, opened, containers in cases: - with self.subTest(enabled=enabled, opened=opened, containers=containers), \ - patch.object(cleanup, "docker_state", side_effect=containers), \ - patch.object(cleanup, "open_files", return_value=opened): + with ( + self.subTest(enabled=enabled, opened=opened, containers=containers), + patch.object(cleanup, "docker_state", side_effect=containers), + patch.object(cleanup, "open_files", return_value=opened), + ): action["retire_idle_sessions"] = enabled with self.assertRaises(Refused): cleanup.apply_action(self.repo, self.output, {"scope": {}}, action) @@ -394,8 +473,7 @@ def test_nested_git_metadata_is_preserved_in_worktree_manifest_mapping(self): result = self.apply(self.action(tree, "worktree")) manifest = json.loads(gzip.open(result["manifest"], "rb").read()) row = next(r for r in manifest["rows"] if r["path"] == "dependency/.git/config") - self.assertEqual(self.git("show", result["tree"] + ":" + row["archive_path"]), - b"nested repository recovery\n") + self.assertEqual(self.git("show", result["tree"] + ":" + row["archive_path"]), b"nested repository recovery\n") def test_retirement_allows_idle_agent_cwd_but_preserves_busy_agent_and_open_files(self): sessions = self.root / ".claude/sessions" @@ -420,8 +498,7 @@ def test_retirement_drops_only_exited_consumers_from_old_lsof_snapshot(self): def test_retirement_keeps_handles_of_virtual_machines_without_docker_backing(self): opened = [(4242, str(self.repo / "in-use"))] - with patch("cleanup_retirement.Path.home", return_value=self.root), \ - patch("cleanup_retirement.run") as process: + with patch("cleanup_retirement.Path.home", return_value=self.root), patch("cleanup_retirement.run") as process: process.return_value.stdout = b"4242 1 S /System/com.apple.Virtualization.VirtualMachine\n" self.assertEqual(retirement_open_files(self.repo, opened, True), opened) @@ -430,11 +507,16 @@ def test_worktree_captures_current_edits_under_the_same_planned_root(self): self.git("worktree", "add", "--detach", str(tree), "HEAD") info = tree.stat() action = self.action(tree, "worktree") - action.update(snapshot={"device": info.st_dev, "inode": info.st_ino}, - git_head=git(tree, "rev-parse", "HEAD").decode().strip(), - staged_sha256=hashlib.sha256(b"").hexdigest()) + action.update( + snapshot={"device": info.st_dev, "inode": info.st_ino}, + git_head=git(tree, "rev-parse", "HEAD").decode().strip(), + staged_sha256=hashlib.sha256(b"").hexdigest(), + ) (tree / "new-helper").write_text("latest uncommitted contents\n") - with patch.object(cleanup, "docker_state", return_value=[]), patch.object(cleanup, "open_files", return_value=[]): + with ( + patch.object(cleanup, "docker_state", return_value=[]), + patch.object(cleanup, "open_files", return_value=[]), + ): result = cleanup.apply_action(self.repo, self.output, {"scope": {}, "pass_id": "fixture"}, action) self.assertEqual(self.git("show", result["tree"] + ":new-helper"), b"latest uncommitted contents\n") self.assertFalse(tree.exists()) @@ -445,7 +527,10 @@ def test_replaced_worktree_root_is_preserved(self): info = tree.stat() action = self.action(tree, "worktree") action["snapshot"] = {"device": info.st_dev, "inode": info.st_ino + 1} - with patch.object(cleanup, "docker_state", return_value=[]), patch.object(cleanup, "open_files", return_value=[]): + with ( + patch.object(cleanup, "docker_state", return_value=[]), + patch.object(cleanup, "open_files", return_value=[]), + ): with self.assertRaisesRegex(Refused, "root was replaced"): cleanup.apply_action(self.repo, self.output, {"scope": {}}, action) self.assertTrue((tree / "tracked").exists()) @@ -465,6 +550,7 @@ def test_archive_preserves_extended_attributes(self): result = self.apply(self.action(source)) manifest = json.loads(gzip.open(result["manifest"], "rb").read()) import base64 + row = next(row for row in manifest["rows"] if row["path"] == "file") self.assertEqual(base64.b64decode(row["xattrs_base64"][key]), b"attribute bytes") @@ -477,8 +563,12 @@ def test_docker_desktop_aliases_protect_native_host_paths(self): with patch("cleanup_state.sys.platform", "darwin"): with self.assertRaisesRegex(Refused, "bind overlapping"): ensure_idle([source], [], [container], []) - info = {"Id": "fixture", "Name": "/active", "Mounts": [mount], - "State": {"Running": False, "Status": "exited"}} + info = { + "Id": "fixture", + "Name": "/active", + "Mounts": [mount], + "State": {"Running": False, "Status": "exited"}, + } with self.assertRaisesRegex(Refused, "protected path"): guard_container(info, {}, {"protected_paths": [str(source)]}) @@ -490,13 +580,19 @@ def test_explicit_cache_discovery_includes_primary_and_limits_scope(self): (target / "debug/deps").mkdir(parents=True) (target / ".rustc_info.json").write_text("{}") registered = {str(p): {} for p in (self.repo, other)} - scope, _ = discover(self.repo, {}, registered, [], [], "user authorized caches", - caches_only=True, cache_worktrees=[str(self.repo)]) - self.assertEqual([a["path"] for a in scope["actions"]], - [str(self.repo / "core/target/debug")]) + scope, _ = discover( + self.repo, + {}, + registered, + [], + [], + "user authorized caches", + caches_only=True, + cache_worktrees=[str(self.repo)], + ) + self.assertEqual([a["path"] for a in scope["actions"]], [str(self.repo / "core/target/debug")]) default, _ = discover(self.repo, {}, registered, [], [], "user authorized caches", caches_only=True) - self.assertEqual([a["path"] for a in default["actions"]], - [str(other / "core/target/debug")]) + self.assertEqual([a["path"] for a in default["actions"]], [str(other / "core/target/debug")]) def test_archive_across_hash_batches_preserves_every_file(self): source = self.root / "many-files" @@ -506,8 +602,7 @@ def test_archive_across_hash_batches_preserves_every_file(self): with patch("cleanup_archive.HASH_BATCH_BYTES", 8192): result = self.apply(self.action(source)) manifest = json.loads(gzip.open(result["manifest"], "rb").read()) - self.assertEqual(sum(row.get("target") is None and row["path"] != "." - for row in manifest["rows"]), 4097) + self.assertEqual(sum(row.get("target") is None and row["path"] != "." for row in manifest["rows"]), 4097) self.assertEqual(self.git("show", result["tree"] + ":0"), b"0") self.assertEqual(self.git("show", result["tree"] + ":4096"), b"4096") self.assertFalse(source.exists()) @@ -549,12 +644,22 @@ def test_parallel_archives_keep_independent_recovery_indexes(self): action = self.action(source) action["snapshot"] = inventory(source)[1] actions.append(action) - plan = {"version": 1, "pass_directory": str(self.output), "repository": str(self.repo), - "scope": {}, "actions": actions, "protected_presence": {}, "blocked": []} + plan = { + "version": 1, + "pass_directory": str(self.output), + "repository": str(self.repo), + "scope": {}, + "actions": actions, + "protected_presence": {}, + "blocked": [], + } path = self.output / "plan.json" path.write_text(json.dumps(plan)) args = type("Args", (), {"plan": str(path), "jobs": 2})() - with patch.object(cleanup, "docker_state", return_value=[]), patch.object(cleanup, "open_files", return_value=[]): + with ( + patch.object(cleanup, "docker_state", return_value=[]), + patch.object(cleanup, "open_files", return_value=[]), + ): with contextlib.redirect_stdout(io.StringIO()): self.assertEqual(cleanup.apply_plan(args), 0) report = json.loads((self.output / "report.json").read_text()) @@ -566,9 +671,15 @@ def test_parallel_archives_keep_independent_recovery_indexes(self): self.assertEqual(report["uncertain_actions"], []) def test_interrupted_attempt_is_uncertain_and_never_requeued(self): - plan = {"version": 1, "pass_directory": str(self.output), "repository": str(self.repo), - "scope": {}, "actions": [{"kind": "archive-file"}, {"kind": "archive-tree"}], - "protected_presence": {}, "blocked": []} + plan = { + "version": 1, + "pass_directory": str(self.output), + "repository": str(self.repo), + "scope": {}, + "actions": [{"kind": "archive-file"}, {"kind": "archive-tree"}], + "protected_presence": {}, + "blocked": [], + } path = self.output / "plan.json" path.write_text(json.dumps(plan)) args = type("Args", (), {"plan": str(path)})() @@ -583,6 +694,7 @@ def test_interrupted_attempt_is_uncertain_and_never_requeued(self): def test_protected_cargo_volume_never_starts_helper(self): import cleanup_volume_cache + action = {"kind": "docker-cargo-cache", "name": "pinned"} with patch.object(cleanup_volume_cache, "helper") as helper: with self.assertRaisesRegex(Refused, "Protected volume"): @@ -591,10 +703,18 @@ def test_protected_cargo_volume_never_starts_helper(self): def test_changed_cargo_volume_preserved_before_helper(self): import cleanup_volume_cache - action = {"kind": "docker-cargo-cache", "name": "cache", - "identity": {"CreatedAt": "old"}, - "helper_sha256": hashlib.sha256(cleanup_volume_cache.SCRIPT.read_bytes()).hexdigest()} - with patch.object(cleanup_volume_cache, "docker_state", return_value=[]), patch.object(cleanup_volume_cache, "volume_identity", return_value={"CreatedAt": "new"}), patch.object(cleanup_volume_cache, "helper") as helper: + + action = { + "kind": "docker-cargo-cache", + "name": "cache", + "identity": {"CreatedAt": "old"}, + "helper_sha256": hashlib.sha256(cleanup_volume_cache.SCRIPT.read_bytes()).hexdigest(), + } + with ( + patch.object(cleanup_volume_cache, "docker_state", return_value=[]), + patch.object(cleanup_volume_cache, "volume_identity", return_value={"CreatedAt": "new"}), + patch.object(cleanup_volume_cache, "helper") as helper, + ): with self.assertRaisesRegex(Refused, "identity changed"): cleanup_volume_cache.clear(action, {}, self.output, lambda *a, **k: None) helper.assert_not_called() @@ -602,33 +722,49 @@ def test_changed_cargo_volume_preserved_before_helper(self): def test_nested_cargo_target_mount_stays_inside_selected_directory(self): import cleanup_volume_cache from types import SimpleNamespace - for mode in ('plan', 'apply'): + + for mode in ("plan", "apply"): calls = [] + def run(args, **kwargs): calls.append(args) - data = (b'[{"State":{"Running":false,"ExitCode":0}}]' - if args[1] == 'inspect' else b'{"version":1}' - if args[1] == 'start' else b'fixture') + data = ( + b'[{"State":{"Running":false,"ExitCode":0}}]' + if args[1] == "inspect" + else b'{"version":1}' + if args[1] == "start" + else b"fixture" + ) return SimpleNamespace(stdout=data) - with patch.object(cleanup_volume_cache, 'run', side_effect=run), patch.object(cleanup_volume_cache, 'docker_state', return_value=[]): - result = cleanup_volume_cache.helper('cache', 'pinned-image', self.output, - mode, cache={'version': 1} if mode == 'apply' else None, - target_subdir='plain') - self.assertEqual(result, {'version': 1}) + + with ( + patch.object(cleanup_volume_cache, "run", side_effect=run), + patch.object(cleanup_volume_cache, "docker_state", return_value=[]), + ): + result = cleanup_volume_cache.helper( + "cache", + "pinned-image", + self.output, + mode, + cache={"version": 1} if mode == "apply" else None, + target_subdir="plain", + ) + self.assertEqual(result, {"version": 1}) create = calls[0] - mounts = [create[i + 1] for i, x in enumerate(create) if x == '--mount'] - self.assertIn('type=volume,src=cache,dst=/target,volume-subpath=plain' - + (',readonly' if mode == 'plan' else ''), mounts) - self.assertFalse(any(x == 'type=volume,src=cache,dst=/target' for x in mounts)) + mounts = [create[i + 1] for i, x in enumerate(create) if x == "--mount"] + self.assertIn( + "type=volume,src=cache,dst=/target,volume-subpath=plain" + (",readonly" if mode == "plan" else ""), + mounts, + ) + self.assertFalse(any(x == "type=volume,src=cache,dst=/target" for x in mounts)) def test_nested_cargo_target_rejects_escape_before_container_creation(self): import cleanup_volume_cache - with patch.object(cleanup_volume_cache, 'run') as run: - for path in ('', '..', '.', '../other', '/plain', 'plain/child', - 'plain,readonly', 1): - with self.assertRaisesRegex(Refused, 'one literal directory name'): - cleanup_volume_cache.helper('cache', 'image', self.output, - 'plan', target_subdir=path) + + with patch.object(cleanup_volume_cache, "run") as run: + for path in ("", "..", ".", "../other", "/plain", "plain/child", "plain,readonly", 1): + with self.assertRaisesRegex(Refused, "one literal directory name"): + cleanup_volume_cache.helper("cache", "image", self.output, "plan", target_subdir=path) run.assert_not_called() diff --git a/tools/test_decode_observations.py b/tools/test_decode_observations.py index db44b84b1..b476f73c6 100644 --- a/tools/test_decode_observations.py +++ b/tools/test_decode_observations.py @@ -13,10 +13,20 @@ def row(call, section, index, values, available=True, scope=SCOPE): - return json.dumps({"layerfs_observation": 1, "call": call, "slot": 0, "scope": scope, - "daemon": DAEMON, "section": section, "index": index, - "available": available, "values": values if available else []}, - separators=(",", ":")) + return json.dumps( + { + "layerfs_observation": 1, + "call": call, + "slot": 0, + "scope": scope, + "daemon": DAEMON, + "section": section, + "index": index, + "available": available, + "values": values if available else [], + }, + separators=(",", ":"), + ) def group(call, scope=SCOPE): @@ -52,11 +62,14 @@ def test_joins_traced_calls_by_scope_and_leaves_others_unjoined(self): lines = [*group(1), *group(2), *group(1, scope=other)[:-1]] with tempfile.TemporaryDirectory() as directory: trace = Path(directory) / "trace.jsonl" - trace.write_text(json.dumps({"layerfs_trace": 1, "id": "ab12cd34-7", "scope": SCOPE_HEX}) + "\n" - + "not json\n", encoding="utf-8") + trace.write_text( + json.dumps({"layerfs_trace": 1, "id": "ab12cd34-7", "scope": SCOPE_HEX}) + "\n" + "not json\n", + encoding="utf-8", + ) decoded = list(decode_observations.decode(lines, decode_observations.trace_scopes(trace))) - self.assertEqual([(g["call"], g.get("trace")) for g in decoded], - [(1, "ab12cd34-7"), (2, "ab12cd34-7"), (1, None)]) + self.assertEqual( + [(g["call"], g.get("trace")) for g in decoded], [(1, "ab12cd34-7"), (2, "ab12cd34-7"), (1, None)] + ) self.assertFalse(decoded[2]["complete"], "no footer means incomplete") def test_every_named_section_matches_its_source_width(self): @@ -70,8 +83,7 @@ def test_kernel_opcodes_decodes_old_and_current_lengths(self): old = list(range(45)) new = list(range(47)) lines = [*group(5), row(5, 26, 0, old), *group(6), row(6, 26, 0, new)] - decoded = {g["call"]: g["sections"]["kernel_opcodes"] for g in - decode_observations.decode(lines, {})} + decoded = {g["call"]: g["sections"]["kernel_opcodes"] for g in decode_observations.decode(lines, {})} self.assertEqual(decoded[5]["store_units"], 44) self.assertNotIn("lookup_missing", decoded[5]) self.assertNotIn("served_ns", decoded[5]) @@ -85,10 +97,8 @@ def test_commit_namespace_decodes_old_and_current_lengths(self): old = list(range(31)) new = list(range(35)) split = list(range(47)) - lines = [*group(7), row(7, 15, 0, old), *group(8), row(8, 15, 0, new), - *group(9), row(9, 15, 0, split)] - decoded = {g["call"]: g["sections"]["commit_namespace"] for g in - decode_observations.decode(lines, {})} + lines = [*group(7), row(7, 15, 0, old), *group(8), row(8, 15, 0, new), *group(9), row(9, 15, 0, split)] + decoded = {g["call"]: g["sections"]["commit_namespace"] for g in decode_observations.decode(lines, {})} self.assertEqual(decoded[7]["fresh_points"], 30) for name in ["window_jobs", "normalize_ns", "values_ns", "update_ns"]: self.assertNotIn(name, decoded[7]) diff --git a/tools/test_production_loc.py b/tools/test_production_loc.py index 71b2090c5..45ec774f6 100644 --- a/tools/test_production_loc.py +++ b/tools/test_production_loc.py @@ -23,18 +23,23 @@ def tearDown(self): self._temporary.cleanup() def test_nested_api_members_count_only_product_source(self): - for name in ("core/src/lib.rs", "sdk/src/client.rs", "sdk/tests/route.rs", - "mcp/src/lib.rs", "cli/src/main.rs"): + for name in ("core/src/lib.rs", "sdk/src/client.rs", "sdk/tests/route.rs", "mcp/src/lib.rs", "cli/src/main.rs"): write(self.root, f"core/crates/layerfs-api/{name}", "pub fn value() {}\n") files = production_loc.scope_files(self.root, "core") - self.assertEqual([str(path.relative_to(self.root / "core/crates/layerfs-api")) - for path in files], ["core/src/lib.rs", "sdk/src/client.rs"]) + self.assertEqual( + [str(path.relative_to(self.root / "core/crates/layerfs-api")) for path in files], + ["core/src/lib.rs", "sdk/src/client.rs"], + ) self.assertEqual(production_loc.scan(self.root)["scopes"]["core"]["lines"], 2) def test_api_counts_the_binding_and_each_package_by_its_name(self): write(self.root, "api/package.json", '{"name": "layerfs-js"}\n') - for folder, name in (("agent/sdk", "@layerfs/agent"), ("agent/cli", "@layerfs/cli"), - ("backend", "@layerfs/backend"), ("runtime", "@layerfs/runtime")): + for folder, name in ( + ("agent/sdk", "@layerfs/agent"), + ("agent/cli", "@layerfs/cli"), + ("backend", "@layerfs/backend"), + ("runtime", "@layerfs/runtime"), + ): write(self.root, f"api/{folder}/package.json", f'{{"name": "{name}"}}\n') write(self.root, f"api/{folder}/src/index.js", "export const value = 1;\n// a comment\n") write(self.root, f"api/{folder}/test/index.test.mjs", "export const test = 1;\n") @@ -46,27 +51,57 @@ def test_api_counts_the_binding_and_each_package_by_its_name(self): write(self.root, "api/binding/tests/unit/admission.rs", "pub fn test() {}\n") write(self.root, "api/binding/examples/restore_record.rs", "pub fn example() {}\n") files = production_loc.scope_files(self.root, "api") - self.assertEqual(sorted(str(path.relative_to(self.root / "api")) for path in files), - ["agent/cli/src/index.js", "agent/sdk/src/index.js", "backend/src/index.js", - "backend/src/store/get.js", "binding/src/lib.rs", "runtime/src/index.js"]) + self.assertEqual( + sorted(str(path.relative_to(self.root / "api")) for path in files), + [ + "agent/cli/src/index.js", + "agent/sdk/src/index.js", + "backend/src/index.js", + "backend/src/store/get.js", + "binding/src/lib.rs", + "runtime/src/index.js", + ], + ) scope = production_loc.scan(self.root)["scopes"]["api"] - self.assertEqual((scope["lines"], scope["crates"]), (6, { - "@layerfs/agent": 1, "@layerfs/backend": 2, "@layerfs/cli": 1, - "@layerfs/runtime": 1, "layerfs-node": 1})) + self.assertEqual( + (scope["lines"], scope["crates"]), + ( + 6, + { + "@layerfs/agent": 1, + "@layerfs/backend": 2, + "@layerfs/cli": 1, + "@layerfs/runtime": 1, + "layerfs-node": 1, + }, + ), + ) def test_api_still_counts_the_layouts_before_the_move(self): """sdk/js and cli/ (the former `sdk` and `cli` scopes) count as `api`.""" - for name in ("native/src/lib.rs", "src/index.js", "operator/index.js", - "packages/agent/src/index.js", "packages/backend/src/store/get.js", - "packages/agent/test/trace.mjs", "packages/agent/test/fake.js"): - write(self.root, f"sdk/js/{name}", "export const value = 1;\n" if name.endswith(".js") else "pub fn value() {}\n") + for name in ( + "native/src/lib.rs", + "src/index.js", + "operator/index.js", + "packages/agent/src/index.js", + "packages/backend/src/store/get.js", + "packages/agent/test/trace.mjs", + "packages/agent/test/fake.js", + ): + write( + self.root, + f"sdk/js/{name}", + "export const value = 1;\n" if name.endswith(".js") else "pub fn value() {}\n", + ) write(self.root, "cli/src/main.js", "export const value = 1;\n// a comment\n") write(self.root, "cli/src/sub/argv.js", "export const other = 2;\n") write(self.root, "cli/test/cli.test.mjs", "export const test = 1;\n") write(self.root, "cli/node_modules/x/src/index.js", "export const dep = 1;\n") crates = production_loc.scan(self.root)["scopes"]["api"]["crates"] - self.assertEqual(crates, {"@layerfs/agent": 1, "@layerfs/backend": 1, "@layerfs/cli": 2, - "@layerfs/sdk": 2, "layerfs-node": 1}) + self.assertEqual( + crates, + {"@layerfs/agent": 1, "@layerfs/backend": 1, "@layerfs/cli": 2, "@layerfs/sdk": 2, "layerfs-node": 1}, + ) def test_blank_and_comment_lines_do_not_count(self): path = write( @@ -116,7 +151,7 @@ def test_a_predicate_that_merely_mentions_test_is_production_code(self): "#[cfg(not(test))]\npub fn production_only() -> u8 {\n 1\n}\n\n" '#[cfg(any(test, feature = "test-instrumentation"))]\n' "pub fn instrumented() -> u8 {\n 2\n}\n\n" - "#[cfg(any(debug_assertions, feature = \"test-instrumentation\"))]\n" + '#[cfg(any(debug_assertions, feature = "test-instrumentation"))]\n' "pub fn counted() -> u8 {\n 3\n}\n", ) self.assertEqual(production_loc.counted_lines(path), 12) @@ -207,19 +242,13 @@ def test_tests_examples_and_benches_are_excluded(self): write(self.root, "crates/example/tests/it.rs", "pub const T: u8 = 1;\n") write(self.root, "crates/example/examples/demo.rs", "pub const E: u8 = 1;\n") write(self.root, "crates/example/benches/bench.rs", "pub const B: u8 = 1;\n") - relative = [ - str(path.relative_to(self.root)) - for path in production_loc.scope_files(self.root, "reference") - ] + relative = [str(path.relative_to(self.root)) for path in production_loc.scope_files(self.root, "reference")] self.assertEqual(relative, ["crates/example/src/lib.rs"]) def test_target_directories_are_excluded(self): write(self.root, "crates/example/src/lib.rs", "pub const N: u8 = 1;\n") write(self.root, "crates/example/target/generated.rs", "pub const G: u8 = 1;\n") - relative = [ - str(path.relative_to(self.root)) - for path in production_loc.scope_files(self.root, "reference") - ] + relative = [str(path.relative_to(self.root)) for path in production_loc.scope_files(self.root, "reference")] self.assertEqual(relative, ["crates/example/src/lib.rs"]) def test_scan_separates_scopes_and_reports_a_combined_total(self): diff --git a/tools/test_worktree.py b/tools/test_worktree.py index 2e8e02d2d..5c9b8ad05 100644 --- a/tools/test_worktree.py +++ b/tools/test_worktree.py @@ -26,15 +26,28 @@ def setUp(self): self.root = Path(self.temp.name).resolve() self.repo = self.root / "checkout" self.repo.mkdir() - for folder in ("core/crates/example/src", "core/vendor", ".cargo", "api/agent/sdk/src", "api/agent/sdk/test", "api/backend/src", "api/runtime/src", "api/binding/src"): + for folder in ( + "core/crates/example/src", + "core/vendor", + ".cargo", + "api/agent/sdk/src", + "api/agent/sdk/test", + "api/backend/src", + "api/runtime/src", + "api/binding/src", + ): (self.repo / folder).mkdir(parents=True) for name in ("core/Cargo.toml", "core/Cargo.lock", ".cargo/config.toml", "api/package.json"): (self.repo / name).write_text("{}\n") (self.repo / "api/agent/sdk/src/index.js").write_text("export const value = 1;\n") - (self.repo / "api/agent/sdk/package.json").write_text('{"name": "@layerfs/agent", "exports": {".": "./src/index.js", "./package.json": "./package.json"}}\n') + (self.repo / "api/agent/sdk/package.json").write_text( + '{"name": "@layerfs/agent", "exports": {".": "./src/index.js", "./package.json": "./package.json"}}\n' + ) for name in ("backend", "runtime"): (self.repo / f"api/{name}/src/index.js").write_text('export * from "@layerfs/agent";\n') - (self.repo / f"api/{name}/package.json").write_text(f'{{"name": "@layerfs/{name}", "exports": {{".": "./src/index.js"}}}}\n') + (self.repo / f"api/{name}/package.json").write_text( + f'{{"name": "@layerfs/{name}", "exports": {{".": "./src/index.js"}}}}\n' + ) (self.repo / "api/agent/cli/src").mkdir(parents=True) (self.repo / "api/agent/cli/src/layerfs.js").write_text('import "@layerfs/agent";\n') (self.repo / "api/agent/cli/package.json").write_text('{"name": "@layerfs/cli"}\n') @@ -43,8 +56,14 @@ def setUp(self): git(self.repo, "config", "user.email", "fixture@example.invalid") git(self.repo, "add", ".") git(self.repo, "commit", "-qm", "fixture") - args = SimpleNamespace(worktree=str(self.repo), root=str(self.root / "state"), owner="fixture", - build_image="build", runtime_image="runtime", sandbox_image="sandbox") + args = SimpleNamespace( + worktree=str(self.repo), + root=str(self.root / "state"), + owner="fixture", + build_image="build", + runtime_image="runtime", + sandbox_image="sandbox", + ) with patch.object(resources, "image_id", side_effect=lambda image: "sha256:" + image): with contextlib.redirect_stdout(io.StringIO()): resources.register(args) @@ -69,7 +88,9 @@ def test_linux_mounts_do_not_use_host_target_or_shared_home(self): self.assertIn("type=bind,src=" + str(self.repo) + ",dst=/work,readonly", command) self.assertIn("CARGO_TARGET_DIR=/work/core/target", command) self.assertIn("CARGO_HOME=/cargo-home", command) - self.assertIn("type=volume,src=" + self.record["linux_target_volume"] + ",dst=/work/core/target,volume-nocopy", command) + self.assertIn( + "type=volume,src=" + self.record["linux_target_volume"] + ",dst=/work/core/target,volume-nocopy", command + ) self.assertIn("sha256:build", command) def test_explicit_flags_cannot_drop_repository_profile(self): @@ -79,22 +100,32 @@ def test_explicit_flags_cannot_drop_repository_profile(self): def test_container_failure_cannot_be_hidden_by_attach_exit_zero(self): calls = [] + def run(cmd, *args, **kwargs): calls.append(cmd) - content = b"id\n" if cmd[1] == "create" else json.dumps([{"State": {"Running": False, "ExitCode": 7}}]).encode() + content = ( + b"id\n" if cmd[1] == "create" else json.dumps([{"State": {"Running": False, "ExitCode": 7}}]).encode() + ) return SimpleNamespace(stdout=content) + with patch.object(resources, "run", side_effect=run), patch.object(resources.subprocess, "Popen") as popen: popen.return_value.wait.return_value = 0 with self.assertRaises(Refused): - resources.execute(["docker", "run", "--rm", "image", "false"], self.root, log=self.root / "container.log") + resources.execute( + ["docker", "run", "--rm", "image", "false"], self.root, log=self.root / "container.log" + ) self.assertFalse(any(cmd[1] == "rm" for cmd in calls)) def test_cargo_scope_and_frozen_harness_guards(self): command = builds.cargo_arguments(["--", "test", "-p", "layerfs-storage", "--no-run"]) self.assertIn("--offline", command) self.assertIn("--locked", command) - for args in (["build", "--workspace"], ["build", "-p", "fs-bench-pro"], - ["build", "-p", "layerfs-node", "--target-dir=/shared"], ["test"]): + for args in ( + ["build", "--workspace"], + ["build", "-p", "fs-bench-pro"], + ["build", "-p", "layerfs-node", "--target-dir=/shared"], + ["test"], + ): with self.assertRaises(Refused): builds.cargo_arguments(args) @@ -119,9 +150,11 @@ def build_bundle(self, mutate=False): (target / "layerfs-daemon").chmod(0o755) (target / "liblayerfs_node.dylib").write_bytes(b"addon") args = SimpleNamespace(resources=str(self.path), platform="macos", profile="debug", jobs=2) + def execute(*a, **kw): if mutate: (self.repo / "api/agent/sdk/src/index.js").write_text("changed\n") + with patch.object(builds, "execute", side_effect=execute), patch.object(builds, "environment", return_value={}): with contextlib.redirect_stdout(io.StringIO()): builds.build(args) @@ -131,16 +164,33 @@ def test_bundle_pins_sdk_and_binary_and_detects_mutation(self): bundle = self.build_bundle() manifest = builds.verify_bundle(bundle) # the bundle mirrors the repository layout under api/ - for name in ("api/package.json", "api/agent/sdk/src/index.js", "api/backend/src/index.js", "api/runtime/src/index.js"): + for name in ( + "api/package.json", + "api/agent/sdk/src/index.js", + "api/backend/src/index.js", + "api/runtime/src/index.js", + ): self.assertIn(name, manifest["files"]) self.assertFalse(any(name.startswith("api/binding/") for name in manifest["files"])) # each package resolves by name through a shim, not a symlink - self.assertEqual((bundle / "api/node_modules/@layerfs/agent/index.js").read_text(), 'export * from "../../../agent/sdk/src/index.js";\n') - self.assertEqual(json.loads((bundle / "api/node_modules/@layerfs/agent/package.json").read_text())["exports"], {".": "./index.js"}) - self.assertEqual((bundle / "api/node_modules/@layerfs/backend/index.js").read_text(), 'export * from "../../../backend/src/index.js";\n') + self.assertEqual( + (bundle / "api/node_modules/@layerfs/agent/index.js").read_text(), + 'export * from "../../../agent/sdk/src/index.js";\n', + ) + self.assertEqual( + json.loads((bundle / "api/node_modules/@layerfs/agent/package.json").read_text())["exports"], + {".": "./index.js"}, + ) + self.assertEqual( + (bundle / "api/node_modules/@layerfs/backend/index.js").read_text(), + 'export * from "../../../backend/src/index.js";\n', + ) # the layerfs command, which reaches the agent through its own shim self.assertIn("api/agent/cli/src/layerfs.js", manifest["files"]) - self.assertEqual((bundle / "api/agent/cli/node_modules/@layerfs/agent/index.js").read_text(), 'export * from "../../../../sdk/src/index.js";\n') + self.assertEqual( + (bundle / "api/agent/cli/node_modules/@layerfs/agent/index.js").read_text(), + 'export * from "../../../../sdk/src/index.js";\n', + ) (self.repo / "api/agent/sdk/src/index.js").write_text("later edit\n") self.assertEqual((bundle / "api/agent/sdk/src/index.js").read_text(), "export const value = 1;\n") file = bundle / "layerfs.node" @@ -157,9 +207,11 @@ def test_changed_build_source_cannot_publish_sealed_bundle(self): def test_teardown_never_removes_other_names(self): record = {"prefix": "owned-unique-", "client_id": "id", "store_volume": "owned-volume"} commands = [] + def bounded(command, *args, **kwargs): commands.append(command) - return b'id\n' if command[1] == "ps" else json.dumps([{"Id": "id", "Name": "/other-owner"}]).encode() + return b"id\n" if command[1] == "ps" else json.dumps([{"Id": "id", "Name": "/other-owner"}]).encode() + with patch.object(runs, "bounded", side_effect=bounded): with self.assertRaises(Refused): runs.finish(record, True, time.monotonic() + 20) @@ -167,11 +219,13 @@ def bounded(command, *args, **kwargs): def test_docker_short_and_full_ids_do_not_duplicate_teardown(self): full = "a" * 64 + def bounded(cmd, *args): if cmd[1] == "ps": return ((full if "--no-trunc" in cmd else full[:12]) + "\n").encode() records = [{"Id": full, "Name": "/mine-client"} for _ in cmd[2:]] return json.dumps(records).encode() + with patch.object(runs, "bounded", side_effect=bounded): result = runs.inspect_owned("mine-", full, time.monotonic() + 20) self.assertEqual(len(result), 1) @@ -180,7 +234,10 @@ def test_failed_run_stops_own_container_but_retains_state(self): record = {"prefix": "mine-", "client_id": "id", "store_volume": "mine-store"} calls = [] containers = [{"Id": "id", "Name": "/mine-client", "State": {"Running": True}}] - with patch.object(runs, "inspect_owned", return_value=containers), patch.object(runs, "bounded", side_effect=lambda cmd, *a: calls.append(cmd)): + with ( + patch.object(runs, "inspect_owned", return_value=containers), + patch.object(runs, "bounded", side_effect=lambda cmd, *a: calls.append(cmd)), + ): runs.finish(record, False, time.monotonic() + 20) self.assertEqual(calls, [["docker", "kill", "id"]]) self.assertFalse(record["released"]) @@ -189,27 +246,45 @@ def test_successful_run_removes_only_acknowledged_resources(self): record = {"prefix": "mine-", "client_id": "id", "store_volume": "mine-store"} calls = [] containers = [{"Id": "id", "Name": "/mine-client", "State": {"Running": False}}] + def bounded(cmd, *a): calls.append(cmd) return b"" - with patch.object(runs, "inspect_owned", return_value=containers), patch.object(runs, "bounded", side_effect=bounded), patch.object(runs, "check_store"): + + with ( + patch.object(runs, "inspect_owned", return_value=containers), + patch.object(runs, "bounded", side_effect=bounded), + patch.object(runs, "check_store"), + ): runs.finish(record, True, time.monotonic() + 20) self.assertEqual(calls[0], ["docker", "rm", "id"]) self.assertEqual(calls[-1], ["docker", "volume", "rm", "mine-store"]) self.assertTrue(record["released"]) def test_recreated_or_foreign_store_is_preserved(self): - state = {"prefix": "mine-token-", "run_id": "token", "worktree_id": "worktree", - "store_volume": "store", "volume_created_at": "old"} - for info in [{"Labels": {}}, {"Labels": {resources.LABEL: "worktree", "layerfs.dev.run": "token"}, "CreatedAt": "new"}]: + state = { + "prefix": "mine-token-", + "run_id": "token", + "worktree_id": "worktree", + "store_volume": "store", + "volume_created_at": "old", + } + for info in [ + {"Labels": {}}, + {"Labels": {resources.LABEL: "worktree", "layerfs.dev.run": "token"}, "CreatedAt": "new"}, + ]: with patch.object(runs, "bounded", return_value=json.dumps([info]).encode()): with self.assertRaises(Refused): runs.check_store(state, time.monotonic() + 20) def test_host_timeout_cancels_owned_process_group(self): pidfile = self.root / "child.pid" - script = ('import subprocess,time,pathlib; p=subprocess.Popen(["' + sys.executable + '","-c","import time; time.sleep(60)"]); ' - 'pathlib.Path(' + repr(str(pidfile)) + ').write_text(str(p.pid)); time.sleep(60)') + script = ( + 'import subprocess,time,pathlib; p=subprocess.Popen(["' + + sys.executable + + '","-c","import time; time.sleep(60)"]); ' + "pathlib.Path(" + repr(str(pidfile)) + ").write_text(str(p.pid)); time.sleep(60)" + ) with self.assertRaises(subprocess.TimeoutExpired): resources.execute([sys.executable, "-c", script], self.root, timeout=0.5, log=self.root / "timeout.log") child = int(pidfile.read_text()) diff --git a/tools/worktree.py b/tools/worktree.py index 4bfb787de..d12ca6781 100644 --- a/tools/worktree.py +++ b/tools/worktree.py @@ -29,7 +29,9 @@ def parser(): if name in {"seed", "build"}: command.add_argument("--profile", choices=("debug", "release"), default="debug") if name == "prepare": - command.add_argument("--dependency-seed", help="Closed Cargo home; copies registry/git only, never credentials") + command.add_argument( + "--dependency-seed", help="Closed Cargo home; copies registry/git only, never credentials" + ) if name == "seed": command.add_argument("--source-resources", required=True) if name == "build": diff --git a/tools/worktree_build.py b/tools/worktree_build.py index 1bacbfc5a..51677f0bc 100644 --- a/tools/worktree_build.py +++ b/tools/worktree_build.py @@ -6,9 +6,22 @@ import shutil from cleanup_state import Refused, ensure_idle, open_files, docker_state, run -from worktree_resources import (TOOLCHAIN, closed_volumes, copy_dependencies, digest, - docker_build, environment, execute, inputs, lease, - load, operation, timestamp, volume, write) +from worktree_resources import ( + TOOLCHAIN, + closed_volumes, + copy_dependencies, + digest, + docker_build, + environment, + execute, + inputs, + lease, + load, + operation, + timestamp, + volume, + write, +) def prepare(args): @@ -29,18 +42,27 @@ def prepare(args): closed_volumes([record["linux_cargo_volume"], record["linux_target_volume"]]) if args.dependency_seed: seed = Path(args.dependency_seed).resolve() - ensure_idle([seed / n for n in ("registry", "git") if (seed / n).exists()], - [], docker_state(), open_files()) - script = ('set -eu; for d in registry git; do if [ -d "/seed/$d" ]; then ' - 'test ! -e "/cargo-home/$d"; cp -a "/seed/$d" "/cargo-home/$d"; fi; done') - command = docker_build(record, ["sh", "-c", script], - ["--mount", f"type=bind,src={seed},dst=/seed,readonly"]) + ensure_idle( + [seed / n for n in ("registry", "git") if (seed / n).exists()], [], docker_state(), open_files() + ) + script = ( + 'set -eu; for d in registry git; do if [ -d "/seed/$d" ]; then ' + 'test ! -e "/cargo-home/$d"; cp -a "/seed/$d" "/cargo-home/$d"; fi; done' + ) + command = docker_build( + record, ["sh", "-c", script], ["--mount", f"type=bind,src={seed},dst=/seed,readonly"] + ) execute(command, record["worktree"], log=receipt / "seed.log") - command = docker_build(record, ["cargo", "+" + TOOLCHAIN, "fetch", "--manifest-path", - "core/Cargo.toml", "--locked"], network="default") + command = docker_build( + record, + ["cargo", "+" + TOOLCHAIN, "fetch", "--manifest-path", "core/Cargo.toml", "--locked"], + network="default", + ) execute(command, record["worktree"], log=receipt / "output.log") - write(receipt / "receipt.json", {"status": "prepared", "at": timestamp(), "resources": record, - "dependency_seed": args.dependency_seed}) + write( + receipt / "receipt.json", + {"status": "prepared", "at": timestamp(), "resources": record, "dependency_seed": args.dependency_seed}, + ) except BaseException as error: write(receipt / "receipt.json", {"status": "failed", "error": str(error), "at": timestamp()}) raise @@ -60,8 +82,16 @@ def cargo_arguments(arguments): raise Refused("fs-bench-pro is frozen") if not any(a in {"-p", "--package"} or a.startswith("--package=") for a in arguments): raise Refused("Cargo invocation needs explicit package selection") - return ["cargo", "+" + TOOLCHAIN, arguments[0], "--manifest-path", "core/Cargo.toml", - "--locked", "--offline", *arguments[1:]] + return [ + "cargo", + "+" + TOOLCHAIN, + arguments[0], + "--manifest-path", + "core/Cargo.toml", + "--locked", + "--offline", + *arguments[1:], + ] def cargo(args): @@ -76,16 +106,24 @@ def cargo(args): if args.platform == "linux": closed_volumes([record["linux_cargo_volume"], record["linux_target_volume"]]) if is_test: - compile_command = command[:command.index("--")] if "--" in command else list(command) + compile_command = command[: command.index("--")] if "--" in command else list(command) compile_command.append("--no-run") compile_command = docker_build(record, compile_command) if args.platform == "linux" else compile_command - execute(compile_command, record["worktree"], - environment(record, "macos") if args.platform == "macos" else None, - log=receipt / "compile.log") + execute( + compile_command, + record["worktree"], + environment(record, "macos") if args.platform == "macos" else None, + log=receipt / "compile.log", + ) if args.platform == "linux": command = docker_build(record, command) - execute(command, record["worktree"], environment(record, "macos") if args.platform == "macos" else None, - timeout=args.timeout if is_test else None, log=receipt / "output.log") + execute( + command, + record["worktree"], + environment(record, "macos") if args.platform == "macos" else None, + timeout=args.timeout if is_test else None, + log=receipt / "output.log", + ) outcome = "passed" except BaseException as error: outcome = "failed" @@ -134,7 +172,9 @@ def shim(node_modules, package): real = os.path.relpath(package / target, shim) (shim / file).write_text(f'export * from "{real}";\n') exports[subpath] = f"./{file}" - (shim / "package.json").write_text(json.dumps({"name": name, "type": "module", "exports": exports}, indent=2) + "\n") + (shim / "package.json").write_text( + json.dumps({"name": name, "type": "module", "exports": exports}, indent=2) + "\n" + ) def verify_bundle(path): @@ -167,19 +207,37 @@ def build(args): bundle.mkdir(mode=0o700) before = inputs(record) write(receipt / "inputs.json", before) - command = ["cargo", "+" + TOOLCHAIN, "build", "--manifest-path", "core/Cargo.toml", "--locked", "--offline", - "-p", "layerfs-daemon", "-p", "layerfs-node", "-j", str(args.jobs)] + command = [ + "cargo", + "+" + TOOLCHAIN, + "build", + "--manifest-path", + "core/Cargo.toml", + "--locked", + "--offline", + "-p", + "layerfs-daemon", + "-p", + "layerfs-node", + "-j", + str(args.jobs), + ] if args.profile == "release": command.append("--release") try: if args.platform == "linux": closed_volumes([record["linux_cargo_volume"], record["linux_target_volume"]]) execute(docker_build(record, command), record["worktree"], log=receipt / "output.log") - script = ('set -eu; cp "/work/core/target/$1/layerfs-daemon" /export/layerfs-daemon; ' - 'cp "/work/core/target/$1/liblayerfs_node.so" /export/layerfs.node; ' - 'chown "$2:$3" /export/layerfs-daemon /export/layerfs.node') - export = docker_build(record, ["sh", "-c", script, "sh", args.profile, str(os.getuid()), str(os.getgid())], - ["--mount", f"type=bind,src={bundle},dst=/export"]) + script = ( + 'set -eu; cp "/work/core/target/$1/layerfs-daemon" /export/layerfs-daemon; ' + 'cp "/work/core/target/$1/liblayerfs_node.so" /export/layerfs.node; ' + 'chown "$2:$3" /export/layerfs-daemon /export/layerfs.node' + ) + export = docker_build( + record, + ["sh", "-c", script, "sh", args.profile, str(os.getuid()), str(os.getgid())], + ["--mount", f"type=bind,src={bundle},dst=/export"], + ) execute(export, record["worktree"], log=receipt / "export.log") else: execute(command, record["worktree"], environment(record, "macos"), log=receipt / "output.log") @@ -190,10 +248,20 @@ def build(args): if inputs(record) != before: raise Refused("Source changed during build; partial bundle is not eligible for E2E") files = {str(p.relative_to(bundle)): digest(p) for p in bundle.rglob("*") if p.is_file()} - manifest = {"version": 1, "status": "sealed", "build_id": build_id, "at": timestamp(), - "worktree_id": record["id"], "platform": args.platform, "profile": args.profile, - "source": {"head": before["head"], "sha256": before["sha256"]}, - "toolchain": TOOLCHAIN, "images": record["images"], "command": command, "files": files} + manifest = { + "version": 1, + "status": "sealed", + "build_id": build_id, + "at": timestamp(), + "worktree_id": record["id"], + "platform": args.platform, + "profile": args.profile, + "source": {"head": before["head"], "sha256": before["sha256"]}, + "toolchain": TOOLCHAIN, + "images": record["images"], + "command": command, + "files": files, + } write(bundle / "manifest.json", manifest) verify_bundle(bundle) for path in bundle.rglob("*"): @@ -221,10 +289,15 @@ def seed(args): for owner in (record, source): volume(owner, "linux_target_volume") closed_volumes([record["linux_target_volume"], source["linux_target_volume"]]) - script = ('set -eu; test ! -e "/work/core/target/$1"; cp -a "/seed/$1" "/work/core/target/$1"; ' - 'for f in .rustc_info.json CACHEDIR.TAG; do if [ -f "/seed/$f" ]; then cp -a "/seed/$f" /work/core/target/; fi; done') - command = docker_build(record, ["sh", "-c", script, "sh", args.profile], - ["--mount", f"type=volume,src={source['linux_target_volume']},dst=/seed,readonly"]) + script = ( + 'set -eu; test ! -e "/work/core/target/$1"; cp -a "/seed/$1" "/work/core/target/$1"; ' + 'for f in .rustc_info.json CACHEDIR.TAG; do if [ -f "/seed/$f" ]; then cp -a "/seed/$f" /work/core/target/; fi; done' + ) + command = docker_build( + record, + ["sh", "-c", script, "sh", args.profile], + ["--mount", f"type=volume,src={source['linux_target_volume']},dst=/seed,readonly"], + ) execute(command, record["worktree"], log=receipt / "output.log") else: environment(record, "macos") @@ -238,6 +311,8 @@ def seed(args): for name in (".rustc_info.json", "CACHEDIR.TAG"): if (original / name).is_file(): shutil.copy2(original / name, dest / name) - write(receipt / "receipt.json", {"status": "seeded", "source": source["id"], - "profile": args.profile, "at": timestamp()}) + write( + receipt / "receipt.json", + {"status": "seeded", "source": source["id"], "profile": args.profile, "at": timestamp()}, + ) print(receipt) diff --git a/tools/worktree_resources.py b/tools/worktree_resources.py index d9b10d79b..6e38e5f25 100644 --- a/tools/worktree_resources.py +++ b/tools/worktree_resources.py @@ -68,13 +68,19 @@ def register(args): if overlaps(state, source): raise Refused("Resource state must be outside the worktree") images = {k: image_id(getattr(args, k + "_image")) for k in ("build", "runtime", "sandbox")} - record = {"version": 1, "id": identity(source), "owner": args.owner, - "worktree": str(source), "state": str(state), "toolchain": TOOLCHAIN, - "macos_target": str(source / "core/target"), - "macos_cargo_home": str(state / "cargo-home-macos"), - "linux_target_volume": "lfs-dev-" + identity(source) + "-target", - "linux_cargo_volume": "lfs-dev-" + identity(source) + "-cargo", - "images": images} + record = { + "version": 1, + "id": identity(source), + "owner": args.owner, + "worktree": str(source), + "state": str(state), + "toolchain": TOOLCHAIN, + "macos_target": str(source / "core/target"), + "macos_cargo_home": str(state / "cargo-home-macos"), + "linux_target_volume": "lfs-dev-" + identity(source) + "-target", + "linux_cargo_volume": "lfs-dev-" + identity(source) + "-cargo", + "images": images, + } path = state / "resources.json" if path.exists(): if load(path) != record: @@ -108,10 +114,20 @@ def lease(record, platform): def volume(record, key): name = record[key] - names = run(["docker", "volume", "ls", "--format", "{{.Name}}"] ).stdout.decode().splitlines() + names = run(["docker", "volume", "ls", "--format", "{{.Name}}"]).stdout.decode().splitlines() if name not in names: - run(["docker", "volume", "create", "--label", f"{LABEL}={record['id']}", - "--label", "layerfs.dev.role=" + key, name]) + run( + [ + "docker", + "volume", + "create", + "--label", + f"{LABEL}={record['id']}", + "--label", + "layerfs.dev.role=" + key, + name, + ] + ) info = json.loads(run(["docker", "volume", "inspect", name]).stdout)[0] labels = info.get("Labels") or {} if labels.get(LABEL) != record["id"] or labels.get("layerfs.dev.role") != key or info["Driver"] != "local": @@ -134,8 +150,7 @@ def environment(record, platform): if platform == "macos": if os.uname().sysname != "Darwin": raise Refused("macos builds require a macOS host") - return {**os.environ, "CARGO_HOME": record["macos_cargo_home"], - "CARGO_TARGET_DIR": record["macos_target"]} + return {**os.environ, "CARGO_HOME": record["macos_cargo_home"], "CARGO_TARGET_DIR": record["macos_target"]} return {"CARGO_HOME": "/cargo-home", "CARGO_TARGET_DIR": "/work/core/target"} @@ -144,13 +159,31 @@ def docker_build(record, command, extra=(), network="none"): volume(record, key) Path(record["macos_target"]).mkdir(parents=True, exist_ok=True) env = environment(record, "linux") - return ["docker", "run", "--rm", "--pull", "never", "--network", network, - "--label", f"{LABEL}={record['id']}", - "--mount", f"type=bind,src={record['worktree']},dst=/work,readonly", - "--mount", f"type=volume,src={record['linux_target_volume']},dst=/work/core/target,volume-nocopy", - "--mount", f"type=volume,src={record['linux_cargo_volume']},dst=/cargo-home,volume-nocopy", - *[v for k, value in env.items() for v in ("-e", k + "=" + value)], - *extra, "-w", "/work", "--entrypoint", command[0], record["images"]["build"], *command[1:]] + return [ + "docker", + "run", + "--rm", + "--pull", + "never", + "--network", + network, + "--label", + f"{LABEL}={record['id']}", + "--mount", + f"type=bind,src={record['worktree']},dst=/work,readonly", + "--mount", + f"type=volume,src={record['linux_target_volume']},dst=/work/core/target,volume-nocopy", + "--mount", + f"type=volume,src={record['linux_cargo_volume']},dst=/cargo-home,volume-nocopy", + *[v for k, value in env.items() for v in ("-e", k + "=" + value)], + *extra, + "-w", + "/work", + "--entrypoint", + command[0], + record["images"]["build"], + *command[1:], + ] def execute(command, cwd, env=None, timeout=None, log=None): @@ -163,9 +196,10 @@ def execute(command, cwd, env=None, timeout=None, log=None): write(Path(log).parent / "container.json", {"id": container, "create": create}) command = ["docker", "start", "--attach", container] try: - with (Path(log).open("wb") if log else open(os.devnull, "wb")) as stream: - process = subprocess.Popen(command, cwd=cwd, env=env, stdout=stream, stderr=subprocess.STDOUT, - start_new_session=True) + with Path(log).open("wb") if log else open(os.devnull, "wb") as stream: + process = subprocess.Popen( + command, cwd=cwd, env=env, stdout=stream, stderr=subprocess.STDOUT, start_new_session=True + ) try: status = process.wait(timeout=timeout) except BaseException: @@ -188,9 +222,21 @@ def execute(command, cwd, env=None, timeout=None, log=None): def inputs(record): root = Path(record["worktree"]) - selected = [".cargo", "core/Cargo.toml", "core/Cargo.lock", "core/crates", "core/vendor", - "api/agent/sdk", "api/backend", "api/runtime", "api/binding", "api/package.json", - "api/agent/cli/src", "api/agent/cli/test", "api/agent/cli/package.json"] + selected = [ + ".cargo", + "core/Cargo.toml", + "core/Cargo.lock", + "core/crates", + "core/vendor", + "api/agent/sdk", + "api/backend", + "api/runtime", + "api/binding", + "api/package.json", + "api/agent/cli/src", + "api/agent/cli/test", + "api/agent/cli/package.json", + ] result = {} for relative in selected: source = root / relative @@ -201,16 +247,20 @@ def inputs(record): for base, dirs, files in os.walk(source): if any((Path(base) / d).is_symlink() for d in dirs): raise Refused("Source directory symlinks need explicit handling: " + base) - dirs[:] = [d for d in dirs if not d.startswith("target") - and d not in {"node_modules", ".git", "__pycache__"}] + dirs[:] = [ + d for d in dirs if not d.startswith("target") and d not in {"node_modules", ".git", "__pycache__"} + ] paths.extend(Path(base) / name for name in sorted(files)) for path in paths: if path.is_symlink(): raise Refused("Build inputs require explicit handling of source symlinks: " + str(path)) result[str(path.relative_to(root))] = digest(path) encoded = json.dumps(result, sort_keys=True).encode() - return {"head": git(root, "rev-parse", "HEAD").decode().strip(), - "files": result, "sha256": hashlib.sha256(encoded).hexdigest()} + return { + "head": git(root, "rev-parse", "HEAD").decode().strip(), + "files": result, + "sha256": hashlib.sha256(encoded).hexdigest(), + } def copy_dependencies(seed, destination): diff --git a/tools/worktree_run.py b/tools/worktree_run.py index b47a9b025..93370c42c 100644 --- a/tools/worktree_run.py +++ b/tools/worktree_run.py @@ -59,7 +59,9 @@ def finish(record, success, deadline): record["released"] = True else: record["released"] = False - record["release_condition"] = "Owner finishes diagnosis; explicitly release exact stopped containers and Store volume" + record["release_condition"] = ( + "Owner finishes diagnosis; explicitly release exact stopped containers and Store volume" + ) def check_store(record, deadline): @@ -92,20 +94,39 @@ def launch(args): prefix = "lfs-e2e-" + resources["id"] + "-" + token + "-" directory = Path(resources["state"]) / "runs" / token directory.mkdir(mode=0o700) - state = {"version": 1, "status": "starting", "at": timestamp(), "worktree_id": resources["id"], - "run_id": token, - "bundle": str(bundle), "bundle_manifest_sha256": digest(bundle / "manifest.json"), - "prefix": prefix, "store_volume": prefix + "store", "command": command, - "timeout_seconds": args.timeout, "store_profile": "Disposable (SDK native route)", - "released": False} + state = { + "version": 1, + "status": "starting", + "at": timestamp(), + "worktree_id": resources["id"], + "run_id": token, + "bundle": str(bundle), + "bundle_manifest_sha256": digest(bundle / "manifest.json"), + "prefix": prefix, + "store_volume": prefix + "store", + "command": command, + "timeout_seconds": args.timeout, + "store_profile": "Disposable (SDK native route)", + "released": False, + } receipt = directory / "receipt.json" write(receipt, state) # Refuse an existing exact name, so create can never adopt someone else's mutable state. - names = run(["docker", "volume", "ls", "--format", "{{.Name}}"] ).stdout.decode().splitlines() + names = run(["docker", "volume", "ls", "--format", "{{.Name}}"]).stdout.decode().splitlines() if state["store_volume"] in names: raise Refused("Run volume already exists") - run(["docker", "volume", "create", "--label", f"{LABEL}={resources['id']}", - "--label", "layerfs.dev.run=" + token, state["store_volume"]]) + run( + [ + "docker", + "volume", + "create", + "--label", + f"{LABEL}={resources['id']}", + "--label", + "layerfs.dev.run=" + token, + state["store_volume"], + ] + ) info = json.loads(run(["docker", "volume", "inspect", state["store_volume"]]).stdout)[0] state["volume_created_at"] = info.get("CreatedAt") write(receipt, state) @@ -114,25 +135,53 @@ def launch(args): raise Refused("E2E needs a local Unix Docker daemon") # Docker Desktop's context endpoint is a macOS proxy socket. Containers need # the Linux engine socket, as the existing SDK verification helper uses. - socket = "/var/run/docker.sock" if os.uname().sysname == "Darwin" else endpoint[len("unix://"):] - env = {"LAYERFS_HOME": "/layerfs-store/sdk-home", "LAYERFS_STORE_PATH": "/layerfs-store/global/store.sqlite", - "LAYERFS_DOCKER": "/var/run/docker.sock", "LAYERFS_IMAGE": resources["images"]["sandbox"], - "LAYERFS_VOLUME": state["store_volume"], "LAYERFS_DAEMON": str(bundle / "layerfs-daemon"), - "LAYERFS_NATIVE": str(bundle / "layerfs.node"), "LAYERFS_CONTAINER_PREFIX": prefix, - "LAYERFS_CONSTRUCTION_WORKERS": "1", "LAYERFS_TRACE": "/receipts/trace.jsonl", - "LAYERFS_DAEMON_HOST": "host.docker.internal" if os.uname().sysname == "Darwin" else "127.0.0.1"} + socket = "/var/run/docker.sock" if os.uname().sysname == "Darwin" else endpoint[len("unix://") :] + env = { + "LAYERFS_HOME": "/layerfs-store/sdk-home", + "LAYERFS_STORE_PATH": "/layerfs-store/global/store.sqlite", + "LAYERFS_DOCKER": "/var/run/docker.sock", + "LAYERFS_IMAGE": resources["images"]["sandbox"], + "LAYERFS_VOLUME": state["store_volume"], + "LAYERFS_DAEMON": str(bundle / "layerfs-daemon"), + "LAYERFS_NATIVE": str(bundle / "layerfs.node"), + "LAYERFS_CONTAINER_PREFIX": prefix, + "LAYERFS_CONSTRUCTION_WORKERS": "1", + "LAYERFS_TRACE": "/receipts/trace.jsonl", + "LAYERFS_DAEMON_HOST": "host.docker.internal" if os.uname().sysname == "Darwin" else "127.0.0.1", + } # The same absolute bundle path is visible to host, client and Docker API callers. script = 'umask 077; mkdir -p /layerfs-store/sdk-home /layerfs-store/global; umask 022; exec "$@"' - create = ["docker", "create", "--pull", "never", "--name", prefix + "client", - "--label", f"{LABEL}={resources['id']}", "--label", "layerfs.dev.run=" + token, - "--mount", f"type=bind,src={bundle},dst={bundle},readonly", - "--mount", f"type=bind,src={directory},dst=/receipts", - "--mount", f"type=bind,src={socket},dst=/var/run/docker.sock", - "--mount", f"type=volume,src={state['store_volume']},dst=/layerfs-store,volume-nocopy", - *[v for k, value in env.items() for v in ("-e", k + "=" + value)], - *(["--network", "host"] if os.uname().sysname == "Linux" else []), - "-w", str(bundle), "--entrypoint", "/bin/sh", resources["images"]["runtime"], - "-c", script, "sh", *command] + create = [ + "docker", + "create", + "--pull", + "never", + "--name", + prefix + "client", + "--label", + f"{LABEL}={resources['id']}", + "--label", + "layerfs.dev.run=" + token, + "--mount", + f"type=bind,src={bundle},dst={bundle},readonly", + "--mount", + f"type=bind,src={directory},dst=/receipts", + "--mount", + f"type=bind,src={socket},dst=/var/run/docker.sock", + "--mount", + f"type=volume,src={state['store_volume']},dst=/layerfs-store,volume-nocopy", + *[v for k, value in env.items() for v in ("-e", k + "=" + value)], + *(["--network", "host"] if os.uname().sysname == "Linux" else []), + "-w", + str(bundle), + "--entrypoint", + "/bin/sh", + resources["images"]["runtime"], + "-c", + script, + "sh", + *command, + ] success = False deadline = time.monotonic() + 120 try: @@ -141,10 +190,13 @@ def launch(args): state["status"] = "running" write(receipt, state) with (directory / "output.log").open("wb") as output: - process = subprocess.Popen(["docker", "start", "--attach", state["client_id"]], - stdout=output, stderr=subprocess.STDOUT) + process = subprocess.Popen( + ["docker", "start", "--attach", state["client_id"]], stdout=output, stderr=subprocess.STDOUT + ) try: - state["attach_exit_code"] = process.wait(timeout=min(args.timeout, max(0, deadline - time.monotonic() - 20))) + state["attach_exit_code"] = process.wait( + timeout=min(args.timeout, max(0, deadline - time.monotonic() - 20)) + ) except BaseException: process.kill() process.wait() @@ -187,8 +239,10 @@ def export_scope(args, resources): for profile in ("debug", "release"): target = Path(resources["macos_target"]) if (target / profile).exists(): - actions.append({**base, "kind": "cargo-cache", "path": str(target / profile), "target_root": str(target)}) - names = run(["docker", "volume", "ls", "--format", "{{.Name}}"] ).stdout.decode().splitlines() + actions.append( + {**base, "kind": "cargo-cache", "path": str(target / profile), "target_root": str(target)} + ) + names = run(["docker", "volume", "ls", "--format", "{{.Name}}"]).stdout.decode().splitlines() for key in ("linux_target_volume", "linux_cargo_volume"): if resources[key] in names: info = json.loads(run(["docker", "volume", "inspect", resources[key]]).stdout)[0] @@ -201,8 +255,13 @@ def export_scope(args, resources): disposed = any(json.loads(p.read_text()).get("released") for p in receipt.parent.glob("disposal-*.json")) if not run_state.get("released") and not disposed: protected_volumes.append(run_state["store_volume"]) - scope = {"version": 1, "protected_paths": [str(state)], "protected_volumes": protected_volumes, - "scan_roots": [], "actions": actions} + scope = { + "version": 1, + "protected_paths": [str(state)], + "protected_volumes": protected_volumes, + "scan_roots": [], + "actions": actions, + } destination = Path(args.out).resolve() if destination.exists(): raise Refused("Scope output already exists")