From b77a8b54b26a41dfb8dc9f9e12cc4f091b206cdf Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 18 Oct 2026 00:40:30 -0700 Subject: [PATCH 01/89] simplify(release): retire unpublished-crate distribution --- .github/ISSUE_TEMPLATE/bug_report.yml | 2 +- .github/workflows/ci.yml | 17 +- .github/workflows/distribution-acceptance.yml | 106 -- .github/workflows/hawk.yml | 3 +- README.md | 17 +- crates/tracedecay-application/Cargo.toml | 1 - crates/tracedecay-capture/Cargo.toml | 1 - crates/tracedecay-cli/Cargo.toml | 12 - crates/tracedecay-cli/build.rs | 36 +- crates/tracedecay-code-extraction/Cargo.toml | 9 - crates/tracedecay-code-index/Cargo.toml | 1 - crates/tracedecay-global-db/Cargo.toml | 1 - crates/tracedecay-graph-db/Cargo.toml | 1 - crates/tracedecay-graph-query/Cargo.toml | 1 - crates/tracedecay-lcm/Cargo.toml | 1 - crates/tracedecay-privacy/Cargo.toml | 1 - crates/tracedecay-query/Cargo.toml | 1 - crates/tracedecay-search-eval/Cargo.toml | 1 - crates/tracedecay-session-memory/Cargo.toml | 1 - .../Cargo.toml | 1 - crates/tracedecay-sessions/Cargo.toml | 1 - crates/tracedecay-temporal-query/Cargo.toml | 1 - crates/tracedecay/Cargo.toml | 53 - docs/RELEASE-AUTOMATION.md | 19 +- scripts/BUILD.bazel | 14 + scripts/bazel/rust_test.bzl | 5 - scripts/check-distribution-acceptance.sh | 998 ------------------ scripts/check-distribution-feature-wiring.py | 261 ----- scripts/check-packaged-mcp-stdio.py | 345 ------ scripts/resolve-installed-binary.py | 28 - scripts/resolve-release-source-profile.py | 93 -- .../test-check-distribution-clean-source.sh | 52 - .../test-check-distribution-feature-wiring.py | 305 ------ scripts/test-check-distribution-reuse.sh | 260 ----- scripts/test-check-distribution-snapshot.sh | 197 ---- scripts/test-check-packaged-lsp-bridge.py | 0 scripts/test-check-packaged-mcp-stdio.py | 171 --- scripts/test-resolve-installed-binary.py | 71 -- .../test-resolve-release-source-profile.py | 166 --- scripts/test-rust-cache-lineage.py | 6 +- 40 files changed, 48 insertions(+), 3212 deletions(-) delete mode 100644 .github/workflows/distribution-acceptance.yml delete mode 100755 scripts/check-distribution-acceptance.sh delete mode 100644 scripts/check-distribution-feature-wiring.py delete mode 100755 scripts/check-packaged-mcp-stdio.py delete mode 100755 scripts/resolve-installed-binary.py delete mode 100755 scripts/resolve-release-source-profile.py delete mode 100644 scripts/test-check-distribution-clean-source.sh delete mode 100644 scripts/test-check-distribution-feature-wiring.py delete mode 100755 scripts/test-check-distribution-reuse.sh delete mode 100644 scripts/test-check-distribution-snapshot.sh mode change 100644 => 100755 scripts/test-check-packaged-lsp-bridge.py delete mode 100755 scripts/test-check-packaged-mcp-stdio.py delete mode 100755 scripts/test-resolve-installed-binary.py delete mode 100755 scripts/test-resolve-release-source-profile.py diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 2fa543ae2b..1766feade5 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -33,7 +33,7 @@ body: attributes: label: Installation method options: - - cargo install + - Built from source (Bazel) - Homebrew (brew) - Scoop - Built from source diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9e51c8bd34..66cb48141c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -183,17 +183,10 @@ jobs: run: | python3 scripts/test-check-packaged-lsp-bridge.py python3 scripts/test-build-mcpb.py - python3 scripts/test-resolve-installed-binary.py - python3 scripts/test-check-packaged-mcp-stdio.py - python3 scripts/test-check-distribution-feature-wiring.py - python3 scripts/test-resolve-release-source-profile.py python3 scripts/test-package-release-archive.py python3 scripts/test-plan-release-recovery.py python3 scripts/test-check-release-artifacts.py python3 scripts/test-neutralize-release-pr-closing-keywords.py - bash scripts/test-check-distribution-clean-source.sh - bash scripts/test-check-distribution-snapshot.sh - bash scripts/test-check-distribution-reuse.sh bash tests/macos_stable_codesign_test.sh # Exercise the release guard scripts when admitting proposed changes. @@ -671,6 +664,16 @@ jobs: "$binary" --version "$binary" --help >/dev/null + - name: Exercise shipped MCP and LSP transports + shell: bash + run: | + set -euo pipefail + binary="$GITHUB_WORKSPACE/target/x86_64-unknown-linux-gnu/release/tracedecay" + smoke_work="$(mktemp -d)" + trap 'rm -rf -- "$smoke_work"' EXIT + TRACEDECAY_BIN="$binary" bazel run //scripts:mcp-conformance-smoke.sh + bazel run //scripts:packaged_lsp_bridge -- "$binary" "$smoke_work/lsp" + dashboard: name: Dashboard needs: [scope-gate, gates] diff --git a/.github/workflows/distribution-acceptance.yml b/.github/workflows/distribution-acceptance.yml deleted file mode 100644 index ed7b7e0975..0000000000 --- a/.github/workflows/distribution-acceptance.yml +++ /dev/null @@ -1,106 +0,0 @@ -# Packaged-crate distribution battery. Visibility only; not a ship gate. -# -# GitHub release assets are the just-built production binary, its archive, -# and the MCPB that wraps that same binary. `cargo package`, extracted-crate -# rebuilds, nextest, and MCP inspector dogfood prove the unpublished crate -# graph, not those assets. Beta.40 run 35271070760 spent ~24m of the aarch64 -# ship job on that battery and then failed before MCPB. This workflow runs -# the battery on one free ubuntu-latest runner when someone dispatches it. -# -name: Distribution acceptance - -# On demand only. Nothing in this repository runs on a timer: a scheduled -# run spends a free runner slot whether or not anyone will read the result. -on: - workflow_dispatch: - -permissions: - contents: read - -concurrency: - group: distribution-acceptance-${{ github.ref }} - cancel-in-progress: true - -jobs: - distribution-acceptance: - name: Distribution acceptance (x86_64-linux) - if: github.repository == 'ScriptedAlchemy/tracedecay' - runs-on: ubuntu-latest - timeout-minutes: 240 - env: - TARGET: x86_64-unknown-linux-gnu - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - node-version: "26.12.0" - - - uses: ./.github/actions/setup-pnpm - - - uses: ./.github/actions/setup-bazel - with: - cache-key: distribution-acceptance-linux - test-tools: "true" - - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: "3.15.0" - - - name: Build dashboard - working-directory: dashboard - run: pnpm run build - - - name: Export dashboard bundle digest for skip-mode builds - shell: bash - run: | - set -euo pipefail - digest="$(python3 scripts/check-dashboard-bundle.py dashboard/app-dist --print-digest)" - echo "TRACEDECAY_SKIP_DASHBOARD_BUILD=1" >> "$GITHUB_ENV" - echo "TRACEDECAY_DASHBOARD_BUNDLE_SHA256=${digest}" >> "$GITHUB_ENV" - - # rust-toolchain.toml is the only compiler this job may install. See - # release-beta.yml: floating stable fragments rust-cache's env key. - - name: Resolve pinned toolchain - id: pin - shell: bash - run: | - python3 -c 'import pathlib, tomllib; print("toolchain=" + tomllib.loads(pathlib.Path("rust-toolchain.toml").read_text(encoding="utf-8"))["toolchain"]["channel"])' >> "$GITHUB_OUTPUT" - - - uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec # stable - with: - toolchain: ${{ steps.pin.outputs.toolchain }} - targets: ${{ env.TARGET }} - - # Own lineage. release-beta stores the `--target` layout. This battery - # compiles the packaged CLI into the host layout (`target/dist`), so - # that shared key restored nothing this job reads, and an exact-key hit - # would have skipped saving this job's artifacts. Dependency artifacts - # only: extracted packages live under a fresh temporary path every run. - # The workspace roots keep the vendored dependencies (see - # scripts/check-rust-cache-lineage.py). - - name: Cache Rust build - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - with: - shared-key: distribution-acceptance-${{ env.TARGET }} - workspaces: | - crates -> ../target - . -> target/rust-cache-lockfile-key - cache-workspace-crates: "false" - cache-on-failure: true - - # The acceptance script runs the packaged Rust-grammar proof and the - # focused packaged-CLI integration journeys through nextest. - - name: Install cargo-nextest - uses: taiki-e/install-action@9252097d36d2d3618ad98f5d873696a962b6897c # nextest - - - name: Run distribution acceptance - run: | - set -euo pipefail - cargo fetch --locked --target ${{ env.TARGET }} - lockfile_sha="$(python3 -c 'import hashlib; print(hashlib.sha256(open("Cargo.lock", "rb").read()).hexdigest())')" - CARGO_NET_OFFLINE=true bash scripts/check-distribution-acceptance.sh - actual_lockfile_sha="$(python3 -c 'import hashlib; print(hashlib.sha256(open("Cargo.lock", "rb").read()).hexdigest())')" - test "$actual_lockfile_sha" = "$lockfile_sha" diff --git a/.github/workflows/hawk.yml b/.github/workflows/hawk.yml index ef67951be6..d995692af7 100644 --- a/.github/workflows/hawk.yml +++ b/.github/workflows/hawk.yml @@ -6,8 +6,7 @@ # softening -D warnings. name: Hawk -# On demand only; see distribution-acceptance.yml for why nothing runs on a -# timer here. +# On demand only; an optional diagnostic should run when someone will read it. on: workflow_dispatch: diff --git a/README.md b/README.md index fb0965ed87..abe6f8b1ce 100644 --- a/README.md +++ b/README.md @@ -269,9 +269,9 @@ bazel build //... bazel test //... --test_output=errors ``` -Use Cargo when you need to validate Cargo packaging, a feature combination that -has no Bazel target, or a platform-specific workflow. The release workflows -continue to build the published Cargo artifact. +Bazel owns workspace builds, tests, checks, and code generation. Release +workflows package its production CLI as binary archives and MCP bundles; +Rust workspace crates are private and are not published to crates.io. On macOS `tracedecay` is ad-hoc signed as `dev.tracedecay.cli` with designated requirement `identifier "dev.tracedecay.cli"`. macOS TCC keys @@ -280,14 +280,9 @@ requirement is the binary's cdhash, so an Allow does not survive the next rebuild; the linker default identifier (`tracedecay-`) changes every build as well, and the daemon blocks in `open()` until the prompt is answered. A release build strips after linking, which mints that identifier -again from the deps filename. Export -`CARGO_BUILD_RUSTC_WORKSPACE_WRAPPER="$PWD/scripts/macos-rustc-wrapper.sh"` -from the checkout root (the macOS CI and release jobs do) and the wrapper -signs the product once rustc has finished, so `cargo build --release` and -`cargo install --path crates/tracedecay-cli` keep the stable identity on -the binary cargo copies into place. The committed Cargo config cannot set -it, because Cargo has no per-target wrapper and Windows cannot run the -script. `install.sh` and `tracedecay update` +again from the deps filename. The shared release build action applies +`scripts/macos-stable-codesign.sh` to the Bazel-built binary before packaging. +`install.sh` and `tracedecay update` apply the same identity to the installed file after the archive checksum check when the binary is unsigned or ad-hoc signed, including an ad-hoc signature whose identifier already matches but whose requirement is still diff --git a/crates/tracedecay-application/Cargo.toml b/crates/tracedecay-application/Cargo.toml index e115500c39..113321fcd2 100644 --- a/crates/tracedecay-application/Cargo.toml +++ b/crates/tracedecay-application/Cargo.toml @@ -6,7 +6,6 @@ edition.workspace = true license = "MIT" repository = "https://github.com/ScriptedAlchemy/tracedecay" description = "TraceDecay product use cases shared by CLI, MCP, HTTP, hooks, and daemon adapters" -include = ["/src/**"] [features] default = [] diff --git a/crates/tracedecay-capture/Cargo.toml b/crates/tracedecay-capture/Cargo.toml index 01d512ad09..ed46ce051d 100644 --- a/crates/tracedecay-capture/Cargo.toml +++ b/crates/tracedecay-capture/Cargo.toml @@ -10,7 +10,6 @@ description = "Bounded event capture primitives for TraceDecay" readme = "README.md" keywords = ["ai", "code-intelligence", "observability", "capture"] categories = ["development-tools", "parsing"] -include = ["/src/**", "/tests/**", "/README.md"] [dependencies] chrono = { version = "0.4.45", default-features = false, features = ["std"] } diff --git a/crates/tracedecay-cli/Cargo.toml b/crates/tracedecay-cli/Cargo.toml index 1f2f7cfb21..baa85bf6c5 100644 --- a/crates/tracedecay-cli/Cargo.toml +++ b/crates/tracedecay-cli/Cargo.toml @@ -6,18 +6,6 @@ edition.workspace = true description = "TraceDecay command-line interface and daemon binary" license = "MIT" repository = "https://github.com/ScriptedAlchemy/tracedecay" -# Packaged-crate whitelist. `/dashboard/app-dist/**` is staged into the crate -# directory by release packaging (the distribution gate validates it); the -# packaged-crate mode in build.rs embeds it from there, mirroring the old -# root-crate arrangement. -include = [ - "/build.rs", - "/build-support/**", - "/src/**", - "/tests/**", - "/dashboard/app-dist/**", - "!/src/resources/logo.png", -] [[bin]] name = "tracedecay" diff --git a/crates/tracedecay-cli/build.rs b/crates/tracedecay-cli/build.rs index 028e9b36a0..7f9e3e05ac 100644 --- a/crates/tracedecay-cli/build.rs +++ b/crates/tracedecay-cli/build.rs @@ -110,22 +110,6 @@ impl EmbeddedDashboard { ) } - fn packaged( - asset_paths: Vec, - digest_hex: String, - app_dist: &Path, - out_dir: &Path, - ) -> Result> { - Self::from_sources( - &asset_paths, - "CARGO_MANIFEST_DIR", - "/dashboard/app-dist", - app_dist, - out_dir, - digest_hex, - ) - } - fn from_sources( asset_paths: &[String], identity_include_env: &'static str, @@ -233,30 +217,18 @@ impl EmbeddedDashboard { /// `TRACEDECAY_SKIP_DASHBOARD_BUILD` is set the prebuilt `dashboard/app-dist` /// is staged instead and must match the digest /// `TRACEDECAY_DASHBOARD_BUNDLE_SHA256` names, so a skip can never embed -/// unproven bytes. Packaged crates carry a staged `dashboard/app-dist` whose -/// integrity Cargo's package checksums already guarantee. A missing or invalid -/// bundle always fails the build; there is no empty-assets fallback. +/// unproven bytes. A missing or invalid bundle always fails the build; +/// there is no empty-assets fallback. fn embed_dashboard( manifest_dir: &Path, out_dir: &Path, ) -> Result> { - let package_local_dashboard = manifest_dir.join("dashboard"); - if package_local_dashboard.is_dir() { - // Packaged-crate mode: release packaging staged the bundle into the - // crate directory and Cargo's checksums are the integrity authority. - let app_dist = package_local_dashboard.join("app-dist"); - let asset_paths = dashboard_manifest::dashboard_asset_paths(&app_dist)?; - let digest_hex = dashboard_bundle::bundle_digest(&app_dist, &asset_paths)?; - return EmbeddedDashboard::packaged(asset_paths, digest_hex, &app_dist, out_dir); - } - let repository_root = manifest_dir.join(REPOSITORY_ROOT_FROM_CRATE); let dashboard = repository_root.join("dashboard"); if !dashboard.join("package.json").is_file() { return Err(format!( - "no dashboard to embed: {} has no package-local dashboard/ and {} has no \ - package.json; a tracedecay binary cannot build without its dashboard bundle", - manifest_dir.display(), + "no dashboard to embed: {} has no package.json; a tracedecay binary cannot \ + build without its dashboard bundle", dashboard.display(), ) .into()); diff --git a/crates/tracedecay-code-extraction/Cargo.toml b/crates/tracedecay-code-extraction/Cargo.toml index 13a6478f9d..b7866714f8 100644 --- a/crates/tracedecay-code-extraction/Cargo.toml +++ b/crates/tracedecay-code-extraction/Cargo.toml @@ -7,15 +7,6 @@ license = "MIT" repository = "https://github.com/ScriptedAlchemy/tracedecay" description = "Tree-sitter language extraction for TraceDecay" build = "build.rs" -include = [ - "/src/**", - "/fixtures/**", - "/tests/**", - "/vendor/tree-sitter-wgsl/src/**", - "/vendor/tree-sitter-wgsl/LICENSE", - "/vendor/tree-sitter-wgsl/PROVENANCE.md", - "/build.rs", -] [features] default = ["lite", "medium", "full"] diff --git a/crates/tracedecay-code-index/Cargo.toml b/crates/tracedecay-code-index/Cargo.toml index 3391129fee..b295f1159b 100644 --- a/crates/tracedecay-code-index/Cargo.toml +++ b/crates/tracedecay-code-index/Cargo.toml @@ -6,7 +6,6 @@ edition.workspace = true license = "MIT" repository = "https://github.com/ScriptedAlchemy/tracedecay" description = "In-process structural code search for TraceDecay" -include = ["/src/**"] [features] default = [ diff --git a/crates/tracedecay-global-db/Cargo.toml b/crates/tracedecay-global-db/Cargo.toml index 00a13a194b..8113fe17c1 100644 --- a/crates/tracedecay-global-db/Cargo.toml +++ b/crates/tracedecay-global-db/Cargo.toml @@ -6,7 +6,6 @@ edition.workspace = true license = "MIT" description = "TraceDecay user-level global database: project registry, observations, session temporal store" repository = "https://github.com/ScriptedAlchemy/tracedecay" -include = ["/src/**"] [features] # Explicit opt-in (same policy as the query kernel): exposes the registered diff --git a/crates/tracedecay-graph-db/Cargo.toml b/crates/tracedecay-graph-db/Cargo.toml index 57b03a6a9c..dc92baceb0 100644 --- a/crates/tracedecay-graph-db/Cargo.toml +++ b/crates/tracedecay-graph-db/Cargo.toml @@ -6,7 +6,6 @@ edition.workspace = true license = "MIT" description = "Embedded graph storage boundary for TraceDecay" repository = "https://github.com/ScriptedAlchemy/tracedecay" -include = ["/src/**"] [features] test-helpers = [] diff --git a/crates/tracedecay-graph-query/Cargo.toml b/crates/tracedecay-graph-query/Cargo.toml index cc7cd47204..d332ee0f05 100644 --- a/crates/tracedecay-graph-query/Cargo.toml +++ b/crates/tracedecay-graph-query/Cargo.toml @@ -6,7 +6,6 @@ edition.workspace = true license = "MIT" repository = "https://github.com/ScriptedAlchemy/tracedecay" description = "Generation-pinned verified code-graph queries and code-index-backed source readers" -include = ["/src/**"] [dependencies] serde_json = "1" diff --git a/crates/tracedecay-lcm/Cargo.toml b/crates/tracedecay-lcm/Cargo.toml index 83348f67f1..3513a4c86c 100644 --- a/crates/tracedecay-lcm/Cargo.toml +++ b/crates/tracedecay-lcm/Cargo.toml @@ -6,7 +6,6 @@ edition.workspace = true license = "MIT" description = "Lossless context memory engine: contracts, raw ingest, summary DAG, payload authority, retrieval, GC, and retention" repository = "https://github.com/ScriptedAlchemy/tracedecay" -include = ["/src/**", "/tests/**"] [features] # Repo-wide `test-helpers` convention: exposes this crate's test-only helper diff --git a/crates/tracedecay-privacy/Cargo.toml b/crates/tracedecay-privacy/Cargo.toml index 4f371b09bc..833f6cba4b 100644 --- a/crates/tracedecay-privacy/Cargo.toml +++ b/crates/tracedecay-privacy/Cargo.toml @@ -6,7 +6,6 @@ edition.workspace = true license = "MIT" repository = "https://github.com/ScriptedAlchemy/tracedecay" description = "TraceDecay privacy authority: credential detection, record sanitization, and protected structural identifiers" -include = ["/src/**"] [features] default = [] diff --git a/crates/tracedecay-query/Cargo.toml b/crates/tracedecay-query/Cargo.toml index 362fd8d088..2a4a29cf81 100644 --- a/crates/tracedecay-query/Cargo.toml +++ b/crates/tracedecay-query/Cargo.toml @@ -6,7 +6,6 @@ edition.workspace = true license = "MIT" repository = "https://github.com/ScriptedAlchemy/tracedecay" description = "TraceDecay exact/lexical/graph code retrieval query kernel and search-quality candidate/profile inputs" -include = ["/src/**", "/assets/**"] [dependencies] # Raw deflate for lexical row text; already in the tree through diff --git a/crates/tracedecay-search-eval/Cargo.toml b/crates/tracedecay-search-eval/Cargo.toml index 665997bc01..9e5835f145 100644 --- a/crates/tracedecay-search-eval/Cargo.toml +++ b/crates/tracedecay-search-eval/Cargo.toml @@ -6,7 +6,6 @@ edition.workspace = true license = "MIT" description = "TraceDecay search-quality evaluator: exact/lexical/graph candidate generation and direct evaluation" repository = "https://github.com/ScriptedAlchemy/tracedecay" -include = ["/assets/**", "/src/**"] [dependencies] clap = { version = "4.6", features = ["derive"] } diff --git a/crates/tracedecay-session-memory/Cargo.toml b/crates/tracedecay-session-memory/Cargo.toml index af4a96fc80..753b8ffe52 100644 --- a/crates/tracedecay-session-memory/Cargo.toml +++ b/crates/tracedecay-session-memory/Cargo.toml @@ -6,7 +6,6 @@ edition.workspace = true license = "MIT" repository = "https://github.com/ScriptedAlchemy/tracedecay" description = "Session retrieval, project memory, provider usage, and runtime telemetry use cases shared by CLI, MCP, and host-admission adapters" -include = ["/src/**"] [features] default = [] diff --git a/crates/tracedecay-session-temporal-store/Cargo.toml b/crates/tracedecay-session-temporal-store/Cargo.toml index 8339294bfb..7ff4d117c8 100644 --- a/crates/tracedecay-session-temporal-store/Cargo.toml +++ b/crates/tracedecay-session-temporal-store/Cargo.toml @@ -6,7 +6,6 @@ edition.workspace = true license = "MIT" description = "Session-temporal projection, retrieval, and LCM hydration store" repository = "https://github.com/ScriptedAlchemy/tracedecay" -include = ["/src/**"] [features] # Exposes package-owned fixture constructors to dependent package tests only. diff --git a/crates/tracedecay-sessions/Cargo.toml b/crates/tracedecay-sessions/Cargo.toml index 7174b5783e..9ac8268d1d 100644 --- a/crates/tracedecay-sessions/Cargo.toml +++ b/crates/tracedecay-sessions/Cargo.toml @@ -6,7 +6,6 @@ edition.workspace = true license = "MIT" repository = "https://github.com/ScriptedAlchemy/tracedecay" description = "TraceDecay session parsing and retrieval primitives" -include = ["/src/**", "/tests/**"] [features] # Repo-wide `test-helpers` convention: exposes this crate's test-only helper diff --git a/crates/tracedecay-temporal-query/Cargo.toml b/crates/tracedecay-temporal-query/Cargo.toml index fc077a607d..6428fa2f9e 100644 --- a/crates/tracedecay-temporal-query/Cargo.toml +++ b/crates/tracedecay-temporal-query/Cargo.toml @@ -6,7 +6,6 @@ edition.workspace = true license = "MIT" repository = "https://github.com/ScriptedAlchemy/tracedecay" description = "TraceDecay temporal retrieval and context assembly kernel" -include = ["/src/**"] [dependencies] hex = "0.4" diff --git a/crates/tracedecay/Cargo.toml b/crates/tracedecay/Cargo.toml index 68b1fc93ce..ca69e473e4 100644 --- a/crates/tracedecay/Cargo.toml +++ b/crates/tracedecay/Cargo.toml @@ -11,59 +11,6 @@ homepage = "https://github.com/ScriptedAlchemy/tracedecay" readme = "../../README.md" keywords = ["code-intelligence", "knowledge-graph", "mcp", "tree-sitter", "semantic-search"] categories = ["development-tools", "command-line-utilities"] -# Explicit source-archive whitelist retained for local distribution validation. -# Frontend sources and the prebuilt dashboard/app-dist are deliberately -# excluded: the CLI package stages and embeds the dashboard bundle and -# registers it into this library at process start. -include = [ - "/CHANGELOG.md", - "/LICENSE", - "/README.md", - "/rust-toolchain.toml", - "/.cargo/config.toml", - "/src/**", - "/benches/**", - "/examples/**", - "/tests/fixtures/analytics/**", - "/tests/fixtures/packaged_host_events/**", - "/tests/fixtures/provider_normalization/**", - "/tests/fixtures/redundancy_eval_labeled.json", - "/tests/fixtures/transcript_golden/cline_like/**", - "/benchmark_data/claude-observation/**", - "/benchmark_data/session-temporal/**", - "/benchmark_data/query-code-index/**", - "/scripts/run-session-temporal-benchmark.sh", - "/plugin/.claude-plugin/**", - "/plugin/.cursor-plugin/**", - "/plugin/.codex-plugin/**", - "/plugin/.kimi-plugin/**", - "/plugin/.lsp.json", - "/plugin/.mcp.json", - "/plugin/plugin.json", - "/plugin/mcp.json", - "/plugin/README*.md", - "/plugin/chatgpt-extension/README.md", - "/plugin/chatgpt-extension/assets/icon.svg", - "/plugin/chatgpt-extension/embedded/app.html", - "/plugin/chatgpt-extension/embedded/server.mjs", - "/plugin/agents/**", - "/plugin/commands/**", - "/plugin/hooks/**", - "/plugin/mcp-cursor.json", - "/plugin/opencode/**", - "/plugin/overlays/**", - "/plugin/rules/**", - "/plugin/skills/**", - "/plugin/cursor-native-extension/package.json", - "/plugin/cursor-native-extension/README.md", - "/plugin/cursor-native-extension/LICENSE", - "/plugin/cursor-native-extension/embedded/extension.js", - "/vendor/**", - "/dashboard/hermes-wrapper/manifest.json", - "/dashboard/hermes-wrapper/plugin_api.py", - "/dashboard/hermes-wrapper/embed_proxy.py", - "/dashboard/hermes-wrapper/src/**", -] # Measured aggregate suites exceed the native five-minute medium limit. [package.metadata.bazel.test_sizes] diff --git a/docs/RELEASE-AUTOMATION.md b/docs/RELEASE-AUTOMATION.md index 62d7ea697b..1bae57a219 100644 --- a/docs/RELEASE-AUTOMATION.md +++ b/docs/RELEASE-AUTOMATION.md @@ -97,16 +97,15 @@ assets. It builds the production binary, smokes `--version`/`--help`, and packages the MCPB and archive from that same file. A source-tree inventory or a release-PR path policy is not acceptance. -`scripts/check-distribution-acceptance.sh` proves the unpublished crate -graph (`cargo package`, extracted-crate rebuilds, nextest, MCP inspector). -That is not required to stamp the binary assets. The battery runs on demand -only, on one free `ubuntu-latest` runner: -`gh workflow run distribution-acceptance.yml --ref master`. Nothing in this -repository runs on a timer. The battery's production binary is the CLI it -release-builds from the extracted package; it does not release-build the -source tree first (that compile is `shipped-cli` in `ci.yml`). A caller that -already holds a source-tree production binary may pass -`--reuse-release-binary` to prove that binary reports the same commit. +Rust workspace crates remain unpublished. There is no packaged-crate +reconstruction or downstream Cargo consumer acceptance path. Bazel builds +and tests the supported feature graph directly; source-package publication +requires a separate product decision with a real consumer. + +The `shipped-cli` CI job also exercises MCP stdio and the LSP initialize +handshake against its production binary in isolated temporary profiles. +Archive and MCPB verification stays attached to the exact release assets; +SDK conformance stays attached to the npm tarball. Host install, update, and uninstall journeys, and recorded native host events as historical-ingestion evidence, stay product contracts. They do diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index a0a59c325d..002fb46be2 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -53,14 +53,28 @@ py_test( main = "test-check-ambient-tokio.py", ) +py_binary( + name = "packaged_lsp_bridge", + srcs = [ + "check-packaged-lsp-bridge.py", + "lib/__init__.py", + "lib/portable_process.py", + "lib/process_control.py", + ], + main = "check-packaged-lsp-bridge.py", +) + exports_files( [ "bazel/rust_test.bzl", + "test-rust-cache-lineage.py", + "test-check-packaged-lsp-bridge.py", "bazel/gen_builds.py", "bazel/rust_test_environment_probe.txt", "check-dashboard-bundle.py", "check-github-hosted-runners.py", "run-session-temporal-benchmark.sh", + "mcp-conformance-smoke.sh", "test-check-github-hosted-runners.py", "test-gen-builds-default-alias.py", "test-measure-ranked-tail-trim.py", diff --git a/scripts/bazel/rust_test.bzl b/scripts/bazel/rust_test.bzl index 93c6d83ff7..9f37b872fb 100644 --- a/scripts/bazel/rust_test.bzl +++ b/scripts/bazel/rust_test.bzl @@ -97,18 +97,13 @@ def _isolated_rust_test_impl(ctx): } for variable, file in {"CARGO": toolchain.cargo, "RUSTC": toolchain.rustc, "RUSTDOC": toolchain.rust_doc}.items(): paths[variable] = _runfiles_path(file, workspace) - # Distribution acceptance explicitly supplies its extracted CLI through - # --test_env. Every other runfile, and the default CLI, stays declared here. if windows: runfiles_env = "\n".join([ - ('if not defined {} '.format(variable) if variable == "TRACEDECAY_TEST_BIN" else "") + 'set "{}=%TEST_SRCDIR%/{}"'.format(variable, path) for variable, path in paths.items() ]) else: runfiles_env = " ".join([ - '"{}=${{{}:-$TEST_SRCDIR/{}}}"'.format(variable, variable, path) - if variable == "TRACEDECAY_TEST_BIN" else '"{}=$TEST_SRCDIR/{}"'.format(variable, path) for variable, path in paths.items() ]) diff --git a/scripts/check-distribution-acceptance.sh b/scripts/check-distribution-acceptance.sh deleted file mode 100755 index 9a2b686071..0000000000 --- a/scripts/check-distribution-acceptance.sh +++ /dev/null @@ -1,998 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -script_path=${BASH_SOURCE[0]} -repo=$(cd -- "$(dirname -- "$script_path")/.." && pwd -P) -reuse_release_binary="" -skip_packaged_runtime_battery=false - -usage() { - cat <<'EOF' -Usage: scripts/check-distribution-acceptance.sh [OPTIONS] - -Build and exercise the release distribution from packaged crate archives. -The gate packages every workspace crate, extracts the produced .crate archives -into an isolated temporary directory, builds the packaged CLI with -`--profile dist`, and tests the packaged library and that CLI. The -production binary this gate proves is the one it builds from the extracted -package; it never builds the source tree. - -Options: - --repo PATH Repository root (default: parent of this script) - --reuse-release-binary PATH Also prove this already-built production - binary reports the staged source commit - --skip-packaged-runtime-battery After packaging and manifest checks, skip - extracted-crate rebuilds, nextest, cargo - install, and MCP inspector dogfood; requires - --reuse-release-binary as the smoked binary - -h, --help Show this help -EOF -} - -die() { - echo "distribution acceptance: $*" >&2 - exit 1 -} - -require_command() { - command -v "$1" >/dev/null 2>&1 || die "required command not found: $1" -} - -resolve_clean_source_head() { - local source_repo=$1 - local expected_sha=${2:-} - local source_git_root - local source_git_sha - local source_drift - local first_drift - - if ! source_git_root=$(git -C "$source_repo" rev-parse --show-toplevel 2>/dev/null); then - die "source repository is not a git worktree: $source_repo" - fi - source_git_root=$(cd -- "$source_git_root" && pwd -P) - [[ $source_git_root == "$source_repo" ]] || - die "source repository must be its own git worktree: $source_repo (git reported $source_git_root)" - if ! source_git_sha=$(git -C "$source_repo" rev-parse --verify 'HEAD^{commit}' 2>/dev/null); then - die "source repository HEAD does not resolve to a commit: $source_repo" - fi - [[ ${#source_git_sha} -eq 40 && $source_git_sha != *[!0-9a-f]* ]] || - die "source repository HEAD is not a full lowercase git sha: $source_git_sha" - if [[ -n $expected_sha && $source_git_sha != "$expected_sha" ]]; then - die "source repository HEAD moved during distribution acceptance: captured $expected_sha, now $source_git_sha" - fi - # Ignored build outputs and the workflow-owned auxiliary checkout are not - # release source. All other tracked, staged, and untracked drift cannot be - # stamped as the clean release-env commit. - if ! source_drift=$(git --no-optional-locks -C "$source_repo" status \ - --porcelain=v1 \ - --untracked-files=all \ - -- . ':(exclude).release-automation' 2>/dev/null); then - die "failed to verify source repository cleanliness: $source_repo" - fi - if [[ -n $source_drift ]]; then - first_drift=${source_drift%%$'\n'*} - die "source repository has tracked or untracked drift ($first_drift); commit or remove it before distribution acceptance" - fi - - printf '%s\n' "$source_git_sha" -} - -assert_binary_source_sha() { - local binary=$1 - local product_version=$2 - local source_git_sha=$3 - local build_kind=$4 - local reported_version - local expected_version="tracedecay ${product_version}+${source_git_sha}" - - if ! reported_version=$("$binary" --version); then - die "$build_kind tracedecay binary failed to report its version: $binary" - fi - [[ $reported_version == "$expected_version" ]] || - die "$build_kind tracedecay binary reported $reported_version; expected $expected_version" -} - -assert_required_assets() { - local root_package=$1 - local cli_package=$2 - local required - local -a root_assets=( - "plugin/.lsp.json" - "plugin/.claude-plugin/plugin.json" - "plugin/.codex-plugin/plugin.json" - "plugin/.cursor-plugin/plugin.json" - "plugin/.kimi-plugin/plugin.json" - "plugin/opencode/tracedecay.ts" - "plugin/opencode/tracedecay-mcp.ts" - "plugin/opencode/opencode.registration.json" - "plugin/cursor-native-extension/embedded/extension.js" - "plugin/plugin.json" - "plugin/mcp.json" - "plugin/chatgpt-extension/embedded/server.mjs" - "plugin/chatgpt-extension/embedded/app.html" - "tests/fixtures/packaged_host_events/claude.json" - "tests/fixtures/packaged_host_events/claude/post_tool_use_write.json" - "tests/fixtures/packaged_host_events/cline-family.json" - "tests/fixtures/packaged_host_events/codex.json" - "tests/fixtures/packaged_host_events/cursor.json" - "tests/fixtures/packaged_host_events/hermes.json" - "tests/fixtures/packaged_host_events/hermes/saved-edit.json" - "tests/fixtures/packaged_host_events/hermes/terminal-receipt.json" - "tests/fixtures/packaged_host_events/kiro.json" - "tests/fixtures/packaged_host_events/kimi-code.json" - "tests/fixtures/packaged_host_events/kimi/post-tool-use-edit.json" - "tests/fixtures/packaged_host_events/opencode/baseline.json" - "tests/fixtures/packaged_host_events/pi.json" - "tests/fixtures/packaged_host_events/droid.json" - "tests/fixtures/provider_normalization/codex/session_meta.input.json" - "tests/fixtures/provider_normalization/codex/agent_message.input.json" - "tests/fixtures/analytics/codex_skill_prose.txt" - "benchmark_data/claude-observation/workload-v1.json" - ) - - for required in "${root_assets[@]}"; do - [[ -f "$root_package/$required" ]] || - die "packaged tracedecay crate is missing $required" - done - # The dashboard bundle ships in the CLI package (its build.rs embeds the - # package-local dashboard/app-dist); the root library package must not - # carry a second copy that could drift from the one the binary serves. - python3 "$repo/scripts/check-dashboard-bundle.py" \ - "$cli_package/dashboard/app-dist" - [[ ! -e "$root_package/dashboard/app-dist" ]] || - die "packaged tracedecay crate must not carry dashboard/app-dist; the CLI package owns the bundle" - - python3 - "$root_package/plugin/.lsp.json" <<'PY' -import json -import pathlib -import sys - -path = pathlib.Path(sys.argv[1]) -try: - value = json.loads(path.read_text(encoding="utf-8")) -except (OSError, json.JSONDecodeError) as error: - raise SystemExit(f"distribution acceptance: invalid packaged JSON {path}: {error}") -if not isinstance(value, dict) or not value: - raise SystemExit( - f"distribution acceptance: packaged JSON must be a non-empty object: {path}" - ) -PY - python3 "$repo/scripts/check-packaged-plugin-manifests.py" \ - "$root_package/plugin" -} - -verify_feature_wiring() { - local source_manifest=$1 - local packaged_manifest=$2 - local code_index_source_manifest=$3 - local code_index_packaged_manifest=$4 - local extraction_source_manifest=$5 - local extraction_packaged_manifest=$6 - local cli_source_manifest=$7 - local cli_packaged_manifest=$8 - local cargo_config=$9 - # Manifest and layering rules only. Per-language `cargo check` isolation is - # a source-graph property, not a packaging proof: it does not use the - # just-built release binary, and on the extracted tree it serializes one - # compile per `lang-*` feature (37 today). The feature-wiring script still - # exposes `--check-extraction-manifest` for dedicated CI. - python3 "$repo/scripts/check-distribution-feature-wiring.py" \ - --root-source "$source_manifest" \ - --root-packaged "$packaged_manifest" \ - --code-index-source "$code_index_source_manifest" \ - --code-index-packaged "$code_index_packaged_manifest" \ - --extraction-source "$extraction_source_manifest" \ - --extraction-packaged "$extraction_packaged_manifest" \ - --cli-source "$cli_source_manifest" \ - --cli-packaged "$cli_packaged_manifest" \ - --cargo-config "$cargo_config" \ - --offline -} - -read_workspace_product_version() { - python3 - "$1" <<'PY' -import sys - -# Same rule the build script applies: the one literal `version` inside -# `[workspace.package]`. An inherited or absent value is not a product version. -in_table = False -value = None -with open(sys.argv[1], encoding="utf-8") as handle: - for line in handle: - line = line.strip() - if line.startswith("#"): - continue - if line.startswith("["): - in_table = line == "[workspace.package]" - continue - if in_table and line.startswith("version"): - _, _, raw = line.partition("=") - raw = raw.strip() - if raw.startswith('"') and raw.endswith('"'): - value = raw[1:-1] -print(value or "") -PY -} - -while (($#)); do - case "$1" in - --repo) - [[ $# -ge 2 ]] || die "--repo requires a path" - repo=$2 - shift 2 - ;; - --reuse-release-binary) - [[ $# -ge 2 ]] || die "--reuse-release-binary requires a path" - reuse_release_binary=$2 - shift 2 - ;; - --skip-packaged-runtime-battery) - skip_packaged_runtime_battery=true - shift - ;; - -h|--help) - usage - exit 0 - ;; - *) - usage >&2 - die "unknown argument: $1" - ;; - esac -done - -require_command cargo -require_command cmp -require_command git -require_command python3 -require_command rustc -require_command tar - -repo=$(cd -- "$repo" && pwd -P) -[[ -f "$repo/Cargo.toml" ]] || die "Cargo.toml not found under $repo" -if [[ -n $reuse_release_binary ]]; then - [[ -e $reuse_release_binary ]] || - die "reuse-release-binary is missing: $reuse_release_binary" - reuse_release_binary=$(cd -- "$(dirname -- "$reuse_release_binary")" && pwd -P)/$(basename -- "$reuse_release_binary") - [[ -f $reuse_release_binary ]] || - die "reuse-release-binary is not a file: $reuse_release_binary" -fi -if [[ $skip_packaged_runtime_battery == true && -z $reuse_release_binary ]]; then - die "--skip-packaged-runtime-battery requires --reuse-release-binary" -fi -source_git_sha=$(resolve_clean_source_head "$repo") -for fixture in \ - claude.json \ - claude/post_tool_use_write.json \ - codex.json \ - cline-family.json \ - cursor.json \ - hermes.json \ - hermes/saved-edit.json \ - hermes/terminal-receipt.json \ - kiro.json \ - kimi-code.json \ - kimi/post-tool-use-edit.json \ - opencode/baseline.json \ - pi.json \ - droid.json; do - cmp -s \ - "$repo/crates/tracedecay-hooks/fixtures/host_events/$fixture" \ - "$repo/tests/fixtures/packaged_host_events/$fixture" || - die "packaged host-event fixture copy differs from its authority: $fixture" -done -product_version=$(read_workspace_product_version "$repo/Cargo.toml") -[[ -n $product_version ]] || - die "$repo/Cargo.toml must declare a literal version in [workspace.package]" - -work=$(mktemp -d "${TMPDIR:-/tmp}/tracedecay-distribution.XXXXXX") -trap 'rm -rf -- "$work"' EXIT - -host_target="" -while IFS= read -r rustc_version_line; do - if [[ $rustc_version_line == "host: "* ]]; then - host_target=${rustc_version_line#host: } - host_target=${host_target%$'\r'} - fi -done < <(rustc -vV) -[[ -n $host_target ]] || die "rustc did not report its host target" -release_profile_output="$work/release-profile-output" -python3 "$repo/scripts/resolve-release-source-profile.py" \ - --source "$repo" \ - --target "$host_target" \ - --github-output "$release_profile_output" -release_profile="" -release_cargo_features="" -while IFS='=' read -r output_name output_value; do - case "$output_name" in - profile) release_profile=$output_value ;; - cargo_features) release_cargo_features=$output_value ;; - esac -done < "$release_profile_output" -[[ $release_profile == production ]] || - die "distribution acceptance requires a production release profile" -[[ -n $release_cargo_features ]] || - die "production release profile omitted Cargo features" -release_cli_cargo_args=( - --no-default-features - --features "$release_cargo_features" -) - -# No source-tree release build here. The packaged CLI compiled from the -# extracted package below is the production binary this gate proves, and the -# source-tree production compile is `shipped-cli` in ci.yml. A caller that -# already holds a source-tree production binary may hand it in to prove it -# was built from this exact commit; nothing else in the battery reads it. -if [[ -n $reuse_release_binary ]]; then - echo "distribution acceptance: reusing the just-built production binary" - assert_binary_source_sha \ - "$reuse_release_binary" \ - "$product_version" \ - "$source_git_sha" \ - "reused-release" -fi - -echo "distribution acceptance: staging the product package tree" -# `tracedecay` is `crates/tracedecay`, but the assets it ships, host plugins, -# vendored payloads, benchmark corpora, and the packaged fixtures, are -# repository-root directories shared with the whole workspace. Cargo packs -# only what lives inside the package directory, so packaging the checkout -# as-is yields a `tracedecay` archive with none of them, and the `include` -# whitelist in `crates/tracedecay/Cargo.toml` silently matches nothing. Stage -# a copy of the workspace, place exactly those root assets beside the product -# manifest, and pack from the staged tree. The dashboard bundle is different: -# it ships in the CLI package (`crates/tracedecay-cli`, whose `include` names -# `/dashboard/app-dist/**`), and its build.rs embeds a package-local -# `dashboard/app-dist` when one exists, so the bundle is staged beside the -# CLI manifest below rather than beside the library manifest. -staged="$work/staged" -mkdir -p -- "$staged" -resolve_clean_source_head "$repo" "$source_git_sha" >/dev/null -tar -C "$repo" \ - --exclude=./target \ - --exclude=./.git \ - --exclude=./.codex-worktrees \ - --exclude=./.claude-worktrees \ - --exclude=node_modules \ - --exclude=.pnpm \ - -cf - . | tar -xf - -C "$staged" -resolve_clean_source_head "$repo" "$source_git_sha" >/dev/null - -staged_product="$staged/crates/tracedecay" -[[ -f "$staged_product/Cargo.toml" ]] || - die "staged workspace is missing the product manifest at crates/tracedecay" - -# Exactly the root entries the product `include` whitelist names. Keep this in -# sync with `include` in crates/tracedecay/Cargo.toml: a missing asset is a -# packaging defect, never something to skip past. -declare -a staged_root_assets=( - "CHANGELOG.md" - "LICENSE" - "README.md" - "rust-toolchain.toml" - ".cargo/config.toml" - "plugin" - "vendor" - "benchmark_data" - "dashboard/hermes-wrapper" - "tests/fixtures" - "scripts/run-session-temporal-benchmark.sh" -) -# A package directory may already carry its own entry at the destination path, -# as `crates/tracedecay/tests/fixtures` does. `cp -a` merges a directory into -# an existing directory of the same name, which would leave the package-local -# asset a superset of the root one. Clear the destination so the staged asset -# is exactly the root snapshot the assertion below demands. -for asset in "${staged_root_assets[@]}"; do - [[ -e "$staged/$asset" ]] || - die "product package asset is missing from the staged source tree: $asset" - mkdir -p -- "$staged_product/$(dirname -- "$asset")" - rm -rf -- "$staged_product/$asset" - cp -a -- "$staged/$asset" "$staged_product/$(dirname -- "$asset")/" -done - -# The CLI package carries the prebuilt dashboard bundle; its `include` -# whitelist names `/dashboard/app-dist/**` and its build.rs embeds the -# package-local copy in packaged-crate mode. -staged_cli_crate="$staged/crates/tracedecay-cli" -[[ -f "$staged_cli_crate/Cargo.toml" ]] || - die "staged workspace is missing the CLI manifest at crates/tracedecay-cli" -declare -a staged_cli_assets=( - "dashboard/app-dist" -) -for asset in "${staged_cli_assets[@]}"; do - [[ -e "$staged/$asset" ]] || - die "CLI package asset is missing from the staged source tree: $asset" - mkdir -p -- "$staged_cli_crate/$(dirname -- "$asset")" - rm -rf -- "$staged_cli_crate/$asset" - cp -a -- "$staged/$asset" "$staged_cli_crate/$(dirname -- "$asset")/" -done - -# The manifest reads its readme from the workspace root, which is outside the -# package. The staged tree carries its own copy, so point at that one; nothing -# else about the staged manifest may differ, and the feature-wiring comparison -# below is run against the source manifest to prove it. -python3 - "$staged_product/Cargo.toml" <<'PY' -import pathlib -import sys - -path = pathlib.Path(sys.argv[1]) -text = path.read_text(encoding="utf-8") -old = 'readme = "../../README.md"' -new = 'readme = "README.md"' -if text.count(old) != 1: - raise SystemExit( - "distribution acceptance: staged product manifest does not declare the " - f"expected workspace-root readme ({old})" - ) -path.write_text(text.replace(old, new), encoding="utf-8") -PY - -# Prove the package-local copies are byte-identical to the validated staged -# snapshot, so no later live-checkout change can enter the archive. -assert_staged_assets_identical() { - local source_dir=$1 - local staged_dir=$2 - shift 2 - python3 - "$source_dir" "$staged_dir" "$@" <<'PY' -import hashlib -import pathlib -import sys - -source_root = pathlib.Path(sys.argv[1]) -staged = pathlib.Path(sys.argv[2]) - - -def digest(root: pathlib.Path) -> str: - if root.is_file(): - return hashlib.sha256(root.read_bytes()).hexdigest() - accumulator = hashlib.sha256() - for path in sorted(p for p in root.rglob("*") if p.is_file()): - accumulator.update(str(path.relative_to(root)).encode("utf-8")) - accumulator.update(path.read_bytes()) - return accumulator.hexdigest() - - -for asset in sys.argv[3:]: - source, copied = source_root / asset, staged / asset - if not copied.exists(): - raise SystemExit(f"distribution acceptance: staged asset is missing: {asset}") - if digest(source) != digest(copied): - raise SystemExit( - f"distribution acceptance: staged asset differs from its snapshot: {asset}" - ) -print(f"distribution acceptance: staged {len(sys.argv) - 3} package asset entries") -PY -} -assert_staged_assets_identical "$staged" "$staged_product" "${staged_root_assets[@]}" -assert_staged_assets_identical "$staged" "$staged_cli_crate" "${staged_cli_assets[@]}" - -echo "distribution acceptance: packaging every workspace crate" -# Every workspace member is publish = false: releases ship GitHub-release -# artifacts, never crates.io uploads. Packaging is used here only to produce -# the per-crate source trees the acceptance battery runs against, so skip the -# per-package lockfile: generating it would resolve the unpublished internal -# dependencies against crates.io and fail. Nothing downstream reads the -# embedded lock, every extracted tree resolves through the [patch.crates-io] -# path overlay below, and the install step builds from a path, not an archive. -cargo package \ - --manifest-path "$staged/Cargo.toml" \ - --workspace \ - --allow-dirty \ - --no-verify \ - --exclude-lockfile - -metadata="$work/metadata.json" -cargo metadata \ - --manifest-path "$staged/Cargo.toml" \ - --format-version 1 \ - --no-deps >"$metadata" - -target_directory=$(python3 - "$metadata" <<'PY' -import json -import sys - -with open(sys.argv[1], encoding="utf-8") as handle: - print(json.load(handle)["target_directory"]) -PY -) - -package_table="$work/packages.tsv" -python3 - "$metadata" >"$package_table" <<'PY' -import json -import sys - -with open(sys.argv[1], encoding="utf-8") as handle: - metadata = json.load(handle) -members = set(metadata["workspace_members"]) -for package in metadata["packages"]: - if package["id"] in members: - print(f'{package["name"]}\t{package["version"]}') -PY - -# `tracedecay-agent-hosts` stamps the product version, and that value now lives -# in `[workspace.package]` at the workspace root, two directories above each -# crate. The sibling symlinks below already reproduce the `crates/` shape -# a build script sees in the repository; this reproduces the workspace root -# that sits above it, so the battery resolves the same version the repository -# build does instead of failing closed on a missing manifest. `exclude` keeps -# every extracted tree a standalone build rather than a member of this stub. -package_root="$work/packages" -packages="$package_root/crates" -mkdir -p -- "$packages" -cat >"$package_root/Cargo.toml" <`, so the stub root has to present -# the same trees. They are linked from the staged copy rather than the live -# checkout so the battery reads exactly what was packaged. -for root_asset in plugin tests dashboard benchmark_data; do - [[ -e "$staged/$root_asset" ]] || - die "staged workspace is missing the root asset tree $root_asset" - ln -sfn -- "$staged/$root_asset" "$package_root/$root_asset" -done -declare -A package_dirs=() -while IFS=$'\t' read -r name version; do - archive="$target_directory/package/$name-$version.crate" - [[ -f "$archive" ]] || - die "cargo package did not produce $archive" - tar -xzf "$archive" -C "$packages" - directory="$packages/$name-$version" - [[ -f "$directory/Cargo.toml" ]] || - die "package archive did not contain $name-$version/Cargo.toml" - package_dirs["$name"]=$directory - # Each extracted tree is its own Cargo root, so without a lockfile Cargo - # would re-resolve every registry dependency. Offline that refuses a - # version crates.io has yanked since the workspace locked it (bisync 0.3.0 - # under gix-protocol did exactly this). The checked-in workspace lockfile is - # the release resolution; every packaged crate builds against it. - cp -- "$staged/Cargo.lock" "$directory/Cargo.lock" - # Extracted archives are named `-`, but in the workspace every - # crate sits at `crates/`. Sources that reach a sibling crate by - # relative `#[path]`, e.g. `tracedecay/src/daemon.rs` includes scheduler - # test modules from `tracedecay-code-index-runtime`, resolve against that - # unversioned shape. `cargo package` cannot carry a file from outside the - # package, so give the battery the same sibling layout the workspace has - # rather than a copy that could drift from it. - ln -sfn -- "$name-$version" "$packages/$name" -done <"$package_table" - -for required_package in \ - tracedecay \ - tracedecay-cli \ - tracedecay-agent-hosts \ - tracedecay-contracts \ - tracedecay-api \ - tracedecay-tool-catalog \ - tracedecay-code-index \ - tracedecay-code-index-runtime \ - tracedecay-code-extraction; do - [[ -n ${package_dirs[$required_package]:-} ]] || - die "workspace package required by the distribution gate was not produced: $required_package" -done -root_package=${package_dirs[tracedecay]} -cli_package=${package_dirs[tracedecay-cli]} -agent_hosts_package=${package_dirs[tracedecay-agent-hosts]} -code_index_package=${package_dirs[tracedecay-code-index]} -code_extraction_package=${package_dirs[tracedecay-code-extraction]} -catalog_package=${package_dirs[tracedecay-tool-catalog]} -contracts_package=${package_dirs[tracedecay-contracts]} - -assert_required_assets "$root_package" "$cli_package" -[[ ! -e "$package_root/.git" && ! -L "$package_root/.git" ]] || - die "extracted package workspace unexpectedly contains .git metadata: $package_root/.git" - -patch_config="$work/packaged-crates.toml" -python3 - "$metadata" "$packages" "$staged/Cargo.toml" >"$patch_config" <<'PY' -import json -import pathlib -import sys -import tomllib - -with open(sys.argv[1], encoding="utf-8") as handle: - metadata = json.load(handle) -members = set(metadata["workspace_members"]) -packages = pathlib.Path(sys.argv[2]) -with open(sys.argv[3], "rb") as handle: - workspace_manifest = tomllib.load(handle) -print("[patch.crates-io]") -for package in sorted(metadata["packages"], key=lambda value: value["name"]): - if package["id"] not in members: - continue - path = packages / f'{package["name"]}-{package["version"]}' - print(f'{json.dumps(package["name"])} = {{ path = {json.dumps(str(path))} }}') -# Packaged crates resolve outside the workspace, so the workspace manifest's -# patches must be re-applied here for the extracted trees to see the same -# patched dependencies: git patches (e.g. the pinned tree-sitter-rust fork) -# verbatim, and path patches against the -# staged workspace root, which carries the same `vendor` tree. -staged_root = pathlib.Path(sys.argv[3]).parent -for name, spec in workspace_manifest.get("patch", {}).get("crates-io", {}).items(): - if not isinstance(spec, dict): - continue - if "git" in spec: - entry = f'{json.dumps(name)} = {{ git = {json.dumps(spec["git"])}' - for key in ("rev", "branch", "tag"): - if key in spec: - entry += f", {key} = {json.dumps(spec[key])}" - entry += " }" - print(entry) - elif "path" in spec: - patched = staged_root / spec["path"] - if not (patched / "Cargo.toml").is_file(): - raise SystemExit( - f"distribution acceptance: workspace path patch {name} is not staged at {patched}" - ) - print(f'{json.dumps(name)} = {{ path = {json.dumps(str(patched))} }}') -# Extracted packages do not see workspace Cargo.toml profiles. Copy -# [profile.dist] from that manifest so `--profile dist` matches shipping. -dist_profile = workspace_manifest.get("profile", {}).get("dist") -if not isinstance(dist_profile, dict): - raise SystemExit( - "distribution acceptance: workspace Cargo.toml omitted [profile.dist]" - ) -required = ("inherits", "lto", "codegen-units", "strip") -missing = [key for key in required if key not in dist_profile] -if missing: - raise SystemExit( - "distribution acceptance: [profile.dist] missing " + ", ".join(missing) - ) -print() -print("[profile.dist]") -for key, value in dist_profile.items(): - if isinstance(value, bool): - rendered = "true" if value else "false" - elif isinstance(value, int): - rendered = str(value) - elif isinstance(value, str): - rendered = json.dumps(value) - else: - raise SystemExit( - f"distribution acceptance: [profile.dist] {key} has unsupported type" - ) - print(f"{key} = {rendered}") -PY - -verify_feature_wiring \ - "$repo/crates/tracedecay/Cargo.toml" \ - "$root_package/Cargo.toml" \ - "$repo/crates/tracedecay-code-index/Cargo.toml" \ - "$code_index_package/Cargo.toml" \ - "$repo/crates/tracedecay-code-extraction/Cargo.toml" \ - "$code_extraction_package/Cargo.toml" \ - "$repo/crates/tracedecay-cli/Cargo.toml" \ - "$cli_package/Cargo.toml" \ - "$patch_config" - -if [[ $skip_packaged_runtime_battery == true ]]; then - echo "distribution acceptance: skipping packaged runtime battery" - "$reuse_release_binary" --help >/dev/null || - die "reused release binary failed --help: $reuse_release_binary" - echo "distribution acceptance passed" - exit 0 -fi - -echo "distribution acceptance: compiling packaged CLI with dist profile" -# Checkout rustflags skip LLVM bitcode for non-LTO graphs. A process -# RUSTFLAGS replaces those flags; thin LTO needs the bitcode back. -TRACEDECAY_RELEASE_GIT_SHA="$source_git_sha" \ - RUSTFLAGS="${RUSTFLAGS:+$RUSTFLAGS }-C embed-bitcode=yes" \ - cargo build \ - --manifest-path "$cli_package/Cargo.toml" \ - --profile dist \ - "${release_cli_cargo_args[@]}" \ - --bin tracedecay \ - --config "$patch_config" -executable_suffix="" -if [[ ${OS:-} == "Windows_NT" ]]; then - executable_suffix=".exe" -fi -packaged_cli_bin="$target_directory/dist/tracedecay${executable_suffix}" -[[ -x $packaged_cli_bin ]] || - die "packaged tracedecay CLI build did not produce $packaged_cli_bin" -assert_binary_source_sha \ - "$packaged_cli_bin" \ - "$product_version" \ - "$source_git_sha" \ - "extracted-package" - -echo "distribution acceptance: testing packaged patched Rust grammar" -cargo nextest run \ - --manifest-path "$code_extraction_package/Cargo.toml" \ - --all-features \ - --config "$patch_config" \ - --test main \ - -E 'test(/^rust::/)' \ - --no-tests=fail - -# The extracted CLI build above compiles the complete packaged production -# graph. Query, root-library, and LSP source suites run in exhaustive Linux CI; -# rerunning them here added no archive assertion. The MCP harness stays focused -# on the modules that spawn TRACEDECAY_TEST_BIN, so it proves the packaged CLI -# without rerunning unrelated source-library tests. Run it from the verified -# checkout through the canonical isolated Bazel launcher. -echo "distribution acceptance: checking packaged CLI integration behavior" -resolve_clean_source_head "$repo" "$source_git_sha" >/dev/null -# The extracted binary is outside Bazel's declared runfiles. Run locally and -# disable result caching so its current bytes are exercised on every invocation. -# Check each filter independently: libtest otherwise accepts zero matches. -require_command bazel -for filter in \ - 'mcp_cli_serve_test::' \ - 'serve_template_path_test::' \ - 'mcp_handler_test::lcm_test::lcm_status_cli_bridge_accepts_json_args'; do - ( - cd "$repo" - test_args=(--test_arg="$filter") - expected_count=nonzero - if [[ $filter == mcp_handler_test::* ]]; then - test_args+=(--test_arg=--exact) - expected_count=1 - fi - REQUIRE_EXACT_TEST_COUNT="$expected_count" scripts/require-exact-test.sh \ - bazel test --config=ci //crates/tracedecay:mcp_suite \ - --test_strategy=standalone \ - --nocache_test_results \ - --test_output=all \ - --test_env="TRACEDECAY_TEST_BIN=$packaged_cli_bin" \ - "${test_args[@]}" - ) -done -resolve_clean_source_head "$repo" "$source_git_sha" >/dev/null - -install_root="$work/install" -echo "distribution acceptance: staging the packaged CLI as the installed binary" -# `cargo install --path` rebuilds the same extracted CLI we just compiled. -# Copy that artifact into the cargo-install layout so later MCP/LSP checks -# exercise the packaged binary without a third dist compile. -mkdir -p -- "$install_root/bin" -cp -- "$packaged_cli_bin" \ - "$install_root/bin/tracedecay${executable_suffix}" -chmod +x "$install_root/bin/tracedecay${executable_suffix}" - -consumer="$work/library-consumer" -mkdir -p -- "$consumer/src" -python3 - "$root_package/Cargo.toml" "$root_package" "$catalog_package" "$agent_hosts_package" \ - "$contracts_package" \ - >"$consumer/Cargo.toml" <<'PY' -import json -import sys -import tomllib -from pathlib import Path - -with Path(sys.argv[1]).open("rb") as handle: - manifest = tomllib.load(handle) -if "production" not in manifest.get("features", {}): - raise SystemExit("packaged tracedecay manifest omitted the production feature") -# The consumer proves the packaged catalog composes and the host bundles -# verify; neither depends on optimization. Its feature unification differs -# from the packaged CLI's (the CLI's own dependencies light extra features on -# mio, hyper, tower, ...), so under --release it recompiled the whole -# workspace spine: 18m measured on ubuntu-latest for a crate that itself -# compiles in seconds. Unoptimized and without debuginfo the same graph -# compiles in ~3m on the same hardware. -print("""[package] -name = "tracedecay-distribution-consumer" -version = "0.0.0" -edition = "2024" - -[profile.dev] -opt-level = 0 -debug = 0 - -[dependencies]""") -print( - "tracedecay = { path = " - + json.dumps(sys.argv[2]) - + ', default-features = false, features = ["production"] }' -) -print( - "tracedecay-tool-catalog = { path = " - + json.dumps(sys.argv[3]) - + " }" -) -print( - "tracedecay-agent-hosts = { path = " - + json.dumps(sys.argv[4]) - + " }" -) -print( - "tracedecay-contracts = { path = " - + json.dumps(sys.argv[5]) - + " }" -) -PY -# The host bundle generators sign each bundle with the commit that produced -# it; the packaged product's source head is that commit. -printf 'const GENERATOR_COMMIT: &str = "%s";\n' "$source_git_sha" >"$consumer/src/main.rs" -cat >>"$consumer/src/main.rs" <<'RS' -use std::collections::BTreeSet; - -use tracedecay_contracts::catalog_composition::build_application_catalog_snapshot; -use tracedecay_agent_hosts::agents::host_bundle_registry::{ - RECEIPT_BACKED_HOST_KINDS, default_components, verified_embedded_default_host_component_set, - verified_embedded_host_bundle, -}; -use tracedecay_agent_hosts::agents::host_bundle::stock_host_kinds; -use tracedecay_tool_catalog::{AvailabilityContract, CapabilityId}; - -const REQUIRED_CAPABILITIES: [&str; 10] = [ - "capability.application.feedback.diagnostics", - "capability.application.feedback.get", - "capability.application.feedback.expand", - "capability.application.feedback.list", - "capability.application.feedback.impact", - "capability.application.feedback.affected-tests", - "capability.application.feedback.test-results", - "capability.application.feedback.github-review-ingest", - "capability.application.feedback.ci-failure-localize", - "capability.application.feedback.proximity", -]; - -fn main() { - let snapshot = build_application_catalog_snapshot() - .expect("packaged application catalog must compose"); - for raw_id in REQUIRED_CAPABILITIES { - let id = CapabilityId::new(raw_id).expect("required capability ID must be valid"); - let capability = snapshot - .capability(&id) - .unwrap_or_else(|| panic!("packaged catalog omitted {raw_id}")); - assert!( - matches!(capability.availability(), AvailabilityContract::Available), - "packaged catalog capability is not callable: {raw_id}" - ); - } - - let supported_hosts = stock_host_kinds() - .into_iter() - .filter(|host| !default_components(*host).is_empty()) - .collect::>(); - let receipt_backed_hosts = RECEIPT_BACKED_HOST_KINDS - .into_iter() - .collect::>(); - assert_eq!( - supported_hosts, - receipt_backed_hosts, - "packaged default host bundle inventory changed" - ); - for host in RECEIPT_BACKED_HOST_KINDS { - let components = default_components(host); - let component_set = verified_embedded_default_host_component_set(host, 0, GENERATOR_COMMIT) - .expect("default packaged host component set must verify"); - assert_eq!(component_set.component_set.components.len(), components.len()); - for component in components { - let bundle = verified_embedded_host_bundle(host, component, 0, GENERATOR_COMMIT) - .expect("packaged host bundle must be callable"); - bundle - .manifest - .validate_structure() - .expect("packaged host bundle manifest must validate"); - assert!(!bundle.contents.is_empty(), "packaged host bundle has no assets"); - } - } -} -RS - -# Deliberately no lockfile. Unlike the extracted packages, which are each -# their own Cargo root and need the release resolution, this consumer is a -# downstream crate that has never seen our lockfile. Letting it resolve from -# scratch is the only check that the published dependency set is resolvable -# at all, which is what caught the yanked bisync 0.3.0 under gix-protocol. -echo "distribution acceptance: calling packaged catalog and host bundles" -CARGO_NET_OFFLINE=true cargo run \ - --manifest-path "$consumer/Cargo.toml" \ - --bin tracedecay-distribution-consumer \ - --config "$patch_config" - -test_api_probe="$work/test-api-probe" -mkdir -p -- "$test_api_probe/src" -cat >"$test_api_probe/Cargo.toml" <"$test_api_probe/src/main.rs" <<'RS' -use tracedecay::mcp::McpServer; - -fn main() { - let _ = McpServer::has_project_session_retrieval_service_for_test; -} -RS -cp -- "$staged/Cargo.lock" "$test_api_probe/Cargo.lock" -echo "distribution acceptance: proving production package omits test APIs" -test_api_stderr="$work/test-api-probe.stderr" -# Same production features and lockfile as the packaged CLI dist build, -# which already compiled this graph into the shared dist directory (repo -# `target-dir`). Only the probe itself is compiled. A dev-profile check paid -# a second metadata compile of the whole graph. The expected refusal is still -# E0599 on the test-only associated item. -if CARGO_NET_OFFLINE=true \ - RUSTFLAGS="${RUSTFLAGS:+$RUSTFLAGS }-C embed-bitcode=yes" \ - cargo check \ - --manifest-path "$test_api_probe/Cargo.toml" \ - --profile dist \ - --config "$patch_config" \ - 2>"$test_api_stderr"; then - die "production package exposed test-transport APIs" -fi -# rustc words this refusal differently across releases ("no function or -# associated item named" before 1.97, "no associated function or constant -# named" from 1.97), so match the error code and the probed name. -grep -Eq "error\[E0599\].*has_project_session_retrieval_service_for_test" \ - "$test_api_stderr" || { - echo "distribution acceptance: test API probe stderr follows" >&2 - tail -n 60 -- "$test_api_stderr" >&2 - die "test API probe failed for an unexpected reason" -} - -binary=$(python3 "$repo/scripts/resolve-installed-binary.py" \ - "$install_root" \ - "${RUNNER_OS:-}") -assert_binary_source_sha \ - "$binary" \ - "$product_version" \ - "$source_git_sha" \ - "installed extracted-package" - -echo "distribution acceptance: exercising installed MCP behavior" -if [[ ${RUNNER_OS:-} == Windows ]]; then - python3 "$repo/scripts/check-packaged-mcp-stdio.py" \ - "$binary" \ - "$work/mcp-stdio" -else - TRACEDECAY_BIN="$binary" "$repo/scripts/mcp-conformance-smoke.sh" -fi - -lsp_servers="$work/lsp-servers.json" -"$binary" lsp servers --json >"$lsp_servers" -python3 - "$lsp_servers" <<'PY' -import json -import sys -from pathlib import Path - -inventory = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) -if not isinstance(inventory, dict) or inventory.get("resolution") not in {"daemon", "cli_path"}: - raise SystemExit("distribution acceptance: lsp servers omitted its availability resolution") -value = inventory.get("servers") -if not isinstance(value, list) or not value: - raise SystemExit("distribution acceptance: lsp servers returned an empty inventory") -required_languages = {"rust", "typescript", "javascript", "python", "go", "c", "cpp"} -languages = set() -for server in value: - if ( - not isinstance(server, dict) - or not isinstance(server.get("language"), str) - or not isinstance(server.get("language_id"), str) - or not isinstance(server.get("command"), str) - or not isinstance(server.get("available"), bool) - or not isinstance(server.get("extensions"), list) - or not server["extensions"] - ): - raise SystemExit("distribution acceptance: invalid lsp server entry") - languages.add(server["language"]) -missing = sorted(required_languages - languages) -if missing: - raise SystemExit( - "distribution acceptance: lsp inventory omitted " + ", ".join(missing) - ) -PY - -python3 "$repo/scripts/check-packaged-lsp-bridge.py" \ - "$binary" \ - "$work/lsp-bridge" - -echo "distribution acceptance passed" diff --git a/scripts/check-distribution-feature-wiring.py b/scripts/check-distribution-feature-wiring.py deleted file mode 100644 index 7e4245f94c..0000000000 --- a/scripts/check-distribution-feature-wiring.py +++ /dev/null @@ -1,261 +0,0 @@ -#!/usr/bin/env python3 -"""Validate source and packaged Cargo feature ownership for distribution builds. - -The checks are public-contract and layering rules only: the packaged manifest -must carry the source feature set, optional native dependencies must stay -optional and feature-wired, and the supported `lang-*` surface must match the -extraction owner. How a feature is forwarded through intermediate crates is -Cargo's job; resolved behavior is proven by the packaged-artifact builds and -launch checks in check-distribution-acceptance.sh. Per-language isolation -compiles remain available via `--check-extraction-manifest` and are not -part of the release acceptance gate. -""" - -from __future__ import annotations - -import argparse -from pathlib import Path -import subprocess -import tomllib - - -REQUIRED_ROOT_FEATURES = { - "full", - "token-counting", - "test-transport", -} -REQUIRED_CLI_FEATURE_MEMBERS = { - "production": {"tracedecay/production"}, -} - - -def load(path: Path) -> dict: - with path.open("rb") as handle: - return tomllib.load(handle) - - -def optional_dependencies(manifest: dict) -> set[str]: - names: set[str] = set() - - def collect(table: object) -> None: - if not isinstance(table, dict): - return - dependencies = table.get("dependencies") - if isinstance(dependencies, dict): - for name, spec in dependencies.items(): - if isinstance(spec, dict) and spec.get("optional") is True: - names.add(name) - - collect(manifest) - for target in manifest.get("target", {}).values(): - collect(target) - return names - - -def require_matching_features(name: str, source: dict, packaged: dict) -> dict: - source_features = source.get("features", {}) - packaged_features = packaged.get("features", {}) - if source_features != packaged_features: - raise SystemExit( - f"distribution acceptance: packaged {name} feature wiring differs from Cargo.toml" - ) - return packaged_features - - -def require_optional_dependencies_wired(name: str, manifest: dict, features: dict) -> None: - references = { - item - for members in features.values() - for item in members - if isinstance(item, str) - } - unwired = sorted( - dependency - for dependency in optional_dependencies(manifest) - if f"dep:{dependency}" not in references and dependency not in features - ) - if unwired: - raise SystemExit( - f"distribution acceptance: {name} optional dependencies are not feature-wired: " - + ", ".join(unwired) - ) - - -def language_feature_names(features: dict) -> set[str]: - return {name for name in features if name.startswith("lang-")} - - -def require_language_surface( - name: str, - features: dict, - authority_features: set[str], -) -> None: - """The public `lang-*` surface must advertise exactly the owner's languages.""" - actual_features = language_feature_names(features) - if actual_features != authority_features: - missing = sorted(authority_features - actual_features) - extra = sorted(actual_features - authority_features) - details = [] - if missing: - details.append("missing " + ", ".join(missing)) - if extra: - details.append("extra " + ", ".join(extra)) - raise SystemExit( - f"distribution acceptance: {name} language features differ from " - "tracedecay-code-extraction: " + "; ".join(details) - ) - - -def require_isolated_language_features_compile( - manifest_path: Path, - authority_features: set[str], - cargo_config: Path | None, - offline: bool, -) -> None: - manifest = load(manifest_path) - package_name = manifest.get("package", {}).get("name") - if not isinstance(package_name, str): - raise SystemExit( - "distribution acceptance: extraction build manifest has no package name" - ) - - build_features = language_feature_names(manifest.get("features", {})) - if build_features != authority_features: - raise SystemExit( - "distribution acceptance: extraction build manifest language features " - "differ from the packaged authority" - ) - - for feature in sorted(authority_features): - command = [ - "cargo", - "check", - "--manifest-path", - str(manifest_path), - "--package", - package_name, - "--lib", - "--no-default-features", - "--features", - feature, - ] - if cargo_config is not None: - command.extend(["--config", str(cargo_config)]) - if offline: - command.append("--offline") - completed = subprocess.run( - command, check=False, capture_output=True, text=True - ) - if completed.returncode != 0: - details = completed.stderr.strip() or completed.stdout.strip() - raise SystemExit( - f"distribution acceptance: {feature} does not compile in isolation" - + (f"\n{details}" if details else "") - ) - - -def validate( - root_source: dict, - root_packaged: dict, - code_index_source: dict, - code_index_packaged: dict, - extraction_source: dict, - extraction_packaged: dict, - cli_source: dict, - cli_packaged: dict, -) -> None: - root_features = require_matching_features("root", root_source, root_packaged) - missing = sorted(REQUIRED_ROOT_FEATURES - root_features.keys()) - if missing: - raise SystemExit( - "distribution acceptance: source manifest is missing required features: " - + ", ".join(missing) - ) - require_optional_dependencies_wired("root", root_packaged, root_features) - - require_matching_features( - "tracedecay-code-index", code_index_source, code_index_packaged - ) - extraction_features = require_matching_features( - "tracedecay-code-extraction", extraction_source, extraction_packaged - ) - require_language_surface( - "root", root_features, language_feature_names(extraction_features) - ) - require_optional_dependencies_wired( - "tracedecay-code-extraction", extraction_packaged, extraction_features - ) - - cli_features = require_matching_features( - "tracedecay-cli", cli_source, cli_packaged - ) - missing_cli = sorted( - REQUIRED_CLI_FEATURE_MEMBERS.keys() - cli_features.keys() - ) - if missing_cli: - raise SystemExit( - "distribution acceptance: tracedecay-cli is missing required features: " - + ", ".join(missing_cli) - ) - for feature, expected in REQUIRED_CLI_FEATURE_MEMBERS.items(): - members = cli_features.get(feature) - # Extra crate passthroughs are allowed; the contract is the required - # release members, not an exhaustive crate inventory. - if not isinstance(members, list) or not expected.issubset(members): - raise SystemExit( - f"distribution acceptance: tracedecay-cli {feature} must enable " - + ", ".join(sorted(expected)) - ) - require_optional_dependencies_wired( - "tracedecay-cli", cli_packaged, cli_features - ) - - -def main() -> int: - repo = Path(__file__).resolve().parent.parent - root_manifest = repo / "crates/tracedecay/Cargo.toml" - code_index_manifest = repo / "crates/tracedecay-code-index/Cargo.toml" - extraction_manifest = repo / "crates/tracedecay-code-extraction/Cargo.toml" - cli_manifest = repo / "crates/tracedecay-cli/Cargo.toml" - parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter - ) - # Source manifests default to this checkout; packaged manifests must be - # the extracted `.crate` trees, so they have no default, comparing a - # manifest with itself is not package verification. - parser.add_argument("--root-source", type=Path, default=root_manifest) - parser.add_argument("--root-packaged", type=Path, required=True) - parser.add_argument("--code-index-source", type=Path, default=code_index_manifest) - parser.add_argument("--code-index-packaged", type=Path, required=True) - parser.add_argument("--extraction-source", type=Path, default=extraction_manifest) - parser.add_argument("--extraction-packaged", type=Path, required=True) - parser.add_argument("--cli-source", type=Path, default=cli_manifest) - parser.add_argument("--cli-packaged", type=Path, required=True) - parser.add_argument("--check-extraction-manifest", type=Path) - parser.add_argument("--cargo-config", type=Path) - parser.add_argument("--offline", action="store_true") - arguments = parser.parse_args() - extraction_packaged = load(arguments.extraction_packaged) - validate( - load(arguments.root_source), - load(arguments.root_packaged), - load(arguments.code_index_source), - load(arguments.code_index_packaged), - load(arguments.extraction_source), - extraction_packaged, - load(arguments.cli_source), - load(arguments.cli_packaged), - ) - if arguments.check_extraction_manifest is not None: - require_isolated_language_features_compile( - arguments.check_extraction_manifest, - language_feature_names(extraction_packaged.get("features", {})), - arguments.cargo_config, - arguments.offline, - ) - print("distribution feature wiring is valid") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/check-packaged-mcp-stdio.py b/scripts/check-packaged-mcp-stdio.py deleted file mode 100755 index a935f10acb..0000000000 --- a/scripts/check-packaged-mcp-stdio.py +++ /dev/null @@ -1,345 +0,0 @@ -#!/usr/bin/env python3 -"""Cross-platform installed-binary MCP stdio transport acceptance.""" - -from __future__ import annotations - -import json -import os -from pathlib import Path -import shutil -import subprocess -import sys -import time - -from lib.process_control import terminate -from lib.portable_process import dies_with_this_process - - -# Default MCP Inspector pin, shared with scripts/mcp-conformance-smoke.sh -# via scripts/lib/inspector_version, override both through INSPECTOR_VERSION. -INSPECTOR_VERSION = os.environ.get( - "INSPECTOR_VERSION", - Path(__file__) - .resolve() - .with_name("lib") - .joinpath("inspector_version") - .read_text(encoding="utf-8") - .strip(), -) -REQUIRED_TOOLS = { - "tracedecay_search", - "tracedecay_diagnostics", - "tracedecay_impact", - "tracedecay_affected", - "tracedecay_test_map", -} - - -def run( - arguments: list[str], - *, - cwd: Path, - environment: dict[str, str], - check: bool = True, -) -> subprocess.CompletedProcess[str]: - return subprocess.run( - arguments, - cwd=cwd, - env=environment, - check=check, - capture_output=True, - text=True, - timeout=60, - ) - - -def inspect( - npx: str, - binary: Path, - fixture: Path, - environment: dict[str, str], - *arguments: str, - check: bool = True, -) -> subprocess.CompletedProcess[str]: - return run( - [ - npx, - "-y", - f"@modelcontextprotocol/inspector@{INSPECTOR_VERSION}", - "--cli", - str(binary), - "serve", - "-p", - str(fixture), - *arguments, - ], - cwd=fixture, - environment=environment, - check=check, - ) - - -def daemon_command( - binary: Path, socket_path: Path, *, platform_name: str = os.name -) -> list[str]: - command = [str(binary), "daemon", "run"] - if platform_name != "nt": - command.extend(["--socket", str(socket_path)]) - return command - - -def daemon_environment( - environment: dict[str, str], - socket_path: Path, - *, - platform_name: str = os.name, -) -> dict[str, str]: - configured = environment.copy() - configured.pop("TRACEDECAY_DAEMON_SOCKET", None) - if platform_name != "nt": - configured["TRACEDECAY_DAEMON_SOCKET"] = str(socket_path) - return configured - - -def daemon_endpoint_is_ready(returncode: int, stdout: str) -> bool: - return returncode == 0 and "(connectable)" in stdout - - -def wait_for_daemon( - daemon: subprocess.Popen[bytes], - binary: Path, - fixture: Path, - environment: dict[str, str], -) -> None: - deadline = time.monotonic() + 15 - while time.monotonic() < deadline: - if daemon.poll() is not None: - raise SystemExit( - f"packaged daemon exited before init with {daemon.returncode}" - ) - status = run( - [str(binary), "daemon", "status"], - cwd=fixture, - environment=environment, - check=False, - ) - if daemon_endpoint_is_ready(status.returncode, status.stdout): - return - time.sleep(0.05) - raise SystemExit("packaged daemon endpoint did not become ready") - - -def problem_of(payload: dict) -> dict: - """The typed problem a tool-result error carries in structuredContent.""" - structured = payload.get("structuredContent") - problem = structured.get("problem") if isinstance(structured, dict) else None - return problem if isinstance(problem, dict) else {} - - -def main() -> int: - if len(sys.argv) != 3: - raise SystemExit( - "usage: check-packaged-mcp-stdio.py TRACEDecay_BINARY WORK_DIRECTORY" - ) - binary = Path(sys.argv[1]).resolve() - work = Path(sys.argv[2]).resolve() - npx = shutil.which("npx") - if not binary.is_file(): - raise SystemExit(f"installed tracedecay binary is missing: {binary}") - if npx is None: - raise SystemExit("npx is required for MCP stdio acceptance") - - fixture = work / "project" - home = work / "home" - fixture.joinpath("src").mkdir(parents=True, exist_ok=True) - home.mkdir(parents=True, exist_ok=True) - fixture.joinpath("src", "main.rs").write_text( - 'fn main() { println!("hello"); }\n', encoding="utf-8" - ) - run(["git", "init", "--quiet"], cwd=fixture, environment=os.environ.copy()) - run(["git", "add", "src/main.rs"], cwd=fixture, environment=os.environ.copy()) - run( - [ - "git", - "-c", - "user.name=TraceDecay MCP Smoke", - "-c", - "user.email=tracedecay-mcp-smoke@example.invalid", - "commit", - "--quiet", - "-m", - "test: seed MCP smoke fixture", - ], - cwd=fixture, - environment=os.environ.copy(), - ) - - environment = os.environ.copy() - environment.update( - { - "HOME": str(home), - "USERPROFILE": str(home), - "XDG_DATA_HOME": str(home / ".local" / "share"), - "XDG_CONFIG_HOME": str(home / ".config"), - "APPDATA": str(home / "AppData" / "Roaming"), - "LOCALAPPDATA": str(home / "AppData" / "Local"), - } - ) - socket_path = home / ".tracedecay" / "daemon.sock" - socket_path.parent.mkdir(parents=True, exist_ok=True) - environment = daemon_environment(environment, socket_path) - daemon = subprocess.Popen( - daemon_command(binary, socket_path), - cwd=fixture, - env=environment, - stdin=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - preexec_fn=dies_with_this_process(), - ) - try: - wait_for_daemon(daemon, binary, fixture, environment) - run([str(binary), "init"], cwd=fixture, environment=environment) - - first = inspect( - npx, - binary, - fixture, - environment, - "--method", - "tools/list", - ) - second = inspect( - npx, - binary, - fixture, - environment, - "--method", - "tools/list", - ) - if first.stdout != second.stdout: - raise SystemExit( - "MCP tools/list output changed across identical invocations" - ) - payload = json.loads(first.stdout) - tools = payload.get("tools") - if not isinstance(tools, list) or not tools: - raise SystemExit("MCP tools/list returned no tools") - names = { - tool.get("name") - for tool in tools - if isinstance(tool, dict) - and isinstance(tool.get("inputSchema"), dict) - and tool["inputSchema"].get("type") == "object" - } - missing = sorted(REQUIRED_TOOLS - names) - if missing: - raise SystemExit( - "MCP tools/list omitted required tools: " + ", ".join(missing) - ) - - diagnostics = inspect( - npx, - binary, - fixture, - environment, - "--method", - "tools/call", - "--tool-name", - "tracedecay_diagnostics", - check=False, - ) - try: - diagnostic_payload = json.loads(diagnostics.stdout) - except json.JSONDecodeError as error: - raise SystemExit( - "advertised tracedecay_diagnostics dispatch returned no typed payload: " - + diagnostics.stderr.strip() - ) from error - content = diagnostic_payload.get("content") - if not isinstance(content, list): - raise SystemExit("tracedecay_diagnostics omitted typed MCP content") - diagnostic_text = "\n".join( - item.get("text", "") - for item in content - if isinstance(item, dict) - and item.get("type") == "text" - and isinstance(item.get("text"), str) - ) - if not diagnostic_text: - raise SystemExit("tracedecay_diagnostics returned no typed text result") - lowered = diagnostic_text.lower() - if diagnostic_payload.get("isError") is True: - # A fixture without a configured compiler is a typed `unsupported` - # problem; a missing daemon authority is `unavailable`. - if problem_of(diagnostic_payload).get("kind") not in ( - "unavailable", - "unsupported", - ): - raise SystemExit( - "tracedecay_diagnostics error was not a typed unavailable or " - "unsupported problem: " + diagnostic_text - ) - else: - if diagnostics.returncode != 0: - raise SystemExit( - "tracedecay_diagnostics successful payload exited nonzero" - ) - if not any( - marker in lowered - for marker in ("diagnostic", "generation", "status", "unavailable") - ): - raise SystemExit( - "tracedecay_diagnostics success omitted concrete diagnostic state" - ) - - resources = inspect( - npx, - binary, - fixture, - environment, - "--method", - "resources/list", - ) - resource_payload = json.loads(resources.stdout) - if not any( - resource.get("uri") == "tracedecay://status" - for resource in resource_payload.get("resources", []) - if isinstance(resource, dict) - ): - raise SystemExit("MCP resources/list omitted tracedecay://status") - - unknown = inspect( - npx, - binary, - fixture, - environment, - "--method", - "tools/call", - "--tool-name", - "definitely_not_a_tool", - check=False, - ) - try: - unknown_payload = json.loads(unknown.stdout) - except json.JSONDecodeError as error: - raise SystemExit( - "unknown MCP tool returned no typed refusal: " + unknown.stderr.strip() - ) from error - if ( - unknown_payload.get("isError") is not True - or problem_of(unknown_payload).get("code") != "unknown_tool" - ): - raise SystemExit( - "unknown MCP tool was not a typed unknown_tool refusal: " - + unknown.stdout.strip() - ) - finally: - terminate(daemon) - - print("packaged MCP stdio acceptance passed") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/resolve-installed-binary.py b/scripts/resolve-installed-binary.py deleted file mode 100755 index 1bc8917dc6..0000000000 --- a/scripts/resolve-installed-binary.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env python3 -"""Resolve the cargo-installed TraceDecay binary across release platforms.""" - -from __future__ import annotations - -import argparse -import os -from pathlib import Path - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument("install_root", type=Path) - parser.add_argument("runner_os") - arguments = parser.parse_args() - - name = "tracedecay.exe" if arguments.runner_os == "Windows" else "tracedecay" - binary = arguments.install_root.resolve() / "bin" / name - if not binary.is_file(): - raise SystemExit(f"installed binary layout did not produce {binary}") - if arguments.runner_os != "Windows" and not os.access(binary, os.X_OK): - raise SystemExit(f"cargo-installed binary is not executable: {binary}") - print(binary) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/resolve-release-source-profile.py b/scripts/resolve-release-source-profile.py deleted file mode 100755 index bd03055e38..0000000000 --- a/scripts/resolve-release-source-profile.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env python3 -"""Resolve safe Cargo feature arguments for current or historical source tags.""" - -from __future__ import annotations - -import argparse -from pathlib import Path -import subprocess -import sys -import tomllib - - -def expand_local_features( - features: dict[str, list[str]], selected: list[str] -) -> set[str]: - resolved: set[str] = set() - pending = list(selected) - while pending: - feature = pending.pop() - if feature in resolved: - continue - resolved.add(feature) - for member in features.get(feature, []): - if member in features: - pending.append(member) - return resolved - - -def write_output(path: Path | None, name: str, value: str) -> None: - line = f"{name}={value}\n" - if path is None: - sys.stdout.write(line) - return - with path.open("a", encoding="utf-8") as handle: - handle.write(line) - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--source", type=Path, required=True) - parser.add_argument("--target") - parser.add_argument("--github-output", type=Path) - arguments = parser.parse_args() - - source = arguments.source.resolve() - # The product package moved to `crates/tracedecay`; the repository root is - # now a virtual workspace manifest that declares no features at all. - # Reading the root here resolved every relocated tree to `legacy-default`, - # which silently drops `production` from the release feature set and skips - # the production feature-profile check below. - package_manifest = source.joinpath("crates", "tracedecay", "Cargo.toml") - if not package_manifest.is_file(): - raise SystemExit(f"missing product package manifest at {package_manifest}") - with package_manifest.open("rb") as handle: - manifest = tomllib.load(handle) - features = manifest.get("features", {}) - if not isinstance(features, dict): - raise SystemExit("source Cargo.toml has an invalid feature table") - defaults = features.get("default", []) - if not isinstance(defaults, list): - raise SystemExit("source Cargo.toml has an invalid feature table") - - if "production" in features: - checker = Path(__file__).with_name("check-production-feature-profile.py") - subprocess.run( - [sys.executable, str(checker), "--repo", str(source)], - check=True, - ) - profile = "production" - # Release archives ship the production profile only: no optional - # instrumentation feature lanes ride along to be switched off at - # runtime. - cargo_features = "production" - cargo_args = "--no-default-features --features production" - else: - resolved_defaults = expand_local_features(features, defaults) - if "test-transport" in resolved_defaults: - raise SystemExit( - "historical default features enable test-transport; refusing release" - ) - profile = "legacy-default" - cargo_args = "" - cargo_features = "" - - write_output(arguments.github_output, "profile", profile) - write_output(arguments.github_output, "cargo_args", cargo_args) - write_output(arguments.github_output, "cargo_features", cargo_features) - print(f"release source profile: {profile}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/test-check-distribution-clean-source.sh b/scripts/test-check-distribution-clean-source.sh deleted file mode 100644 index 9a937e3495..0000000000 --- a/scripts/test-check-distribution-clean-source.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -root=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P) -gate="$root/scripts/check-distribution-acceptance.sh" -work=$(mktemp -d "${TMPDIR:-/tmp}/tracedecay-clean-source-test.XXXXXX") -trap 'rm -rf -- "$work"' EXIT - -repo="$work/repo" -bin="$work/bin" -mkdir -p -- "$repo/.release-automation" "$bin" -for command in cargo rustc; do - cat >"$bin/$command" <<'SH' -#!/usr/bin/env bash -exit 2 -SH - chmod +x "$bin/$command" -done -cat >"$repo/Cargo.toml" <<'TOML' -[workspace] -members = [] -TOML - -git -C "$repo" init -q -git -C "$repo" config user.name "TraceDecay test" -git -C "$repo" config user.email "test@tracedecay.local" -git -C "$repo" add Cargo.toml -git -C "$repo" commit -qm "test fixture" -git -C "$repo/.release-automation" init -q - -output="$work/output" -if PATH="$bin:$PATH" "$gate" --repo "$repo" >"$output" 2>&1; then - echo "distribution gate unexpectedly accepted the incomplete fixture repository" >&2 - exit 1 -fi - -if grep -Fq "source repository has tracked or untracked drift" "$output"; then - echo "workflow-owned .release-automation checkout was treated as source drift" >&2 - exit 1 -fi -grep -Fq \ - "packaged host-event fixture copy differs from its authority: claude.json" \ - "$output" - -printf 'untracked source\n' >"$repo/untracked.txt" -if PATH="$bin:$PATH" "$gate" --repo "$repo" >"$output" 2>&1; then - echo "distribution gate unexpectedly accepted real untracked source drift" >&2 - exit 1 -fi -grep -Fq "source repository has tracked or untracked drift (?? untracked.txt)" "$output" - -printf 'distribution clean-source regression passed\n' diff --git a/scripts/test-check-distribution-feature-wiring.py b/scripts/test-check-distribution-feature-wiring.py deleted file mode 100644 index 7401b79959..0000000000 --- a/scripts/test-check-distribution-feature-wiring.py +++ /dev/null @@ -1,305 +0,0 @@ -#!/usr/bin/env python3 -"""Regression tests for distribution feature ownership validation.""" - -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import Path -import subprocess -import sys -import tempfile - - -VALIDATOR = Path(__file__).with_name("check-distribution-feature-wiring.py") - -ROOT_MANIFEST = """[package] -name = "tracedecay" -version = "0.1.0-beta.34" - -[dependencies] -tracedecay-code-index = { version = "0.1.0" } -tracedecay-application = { version = "0.1.0" } - -[features] -lite = ["tracedecay-code-index/lite", "tracedecay-code-index-runtime/lite"] -medium = ["tracedecay-code-index/medium"] -full = ["tracedecay-code-index/full", "tracedecay-code-index-runtime/full"] -lang-dart = ["tracedecay-code-index/lang-dart"] -lang-markdown = ["tracedecay-code-index/lang-markdown"] -token-counting = [] -test-transport = [] -""" - -CODE_INDEX_MANIFEST = """[package] -name = "tracedecay-code-index" -version = "0.1.0" - -[dependencies] -tracedecay-code-extraction = { version = "0.1.0" } - -[features] -lite = ["tracedecay-code-extraction/lite", "lang-markdown"] -medium = ["tracedecay-code-extraction/medium"] -full = ["tracedecay-code-extraction/full", "lang-markdown"] -lang-dart = ["tracedecay-code-extraction/lang-dart"] -lang-markdown = ["tracedecay-code-extraction/lang-markdown"] -""" - -EXTRACTION_MANIFEST = """[package] -name = "tracedecay-code-extraction" -version = "0.1.0" - -[features] -lite = ["lang-markdown"] -medium = ["lang-dart"] -full = ["medium", "lang-markdown"] -lang-dart = [] -lang-markdown = [] -""" - -EXTRACTION_BUILD_MANIFEST = """[package] -name = "tracedecay-code-extraction" -version = "0.1.0" -edition = "2024" - -[features] -default = [] -lang-dart = ["dep:dart-grammar"] -lang-markdown = ["dep:markdown-grammar"] - -[dependencies] -dart-grammar = { path = "dart-grammar", optional = true } -markdown-grammar = { path = "markdown-grammar", optional = true } -""" - -EXTRACTION_BUILD_LIB = """#[cfg(feature = "lang-dart")] -pub fn dart() -> dart_grammar::Language { - dart_grammar::language() -} - -#[cfg(feature = "lang-markdown")] -pub fn markdown() -> markdown_grammar::Language { - markdown_grammar::language() -} -""" - -GRAMMAR_MANIFEST = """[package] -name = "{name}" -version = "0.1.0" -edition = "2024" - -[lib] -path = "lib.rs" -""" - -GRAMMAR_LIB = """pub struct Language; - -pub fn language() -> Language { - Language -} -""" - -CLI_MANIFEST = """[package] -name = "tracedecay-cli" -version = "0.1.0" - -[dependencies] -tracedecay = { version = "0.1.0" } - -[features] -default = ["production"] -production = ["tracedecay/production"] -""" - - -@dataclass(frozen=True) -class FixtureResult: - returncode: int - stdout: str - stderr: str - - -def run_fixture( - root_source: str = ROOT_MANIFEST, - root_packaged: str = ROOT_MANIFEST, - code_index_source: str = CODE_INDEX_MANIFEST, - code_index_packaged: str = CODE_INDEX_MANIFEST, - extraction_source: str = EXTRACTION_MANIFEST, - extraction_packaged: str = EXTRACTION_MANIFEST, - cli_source: str = CLI_MANIFEST, - cli_packaged: str = CLI_MANIFEST, - extraction_build_manifest: str | None = None, -) -> FixtureResult: - with tempfile.TemporaryDirectory() as temporary_directory: - root = Path(temporary_directory) - manifests = { - "root-source.toml": root_source, - "root-packaged.toml": root_packaged, - "code-index-source.toml": code_index_source, - "code-index-packaged.toml": code_index_packaged, - "extraction-source.toml": extraction_source, - "extraction-packaged.toml": extraction_packaged, - "cli-source.toml": cli_source, - "cli-packaged.toml": cli_packaged, - } - for name, contents in manifests.items(): - root.joinpath(name).write_text(contents, encoding="utf-8") - command = [ - sys.executable, - str(VALIDATOR), - "--root-source", - str(root / "root-source.toml"), - "--root-packaged", - str(root / "root-packaged.toml"), - "--code-index-source", - str(root / "code-index-source.toml"), - "--code-index-packaged", - str(root / "code-index-packaged.toml"), - "--extraction-source", - str(root / "extraction-source.toml"), - "--extraction-packaged", - str(root / "extraction-packaged.toml"), - "--cli-source", - str(root / "cli-source.toml"), - "--cli-packaged", - str(root / "cli-packaged.toml"), - ] - if extraction_build_manifest is not None: - extraction = root / "extraction-build" - extraction.joinpath("src").mkdir(parents=True) - extraction.joinpath("Cargo.toml").write_text( - extraction_build_manifest, encoding="utf-8" - ) - extraction.joinpath("src/lib.rs").write_text( - EXTRACTION_BUILD_LIB, encoding="utf-8" - ) - for grammar in ("dart-grammar", "markdown-grammar"): - dependency = extraction / grammar - dependency.mkdir() - dependency.joinpath("Cargo.toml").write_text( - GRAMMAR_MANIFEST.format(name=grammar), encoding="utf-8" - ) - dependency.joinpath("lib.rs").write_text( - GRAMMAR_LIB, encoding="utf-8" - ) - command.extend( - ["--check-extraction-manifest", str(extraction / "Cargo.toml")] - ) - completed = subprocess.run( - command, - check=False, - capture_output=True, - text=True, - ) - return FixtureResult(completed.returncode, completed.stdout, completed.stderr) - - -def main() -> int: - extracted_owner = run_fixture() - if extracted_owner.returncode != 0: - raise SystemExit(extracted_owner.stderr) - - cli_without_production = CLI_MANIFEST.replace( - 'production = ["tracedecay/production"]\n', - "", - ) - missing_cli_production = run_fixture( - cli_source=cli_without_production, - cli_packaged=cli_without_production, - ) - if missing_cli_production.returncode == 0: - raise SystemExit("CLI without the production release feature was accepted") - if ( - "tracedecay-cli is missing required features" - not in missing_cli_production.stderr - ): - raise SystemExit("missing CLI production feature failed for an unexpected reason") - - cli_with_miswired_production = CLI_MANIFEST.replace( - 'production = ["tracedecay/production"]', - 'production = []', - ) - miswired_cli_production = run_fixture( - cli_source=cli_with_miswired_production, - cli_packaged=cli_with_miswired_production, - ) - if miswired_cli_production.returncode == 0: - raise SystemExit("CLI with a mountless production feature was accepted") - if ( - "tracedecay-cli production must enable" - not in miswired_cli_production.stderr - ): - raise SystemExit("miswired CLI production feature failed for an unexpected reason") - - # Internal forwarding topology is not a contract: rerouting a tier through - # a different intermediate edge, forwarding a language alias through the - # runtime crate, and adding a local tier member are all accepted without - # touching this validator. - rewired_root = ROOT_MANIFEST.replace( - 'lite = ["tracedecay-code-index/lite", "tracedecay-code-index-runtime/lite"]', - 'lite = ["tracedecay-code-index-runtime/lite"]', - ).replace( - 'lang-dart = ["tracedecay-code-index/lang-dart"]', - 'lang-dart = ["tracedecay-code-index-runtime/lang-dart"]', - ).replace( - 'full = ["tracedecay-code-index/full", "tracedecay-code-index-runtime/full"]', - 'full = ["tracedecay-code-index/full", "tracedecay-code-index-runtime/full", "lang-markdown"]', - ) - rewired_code_index = CODE_INDEX_MANIFEST.replace( - 'lang-dart = ["tracedecay-code-extraction/lang-dart"]\n', "" - ) - rewired = run_fixture( - root_source=rewired_root, - root_packaged=rewired_root, - code_index_source=rewired_code_index, - code_index_packaged=rewired_code_index, - ) - if rewired.returncode != 0: - raise SystemExit( - "behavior-preserving forwarding change was rejected: " + rewired.stderr - ) - - packaged_root_drift = run_fixture( - root_packaged=ROOT_MANIFEST.replace('lang-dart = ["tracedecay-code-index/lang-dart"]\n', "") - ) - if packaged_root_drift.returncode == 0: - raise SystemExit("packaged root manifest dropping a feature was accepted") - if "packaged root feature wiring differs" not in packaged_root_drift.stderr: - raise SystemExit("packaged root drift failed for an unexpected reason") - - extraction_with_new_language = EXTRACTION_MANIFEST.replace( - "lang-markdown = []\n", "lang-markdown = []\nlang-rustdoc = []\n" - ) - unforwarded_language = run_fixture( - extraction_source=extraction_with_new_language, - extraction_packaged=extraction_with_new_language, - ) - if unforwarded_language.returncode == 0: - raise SystemExit("new extraction language without public aliases was accepted") - if "root language features differ" not in unforwarded_language.stderr: - raise SystemExit("unforwarded language failed for an unexpected reason") - - isolated_languages = run_fixture( - extraction_build_manifest=EXTRACTION_BUILD_MANIFEST - ) - if isolated_languages.returncode != 0: - raise SystemExit(isolated_languages.stderr) - - extraction_with_miswired_language = EXTRACTION_BUILD_MANIFEST.replace( - 'lang-dart = ["dep:dart-grammar"]', - 'lang-dart = ["dep:markdown-grammar"]', - ) - miswired_language = run_fixture( - extraction_build_manifest=extraction_with_miswired_language - ) - if miswired_language.returncode == 0: - raise SystemExit("miswired isolated extraction language was accepted") - if "lang-dart does not compile in isolation" not in miswired_language.stderr: - raise SystemExit("miswired extraction language failed for an unexpected reason") - - print("distribution feature wiring fixtures passed") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/test-check-distribution-reuse.sh b/scripts/test-check-distribution-reuse.sh deleted file mode 100755 index 736a647c5a..0000000000 --- a/scripts/test-check-distribution-reuse.sh +++ /dev/null @@ -1,260 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -root=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P) -gate="$root/scripts/check-distribution-acceptance.sh" -work=$(mktemp -d "${TMPDIR:-/tmp}/tracedecay-reuse-test.XXXXXX") -trap 'rm -rf -- "$work"' EXIT - -repo="$work/repo" -bin="$work/bin" -state="$work/state" -mkdir -p -- \ - "$repo/crates/tracedecay" \ - "$repo/crates/tracedecay-cli" \ - "$repo/crates/tracedecay-hooks/fixtures/host_events" \ - "$repo/tests/fixtures/packaged_host_events" \ - "$repo/.cargo" \ - "$repo/plugin" \ - "$repo/vendor" \ - "$repo/benchmark_data" \ - "$repo/dashboard/hermes-wrapper" \ - "$repo/dashboard/app-dist" \ - "$repo/scripts" \ - "$bin" \ - "$state" - -cat >"$repo/Cargo.toml" <<'TOML' -[workspace] -members = ["crates/tracedecay", "crates/tracedecay-cli"] - -[workspace.package] -version = "0.0.0" -TOML -cat >"$repo/crates/tracedecay/Cargo.toml" <<'TOML' -[package] -name = "tracedecay" -version = "0.0.0" -readme = "../../README.md" -TOML -cat >"$repo/crates/tracedecay-cli/Cargo.toml" <<'TOML' -[package] -name = "tracedecay-cli" -version = "0.0.0" -TOML - -for fixture in \ - claude.json \ - claude/post_tool_use_write.json \ - codex.json \ - cline-family.json \ - cursor.json \ - hermes.json \ - hermes/saved-edit.json \ - hermes/terminal-receipt.json \ - kiro.json \ - kimi-code.json \ - kimi/post-tool-use-edit.json \ - opencode/baseline.json \ - pi.json \ - droid.json; do - mkdir -p -- \ - "$repo/crates/tracedecay-hooks/fixtures/host_events/$(dirname -- "$fixture")" \ - "$repo/tests/fixtures/packaged_host_events/$(dirname -- "$fixture")" - printf 'fixture\n' >"$repo/crates/tracedecay-hooks/fixtures/host_events/$fixture" - printf 'fixture\n' >"$repo/tests/fixtures/packaged_host_events/$fixture" -done - -printf 'original readme\n' >"$repo/README.md" -printf 'changelog\n' >"$repo/CHANGELOG.md" -printf 'license\n' >"$repo/LICENSE" -printf 'stable\n' >"$repo/rust-toolchain.toml" -printf '[net]\noffline = true\n' >"$repo/.cargo/config.toml" -printf 'plugin\n' >"$repo/plugin/fixture" -printf 'vendor\n' >"$repo/vendor/fixture" -printf 'benchmark\n' >"$repo/benchmark_data/fixture" -printf 'wrapper\n' >"$repo/dashboard/hermes-wrapper/fixture" -printf 'bundle\n' >"$repo/dashboard/app-dist/fixture" -printf '#!/usr/bin/env bash\n' >"$repo/scripts/run-session-temporal-benchmark.sh" - -git -C "$repo" init -q -git -C "$repo" config user.name "TraceDecay test" -git -C "$repo" config user.email "test@tracedecay.local" -git -C "$repo" add -A -git -C "$repo" commit -qm "test fixture" -source_sha=$(git -C "$repo" rev-parse HEAD) - -real_python=$(command -v python3) - -cat >"$bin/tracedecay" <"$bin/cargo" <<'SH' -#!/usr/bin/env bash -set -euo pipefail -printf '%s\n' "$*" >>"$TEST_STATE/cargo.log" -case "${1:-}" in - build) - if [[ " $* " == *" --workspace "* ]]; then - printf 'workspace-rebuild\n' >>"$TEST_STATE/cargo.log" - exit 3 - fi - exit 0 - ;; - package) - exit 77 - ;; - *) - exit 2 - ;; -esac -SH -cat >"$bin/rustc" <<'SH' -#!/usr/bin/env bash -if [[ ${1:-} == -vV ]]; then - printf 'rustc 1.0.0\nbinary: rustc\nhost: x86_64-unknown-linux-gnu\n' - exit 0 -fi -exit 2 -SH -cat >"$bin/python3" <<'SH' -#!/usr/bin/env bash -set -euo pipefail -case "${1:-}" in - */resolve-release-source-profile.py) - output= - while (($#)); do - if [[ $1 == --github-output ]]; then - output=$2 - break - fi - shift - done - printf 'profile=production\ncargo_features=production\n' >"$output" - ;; - *) - exec "$REAL_PYTHON" "$@" - ;; -esac -SH -chmod +x "$bin/cargo" "$bin/rustc" "$bin/python3" - -run_gate() { - local output=$1 - shift - set +e - PATH="$bin:$PATH" \ - REAL_PYTHON="$real_python" \ - TEST_STATE="$state" \ - TMPDIR="$work" \ - "$gate" --repo "$repo" "$@" >"$output" 2>&1 - local status=$? - set -e - printf '%s\n' "$status" -} - -output="$work/output" - -: >"$state/cargo.log" -status=$(run_gate "$output" --skip-packaged-runtime-battery) -[[ $status -ne 0 ]] || { - cat "$output" >&2 - echo "skip-packaged-runtime-battery without a reused binary was accepted" >&2 - exit 1 -} -grep -Fq -- "--skip-packaged-runtime-battery requires --reuse-release-binary" "$output" || { - cat "$output" >&2 - echo "missing reuse-binary requirement was not reported" >&2 - exit 1 -} -if [[ -s $state/cargo.log ]]; then - echo "cargo ran before the reuse-binary requirement failed" >&2 - exit 1 -fi - -# Without a handed-in binary the gate goes straight to packaging: the -# packaged CLI it builds later is the production binary under test, so a -# source-tree workspace release build here would be a second compile of the -# same graph that nothing reads. -: >"$state/cargo.log" -status=$(run_gate "$output") -[[ $status -eq 77 ]] || { - cat "$output" >&2 - echo "default path did not stop at the controlled cargo package boundary" >&2 - exit 1 -} -if grep -Fxq "workspace-rebuild" "$state/cargo.log"; then - echo "default path invoked a source-tree workspace release build" >&2 - exit 1 -fi -if grep -Eq '^build( |$)' "$state/cargo.log"; then - echo "default path invoked cargo build before packaging" >&2 - exit 1 -fi -if grep -Fq "reusing the just-built production binary" "$output"; then - cat "$output" >&2 - echo "default path claimed to reuse a binary it was never given" >&2 - exit 1 -fi - -: >"$state/cargo.log" -status=$(run_gate "$output" --reuse-release-binary "$bin/tracedecay") -[[ $status -eq 77 ]] || { - cat "$output" >&2 - echo "reuse path did not stop at the controlled cargo package boundary" >&2 - exit 1 -} -grep -Fq "distribution acceptance: reusing the just-built production binary" "$output" || { - cat "$output" >&2 - echo "reuse path did not announce the just-built binary" >&2 - exit 1 -} -if grep -Fxq "workspace-rebuild" "$state/cargo.log"; then - echo "reuse path still invoked the workspace release rebuild" >&2 - exit 1 -fi -grep -Eq '^package( |$)' "$state/cargo.log" || { - echo "reuse path skipped cargo package" >&2 - exit 1 -} - -printf 'wrong version\n' >"$bin/bad-tracedecay" -chmod +x "$bin/bad-tracedecay" -cat >"$bin/bad-tracedecay" <<'SH' -#!/usr/bin/env bash -if [[ ${1:-} == --version ]]; then - printf 'tracedecay 9.9.9+deadbeef\n' - exit 0 -fi -exit 2 -SH -chmod +x "$bin/bad-tracedecay" -: >"$state/cargo.log" -status=$(run_gate "$output" --reuse-release-binary "$bin/bad-tracedecay") -[[ $status -ne 0 ]] || { - cat "$output" >&2 - echo "reuse path accepted a binary with the wrong source sha" >&2 - exit 1 -} -grep -Fq "reused-release tracedecay binary reported" "$output" || { - cat "$output" >&2 - echo "wrong reused binary failed for an unexpected reason" >&2 - exit 1 -} -if [[ -s $state/cargo.log ]]; then - echo "cargo ran after a reused binary failed its source-sha check" >&2 - exit 1 -fi - -printf 'distribution reuse-binary regression passed\n' diff --git a/scripts/test-check-distribution-snapshot.sh b/scripts/test-check-distribution-snapshot.sh deleted file mode 100644 index 5e4d15e870..0000000000 --- a/scripts/test-check-distribution-snapshot.sh +++ /dev/null @@ -1,197 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -root=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P) -gate="$root/scripts/check-distribution-acceptance.sh" -work=$(mktemp -d "${TMPDIR:-/tmp}/tracedecay-snapshot-test.XXXXXX") -trap 'rm -rf -- "$work"' EXIT - -repo="$work/repo" -bin="$work/bin" -state="$work/state" -mkdir -p -- \ - "$repo/crates/tracedecay" \ - "$repo/crates/tracedecay-cli" \ - "$repo/crates/tracedecay-hooks/fixtures/host_events" \ - "$repo/tests/fixtures/packaged_host_events" \ - "$repo/.cargo" \ - "$repo/plugin" \ - "$repo/vendor" \ - "$repo/benchmark_data" \ - "$repo/dashboard/hermes-wrapper" \ - "$repo/dashboard/app-dist" \ - "$repo/scripts" \ - "$bin" \ - "$state" - -cat >"$repo/Cargo.toml" <<'TOML' -[workspace] -members = ["crates/tracedecay", "crates/tracedecay-cli"] - -[workspace.package] -version = "0.0.0" -TOML -cat >"$repo/crates/tracedecay/Cargo.toml" <<'TOML' -[package] -name = "tracedecay" -version = "0.0.0" -readme = "../../README.md" -TOML -cat >"$repo/crates/tracedecay-cli/Cargo.toml" <<'TOML' -[package] -name = "tracedecay-cli" -version = "0.0.0" -TOML - -for fixture in \ - claude.json \ - claude/post_tool_use_write.json \ - codex.json \ - cline-family.json \ - cursor.json \ - hermes.json \ - hermes/saved-edit.json \ - hermes/terminal-receipt.json \ - kiro.json \ - kimi-code.json \ - kimi/post-tool-use-edit.json \ - opencode/baseline.json \ - pi.json \ - droid.json; do - mkdir -p -- \ - "$repo/crates/tracedecay-hooks/fixtures/host_events/$(dirname -- "$fixture")" \ - "$repo/tests/fixtures/packaged_host_events/$(dirname -- "$fixture")" - printf 'fixture\n' >"$repo/crates/tracedecay-hooks/fixtures/host_events/$fixture" - printf 'fixture\n' >"$repo/tests/fixtures/packaged_host_events/$fixture" -done - -printf 'original readme\n' >"$repo/README.md" -printf 'changelog\n' >"$repo/CHANGELOG.md" -printf 'license\n' >"$repo/LICENSE" -printf 'stable\n' >"$repo/rust-toolchain.toml" -printf '[net]\noffline = true\n' >"$repo/.cargo/config.toml" -printf 'plugin\n' >"$repo/plugin/fixture" -printf 'vendor\n' >"$repo/vendor/fixture" -printf 'benchmark\n' >"$repo/benchmark_data/fixture" -printf 'wrapper\n' >"$repo/dashboard/hermes-wrapper/fixture" -printf 'bundle\n' >"$repo/dashboard/app-dist/fixture" -printf '#!/usr/bin/env bash\n' >"$repo/scripts/run-session-temporal-benchmark.sh" - -# The product crate carries its own `tests/fixtures`, which occupies the path -# the root asset of the same name is staged onto. -mkdir -p -- "$repo/crates/tracedecay/tests/fixtures/crate_local" -printf 'crate local\n' >"$repo/crates/tracedecay/tests/fixtures/crate_local/fixture" - -git -C "$repo" init -q -git -C "$repo" config user.name "TraceDecay test" -git -C "$repo" config user.email "test@tracedecay.local" -git -C "$repo" add -A -git -C "$repo" commit -qm "test fixture" - -real_cp=$(command -v cp) -real_python=$(command -v python3) - -# The package boundary is the last point where the gate's staged tree exists, -# so the staged-snapshot assertions run there and record that they did. -cat >"$bin/cargo" <<'SH' -#!/usr/bin/env bash -set -euo pipefail -case "${1:-}" in - build) exit 0 ;; - package) ;; - *) exit 2 ;; -esac -manifest= -while (($#)); do - if [[ $1 == --manifest-path ]]; then - manifest=$2 - break - fi - shift -done -staged=$(dirname -- "$manifest") -grep -Fxq "original readme" "$staged/crates/tracedecay/README.md" || { - echo "packaged asset was copied from the mutated live repository" >&2 - exit 1 -} -grep -Fxq "mutated live readme" "$TEST_REPO/README.md" -product_fixtures="$staged/crates/tracedecay/tests/fixtures" -[[ ! -e "$product_fixtures/crate_local" ]] || { - echo "crate-local content survived beside the staged root asset" >&2 - exit 1 -} -[[ -f "$product_fixtures/packaged_host_events/claude.json" ]] || { - echo "staged root asset is missing from the product package" >&2 - exit 1 -} -printf '%s\n' "$staged" >"$TEST_STATE/staged-checked" -exit 77 -SH -cat >"$bin/rustc" <<'SH' -#!/usr/bin/env bash -if [[ ${1:-} == -vV ]]; then - printf 'rustc 1.0.0\nbinary: rustc\nhost: x86_64-unknown-linux-gnu\n' - exit 0 -fi -exit 2 -SH -cat >"$bin/python3" <<'SH' -#!/usr/bin/env bash -set -euo pipefail -case "${1:-}" in - */resolve-release-source-profile.py) - output= - while (($#)); do - if [[ $1 == --github-output ]]; then - output=$2 - break - fi - shift - done - printf 'profile=production\ncargo_features=production\n' >"$output" - ;; - *) - exec "$REAL_PYTHON" "$@" - ;; -esac -SH -cat >"$bin/cp" <<'SH' -#!/usr/bin/env bash -set -euo pipefail -if [[ ! -e "$TEST_STATE/mutated" ]]; then - : >"$TEST_STATE/mutated" - printf 'mutated live readme\n' >"$TEST_REPO/README.md" -fi -exec "$REAL_CP" "$@" -SH -chmod +x "$bin/cargo" "$bin/rustc" "$bin/python3" "$bin/cp" - -output="$work/output" -set +e -PATH="$bin:$PATH" \ -REAL_CP="$real_cp" \ -REAL_PYTHON="$real_python" \ -TEST_REPO="$repo" \ -TEST_STATE="$state" \ -TMPDIR="$work" \ - "$gate" --repo "$repo" >"$output" 2>&1 -status=$? -set -e -[[ $status -eq 77 ]] || { - cat "$output" >&2 - echo "distribution gate stopped before the controlled cargo package boundary" >&2 - exit 1 -} - -staged=$(<"$state/staged-checked") -[[ $staged == "$work"/tracedecay-distribution.*/staged ]] || { - echo "staged-snapshot assertions did not run inside the gate's work directory: $staged" >&2 - exit 1 -} -leftover=$(find "$work" -maxdepth 1 -name 'tracedecay-distribution.*' -print) -[[ -z $leftover ]] || { - echo "distribution gate left its temporary directory behind: $leftover" >&2 - exit 1 -} - -printf 'distribution staged-snapshot regression passed\n' diff --git a/scripts/test-check-packaged-lsp-bridge.py b/scripts/test-check-packaged-lsp-bridge.py old mode 100644 new mode 100755 diff --git a/scripts/test-check-packaged-mcp-stdio.py b/scripts/test-check-packaged-mcp-stdio.py deleted file mode 100755 index 2c9423b43b..0000000000 --- a/scripts/test-check-packaged-mcp-stdio.py +++ /dev/null @@ -1,171 +0,0 @@ -#!/usr/bin/env python3 -"""Fixture-driven Windows MCP stdio acceptance regression.""" - -from __future__ import annotations - -import importlib.util -import json -import os -from pathlib import Path -import subprocess -import sys -import tempfile - - -SMOKE = Path(__file__).with_name("check-packaged-mcp-stdio.py") -SPEC = importlib.util.spec_from_file_location("check_packaged_mcp_stdio", SMOKE) -if SPEC is None or SPEC.loader is None: - raise RuntimeError(f"could not load {SMOKE}") -MODULE = importlib.util.module_from_spec(SPEC) -SPEC.loader.exec_module(MODULE) -REQUIRED_TOOLS = [ - "tracedecay_search", - "tracedecay_diagnostics", - "tracedecay_impact", - "tracedecay_affected", - "tracedecay_test_map", -] - - -def write_executable(path: Path, source: str) -> None: - path.write_text(source, encoding="utf-8") - path.chmod(path.stat().st_mode | 0o111) - - -def write_python_launcher(path: Path, source: str) -> Path: - if os.name != "nt": - write_executable(path, f"#!/usr/bin/env python3\n{source}") - return path - - python_source = path.with_suffix(".py") - python_source.write_text(source, encoding="utf-8") - launcher = path.with_suffix(".cmd") - command = subprocess.list2cmdline([sys.executable, str(python_source)]) - launcher.write_text(f"@echo off\r\n{command} %*\r\n", encoding="utf-8") - return launcher - - -def main() -> int: - binary_path = Path("tracedecay.exe") - socket_path = Path("home/.tracedecay/daemon.sock") - assert MODULE.daemon_command(binary_path, socket_path, platform_name="nt") == [ - str(binary_path), - "daemon", - "run", - ] - assert "TRACEDECAY_DAEMON_SOCKET" not in MODULE.daemon_environment( - {"TRACEDECAY_DAEMON_SOCKET": "fake"}, socket_path, platform_name="nt" - ) - assert MODULE.daemon_command(binary_path, socket_path, platform_name="posix") == [ - str(binary_path), - "daemon", - "run", - "--socket", - str(socket_path), - ] - assert MODULE.daemon_environment( - {"HOME": "home"}, socket_path, platform_name="posix" - )["TRACEDECAY_DAEMON_SOCKET"] == str(socket_path) - - assert not MODULE.daemon_endpoint_is_ready(0, f"socket: {socket_path} (missing)") - assert not MODULE.daemon_endpoint_is_ready(0, f"socket: {socket_path} (stale)") - assert not MODULE.daemon_endpoint_is_ready( - 1, "endpoint: tcp://127.0.0.1:9 (connectable)" - ) - assert MODULE.daemon_endpoint_is_ready(0, f"socket: {socket_path} (connectable)") - assert MODULE.daemon_endpoint_is_ready( - 0, "endpoint: tcp://127.0.0.1:39181 (connectable)" - ) - - with tempfile.TemporaryDirectory() as temporary_directory: - root = Path(temporary_directory) - bin_directory = root / "bin" - bin_directory.mkdir() - binary = write_python_launcher( - bin_directory / "tracedecay", - """import os -import sys -import time -from pathlib import Path - -args = sys.argv[1:] -ready = Path(os.environ["HOME"]) / ".tracedecay" / "fixture-daemon-ready" -if args[:2] == ["daemon", "run"]: - ready.parent.mkdir(parents=True, exist_ok=True) - ready.write_text("", encoding="utf-8") - while True: - time.sleep(60) -if args == ["daemon", "status"]: - state = "connectable" if ready.exists() else "missing" - print(f"endpoint: fixture ({state})") - raise SystemExit(0) -if args == ["init"]: - raise SystemExit(0) -if args == ["fixture-probe-daemon"]: - raise SystemExit(0 if ready.exists() else 1) -raise SystemExit(1) -""", - ) - - tools = [ - {"name": name, "inputSchema": {"type": "object"}} for name in REQUIRED_TOOLS - ] - write_python_launcher( - bin_directory / "npx", - f"""import json -import subprocess -import sys - -arguments = sys.argv[1:] -binary = arguments[arguments.index("--cli") + 1] -subprocess.run([binary, "fixture-probe-daemon"], check=True) -method = arguments[arguments.index("--method") + 1] -if method == "tools/list": - print(json.dumps({{"tools": {json.dumps(tools)}}})) -elif method == "resources/list": - print(json.dumps({{"resources": [{{"uri": "tracedecay://status"}}]}})) -elif method == "tools/call": - tool = arguments[arguments.index("--tool-name") + 1] - if tool == "tracedecay_diagnostics": - print(json.dumps({{ - "isError": True, - "content": [{{ - "type": "text", - "text": "daemon diagnostic authority is unavailable", - }}], - "structuredContent": {{"problem": {{"kind": "unavailable"}}}}, - }})) - raise SystemExit(2) - elif tool == "definitely_not_a_tool": - print(json.dumps({{ - "isError": True, - "content": [{{"type": "text", "text": "unknown tool"}}], - "structuredContent": {{"problem": {{"code": "unknown_tool"}}}}, - }})) - else: - raise SystemExit(1) -else: - raise SystemExit(1) -""", - ) - - environment = os.environ.copy() - environment["PATH"] = str(bin_directory) + os.pathsep + environment["PATH"] - completed = subprocess.run( - [sys.executable, str(SMOKE), str(binary), str(root / "work")], - env=environment, - check=False, - capture_output=True, - text=True, - ) - if completed.returncode != 0: - raise SystemExit(completed.stderr or completed.stdout) - if "packaged MCP stdio acceptance passed" not in completed.stdout: - raise SystemExit("Windows MCP fixture did not complete the real-tool smoke") - - print("Windows packaged MCP stdio fixture passed") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/test-resolve-installed-binary.py b/scripts/test-resolve-installed-binary.py deleted file mode 100755 index 6801dc59bc..0000000000 --- a/scripts/test-resolve-installed-binary.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python3 -"""Cross-platform fixtures for installed binary path resolution.""" - -from __future__ import annotations - -import os -from pathlib import Path -import subprocess -import sys -import tempfile - - -RESOLVER = Path(__file__).with_name("resolve-installed-binary.py") - - -def resolve(root: Path, runner_os: str) -> subprocess.CompletedProcess[str]: - return subprocess.run( - [sys.executable, str(RESOLVER), str(root), runner_os], - check=False, - capture_output=True, - text=True, - ) - - -def main() -> int: - with tempfile.TemporaryDirectory() as temporary_directory: - root = Path(temporary_directory) - binary_directory = root / "bin" - binary_directory.mkdir() - windows_binary = binary_directory / "tracedecay.exe" - windows_binary.write_bytes(b"MZ") - - windows = resolve(root, "Windows") - if windows.returncode != 0: - raise SystemExit(windows.stderr) - if Path(windows.stdout.strip()) != windows_binary.resolve(): - raise SystemExit("Windows resolver did not return tracedecay.exe") - - unix_binary = binary_directory / "tracedecay" - extensionless = resolve(root, "Linux") - if extensionless.returncode == 0: - raise SystemExit("Unix resolver incorrectly accepted tracedecay.exe") - if extensionless.stderr != ( - f"installed binary layout did not produce {unix_binary.resolve()}\n" - ): - raise SystemExit(f"Unix resolver failed for an unexpected reason: {extensionless.stderr}") - - windows_binary.unlink() - unix_binary.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") - if os.name != "nt": - unix_binary.chmod(0o644) - unexecutable = resolve(root, "Linux") - if unexecutable.stderr != ( - f"cargo-installed binary is not executable: {unix_binary.resolve()}\n" - ): - raise SystemExit( - f"Unix resolver accepted a non-executable binary: {unexecutable.stderr}" - ) - unix_binary.chmod(unix_binary.stat().st_mode | 0o111) - unix = resolve(root, "Linux") - if unix.returncode != 0: - raise SystemExit(unix.stderr) - if Path(unix.stdout.strip()) != unix_binary.resolve(): - raise SystemExit("Unix resolver did not return tracedecay") - - print("installed binary path fixtures passed") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/test-resolve-release-source-profile.py b/scripts/test-resolve-release-source-profile.py deleted file mode 100755 index 2c5ab01790..0000000000 --- a/scripts/test-resolve-release-source-profile.py +++ /dev/null @@ -1,166 +0,0 @@ -#!/usr/bin/env python3 -"""Regression tests for historical-tag release profile resolution.""" - -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import Path -import subprocess -import sys -import tempfile - - -RESOLVER = Path(__file__).with_name("resolve-release-source-profile.py") - - -@dataclass(frozen=True) -class FixtureResult: - returncode: int - stdout: str - stderr: str - github_output: str - - -def run_fixture(manifest: str) -> FixtureResult: - with tempfile.TemporaryDirectory() as temporary_directory: - source = Path(temporary_directory) - # The resolver reads the product package manifest, not the workspace - # root: `crates/tracedecay/Cargo.toml` is where the feature table lives. - product = source.joinpath("crates", "tracedecay") - product.joinpath("src").mkdir(parents=True) - product.joinpath("Cargo.toml").write_text(manifest, encoding="utf-8") - product.joinpath("src", "lib.rs").write_text("", encoding="utf-8") - # The resolver walks the resolved dependency graph with `cargo tree - # --locked`, so the fixture is a real workspace with a lockfile. It - # has no dependencies, so the lockfile resolves offline. - source.joinpath("Cargo.toml").write_text( - '[workspace]\nmembers = ["crates/tracedecay"]\nresolver = "2"\n', - encoding="utf-8", - ) - subprocess.run( - ["cargo", "generate-lockfile", "--offline"], - cwd=source, - check=True, - capture_output=True, - ) - output = source / "github-output.txt" - completed = subprocess.run( - [ - sys.executable, - str(RESOLVER), - "--source", - str(source), - "--github-output", - str(output), - ], - check=False, - capture_output=True, - text=True, - ) - return FixtureResult( - returncode=completed.returncode, - stdout=completed.stdout, - stderr=completed.stderr, - github_output=output.read_text(encoding="utf-8") if output.exists() else "", - ) - - -def main() -> int: - production = run_fixture( - """[package] -name = "tracedecay" -version = "0.1.0" -edition = "2024" - -[features] -default = ["production"] -production = ["token-counting", "lite", "full"] -token-counting = [] -lite = [] -full = [] -""" - ) - if production.returncode != 0: - raise SystemExit(production.stderr) - if production.github_output != ( - "profile=production\n" - "cargo_args=--no-default-features --features production\n" - "cargo_features=production\n" - ): - raise SystemExit( - f"unexpected production profile output: {production.github_output!r}" - ) - - contaminated_production = run_fixture( - """[package] -name = "tracedecay" -version = "0.1.0" -edition = "2024" - -[features] -default = ["production"] -production = ["token-counting", "lite", "full", "test-transport"] -token-counting = [] -lite = [] -full = [] -test-transport = [] -""" - ) - if contaminated_production.returncode == 0: - raise SystemExit("production test-transport contamination was accepted") - if not contaminated_production.stderr.startswith( - "production feature directly enables test-transport\n" - ): - raise SystemExit( - "production contamination failed for an unexpected reason: " - f"{contaminated_production.stderr}" - ) - if contaminated_production.github_output: - raise SystemExit("refused production profile still wrote release outputs") - - legacy = run_fixture( - """[package] -name = "tracedecay" -version = "0.0.67" -edition = "2021" - -[features] -default = ["full", "token-counting"] -full = ["medium"] -medium = [] -token-counting = [] -test-transport = [] -""" - ) - if legacy.returncode != 0: - raise SystemExit(legacy.stderr) - if legacy.github_output != ( - "profile=legacy-default\ncargo_args=\ncargo_features=\n" - ): - raise SystemExit( - f"unexpected historical profile output: {legacy.github_output!r}" - ) - - contaminated = run_fixture( - """[package] -name = "tracedecay" -version = "0.0.1" -edition = "2021" - -[features] -default = ["full"] -full = ["test-transport"] -test-transport = [] -""" - ) - if contaminated.returncode == 0: - raise SystemExit("historical default test-transport contamination was accepted") - if "default features enable test-transport" not in contaminated.stderr: - raise SystemExit("historical contamination failed for an unexpected reason") - - print("historical release source profile fixtures passed") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/test-rust-cache-lineage.py b/scripts/test-rust-cache-lineage.py index cfe21e4a8c..a5ffe327e7 100755 --- a/scripts/test-rust-cache-lineage.py +++ b/scripts/test-rust-cache-lineage.py @@ -17,8 +17,8 @@ LINEAGE_PATH = ROOT / "scripts/rust_cache_lineage.py" CHECKER_PATH = ROOT / "scripts/check-rust-cache-lineage.py" WORKFLOWS = ROOT / ".github/workflows" -# The packaging battery still builds with Cargo behind rust-cache. -FIXTURE = "distribution-acceptance.yml" +# Hawk still uses the Cargo cache for its optional diagnostics. +FIXTURE = "hawk.yml" def load_module(path: Path, name: str) -> ModuleType: @@ -87,7 +87,7 @@ def _assert_rejected_after(self, old: str, new: str, reason: str) -> None: def test_rejects_rooting_rust_cache_at_the_vendored_checkout(self) -> None: self._assert_rejected_after( - "crates -> ../target\n", + "crates -> ../target/hawk\n", ". -> target\n", f"{FIXTURE} rust-cache step must list workspace root 'crates -> ../target'", ) From 7ad21f2c390742dd282a102fe6f2b4e63b2fa0fd Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 18 Oct 2026 01:11:48 -0700 Subject: [PATCH 02/89] fix(dashboard): generate settings PATCH contracts --- .../workflows/dashboard-contract-export.yml | 73 +++++ crates/tracedecay-api/src/configuration.rs | 111 +++++-- .../src/result/envelope.rs | 2 +- .../src/result/evidence.rs | 33 ++- .../src/result/receipt.rs | 2 +- .../src/settings_preview.rs | 3 +- .../src/application_surface.rs | 11 +- .../src/automation_config_api.rs | 36 +-- .../src/contract_schema.rs | 6 +- .../src/settings_api.rs | 88 +++--- .../src/settings_api/tests/wire.rs | 272 ++++++++++++++++++ dashboard/codegen/src/generate.ts | 29 +- dashboard/codegen/tests/generate.test.ts | 45 ++- 13 files changed, 594 insertions(+), 117 deletions(-) create mode 100644 .github/workflows/dashboard-contract-export.yml create mode 100644 crates/tracedecay-dashboard-api/src/settings_api/tests/wire.rs diff --git a/.github/workflows/dashboard-contract-export.yml b/.github/workflows/dashboard-contract-export.yml new file mode 100644 index 0000000000..a4eba1eb74 --- /dev/null +++ b/.github/workflows/dashboard-contract-export.yml @@ -0,0 +1,73 @@ +# Temporary build route: remove after importing the canonical generated JSON. +name: Export dashboard contract artifacts +run-name: Export dashboard contracts for ${{ github.event.pull_request.head.sha }} + +on: + pull_request: + types: [opened, synchronize, reopened] + +permissions: + contents: read + +concurrency: + group: dashboard-contract-export-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + export: + if: github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + timeout-minutes: 90 + env: + CARGO_BAZEL_TIMEOUT: "3600" + EXPORT_DIRECTORY: ${{ runner.temp }}/dashboard-contract-export + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: "26" + + - uses: ./.github/actions/setup-pnpm + + - uses: ./.github/actions/setup-bazel + with: + cache-key: ci-linux-test-v2 + + - name: Export the canonical dashboard schema + run: | + set -euo pipefail + mkdir -p "$EXPORT_DIRECTORY" + bazel run --config=ci --jobs=1 //sdks/codegen:dashboard_schema -- "$EXPORT_DIRECTORY/dashboard-contracts.schema.json" + test -s "$EXPORT_DIRECTORY/dashboard-contracts.schema.json" + + # Preserve the schema before the independent fixture build starts. + - name: Upload the canonical dashboard schema + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: dashboard-schema-${{ github.event.pull_request.head.sha }} + path: ${{ runner.temp }}/dashboard-contract-export/dashboard-contracts.schema.json + if-no-files-found: error + retention-days: 1 + + - name: Serialize the real settings response fixture + run: | + set -euo pipefail + bazel test --config=ci --jobs=1 //crates/tracedecay-dashboard-api:unit_test \ + --test_arg=settings_api::tests::wire::project_patch_serialization_preserves_settlement_and_refusals \ + --test_arg=--exact --test_output=all --cache_test_results=no \ + --nozip_undeclared_test_outputs + fixture=bazel-testlogs/crates/tracedecay-dashboard-api/unit_test/test.outputs/settings-patch.json + test -s "$fixture" + cp -- "$fixture" "$EXPORT_DIRECTORY/settings-patch.json" + + - name: Upload the canonical settings response fixture + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: settings-patch-fixture-${{ github.event.pull_request.head.sha }} + path: ${{ runner.temp }}/dashboard-contract-export/settings-patch.json + if-no-files-found: error + retention-days: 1 diff --git a/crates/tracedecay-api/src/configuration.rs b/crates/tracedecay-api/src/configuration.rs index 4817efb7a1..6443e3dd33 100644 --- a/crates/tracedecay-api/src/configuration.rs +++ b/crates/tracedecay-api/src/configuration.rs @@ -9,8 +9,10 @@ use axum::Json; use axum::http::StatusCode; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use serde_json::{Value, json}; -use tracedecay_contracts::{ApplicationProblemEnvelope, ApplicationProblemKind}; +use serde_json::Value; +use tracedecay_contracts::{ + ApplicationProblemEnvelope, ApplicationProblemKind, SettingsValidationIssueV1, +}; use tracedecay_domain::configuration::{CodeIndexWorkerSelectionV1, ContextScoutIntentSettingsV1}; use tracedecay_tool_catalog::ApplicationSurfaceOperation; @@ -89,8 +91,39 @@ pub struct CodeIndexWorkerSettingsPatch { pub code_index_workers: CodeIndexWorkerSelectionV1, } +/// Configuration refusals carrying the same stable codes as the HTTP routes. +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +#[serde(tag = "code", rename_all = "snake_case", deny_unknown_fields)] +pub enum ConfigurationRefusalV1 { + ConfigurationRevisionConflict { + detail: String, + expected_revision_id: String, + actual_revision_id: String, + }, + ConfigurationAuthorityUnavailable { + detail: String, + }, + ApplicationContractViolation { + detail: String, + }, +} + +/// Every refusal body produced by the dashboard configuration adapter. The +/// untagged wrapper preserves the existing validation, CAS, and application +/// problem wire formats while giving producers and consumers one authority. +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +#[serde(untagged, deny_unknown_fields)] +pub enum DashboardConfigurationRefusalV1 { + Validation { + detail: String, + validation_errors: Vec, + }, + Configuration(ConfigurationRefusalV1), + ApplicationProblem(ApplicationProblemEnvelope), +} + /// Axum-compatible typed error used by dashboard configuration handlers. -pub type DashboardConfigurationRouteErrorV1 = (StatusCode, Json); +pub type DashboardConfigurationRouteErrorV1 = (StatusCode, Json); /// Parse a project settings patch while preserving the dashboard's established /// malformed-payload response shape. @@ -175,13 +208,15 @@ pub fn validate_code_index_worker_settings_patch( } /// Render validation failures using the generated dashboard wire shape. -pub fn settings_validation_error(errors: impl Serialize) -> DashboardConfigurationRouteErrorV1 { +pub fn settings_validation_error( + errors: impl IntoIterator, +) -> DashboardConfigurationRouteErrorV1 { ( StatusCode::BAD_REQUEST, - Json(json!({ - "detail": "settings validation failed", - "validation_errors": errors, - })), + Json(DashboardConfigurationRefusalV1::Validation { + detail: "settings validation failed".to_owned(), + validation_errors: errors.into_iter().collect(), + }), ) } @@ -193,12 +228,13 @@ pub fn configuration_revision_conflict_error( ) -> DashboardConfigurationRouteErrorV1 { ( StatusCode::CONFLICT, - Json(json!({ - "code": "configuration_revision_conflict", - "detail": detail, - "expected_revision_id": expected, - "actual_revision_id": actual, - })), + Json(DashboardConfigurationRefusalV1::Configuration( + ConfigurationRefusalV1::ConfigurationRevisionConflict { + detail: detail.to_owned(), + expected_revision_id: expected.to_owned(), + actual_revision_id: actual.to_owned(), + }, + )), ) } @@ -206,10 +242,11 @@ pub fn configuration_revision_conflict_error( pub fn configuration_authority_unavailable_error() -> DashboardConfigurationRouteErrorV1 { ( StatusCode::SERVICE_UNAVAILABLE, - Json(json!({ - "code": "configuration_authority_unavailable", - "detail": "configuration authority is unavailable", - })), + Json(DashboardConfigurationRefusalV1::Configuration( + ConfigurationRefusalV1::ConfigurationAuthorityUnavailable { + detail: "configuration authority is unavailable".to_owned(), + }, + )), ) } @@ -225,13 +262,17 @@ pub fn configuration_application_problem_error( } else { crate::application_problem_status(problem.problem.kind) }; - let payload = serde_json::to_value(problem) - .unwrap_or_else(|_| json!({ "detail": "configuration mutation was rejected" })); - (status, Json(payload)) + ( + status, + Json(DashboardConfigurationRefusalV1::ApplicationProblem(problem)), + ) } -fn validation_error(field: &str, message: &str) -> Value { - json!({ "field": field, "message": message }) +fn validation_error(field: &str, message: &str) -> SettingsValidationIssueV1 { + SettingsValidationIssueV1 { + field: field.to_owned(), + message: message.to_owned(), + } } fn patch_shape_error(scope: &str, error: &serde_json::Error) -> DashboardConfigurationRouteErrorV1 { @@ -239,10 +280,10 @@ fn patch_shape_error(scope: &str, error: &serde_json::Error) -> DashboardConfigu let field = serde_error_field(&message).unwrap_or_else(|| "patch".to_owned()); ( StatusCode::BAD_REQUEST, - Json(json!({ - "detail": message, - "validation_errors": [{ "field": field, "message": message }], - })), + Json(DashboardConfigurationRefusalV1::Validation { + detail: message.clone(), + validation_errors: vec![SettingsValidationIssueV1 { field, message }], + }), ) } @@ -260,6 +301,7 @@ fn serde_error_field(message: &str) -> Option { #[cfg(test)] mod tests { use super::*; + use serde_json::json; #[test] fn user_worker_patch_round_trips_the_tagged_contract() { @@ -298,7 +340,10 @@ mod tests { let (status, Json(body)) = validate_code_index_worker_settings_patch(&patch).expect_err("zero must be denied"); assert_eq!(status, StatusCode::BAD_REQUEST); - assert_eq!(body["validation_errors"][0]["field"], "code_index_workers"); + assert_eq!( + serde_json::to_value(body).unwrap()["validation_errors"][0]["field"], + "code_index_workers" + ); } #[test] @@ -324,7 +369,10 @@ mod tests { let (status, Json(body)) = admit_user_settings_patch(&invalid, |_| None) .expect_err("zero duration must be denied"); assert_eq!(status, StatusCode::BAD_REQUEST); - assert_eq!(body["validation_errors"][0]["field"], "watcher_debounce"); + assert_eq!( + serde_json::to_value(body).unwrap()["validation_errors"][0]["field"], + "watcher_debounce" + ); } #[test] @@ -338,6 +386,9 @@ mod tests { let (status, Json(body)) = result.expect_err("mixed authority must be rejected"); assert_eq!(status, StatusCode::BAD_REQUEST); - assert_eq!(body["validation_errors"][0]["field"], "code_index_workers"); + assert_eq!( + serde_json::to_value(body).unwrap()["validation_errors"][0]["field"], + "code_index_workers" + ); } } diff --git a/crates/tracedecay-contracts/src/result/envelope.rs b/crates/tracedecay-contracts/src/result/envelope.rs index 28c5b3b297..4a528b6a36 100644 --- a/crates/tracedecay-contracts/src/result/envelope.rs +++ b/crates/tracedecay-contracts/src/result/envelope.rs @@ -84,7 +84,7 @@ impl ResultContractRef { } /// Canonical outcome family for an admitted application operation. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] #[serde(rename_all = "snake_case", tag = "outcome", content = "value")] pub enum ApplicationOutcome { Evidence(EvidencePacket), diff --git a/crates/tracedecay-contracts/src/result/evidence.rs b/crates/tracedecay-contracts/src/result/evidence.rs index 5894d133a1..c69f765ee6 100644 --- a/crates/tracedecay-contracts/src/result/evidence.rs +++ b/crates/tracedecay-contracts/src/result/evidence.rs @@ -15,7 +15,6 @@ use crate::identity::application_identifier; use super::{CancellationObservation, OperationBudgetUsage, OperationReceipt, ResultContractRef}; application_identifier!( - @no_schema EvidenceIdentity => ("evidence identity", 512), ScoreId => ("score id", 512), // Existing authenticated query cursors bind typed scope, access, key, @@ -26,7 +25,7 @@ application_identifier!( /// Exact continuation authority for the enclosing evidence page: an opaque /// cursor bound to the operation and request that minted it. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum PageCursor { Opaque { cursor: OpaqueCursor }, @@ -47,7 +46,9 @@ impl From for PageCursor { } /// Application-level freshness. Missing or partial truth never becomes current. -#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[derive( + Clone, Copy, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq, PartialOrd, Ord, Hash, +)] #[serde(rename_all = "snake_case")] pub enum FreshnessState { Current, @@ -56,7 +57,7 @@ pub enum FreshnessState { } /// Temporal provenance for a packet. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct TemporalState { pub requested_mode: TemporalModeV1, @@ -363,7 +364,9 @@ pub struct Omission { } /// Evidence score semantics. Scores are metadata, never authority. -#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[derive( + Clone, Copy, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq, PartialOrd, Ord, Hash, +)] #[serde(rename_all = "snake_case")] pub enum EvidenceScoreKind { OrdinalRank, @@ -372,7 +375,7 @@ pub enum EvidenceScoreKind { CalibratedInterval, } -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] #[serde(rename_all = "snake_case", tag = "kind")] pub enum EvidenceScoreValue { Ordinal { @@ -387,7 +390,7 @@ pub enum EvidenceScoreValue { }, } -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct EvidenceScore { pub score_id: ScoreId, @@ -399,7 +402,7 @@ pub struct EvidenceScore { } /// Evidence authority is separate from caller authority and cannot grant access. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct EvidenceAuthority { pub evidence_id: EvidenceIdentity, @@ -411,7 +414,9 @@ pub struct EvidenceAuthority { } /// Bounded elapsed-work classification supplied by a retriever. -#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[derive( + Clone, Copy, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq, PartialOrd, Ord, Hash, +)] #[serde(rename_all = "snake_case")] pub enum BudgetClass { WithinBudget, @@ -420,7 +425,9 @@ pub enum BudgetClass { } /// Terminal state reported by one retriever contribution. -#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[derive( + Clone, Copy, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq, PartialOrd, Ord, Hash, +)] #[serde(rename_all = "snake_case")] pub enum RetrieverContributionState { Completed, @@ -434,7 +441,7 @@ pub enum RetrieverContributionState { } /// One source-owned contribution to the packet fold. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct RetrieverContribution { pub retriever_id: RetrieverId, @@ -453,7 +460,7 @@ pub struct RetrieverContribution { /// Stable page state. General cursor bytes stay opaque until authorization is /// revalidated; structural fact cursors retain their canonical ordering type. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct PageState { pub sort_contract_id: SortContractId, @@ -510,7 +517,7 @@ pub struct RetrievalEvidence { } /// Immutable evidence packet consumed by adapters and later planner work. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct EvidencePacket { pub temporal: TemporalState, diff --git a/crates/tracedecay-contracts/src/result/receipt.rs b/crates/tracedecay-contracts/src/result/receipt.rs index 155f9808a5..42173d9252 100644 --- a/crates/tracedecay-contracts/src/result/receipt.rs +++ b/crates/tracedecay-contracts/src/result/receipt.rs @@ -188,7 +188,7 @@ pub enum ReconciliationState { } /// Read-only preview of a future typed effect. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct PreviewResult { pub preview_id: PreviewId, diff --git a/crates/tracedecay-contracts/src/settings_preview.rs b/crates/tracedecay-contracts/src/settings_preview.rs index b193cdd58c..001d51fadc 100644 --- a/crates/tracedecay-contracts/src/settings_preview.rs +++ b/crates/tracedecay-contracts/src/settings_preview.rs @@ -4,10 +4,11 @@ //! operation. This module owns only project patch validation shared before //! adapters invoke that operation. +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; /// One field-level settings validation failure. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] pub struct SettingsValidationIssueV1 { pub field: String, pub message: String, diff --git a/crates/tracedecay-dashboard-api/src/application_surface.rs b/crates/tracedecay-dashboard-api/src/application_surface.rs index 8a2165857e..d6d5a6c9d8 100644 --- a/crates/tracedecay-dashboard-api/src/application_surface.rs +++ b/crates/tracedecay-dashboard-api/src/application_surface.rs @@ -9,7 +9,7 @@ use axum::Router; use axum::extract::Json; use axum::http::StatusCode; use serde_json::Value; -use serde_json::json; +use tracedecay_api::configuration::{ConfigurationRefusalV1, DashboardConfigurationRefusalV1}; use tracedecay_contracts::{ ApplicationContractError, ApplicationOutcome, ApplicationProblemEnvelope, AuthorizedScopeSet, NativeIntegrationSurfaceResultV1, RequestId, @@ -68,10 +68,11 @@ pub(crate) fn configuration_apply_error( } DashboardConfigurationApplyError::ApplicationContractViolation(_) => ( StatusCode::INTERNAL_SERVER_ERROR, - Json(json!({ - "code": "application_contract_violation", - "detail": "the configuration application result violated its contract", - })), + Json(DashboardConfigurationRefusalV1::Configuration( + ConfigurationRefusalV1::ApplicationContractViolation { + detail: "the configuration application result violated its contract".to_owned(), + }, + )), ), } } diff --git a/crates/tracedecay-dashboard-api/src/automation_config_api.rs b/crates/tracedecay-dashboard-api/src/automation_config_api.rs index 680372d3a5..4d704e4283 100644 --- a/crates/tracedecay-dashboard-api/src/automation_config_api.rs +++ b/crates/tracedecay-dashboard-api/src/automation_config_api.rs @@ -14,7 +14,7 @@ use tracedecay_api::configuration::{ DashboardConfigurationRouteErrorV1, configuration_authority_unavailable_error, configuration_revision_conflict_error, settings_validation_error, }; -use tracedecay_contracts::ApplicationOutcome; +use tracedecay_contracts::{ApplicationOutcome, SettingsValidationIssueV1}; use tracedecay_domain::ProjectId; use tracedecay_domain::configuration::{ AUTOMATION_SETTINGS_SETTING_KEY, ConfigurationIdempotencyKey, ConfigurationLayerIdV1, @@ -70,24 +70,26 @@ pub async fn patch_config( request: std::result::Result, JsonRejection>, ) -> ApiResult { let Json(request) = request.map_err(|error| { - settings_validation_error(json!([{ - "field": "request", - "message": error.body_text(), - }])) + settings_validation_error([SettingsValidationIssueV1 { + field: "request".to_owned(), + message: error.body_text(), + }]) })?; let expected_revision = ConfigurationRevisionId::new(request.expected_revision_id).map_err(|_| { - settings_validation_error(json!([{ - "field": "expected_revision_id", - "message": "expected_revision_id must name one canonical configuration revision" - }])) + settings_validation_error([SettingsValidationIssueV1 { + field: "expected_revision_id".to_owned(), + message: "expected_revision_id must name one canonical configuration revision" + .to_owned(), + }]) })?; let idempotency_key = ConfigurationIdempotencyKey::new(request.idempotency_key).map_err(|_| { - settings_validation_error(json!([{ - "field": "idempotency_key", - "message": "idempotency_key must be one non-empty canonical caller-stable value" - }])) + settings_validation_error([SettingsValidationIssueV1 { + field: "idempotency_key".to_owned(), + message: "idempotency_key must be one non-empty canonical caller-stable value" + .to_owned(), + }]) })?; let (current_revision, current, _) = effective_automation_config(&state) .map_err(|_| configuration_authority_unavailable_error())?; @@ -99,10 +101,10 @@ pub async fn patch_config( )); } let candidate = effective_config(¤t, Some(&request.patch)).map_err(|error| { - settings_validation_error(json!([{ - "field": "automation", - "message": error.to_string(), - }])) + settings_validation_error([SettingsValidationIssueV1 { + field: "automation".to_owned(), + message: error.to_string(), + }]) })?; let application_outcome = if candidate == current { diff --git a/crates/tracedecay-dashboard-api/src/contract_schema.rs b/crates/tracedecay-dashboard-api/src/contract_schema.rs index c94f774e50..723cc82be6 100644 --- a/crates/tracedecay-dashboard-api/src/contract_schema.rs +++ b/crates/tracedecay-dashboard-api/src/contract_schema.rs @@ -80,11 +80,13 @@ use super::read_model::{DASHBOARD_SCHEMA_REVISION_V1, DashboardEnvelopeV1}; use super::remote_status_api::RemoteOperationalStatusPayloadV1; use super::savings_api::{SavingsModelsPayloadV1, SavingsOverviewPayloadV1}; use super::settings_api::{ - CodeIndexWorkerSettingsPatch, ProjectSettingsPatch, SettingsPayloadV1, UserSettingsPatch, + CodeIndexWorkerSettingsPatch, ProjectSettingsPatch, ProjectSettingsPatchResponseV1, + SettingsPayloadV1, UserSettingsPatch, }; use super::storage_telemetry_api::StorageTelemetryPayloadV1; use super::work_api::registered_route_contracts as registered_work_route_contracts; use crate::application::feedback::observations::FeedbackObservationReadModelV1; +use tracedecay_api::configuration::DashboardConfigurationRefusalV1; use tracedecay_contracts::code_index_freshness::CodeIndexFreshnessPayloadV1; #[derive(JsonSchema)] @@ -142,6 +144,8 @@ struct DashboardContractCatalogV1 { code_index_freshness: CodeIndexFreshnessPayloadV1, settings: SettingsPayloadV1, settings_project_patch: ProjectSettingsPatch, + settings_project_patch_response: ProjectSettingsPatchResponseV1, + settings_refusal: DashboardConfigurationRefusalV1, settings_user_patch: UserSettingsPatch, settings_code_index_worker_patch: CodeIndexWorkerSettingsPatch, observatory: ObservatoryReadModelV1, diff --git a/crates/tracedecay-dashboard-api/src/settings_api.rs b/crates/tracedecay-dashboard-api/src/settings_api.rs index 5d69f5d0b1..09ee62a729 100644 --- a/crates/tracedecay-dashboard-api/src/settings_api.rs +++ b/crates/tracedecay-dashboard-api/src/settings_api.rs @@ -9,7 +9,7 @@ use axum::Json; use axum::extract::State; use schemars::JsonSchema; use serde::Serialize; -use serde_json::{Value, json}; +use serde_json::Value; use tracedecay_api::configuration::{ DashboardConfigurationRouteErrorV1, PROJECT_SETTINGS_APPLY_OPERATION, SETTINGS_REFRESH_OPERATION, admit_user_settings_patch, configuration_application_problem_error, @@ -20,6 +20,7 @@ use tracedecay_api::configuration::{ }; use tracedecay_contracts::{ ApplicationOutcome, ApplicationProblemEnvelope, ApplicationProblemKind, + SettingsValidationIssueV1, }; use super::DashboardState; @@ -139,7 +140,7 @@ pub struct SettingsPayloadV1 { restart_recommended: Option, } -#[derive(Serialize)] +#[derive(Clone, Debug, Serialize, JsonSchema)] pub struct ProjectSettingsPatchResponseV1 { /// Exact application settlement returned by the configuration binding. /// `None` means the submitted patch was already the current state. @@ -331,10 +332,11 @@ pub async fn patch_project_settings( let pr_autotrack = pr_autotrack_payload(&state)?; let idempotency_key = ConfigurationIdempotencyKey::new(patch.idempotency_key.clone()).map_err(|_| { - settings_validation_error(json!([{ - "field": "idempotency_key", - "message": "idempotency_key must be one non-empty canonical caller-stable value" - }])) + settings_validation_error([SettingsValidationIssueV1 { + field: "idempotency_key".to_owned(), + message: "idempotency_key must be one non-empty canonical caller-stable value" + .to_owned(), + }]) })?; let current = state .cached_runtime_configuration() @@ -402,17 +404,19 @@ pub async fn patch_user_settings( let admitted = admit_user_settings_patch(&patch, parse_duration_millis)?; let idempotency_key = ConfigurationIdempotencyKey::new(patch.idempotency_key.clone()).map_err(|_| { - settings_validation_error(json!([{ - "field": "idempotency_key", - "message": "idempotency_key must be one non-empty canonical caller-stable value" - }])) + settings_validation_error([SettingsValidationIssueV1 { + field: "idempotency_key".to_owned(), + message: "idempotency_key must be one non-empty canonical caller-stable value" + .to_owned(), + }]) })?; let expected_revision = ConfigurationRevisionId::new(patch.expected_revision_id).map_err(|_| { - settings_validation_error(json!([{ - "field": "expected_revision_id", - "message": "expected_revision_id must name one canonical configuration revision" - }])) + settings_validation_error([SettingsValidationIssueV1 { + field: "expected_revision_id".to_owned(), + message: "expected_revision_id must name one canonical configuration revision" + .to_owned(), + }]) })?; let runtime = state .application_invocation_executor @@ -489,17 +493,19 @@ pub async fn patch_code_index_worker_settings( } let idempotency_key = ConfigurationIdempotencyKey::new(patch.idempotency_key.clone()).map_err(|_| { - settings_validation_error(json!([{ - "field": "idempotency_key", - "message": "idempotency_key must be one non-empty canonical caller-stable value" - }])) + settings_validation_error([SettingsValidationIssueV1 { + field: "idempotency_key".to_owned(), + message: "idempotency_key must be one non-empty canonical caller-stable value" + .to_owned(), + }]) })?; let expected_revision = ConfigurationRevisionId::new(patch.expected_revision_id).map_err(|_| { - settings_validation_error(json!([{ - "field": "expected_revision_id", - "message": "expected_revision_id must name one canonical configuration revision" - }])) + settings_validation_error([SettingsValidationIssueV1 { + field: "expected_revision_id".to_owned(), + message: "expected_revision_id must name one canonical configuration revision" + .to_owned(), + }]) })?; let port = state .profile_code_index_worker_settings @@ -545,28 +551,28 @@ pub async fn patch_code_index_worker_settings( fn code_index_worker_admission_errors( selection: &CodeIndexWorkerSelectionV1, status: Option<&CodeIndexWorkerStatusV1>, -) -> Vec { +) -> Vec { let (CodeIndexWorkerSelectionV1::Exact { workers }, Some(status)) = (selection, status) else { return Vec::new(); }; let mut errors = Vec::new(); if *workers > status.available_logical_cpus { - errors.push(json!({ - "field": "code_index_workers", - "message": format!( + errors.push(SettingsValidationIssueV1 { + field: "code_index_workers".to_owned(), + message: format!( "code_index_workers exact mode must request no more than {} available logical CPUs", status.available_logical_cpus, ), - })); + }); } if *workers > status.memory_safe_workers { - errors.push(json!({ - "field": "code_index_workers", - "message": format!( + errors.push(SettingsValidationIssueV1 { + field: "code_index_workers".to_owned(), + message: format!( "code_index_workers exact mode must request no more than {} memory-safe workers", status.memory_safe_workers, ), - })); + }); } errors } @@ -865,6 +871,8 @@ fn project_apply_error( #[cfg(test)] mod tests { + mod wire; + use super::*; use serde_json::json; use tracedecay_configuration::config::registry::ConfigurationRegistry; @@ -1023,10 +1031,10 @@ mod tests { &CodeIndexWorkerSelectionV1::Exact { workers: 5 }, Some(&status), ), - vec![json!({ - "field": "code_index_workers", - "message": "code_index_workers exact mode must request no more than 4 available logical CPUs", - })] + vec![SettingsValidationIssueV1 { + field: "code_index_workers".to_owned(), + message: "code_index_workers exact mode must request no more than 4 available logical CPUs".to_owned(), + }] ); } @@ -1077,10 +1085,12 @@ mod tests { &CodeIndexWorkerSelectionV1::Exact { workers: 5 }, Some(&status), ), - vec![json!({ - "field": "code_index_workers", - "message": "code_index_workers exact mode must request no more than 4 memory-safe workers", - })] + vec![SettingsValidationIssueV1 { + field: "code_index_workers".to_owned(), + message: + "code_index_workers exact mode must request no more than 4 memory-safe workers" + .to_owned(), + }] ); } } diff --git a/crates/tracedecay-dashboard-api/src/settings_api/tests/wire.rs b/crates/tracedecay-dashboard-api/src/settings_api/tests/wire.rs new file mode 100644 index 0000000000..47682ebfde --- /dev/null +++ b/crates/tracedecay-dashboard-api/src/settings_api/tests/wire.rs @@ -0,0 +1,272 @@ +//! Serialized producer fixtures shared with the dashboard behavior tests. + +use axum::Json; +use axum::body::to_bytes; +use axum::response::IntoResponse; +use serde_json::{Value, json}; +use tracedecay_api::configuration::{ + configuration_application_problem_error, configuration_authority_unavailable_error, + configuration_revision_conflict_error, parse_project_settings_patch, +}; +use tracedecay_api::read_model::{ + DashboardCoverageV1, DashboardEnvelopeV1, DashboardLegalActionKindV1, + DashboardLegalActionRefV1, DashboardScopeV1, +}; +use tracedecay_contracts::{ + ApplicationOutcome, ApplicationProblem, ApplicationProblemEnvelope, AuthorityReceipt, + CapabilityGrantId, Deadline, DisclosureClass, EffectId, EffectReceipt, EffectResult, + EffectTermination, IdempotencyKey, OperationBudgetUsage, OperationReceipt, PolicyDecisionRef, + ReconciliationState, RequestId, ResolvedScope, ResultContractRef, +}; +use tracedecay_domain::configuration::{CodeIndexWorkerSelectionV1, ConfigurationRevisionId}; +use tracedecay_domain::{ + ActorId, ComponentVersion, ManifestDigest, ProjectId, RepositoryId, UtcMicros, WorktreeId, +}; +use tracedecay_tool_catalog::{EffectClass, SchemaId, UseCaseId}; + +use super::pinned_defaults; +use crate::settings_api::{ + EnvironmentSettingsPayloadV1, PrAutoTrackPayloadV1, ProjectSettingsPatchResponseV1, + ProjectSettingsPayloadV1, SettingsPayloadV1, StorageSettingsPayloadV1, UserSettingsPayloadV1, + VersionSettingsPayloadV1, automation_settings_payload, project_editable_settings, +}; + +fn settings(revision: &str, max_file_size: u64) -> DashboardEnvelopeV1 { + let pinned = pinned_defaults(&ConfigurationRevisionId::new(revision).unwrap()); + let mut config = project_editable_settings(&pinned); + config.max_file_size = max_file_size; + let payload = SettingsPayloadV1 { + project: ProjectSettingsPayloadV1 { + configuration_snapshot_id: pinned.snapshot().snapshot_id.as_str().to_owned(), + configuration_revision_id: revision.to_owned(), + config, + pr_autotrack: PrAutoTrackPayloadV1 { tracked: vec![] }, + }, + user: UserSettingsPayloadV1 { + configuration_snapshot_id: "user-snapshot-7".to_owned(), + configuration_revision_id: "user-rev-7".to_owned(), + code_index_worker_configuration_snapshot_id: "workers-snapshot-3".to_owned(), + code_index_worker_configuration_revision_id: "workers-rev-3".to_owned(), + upload_enabled: false, + code_index_workers: CodeIndexWorkerSelectionV1::Automatic {}, + code_index_worker_status: None, + watcher_debounce: "15s".to_owned(), + extraction_timeout_secs: 30, + installed_agents: vec![], + }, + automation: automation_settings_payload(&pinned), + environment: EnvironmentSettingsPayloadV1 { + global_accounting_mode: "disabled".to_owned(), + global_accounting_enabled: false, + pricing_offline: true, + variables: vec![], + }, + storage: StorageSettingsPayloadV1 { + project_id: Some("project.settings.scout".to_owned()), + project_root: "/project".to_owned(), + storage_mode: "project_local".to_owned(), + store_root: "/project/.tracedecay".to_owned(), + dashboard_root: "/project".to_owned(), + graph_db: "/project/.tracedecay/graph.db".to_owned(), + memory_db: "/project/.tracedecay/memory.db".to_owned(), + lcm_db: "/project/.tracedecay/lcm.db".to_owned(), + lcm_scope: "project".to_owned(), + savings_db: "/project/.tracedecay/savings.db".to_owned(), + }, + version: VersionSettingsPayloadV1 { + version: "0.1.0".to_owned(), + channel: "stable".to_owned(), + cached_latest_version: None, + }, + resync_recommended: Some(max_file_size != 1_048_576), + restart_recommended: None, + }; + let mut envelope = DashboardEnvelopeV1::ready( + DashboardScopeV1 { + project_id: Some("project.settings.scout".to_owned()), + storage_mode: "project_local".to_owned(), + store_root: "/project/.tracedecay".to_owned(), + }, + DashboardCoverageV1::complete(1, "configuration_control_plane"), + payload, + ) + .with_legal_actions(vec![ + DashboardLegalActionRefV1::new( + DashboardLegalActionKindV1::RequestApply, + "configuration_batch", + ), + DashboardLegalActionRefV1::new(DashboardLegalActionKindV1::Refresh, "configuration_list"), + ]); + envelope.time.observation_time_micros = 100; + envelope.freshness.observed_at_micros = Some(100); + envelope +} + +fn digest(character: char) -> ManifestDigest { + ManifestDigest::new(format!("sha256:{}", character.to_string().repeat(64))).unwrap() +} + +fn effect() -> ApplicationOutcome { + let scope = ResolvedScope::new( + ProjectId::new("project.settings.scout").unwrap(), + RepositoryId::new("repository.settings").unwrap(), + WorktreeId::new("worktree.settings").unwrap(), + None, + ) + .unwrap(); + let idempotency_key = IdempotencyKey::new("settings-idempotency-fixture").unwrap(); + let authority = AuthorityReceipt { + grant_id: CapabilityGrantId::new("grant.settings").unwrap(), + grant_revision: 1, + grant_digest: digest('a'), + authorized_scope_digest: scope.scope_digest.clone(), + disclosure: DisclosureClass::Metadata, + policy: PolicyDecisionRef::new( + "policy.settings", + 1, + digest('b'), + ComponentVersion::new("policy.settings.v1").unwrap(), + ) + .unwrap(), + revalidated_at: UtcMicros(10), + }; + authority.validate_for(&scope).unwrap(); + ApplicationOutcome::Effect( + EffectResult::new( + EffectId::new("effect.configuration.settings").unwrap(), + EffectClass::Administrative, + idempotency_key.clone(), + authority, + digest('c'), + OperationReceipt::completed( + UtcMicros(10), + UtcMicros(20), + Deadline::new(UtcMicros(1_000)).unwrap(), + OperationBudgetUsage::default(), + ) + .unwrap(), + ReconciliationState::Reconciled, + EffectReceipt { + operation: UseCaseId::new("use-case.configuration.batch").unwrap(), + request_id: RequestId::new("request.configuration.settings-fixture").unwrap(), + actor: ActorId::new("actor.settings").unwrap(), + scope, + effect_class: EffectClass::Administrative, + idempotency_key, + input_digest: digest('a'), + expected_state: digest('c'), + policy_digest: digest('b'), + configuration_digest: digest('d'), + catalog_digest: digest('e'), + privacy_digest: digest('f'), + outcome: EffectTermination::Completed, + committed_state: Some(digest('1')), + external_proof: None, + }, + None, + ) + .unwrap(), + ) +} + +async fn http_fixture(response: impl IntoResponse) -> Value { + let response = response.into_response(); + let status = response.status().as_u16(); + let bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + let body: Value = serde_json::from_slice(&bytes).unwrap(); + json!({ "status": status, "body": body }) +} + +#[tokio::test] +async fn project_patch_serialization_preserves_settlement_and_refusals() { + let outcome = effect(); + let expected_receipt = match &outcome { + ApplicationOutcome::Effect(effect) => serde_json::to_value(&effect.receipt).unwrap(), + _ => unreachable!("fixture constructs an effect"), + }; + let changed = http_fixture(Json(ProjectSettingsPatchResponseV1 { + application_outcome: Some(outcome), + current: settings("rev-43", 2_097_152), + })) + .await; + assert_eq!(changed["status"], 200); + assert_eq!(changed["body"]["application_outcome"]["outcome"], "effect"); + assert_eq!( + changed["body"]["application_outcome"]["value"]["receipt"], + expected_receipt + ); + assert_eq!( + changed["body"]["current"]["payload"]["project"]["configuration_revision_id"], + "rev-43" + ); + let unchanged = http_fixture(Json(ProjectSettingsPatchResponseV1 { + application_outcome: None, + current: settings("rev-42", 1_048_576), + })) + .await; + assert!( + unchanged["body"] + .as_object() + .unwrap() + .contains_key("application_outcome") + ); + assert!(unchanged["body"]["application_outcome"].is_null()); + + let idempotency_conflict = ApplicationProblemEnvelope::new( + ResultContractRef::new( + SchemaId::new("schema.configuration.batch.result").unwrap(), + 1, + ) + .unwrap(), + RequestId::new("request.configuration.settings-fixture").unwrap(), + ApplicationProblem::conflict( + "configuration.conflict", + "The idempotency key was already used for a different settings change.", + ), + ) + .unwrap(); + let expected_problem = serde_json::to_value(&idempotency_conflict).unwrap(); + let idempotency = http_fixture(configuration_application_problem_error( + idempotency_conflict, + )) + .await; + assert_eq!(idempotency["status"], 409); + assert_eq!(idempotency["body"], expected_problem); + + let validation = parse_project_settings_patch(json!({ + "expected_revision_id": "rev-42", + "idempotency_key": "settings-idempotency-fixture", + "unknown_setting": true, + })) + .expect_err("unknown fields must be refused"); + let cancelled = ApplicationProblemEnvelope::new( + ResultContractRef::new( + SchemaId::new("schema.configuration.batch.result").unwrap(), + 1, + ) + .unwrap(), + RequestId::new("request.configuration.settings-fixture").unwrap(), + ApplicationProblem::cancelled_before_admission(), + ) + .unwrap(); + let fixtures = json!({ + "read": http_fixture(Json(settings("rev-42", 1_048_576))).await, + "changed": changed, + "unchanged": unchanged, + "revision_conflict": http_fixture(configuration_revision_conflict_error( + "settings changed after this edit began; refresh and retry", "rev-42", "rev-43" + )).await, + "validation": http_fixture(validation).await, + "unavailable": http_fixture(configuration_authority_unavailable_error()).await, + "idempotency_conflict": idempotency, + "cancelled": http_fixture(configuration_application_problem_error(cancelled)).await, + }); + // Bazel preserves this producer output for the frontend's shared fixture. + if let Some(output) = std::env::var_os("TEST_UNDECLARED_OUTPUTS_DIR") { + std::fs::write( + std::path::PathBuf::from(output).join("settings-patch.json"), + format!("{}\n", serde_json::to_string_pretty(&fixtures).unwrap()), + ) + .unwrap(); + } +} diff --git a/dashboard/codegen/src/generate.ts b/dashboard/codegen/src/generate.ts index 3ecd2815cd..1c3d4fc890 100644 --- a/dashboard/codegen/src/generate.ts +++ b/dashboard/codegen/src/generate.ts @@ -167,6 +167,7 @@ function isTaggedUnion(schema: JsonSchema): boolean { interface ResolveCtx { genericParam?: string; genericSchemaVar?: string; + envelopePayloads: ReadonlyMap; } interface Resolved { @@ -197,6 +198,16 @@ function resolveType(schema: JsonSchema, ctx: ResolveCtx): Resolved { if (schema.$ref) { const name = refName(schema.$ref); + const payload = ctx.envelopePayloads.get(name); + if (payload) { + const resolved = resolveType(payload, ctx); + const ts = `DashboardEnvelopeV1<${resolved.ts}>`; + return { + ts, + zod: `z.lazy(() => DashboardEnvelopeV1Schema(${resolved.zod}))`, + ann: typeAnn(ts), + }; + } return { ts: name, zod: `z.lazy(() => ${name}Schema)`, ann: typeAnn(name) }; } @@ -393,14 +404,18 @@ interface EmittedDef { decoders: string; } -function emitNamedDef(name: string, schema: JsonSchema): EmittedDef { +function emitNamedDef( + name: string, + schema: JsonSchema, + envelopePayloads: ReadonlyMap, +): EmittedDef { const genericParam = schema["x-generic"]; const desc = schema.description ? `/** ${schema.description} */\n` : ""; if (genericParam) { // Generic type + decoder factory (DashboardEnvelope). const schemaVar = "payloadSchema"; - const ctx: ResolveCtx = { genericParam, genericSchemaVar: schemaVar }; + const ctx: ResolveCtx = { genericParam, genericSchemaVar: schemaVar, envelopePayloads }; const obj = resolveObject(schema, ctx); return { typeName: name, @@ -415,7 +430,7 @@ function emitNamedDef(name: string, schema: JsonSchema): EmittedDef { }; } - const ctx: ResolveCtx = {}; + const ctx: ResolveCtx = { envelopePayloads }; if (isTaggedUnion(schema)) { const union = resolveTaggedUnion(schema, ctx); @@ -457,12 +472,14 @@ export function generateContracts(bundles: JsonSchema[]): GeneratedContracts { const schemaRevision = revisions[0] as string | number; const omittedNames = new Set(["DashboardPayloadMarkerV1"]); + const envelopePayloads = new Map(); const envelopeIndex = defs.findIndex(([name]) => name === "DashboardEnvelopeV1"); const envelope = defs[envelopeIndex]?.[1]; if (envelope?.properties) { for (const [name, schema] of defs) { if (name !== "DashboardEnvelopeV1" && isMonomorphizedEnvelope(schema, envelope)) { omittedNames.add(name); + envelopePayloads.set(name, schema.properties!.payload!); } } defs[envelopeIndex] = [ @@ -482,13 +499,15 @@ export function generateContracts(bundles: JsonSchema[]): GeneratedContracts { const emitted: EmittedDef[] = []; for (const [name, schema] of defs) { if (omittedNames.has(name)) continue; - const omittedRef = [...localDefinitionRefs(schema)].find((ref) => omittedNames.has(ref)); + const omittedRef = [...localDefinitionRefs(schema)].find( + (ref) => omittedNames.has(ref) && !envelopePayloads.has(ref), + ); if (omittedRef) { throw new Error( `${name} references omitted generated definition ${omittedRef}`, ); } - emitted.push(emitNamedDef(name, schema)); + emitted.push(emitNamedDef(name, schema, envelopePayloads)); } const types = [ diff --git a/dashboard/codegen/tests/generate.test.ts b/dashboard/codegen/tests/generate.test.ts index bcae67b313..d6ba9dd293 100644 --- a/dashboard/codegen/tests/generate.test.ts +++ b/dashboard/codegen/tests/generate.test.ts @@ -164,14 +164,14 @@ describe("contracts generator", () => { expect(generated).toContain("export const DashboardEnvelopeV1MetadataSchema"); }); - it("rejects an emitted contract that still references an omitted envelope instance", () => { + it("rejects an emitted contract that references the untyped payload marker", () => { const bundle = structuredClone(bundles[0]!); bundle.$defs!.EnvelopeConsumer = { - $ref: "#/$defs/DashboardEnvelopeV12", + $ref: "#/$defs/DashboardPayloadMarkerV1", }; expect(() => generateContracts([bundle])).toThrow( - "EnvelopeConsumer references omitted generated definition DashboardEnvelopeV12", + "EnvelopeConsumer references omitted generated definition DashboardPayloadMarkerV1", ); }); @@ -216,6 +216,28 @@ describe("contracts generator", () => { const bundle: JsonSchema = { schemaRevision: "test.1", $defs: { + DashboardPayloadMarkerV1: { type: "null" }, + DashboardEnvelopeV1: { + type: "object", + properties: { + schema_revision: { type: "string" }, + payload: { $ref: "#/$defs/DashboardPayloadMarkerV1" }, + }, + required: ["schema_revision", "payload"], + }, + NodeEnvelope: { + type: "object", + properties: { + schema_revision: { type: "string" }, + payload: { $ref: "#/$defs/Node" }, + }, + required: ["schema_revision", "payload"], + }, + EnvelopeConsumer: { + type: "object", + properties: { current: { $ref: "#/$defs/NodeEnvelope" } }, + required: ["current"], + }, Node: { type: "object", additionalProperties: false, @@ -278,12 +300,15 @@ import { ChoiceSchema, ClosedReadingSchema, DashboardDomainStateV1Schema, + EnvelopeConsumerSchema, NodeSchema, OpenReadingSchema, ResultSchema, StatusSchema, type Choice, type ClosedReading, + type DashboardEnvelopeV1, + type EnvelopeConsumer, type Node, type OpenReading, type Result, @@ -293,6 +318,10 @@ import { const catchInput: z.input = 42; const valid: Node = { id: "root", label: null, metadata: null, child: { id: "leaf", label: "ok", metadata: {} } }; const decoded: Node = NodeSchema.parse(valid); +const nested: EnvelopeConsumer = { current: { schema_revision: "test.1", payload: valid } }; +const envelope: DashboardEnvelopeV1 = EnvelopeConsumerSchema.parse(nested).current; +// @ts-expect-error a nested envelope keeps its payload contract +const invalidNested: EnvelopeConsumer = { current: { schema_revision: "test.1", payload: { id: 42 } } }; const inferred: z.infer = valid; // @ts-expect-error required nullable fields cannot be omitted const missing: Node = { id: "root" }; @@ -315,7 +344,7 @@ function unwrap(result: Result): Node | undefined { default: return assertNever(result); } } -export { catchInput, decoded, inferred, missing, invalidChild, invalidStatus, closedMissing, closed, open, extended, field, status, choice, unwrap }; +export { catchInput, decoded, nested, envelope, invalidNested, inferred, missing, invalidChild, invalidStatus, closedMissing, closed, open, extended, field, status, choice, unwrap }; `); checkContractTypes(directory, ["decoders.ts", "consumer.ts"]); // Node's loader avoids waiting on Vitest's shared transform server. @@ -327,6 +356,14 @@ export { catchInput, decoded, inferred, missing, invalidChild, invalidStatus, cl child: { id: "leaf", label: "ok", metadata: 1 }, }; expect(runtime.NodeSchema!.parse(node)).toEqual(node); + const nested = { current: { schema_revision: "test.1", payload: node } }; + expect(runtime.EnvelopeConsumerSchema!.parse(nested)).toEqual(nested); + expect(runtime.EnvelopeConsumerSchema!.safeParse({ + current: { schema_revision: "test.1", payload: { ...node, id: 42 } }, + }).success).toBe(false); + expect(runtime.EnvelopeConsumerSchema!.safeParse({ + current: { schema_revision: "future", payload: node }, + }).success).toBe(false); expect(runtime.NodeSchema!.safeParse({ id: "root", metadata: 1 }).success).toBe(false); expect(runtime.NodeSchema!.safeParse({ ...node, extra: true }).success).toBe(false); expect(runtime.NodeSchema!.safeParse({ From f5e226e8d84c05880ab3386a097a574be3cb4dfc Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 18 Oct 2026 01:13:17 -0700 Subject: [PATCH 03/89] fix(ci): resolve export paths in the runner steps --- .github/workflows/dashboard-contract-export.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dashboard-contract-export.yml b/.github/workflows/dashboard-contract-export.yml index a4eba1eb74..c9aeab7cda 100644 --- a/.github/workflows/dashboard-contract-export.yml +++ b/.github/workflows/dashboard-contract-export.yml @@ -20,7 +20,6 @@ jobs: timeout-minutes: 90 env: CARGO_BAZEL_TIMEOUT: "3600" - EXPORT_DIRECTORY: ${{ runner.temp }}/dashboard-contract-export steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -40,6 +39,7 @@ jobs: - name: Export the canonical dashboard schema run: | set -euo pipefail + EXPORT_DIRECTORY="$RUNNER_TEMP/dashboard-contract-export" mkdir -p "$EXPORT_DIRECTORY" bazel run --config=ci --jobs=1 //sdks/codegen:dashboard_schema -- "$EXPORT_DIRECTORY/dashboard-contracts.schema.json" test -s "$EXPORT_DIRECTORY/dashboard-contracts.schema.json" @@ -56,6 +56,7 @@ jobs: - name: Serialize the real settings response fixture run: | set -euo pipefail + EXPORT_DIRECTORY="$RUNNER_TEMP/dashboard-contract-export" bazel test --config=ci --jobs=1 //crates/tracedecay-dashboard-api:unit_test \ --test_arg=settings_api::tests::wire::project_patch_serialization_preserves_settlement_and_refusals \ --test_arg=--exact --test_output=all --cache_test_results=no \ From 6fef056b11b2ac843bf82e43be0c70b686c5fbb5 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 12:33:47 +0000 Subject: [PATCH 04/89] fix: remove the worldwide token counter --- .cargo/config.toml | 2 +- README.md | 22 +- SECURITY.md | 2 - crates/tracedecay-api/src/configuration.rs | 19 +- crates/tracedecay-cli/src/cli.rs | 14 - crates/tracedecay-cli/src/cli/help.rs | 23 -- crates/tracedecay-cli/src/cloud.rs | 99 +----- crates/tracedecay-cli/src/commands.rs | 9 +- .../tracedecay-cli/src/commands/settings.rs | 86 ------ crates/tracedecay-cli/src/global.rs | 69 ----- crates/tracedecay-cli/src/main.rs | 90 +----- .../tracedecay-cli/src/native_root_tests.rs | 3 +- crates/tracedecay-cli/src/startup_tests.rs | 27 +- .../tests/core_cli_suite/tool_daemon_test.rs | 65 ++-- .../tests/core_cli_suite/user_config_test.rs | 6 +- .../sweep_outcomes.rs | 2 +- .../src/configuration/user_settings.rs | 28 +- crates/tracedecay-dashboard-api/src/cloud.rs | 18 +- .../src/settings_api.rs | 3 - crates/tracedecay-domain/src/configuration.rs | 11 +- crates/tracedecay-domain/src/lib.rs | 14 +- .../src/configuration/registry.rs | 17 +- .../src/configuration/resolver.rs | 6 +- .../src/configuration/store.rs | 30 +- .../configuration/store/tests/persistence.rs | 125 +++++++- crates/tracedecay-global-db/src/support.rs | 4 +- .../src/user_config.rs | 65 ++-- .../configuration_idempotency_journey_test.rs | 6 +- crates/tracedecay/src/doctor.rs | 220 ++------------ crates/tracedecay/src/doctor/tests.rs | 44 +-- crates/tracedecay/src/mcp/server.rs | 5 - .../tracedecay/src/mcp/server/connection.rs | 44 +-- crates/tracedecay/src/mcp/server/ledger.rs | 284 +++--------------- crates/tracedecay/src/mcp/server/requests.rs | 1 - .../tests/dashboard_api_test/settings.rs | 29 +- .../tests/mcp_suite/mcp_cli_serve_test.rs | 24 +- .../runtime_surface_acceptance.rs | 6 +- .../AnalyticsControls.dom.test.tsx | 131 ++------ .../observatory/AnalyticsControls.tsx | 60 +--- .../observatory/analyticsControls.test.ts | 12 +- .../observatory/analyticsControls.ts | 28 +- .../src/workspaces/observatory/evidence.ts | 2 +- .../settings/SettingsPage.dom.test.tsx | 1 - .../settings/settingsEditorMachine.ts | 1 - .../src/workspaces/settings/settingsModel.ts | 8 +- .../workspaces/settings/settingsRows.test.ts | 2 +- .../src/workspaces/settings/settingsRows.ts | 3 - dashboard/stories/fixtures/data.ts | 1 - docs/REBRAND-COMPATIBILITY-POLICY.md | 11 +- docs/USER-GUIDE.md | 57 ++-- evals/agent_adoption/run.sh | 1 - scripts/mcp-conformance-smoke.sh | 1 - 52 files changed, 437 insertions(+), 1404 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index c12efa73a9..28ad4ef952 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -31,7 +31,7 @@ CXXFLAGS_aarch64_pc_windows_msvc = { value = "-DNDEBUG /utf-8", force = false } # connections. Nothing reads those statistics or sets a heap limit. LIBSQLITE3_FLAGS = { value = "-DSQLITE_MAX_WORKER_THREADS=15 -DSQLITE_DEFAULT_MEMSTATUS=0", force = true } -# Global accounting (savings ledger, worldwide-counter flushes) is ON by +# Global accounting (the savings ledger) is ON by # default for installed binaries, but `cargo test` / `cargo run` would then # write temp-project rows into the developer's real ~/.tracedecay/global.db. # Keep cargo-launched processes hermetic; tests that exercise the ledger diff --git a/README.md b/README.md index fb0965ed87..e4417998e7 100644 --- a/README.md +++ b/README.md @@ -195,8 +195,6 @@ authorized, sanitized, classified records under their remote policy. Network-capable effects are separate from core retrieval: -- Worldwide counter: when opted in, sends the pending aggregate token-savings - amount, and `tracedecay doctor` reads the public total to check reachability. - GitHub release checks and updates: contact GitHub release endpoints. An explicitly configured private GitHub review source can use a read-only credential from the operating-system keyring; missing, ambiguous, @@ -212,19 +210,13 @@ Network-capable effects are separate from core retrieval: path fails closed with typed `unavailable`, `denied`, or `stale-peer` state. These services receive normal transport metadata, including the connection's -source address, in addition to any request payload. The counter request is an -aggregate amount, not repository content, but it is not an anonymity guarantee. -Use a network policy or firewall if a command must make no outbound connection. -An opt-out, denial, timeout, or unavailable remote service must remain visible -as that state; it is not evidence of a zero counter, an up-to-date release, or -current pricing. See [Security](SECURITY.md#network-access) for the complete -outbound-access and credential boundary. - -Disable the worldwide counter with: - -```bash -tracedecay disable-upload-counter -``` +source address, in addition to any request payload. Use a network policy or +firewall if a command must make no outbound connection. A denial, timeout, or +unavailable remote service must remain visible as that state; it is not +evidence of an up-to-date release or current pricing. TraceDecay no longer has +a worldwide token counter: token savings and costs stay local. See +[Security](SECURITY.md#network-access) for the complete outbound-access and +credential boundary. ## Troubleshooting diff --git a/SECURITY.md b/SECURITY.md index 2d0d2dce1d..80992d8d4b 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -67,7 +67,6 @@ Outbound capabilities include: | Configured remote sources and authorities | Policy-authorized retrieval and remote operations | Configured peer authentication and disclosure policy | Admission or authorization failures are reported; review each peer before enabling | | `github.com` | Download binary during `tracedecay upgrade` | None (public releases) | Error shown to user | | `huggingface.co` and Hugging Face artifact hosts | Download missing, revision-pinned semantic-model artifacts when semantic auto-download is enabled | None | Semantic retrieval reports model acquisition state or failure; exact, lexical, and graph retrieval remain available | -| `tracedecay-counter.enzinol.workers.dev` | Aggregate token-savings counter | None | Silently ignored | Provider usage and pricing do not add an outbound connection. `tracedecay cost` reads immutable provider-native usage observations and the deterministic bundled @@ -169,7 +168,6 @@ an old count of unsafe sites is not a security guarantee. The following are **not** security issues: -- The aggregate token counter sending a count to the public Cloudflare Worker endpoint (this is documented behavior and contains no identifying information beyond an approximate country derived from IP by Cloudflare) - The database containing symbol names or file paths from your project (this is core functionality) - The MCP edit tools modifying files (this is opt-in functionality your AI agent invokes through its own tool-approval flow) - `tracedecay_run_affected_tests` compiling and running your project's own test suite (this is the tool's documented purpose) diff --git a/crates/tracedecay-api/src/configuration.rs b/crates/tracedecay-api/src/configuration.rs index 4817efb7a1..dc04cf01dc 100644 --- a/crates/tracedecay-api/src/configuration.rs +++ b/crates/tracedecay-api/src/configuration.rs @@ -72,8 +72,6 @@ pub struct UserSettingsPatch { pub expected_revision_id: String, pub idempotency_key: String, #[serde(default, skip_serializing_if = "Option::is_none")] - pub upload_enabled: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] pub watcher_debounce: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub extraction_timeout_secs: Option, @@ -121,7 +119,6 @@ pub fn parse_code_index_worker_settings_patch( /// write that unsigned value and do not parse the wire string again. #[derive(Debug, Clone, PartialEq, Eq)] pub struct AdmittedUserSettingsPatch { - pub upload_enabled: Option, pub watcher_debounce_ms: Option, pub extraction_timeout_secs: Option, } @@ -154,7 +151,6 @@ pub fn admit_user_settings_patch( } if errors.is_empty() { Ok(AdmittedUserSettingsPatch { - upload_enabled: patch.upload_enabled, watcher_debounce_ms, extraction_timeout_secs: patch.extraction_timeout_secs, }) @@ -332,7 +328,7 @@ mod tests { let result = parse_user_settings_patch(json!({ "expected_revision_id": "configuration.revision.project", "idempotency_key": "configuration.idempotency.fixture", - "upload_enabled": true, + "watcher_debounce": "2s", "code_index_workers": { "mode": "automatic" } })); @@ -340,4 +336,17 @@ mod tests { assert_eq!(status, StatusCode::BAD_REQUEST); assert_eq!(body["validation_errors"][0]["field"], "code_index_workers"); } + + #[test] + fn user_patch_refuses_the_retired_worldwide_counter_upload_field() { + let result = parse_user_settings_patch(json!({ + "expected_revision_id": "configuration.revision.profile", + "idempotency_key": "configuration.idempotency.fixture", + "upload_enabled": true + })); + + let (status, Json(body)) = result.expect_err("a retired field must be refused"); + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(body["validation_errors"][0]["field"], "upload_enabled"); + } } diff --git a/crates/tracedecay-cli/src/cli.rs b/crates/tracedecay-cli/src/cli.rs index 7eb5537359..eaa11bd830 100644 --- a/crates/tracedecay-cli/src/cli.rs +++ b/crates/tracedecay-cli/src/cli.rs @@ -578,20 +578,6 @@ pub enum Commands { #[arg(short, long)] path: Option, }, - /// Disable uploading token counts to the worldwide counter - #[command( - name = "disable-upload-counter", - long_about = DISABLE_UPLOAD_COUNTER_LONG_ABOUT, - after_help = DISABLE_UPLOAD_COUNTER_AFTER_HELP - )] - DisableUploadCounter, - /// Enable uploading token counts to the worldwide counter - #[command( - name = "enable-upload-counter", - long_about = ENABLE_UPLOAD_COUNTER_LONG_ABOUT, - after_help = ENABLE_UPLOAD_COUNTER_AFTER_HELP - )] - EnableUploadCounter, /// Check tracedecay installation, configuration, and agent integration #[command(long_about = DOCTOR_LONG_ABOUT, after_help = DOCTOR_AFTER_HELP)] Doctor { diff --git a/crates/tracedecay-cli/src/cli/help.rs b/crates/tracedecay-cli/src/cli/help.rs index 77bc4f4833..a0d8e2c722 100644 --- a/crates/tracedecay-cli/src/cli/help.rs +++ b/crates/tracedecay-cli/src/cli/help.rs @@ -446,29 +446,6 @@ Examples: Related: tracedecay current-counter, tracedecay gain."; -pub(crate) const DISABLE_UPLOAD_COUNTER_LONG_ABOUT: &str = "\ -Opts this machine out of contributing anonymous token-savings counts to the \ -public worldwide counter. Only aggregate numbers are ever uploaded, never \ -code, paths, or queries, but uploading is entirely optional."; - -pub(crate) const DISABLE_UPLOAD_COUNTER_AFTER_HELP: &str = "\ -Examples: - tracedecay disable-upload-counter Stop contributing counts - -Related: tracedecay enable-upload-counter, tracedecay gain (local ledger -is unaffected)."; - -pub(crate) const ENABLE_UPLOAD_COUNTER_LONG_ABOUT: &str = "\ -Re-enables contributing anonymous token-savings counts to the public \ -worldwide counter after a previous `disable-upload-counter`. Only aggregate \ -numbers are uploaded, never code, paths, or queries."; - -pub(crate) const ENABLE_UPLOAD_COUNTER_AFTER_HELP: &str = "\ -Examples: - tracedecay enable-upload-counter Resume contributing counts - -Related: tracedecay disable-upload-counter, tracedecay gain."; - pub(crate) const DOCTOR_LONG_ABOUT: &str = "\ Checks the binary, PATH, daemon service, project index, and every agent \ integration, printing actionable fixes for anything broken. The running \ diff --git a/crates/tracedecay-cli/src/cloud.rs b/crates/tracedecay-cli/src/cloud.rs index ced9744617..27618d8b48 100644 --- a/crates/tracedecay-cli/src/cloud.rs +++ b/crates/tracedecay-cli/src/cloud.rs @@ -1,7 +1,5 @@ -//! HTTP client for the worldwide counter Cloudflare Worker and GitHub release -//! version checking. +//! HTTP client for GitHub release version checking. //! -//! Counter operations are best-effort: failures surface as `None` / empty. //! Release lookups classify every failure as a [`ReleaseLookupError`]. All //! are synchronous `ureq` calls that block the calling thread for up to their //! own timeout, which an enclosing Tokio deadline cannot cut short. A caller @@ -14,20 +12,14 @@ use serde::de::DeserializeOwned; use tracedecay_application::http_agent::http_agent; use tracedecay_dashboard_api::cloud::ReleaseLookupError; -/// The Cloudflare Worker endpoint URL. -const WORKER_URL: &str = "https://tracedecay-counter.enzinol.workers.dev"; - /// Base URL of the GitHub REST API release lookups are issued against. pub(crate) const GITHUB_API_URL: &str = "https://api.github.com"; const GITHUB_OWNER: &str = "ScriptedAlchemy"; const GITHUB_REPOSITORY: &str = "tracedecay"; -/// Timeout for flush (upload) requests. -const FLUSH_TIMEOUT: Duration = Duration::from_secs(2); - -/// Timeout for opportunistic advisory reads (worldwide counter and the -/// background / `doctor` version probe). Keep this short so a stalled -/// GitHub or worker never blocks those paths. +/// Timeout for opportunistic advisory reads (the background / `doctor` +/// version probe). Keep this short so a stalled GitHub never blocks those +/// paths. const FETCH_TIMEOUT: Duration = Duration::from_secs(1); /// Bound for each GitHub release-metadata read of an explicit `tracedecay @@ -38,53 +30,6 @@ const FETCH_TIMEOUT: Duration = Duration::from_secs(1); /// keep [`FETCH_TIMEOUT`]. pub(crate) const UPGRADE_RELEASE_LOOKUP_TIMEOUT: Duration = Duration::from_secs(30); -/// Response from the worker's POST /increment and GET /total endpoints. -#[derive(serde::Deserialize)] -struct WorkerResponse { - total: u64, -} - -/// Creates a ureq agent with the given timeout. -pub fn agent_with_timeout(timeout: Duration) -> ureq::Agent { - http_agent( - ureq::Agent::config_builder() - .timeout_global(Some(timeout)) - .build(), - ) -} - -/// Uploads pending tokens to the worldwide counter. -/// Returns the new worldwide total on success, or None on any failure. -#[tracing::instrument(name = "cloud.flush_pending", level = "trace", skip_all)] -pub fn flush_pending(amount: u64) -> Option { - if amount == 0 { - return None; - } - let body = serde_json::json!({ "amount": amount }); - let agent = agent_with_timeout(FLUSH_TIMEOUT); - let parsed: WorkerResponse = agent - .post(&format!("{WORKER_URL}/increment")) - .send_json(&body) - .ok()? - .body_mut() - .read_json() - .ok()?; - Some(parsed.total) -} - -/// Fetches the current worldwide total from the worker. -/// Returns None on timeout, network error, or parse failure. -#[tracing::instrument(name = "cloud.fetch_worldwide_total", level = "trace", skip_all)] -pub fn fetch_worldwide_total() -> Option { - fetch_worldwide_total_from(&format!("{WORKER_URL}/total")) -} - -fn fetch_worldwide_total_from(url: &str) -> Option { - let agent = agent_with_timeout(FETCH_TIMEOUT); - let parsed: WorkerResponse = agent.get(url).call().ok()?.body_mut().read_json().ok()?; - Some(parsed.total) -} - /// Response from GitHub releases API (only the fields we need). #[derive(serde::Deserialize)] struct GitHubRelease { @@ -356,16 +301,15 @@ pub fn is_beta() -> bool { tracedecay_dashboard_api::cloud::is_beta(env!("CARGO_PKG_VERSION")) } -/// Admits these ureq implementations into the composition library so MCP -/// flush/version checks can run after the binary starts. Unregistered -/// lookups stay `None` (already best-effort). +/// Admits the ureq release lookup into the composition library so MCP version +/// checks can run after the binary starts. An unregistered lookup stays +/// `None` (already best-effort). pub fn admit_sync_probes() { - tracedecay_dashboard_api::cloud::admit_sync_cloud_probes(flush_pending, fetch_latest_version); + tracedecay_dashboard_api::cloud::admit_sync_cloud_probes(fetch_latest_version); } pub fn doctor_network_probes() -> tracedecay::doctor::AdmittedDoctorNetworkProbes { tracedecay::doctor::AdmittedDoctorNetworkProbes { - fetch_worldwide_total, fetch_latest_version, } } @@ -570,33 +514,6 @@ mod tests { ); } - #[test] - fn advisory_worldwide_total_still_times_out_at_the_one_second_budget() { - let (base, _heads) = stub_after( - Duration::from_millis(1500), - Some(respond( - "200 OK", - "Content-Type: application/json\r\n", - r#"{"total":42}"#, - )), - ); - let started = Instant::now(); - - assert!( - fetch_worldwide_total_from(&format!("{base}/total")).is_none(), - "advisory worldwide-total must keep the one-second budget" - ); - let elapsed = started.elapsed(); - assert!( - elapsed >= FETCH_TIMEOUT, - "advisory worldwide-total returned before its one-second budget: {elapsed:?}" - ); - assert!( - elapsed < Duration::from_millis(1500), - "advisory worldwide-total waited for the delayed answer: {elapsed:?}" - ); - } - fn respond(status: &str, headers: &str, body: &str) -> &'static str { format!( "HTTP/1.1 {status}\r\n{headers}Content-Length: {}\r\nConnection: close\r\n\r\n{body}", diff --git a/crates/tracedecay-cli/src/commands.rs b/crates/tracedecay-cli/src/commands.rs index d86586b06b..be8a552d0c 100644 --- a/crates/tracedecay-cli/src/commands.rs +++ b/crates/tracedecay-cli/src/commands.rs @@ -25,12 +25,11 @@ pub(crate) use index::{handle_init, handle_no_command, handle_sync}; pub(crate) use profile_storage::handle_profile_storage_action; pub(crate) use scope::resolve_project_scope; pub(crate) use settings::{ - canonical_upload_enabled, current_configuration_revision, current_project_setting, - handle_upload_counter, mutate_project_configuration, project_configuration_set, - repeated_query_hints_enabled, report_configuration_receipt, + current_configuration_revision, current_project_setting, mutate_project_configuration, + project_configuration_set, repeated_query_hints_enabled, report_configuration_receipt, }; pub(crate) use stale_store_reset::handle_wipe_stale; pub(crate) use storage::{ - ProfileOfflineAuthority, RestoreWording, annotate_reset_required, handle_list, handle_wipe, - join_outcome_and_restore, process_error_text, take_profile_offline, try_admit_profile_registry, + ProfileOfflineAuthority, RestoreWording, handle_list, handle_wipe, join_outcome_and_restore, + process_error_text, take_profile_offline, try_admit_profile_registry, }; diff --git a/crates/tracedecay-cli/src/commands/settings.rs b/crates/tracedecay-cli/src/commands/settings.rs index 435110761c..e203031ac6 100644 --- a/crates/tracedecay-cli/src/commands/settings.rs +++ b/crates/tracedecay-cli/src/commands/settings.rs @@ -17,7 +17,6 @@ use tracedecay_daemon_protocol::RequestedOutputFormat; use tracedecay_domain::configuration::{ ConfigurationIdempotencyKey, ConfigurationLayerIdV1, ConfigurationRevisionId, ConfigurationValueV1, SettingKey, USER_REPEATED_QUERY_HINTS_SETTING_KEY, - USER_UPLOAD_ENABLED_SETTING_KEY, UserProfileId, }; use tracedecay_domain::{ProjectId, UtcMicros, canonical_sha256}; use tracedecay_tool_catalog::ApplicationSurfaceOperation; @@ -43,25 +42,6 @@ fn cli_configuration_idempotency_key( .map_err(|error| config_error(format!("invalid configuration request key: {error}"))) } -fn cli_user_configuration_idempotency_key( - profile_id: &UserProfileId, - expected_revision: &ConfigurationRevisionId, - mutations: &[ConfigurationDirectMutationRequestV1], -) -> tracedecay_domain::errors::Result { - let digest = canonical_sha256(&( - "tracedecay.cli.user-configuration-mutation.v1", - profile_id, - expected_revision, - mutations, - )) - .map_err(|error| config_error(format!("invalid user configuration mutation: {error}")))?; - let suffix = digest - .hex_suffix() - .ok_or_else(|| config_error("user configuration mutation digest is malformed"))?; - ConfigurationIdempotencyKey::new(format!("configuration.idempotency.cli.user.{suffix}")) - .map_err(|error| config_error(format!("invalid user configuration request key: {error}"))) -} - fn configuration_deadline( operation: ApplicationSurfaceOperation, observed_at: UtcMicros, @@ -210,13 +190,6 @@ async fn resolved_setting( .map_err(|error| config_error(format!("invalid configuration setting: {error}"))) } -/// The profile's worldwide-counter upload setting, wherever the command runs. -pub(crate) async fn canonical_upload_enabled( - profile: &ProfileRoot, -) -> tracedecay_domain::errors::Result { - boolean_setting(&resolved_setting(profile, None, USER_UPLOAD_ENABLED_SETTING_KEY).await?) -} - /// The profile's experimental repeated-query hint setting for `serve`. pub(crate) async fn repeated_query_hints_enabled( profile: &ProfileRoot, @@ -280,31 +253,6 @@ pub(crate) async fn mutate_project_configuration( configuration_effect_receipt(envelope, &idempotency_key).map(Some) } -async fn mutate_user_configuration( - profile: &ProfileRoot, - profile_id: &UserProfileId, - expected_revision: ConfigurationRevisionId, - mutations: Vec, -) -> tracedecay_domain::errors::Result> { - if mutations.is_empty() { - return Ok(None); - } - let idempotency_key = - cli_user_configuration_idempotency_key(profile_id, &expected_revision, &mutations)?; - let envelope = invoke_configuration_surface( - profile, - None, - ApplicationSurfaceOperation::ConfigurationBatch, - ConfigurationWireRequestV1::Batch(ConfigurationBatchRequestV1 { - mutations, - expected_revision, - idempotency_key: idempotency_key.clone(), - }), - ) - .await?; - configuration_effect_receipt(envelope, &idempotency_key).map(Some) -} - fn configuration_effect_receipt( envelope: ApplicationEnvelope, idempotency_key: &ConfigurationIdempotencyKey, @@ -347,37 +295,3 @@ pub(crate) fn report_configuration_receipt(receipt: Option<&EffectReceipt>) { eprintln!("Receipt: {}", receipt.request_id.as_str()); } } - -#[tracing::instrument(name = "cli.settings.upload_counter", level = "trace", skip_all)] -pub(crate) async fn handle_upload_counter( - profile: &ProfileRoot, - enable: bool, -) -> tracedecay_domain::errors::Result<()> { - let current = resolved_setting(profile, None, USER_UPLOAD_ENABLED_SETTING_KEY).await?; - let profile_id = - tracedecay_daemon_identity::profile_identity::load_existing(profile.data_dir())? - .profile_id() - .clone(); - let mutations = if boolean_setting(¤t)? != enable { - vec![ConfigurationDirectMutationRequestV1::Set { - layer: ConfigurationLayerIdV1::UserProfile { - profile_id: profile_id.clone(), - }, - key: current.key, - value: Box::new(ConfigurationValueV1::Boolean(enable)), - }] - } else { - Vec::new() - }; - let receipt = - mutate_user_configuration(profile, &profile_id, current.revision_id, mutations).await?; - if enable { - eprintln!("Worldwide counter upload enabled."); - } else { - eprintln!( - "Worldwide counter upload disabled. You can re-enable with `tracedecay enable-upload-counter`." - ); - } - report_configuration_receipt(receipt.as_ref()); - Ok(()) -} diff --git a/crates/tracedecay-cli/src/global.rs b/crates/tracedecay-cli/src/global.rs index c96a487b65..5ea8e37a9c 100644 --- a/crates/tracedecay-cli/src/global.rs +++ b/crates/tracedecay-cli/src/global.rs @@ -32,55 +32,6 @@ pub(crate) async fn classify_project_storage_with_registry( .await } -/// Returns how many seconds have elapsed since a persisted timestamp. -/// -/// User config timestamps can land in the future because of clock skew, -/// manual edits, or state copied across machines. Clamp those cases to zero so -/// cooldown/version-cache logic degrades gracefully instead of getting stuck on -/// a negative delta. -fn elapsed_since(now: i64, recorded_at: i64) -> i64 { - if recorded_at >= now { - 0 - } else { - now - recorded_at - } -} - -/// Best-effort: try to flush pending tokens to the worldwide counter. -/// -/// `upload_enabled` is the already-resolved canonical user-profile setting; -/// `config` only carries the pending counter and cooldown timestamps. -/// `force` = true on status/sync commands (always attempt), false on others -/// (only flush if stale > 30s). -pub(crate) fn try_flush( - config: &mut tracedecay_session_memory::user_config::UserConfig, - force: bool, - upload_enabled: bool, -) { - if config.pending_upload == 0 || !upload_enabled { - return; - } - let now = tracedecay_runtime_core::tracedecay::current_timestamp(); - - // Cooldown: skip if last flush attempt failed less than 60s ago - if config.last_flush_attempt_at > config.last_upload_at - && elapsed_since(now, config.last_flush_attempt_at) < 60 - { - return; - } - - // Staleness check for non-force commands - if !force && elapsed_since(now, config.last_upload_at) < 30 { - return; - } - - config.last_flush_attempt_at = now; - if crate::cloud::flush_pending(config.pending_upload).is_some() { - config.pending_upload = 0; - config.last_upload_at = now; - } -} - /// Returns the project paths the `wipe` command should act on. /// /// `--all` returns every path tracked in the global DB (including stale rows). @@ -702,24 +653,4 @@ mod gather_tests { vec![std::path::PathBuf::from("/repos/a")] ); } - - #[test] - fn elapsed_since_clamps_future_timestamps() { - assert_eq!(elapsed_since(100, 40), 60); - assert_eq!(elapsed_since(100, 100), 0); - assert_eq!(elapsed_since(100, 140), 0); - } - - #[test] - fn canonical_upload_denial_leaves_pending_tokens_unflushed() { - let mut config = tracedecay_session_memory::user_config::UserConfig { - pending_upload: 42, - ..tracedecay_session_memory::user_config::UserConfig::default() - }; - - try_flush(&mut config, true, false); - - assert_eq!(config.pending_upload, 42); - assert_eq!(config.last_flush_attempt_at, 0); - } } diff --git a/crates/tracedecay-cli/src/main.rs b/crates/tracedecay-cli/src/main.rs index f31f3bf161..3e1280fb3e 100644 --- a/crates/tracedecay-cli/src/main.rs +++ b/crates/tracedecay-cli/src/main.rs @@ -733,7 +733,7 @@ async fn run(cli: Cli) -> tracedecay_domain::errors::Result { } }; - run_startup_preamble(&profile, &command).await; + run_startup_preamble(&profile, &command); dispatch_command(&profile, command, host_bundle).await } @@ -754,28 +754,9 @@ fn command_profile(command: Option<&Commands>) -> tracedecay_domain::errors::Res } #[tracing::instrument(name = "cli.startup.preamble", level = "trace", skip_all)] -async fn run_startup_preamble(profile: &ProfileRoot, command: &Commands) { +fn run_startup_preamble(profile: &ProfileRoot, command: &Commands) { let startup_policy = CommandStartupPolicy::for_command(command); - // Check first-run before any config save creates the file. - let profile_root = profile.data_dir(); - let is_first_run = !tracedecay_session_memory::user_config::UserConfig::exists(profile_root); - - let is_force_flush = matches!(command, Commands::Sync { .. } | Commands::Status { .. }); - match tracedecay_session_memory::user_config::UserConfig::load(profile_root) { - Ok(user_config) => { - flush_worldwide_counter(profile, command, is_force_flush, user_config).await; - } - Err(err) => eprintln!("warning: {err}"), - } - - if is_first_run && startup_policy.runs_startup_maintenance() { - eprintln!( - "note: tracedecay can optionally upload anonymous token savings counts to a worldwide counter.\n\ - \x20 Run `tracedecay enable-upload-counter` to opt in." - ); - } - if startup_policy.runs_agent_install_check() && let Some(home) = profile.home() { @@ -783,52 +764,6 @@ async fn run_startup_preamble(profile: &ProfileRoot, command: &Commands) { } } -async fn flush_worldwide_counter( - profile: &ProfileRoot, - command: &Commands, - is_force_flush: bool, - mut user_config: tracedecay_session_memory::user_config::UserConfig, -) { - let profile_root = profile.data_dir(); - // Skip the worldwide-counter flush on hot startup paths. `try_flush` - // makes a synchronous HTTP call which can add seconds to - // `tracedecay serve` startup on slow networks, long enough to blow the - // MCP client's 30 s `initialize` timeout. The canonical setting lookup is - // only consulted when there are pending tokens to flush: with nothing - // pending the setting cannot change behavior, and probing it on every - // command turned the daemon's transient "runtime still mounting" state - // into per-command stderr noise. A failed lookup on an ordinary command - // is deferred (the next command retries); the flush-bearing commands - // (`sync`, `status`) still surface it, so a persistent failure - // stays visible exactly where the flush is expected to happen. - // The setting belongs to the profile, so the flush does not depend on - // the current directory. - if runs_worldwide_counter_flush(command) && user_config.pending_upload > 0 { - match commands::canonical_upload_enabled(profile).await { - Ok(upload_enabled) => { - global::try_flush(&mut user_config, is_force_flush, upload_enabled); - } - Err(error) if is_force_flush => { - eprintln!( - "warning: canonical worldwide-counter upload setting is unavailable: {}", - commands::annotate_reset_required(error, None) - ); - } - Err(error) => { - tracing::debug!( - "worldwide-counter flush deferred: canonical upload setting unavailable: \ - {error}" - ); - } - } - } - if !is_local_install_command(command) - && let Err(err) = user_config.save_if_exists(profile_root) - { - eprintln!("warning: could not save tracedecay config: {err}"); - } -} - async fn resolve_registered_project_root( profile: &ProfileRoot, project_id: Option, @@ -975,10 +910,7 @@ impl CommandFamily { | Commands::PostUpdate { .. } | Commands::PackageHook { .. } | Commands::Channel { .. } => Self::Update, - Commands::CurrentCounter { .. } - | Commands::ResetCounter { .. } - | Commands::DisableUploadCounter - | Commands::EnableUploadCounter => Self::Configuration, + Commands::CurrentCounter { .. } | Commands::ResetCounter { .. } => Self::Configuration, Commands::Doctor { .. } | Commands::Cost { .. } | Commands::Gain { .. } @@ -1822,12 +1754,6 @@ async fn dispatch_configuration_command( } eprintln!("Local counter reset (was {prev})"); } - Commands::DisableUploadCounter => { - commands::handle_upload_counter(profile, false).await?; - } - Commands::EnableUploadCounter => { - commands::handle_upload_counter(profile, true).await?; - } _ => unreachable!("non-configuration command passed to configuration dispatcher"), } Ok(()) @@ -2023,6 +1949,7 @@ impl CommandStartupPolicy { } } + #[cfg(test)] fn runs_startup_maintenance(self) -> bool { !matches!(self, Self::SkipAll) } @@ -2032,11 +1959,6 @@ impl CommandStartupPolicy { } } -fn runs_worldwide_counter_flush(command: &Commands) -> bool { - !matches!(command, Commands::Init { .. }) - && CommandStartupPolicy::for_command(command).runs_startup_maintenance() -} - #[cfg(test)] fn should_skip_startup_maintenance(command: &Commands) -> bool { !CommandStartupPolicy::for_command(command).runs_startup_maintenance() @@ -2047,10 +1969,6 @@ fn should_skip_agent_install_check(command: &Commands) -> bool { !CommandStartupPolicy::for_command(command).runs_agent_install_check() } -fn is_local_install_command(command: &Commands) -> bool { - matches!(command, Commands::Install { local: true, .. }) -} - fn normalize_tool_reserved_global_flags(cli: &mut Cli) { if !cli.dry_run { return; diff --git a/crates/tracedecay-cli/src/native_root_tests.rs b/crates/tracedecay-cli/src/native_root_tests.rs index 88cfe50643..6f0681009d 100644 --- a/crates/tracedecay-cli/src/native_root_tests.rs +++ b/crates/tracedecay-cli/src/native_root_tests.rs @@ -31,7 +31,7 @@ fn run_on_native_root(profile: &ProfileRoot, args: &[&str]) -> Result (Option, V (output.status.code(), body) } -fn run_upload_counter_command(home: &Path, cwd: &Path, command: &str) { - let output = tracedecay_command_with_home(home) - .current_dir(cwd) - .arg(command) - .stdin(Stdio::null()) - .output() - .expect("upload counter command should run"); - assert!(output.status.success(), "{command} failed: {output:?}"); -} - /// A user-scoped setting belongs to the profile, so it resolves wherever the -/// command runs: a value written inside a project is the one doctor and -/// `configuration_get` report outside any project, and a write made outside a -/// project is the one the project then reads. A project-scoped key still needs -/// a project. +/// command runs: the same value is reported inside and outside any project. A +/// project-scoped key still needs a project, and the retired worldwide-counter +/// upload setting is no longer a registered key. #[test] fn user_settings_resolve_from_the_profile_outside_any_project() { let home = TempDir::new().unwrap(); @@ -4060,40 +4049,22 @@ fn user_settings_resolve_from_the_profile_outside_any_project() { init_committed_git_project_with_cli(&home, &project); let _daemon = spawn_tracedecay_daemon(&home); - run_upload_counter_command(&home, &project, "enable-upload-counter"); - - let doctor = tracedecay_command_with_home(&home) - .current_dir(&outside) - .arg("doctor") - .stdin(Stdio::null()) - .output() - .expect("doctor should run"); - let doctor_stderr = String::from_utf8_lossy(&doctor.stderr); - assert!( - doctor_stderr.contains("Worldwide counter upload enabled"), - "doctor outside a project must report the profile's upload setting:\n{doctor_stderr}" - ); - assert!( - !doctor_stderr.contains("upload setting unavailable") - && !doctor_stderr.contains("canonical configuration is unavailable"), - "doctor outside a project reported the user setting unavailable:\n{doctor_stderr}" - ); - - let (code, enabled) = configuration_get_from(&home, &outside, "user.upload_enabled.v1"); - assert_eq!(code, Some(0), "{enabled}"); - assert_eq!( - enabled["structuredContent"]["outcome"]["value"]["payload"]["effective_value"], - json!({ "kind": "boolean", "value": true }), - "{enabled}" - ); + for cwd in [&outside, &project] { + let (code, hints) = configuration_get_from(&home, cwd, "user.repeated_query_hints.v1"); + assert_eq!(code, Some(0), "{hints}"); + assert_eq!( + hints["structuredContent"]["outcome"]["value"]["payload"]["effective_value"], + json!({ "kind": "boolean", "value": false }), + "the profile setting must resolve from {}: {hints}", + cwd.display() + ); + } - run_upload_counter_command(&home, &outside, "disable-upload-counter"); - let (code, disabled) = configuration_get_from(&home, &project, "user.upload_enabled.v1"); - assert_eq!(code, Some(0), "{disabled}"); - assert_eq!( - disabled["structuredContent"]["outcome"]["value"]["payload"]["effective_value"], - json!({ "kind": "boolean", "value": false }), - "a write made outside a project must be the value the project reads: {disabled}" + let (code, retired) = configuration_get_from(&home, &outside, "user.upload_enabled.v1"); + assert_ne!( + code, + Some(0), + "the retired upload setting must be refused, not read: {retired}" ); let (code, refused) = configuration_get_from(&home, &outside, "index.max_file_size.v1"); diff --git a/crates/tracedecay-cli/tests/core_cli_suite/user_config_test.rs b/crates/tracedecay-cli/tests/core_cli_suite/user_config_test.rs index 381ba9dd64..ce0365ebdd 100644 --- a/crates/tracedecay-cli/tests/core_cli_suite/user_config_test.rs +++ b/crates/tracedecay-cli/tests/core_cli_suite/user_config_test.rs @@ -2,9 +2,9 @@ use tracedecay_session_memory::user_config::UserConfig; #[test] fn missing_fields_use_defaults() { - let toml_str = "pending_upload = 5\n"; + let toml_str = "last_version_check_at = 5\n"; let parsed: UserConfig = toml::from_str(toml_str).unwrap(); - assert_eq!(parsed.pending_upload, 5); - assert_eq!(parsed.last_upload_at, 0); + assert_eq!(parsed.last_version_check_at, 5); + assert_eq!(parsed.last_version_warning_at, 0); assert!(parsed.installed_agents.is_empty()); } diff --git a/crates/tracedecay-cli/tests/host_journeys_suite/host_lifecycle_cli_acceptance/sweep_outcomes.rs b/crates/tracedecay-cli/tests/host_journeys_suite/host_lifecycle_cli_acceptance/sweep_outcomes.rs index f61c947ef8..e447a37183 100644 --- a/crates/tracedecay-cli/tests/host_journeys_suite/host_lifecycle_cli_acceptance/sweep_outcomes.rs +++ b/crates/tracedecay-cli/tests/host_journeys_suite/host_lifecycle_cli_acceptance/sweep_outcomes.rs @@ -518,7 +518,7 @@ fn doctor_fails_a_corrupt_profile_config_with_its_repair() { let _daemon = ProfileDaemon::start(&cli); let config = cli.profile.join("config.toml"); let installed = fs::read_to_string(&config).unwrap(); - fs::write(&config, "pending_upload = 0\n true").unwrap(); + fs::write(&config, "last_version_check_at = 0\n true").unwrap(); let doctor = cli.run(&["doctor"]); let doctor_stderr = stderr(&doctor); diff --git a/crates/tracedecay-configuration/src/configuration/user_settings.rs b/crates/tracedecay-configuration/src/configuration/user_settings.rs index eb490cd0e2..5b3a8c6c10 100644 --- a/crates/tracedecay-configuration/src/configuration/user_settings.rs +++ b/crates/tracedecay-configuration/src/configuration/user_settings.rs @@ -12,8 +12,7 @@ use std::pin::Pin; use tracedecay_automation::config::AutomationConfig; use tracedecay_domain::configuration::{ ConfigurationLayerIdV1, ConfigurationValueV1, SettingKey, - USER_EXTRACTION_TIMEOUT_SECS_SETTING_KEY, USER_UPLOAD_ENABLED_SETTING_KEY, - USER_WATCHER_DEBOUNCE_MS_SETTING_KEY, UserProfileId, + USER_EXTRACTION_TIMEOUT_SECS_SETTING_KEY, USER_WATCHER_DEBOUNCE_MS_SETTING_KEY, UserProfileId, }; use tracedecay_contracts::now_micros; @@ -29,7 +28,6 @@ pub type UserSettingsFuture<'a, T> = pub struct UserSettingsSnapshotV1 { pub configuration_snapshot_id: String, pub configuration_revision_id: String, - pub upload_enabled: bool, pub watcher_debounce: String, pub watcher_debounce_ms: u64, pub extraction_timeout_secs: u64, @@ -40,7 +38,6 @@ pub struct UserSettingsSnapshotV1 { #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct UserSettingsMutationV1 { - pub upload_enabled: Option, /// Positive milliseconds already parsed at the settings boundary. pub watcher_debounce_ms: Option, pub extraction_timeout_secs: Option, @@ -142,14 +139,12 @@ fn user_settings_snapshot( current: &ProfileConfigurationV1, metadata: UserMetadata, ) -> Result { - let upload_enabled = required_bool(current, USER_UPLOAD_ENABLED_SETTING_KEY)?; let watcher_debounce_ms = required_unsigned(current, USER_WATCHER_DEBOUNCE_MS_SETTING_KEY)?; let extraction_timeout_secs = required_unsigned(current, USER_EXTRACTION_TIMEOUT_SECS_SETTING_KEY)?; Ok(UserSettingsSnapshotV1 { configuration_snapshot_id: current.snapshot.snapshot_id.as_str().to_owned(), configuration_revision_id: current.revision_id.as_str().to_owned(), - upload_enabled, watcher_debounce: format_duration_millis(watcher_debounce_ms), watcher_debounce_ms, extraction_timeout_secs, @@ -173,13 +168,6 @@ pub fn plan_user_settings_mutation( let mut mutations = Vec::new(); let mut restart_recommended = false; - if let Some(upload_enabled) = mutation.upload_enabled { - mutations.push(set( - layer.clone(), - USER_UPLOAD_ENABLED_SETTING_KEY, - ConfigurationValueV1::Boolean(upload_enabled), - )?); - } if let Some(watcher_debounce_ms) = mutation.watcher_debounce_ms { mutations.push(set( layer.clone(), @@ -235,16 +223,6 @@ fn setting_key(raw_key: &str) -> Result .map_err(|error| unavailable(format!("registered user setting key: {error}"))) } -fn required_bool( - current: &ProfileConfigurationV1, - raw_key: &str, -) -> Result { - match current.value(raw_key) { - Ok(ConfigurationValueV1::Boolean(value)) => Ok(*value), - _ => Err(unavailable(format!("boolean user setting {raw_key}"))), - } -} - fn required_unsigned( current: &ProfileConfigurationV1, raw_key: &str, @@ -271,7 +249,6 @@ mod tests { UserSettingsSnapshotV1 { configuration_snapshot_id: "configuration.snapshot.fixture".to_owned(), configuration_revision_id: "configuration.revision.fixture".to_owned(), - upload_enabled: false, watcher_debounce: "2s".to_owned(), watcher_debounce_ms: 2_000, extraction_timeout_secs: 60, @@ -288,14 +265,13 @@ mod tests { &snapshot(), profile_id.clone(), UserSettingsMutationV1 { - upload_enabled: Some(true), watcher_debounce_ms: Some(15_000), extraction_timeout_secs: Some(60), }, ) .unwrap(); - assert_eq!(plan.mutations.len(), 3); + assert_eq!(plan.mutations.len(), 2); assert!(plan.restart_recommended); assert!(plan.mutations.iter().any(|mutation| matches!( mutation, diff --git a/crates/tracedecay-dashboard-api/src/cloud.rs b/crates/tracedecay-dashboard-api/src/cloud.rs index 598e6e4ab3..8f436a98e1 100644 --- a/crates/tracedecay-dashboard-api/src/cloud.rs +++ b/crates/tracedecay-dashboard-api/src/cloud.rs @@ -164,27 +164,17 @@ fn http_date(unix_seconds: u64) -> String { httpdate::fmt_http_date(UNIX_EPOCH + Duration::from_secs(unix_seconds)) } -type FlushPending = fn(u64) -> Option; type FetchLatestVersion = fn() -> Result; -static FLUSH_PENDING: OnceLock = OnceLock::new(); static FETCH_LATEST_VERSION: OnceLock = OnceLock::new(); -/// Admits the CLI binary's sync ureq implementations. A second admission is -/// ignored so tests that re-enter process start stay idempotent. Unregistered -/// lookups return `None`. -pub fn admit_sync_cloud_probes( - flush_pending: FlushPending, - fetch_latest_version: FetchLatestVersion, -) { - let _ = FLUSH_PENDING.set(flush_pending); +/// Admits the CLI binary's sync ureq release lookup. A second admission is +/// ignored so tests that re-enter process start stay idempotent. An +/// unregistered lookup returns `None`. +pub fn admit_sync_cloud_probes(fetch_latest_version: FetchLatestVersion) { let _ = FETCH_LATEST_VERSION.set(fetch_latest_version); } -pub fn flush_pending(amount: u64) -> Option { - FLUSH_PENDING.get().and_then(|flush| flush(amount)) -} - /// Set by test harnesses so no spawned CLI or daemon asks GitHub for the /// latest release; the advisory version check then has no answer. pub const NO_RELEASE_CHECK_ENV: &str = "TRACEDECAY_NO_RELEASE_CHECK"; diff --git a/crates/tracedecay-dashboard-api/src/settings_api.rs b/crates/tracedecay-dashboard-api/src/settings_api.rs index 5d69f5d0b1..4056a5a43d 100644 --- a/crates/tracedecay-dashboard-api/src/settings_api.rs +++ b/crates/tracedecay-dashboard-api/src/settings_api.rs @@ -212,7 +212,6 @@ struct UserSettingsPayloadV1 { /// user settings revision above. code_index_worker_configuration_snapshot_id: String, code_index_worker_configuration_revision_id: String, - upload_enabled: bool, code_index_workers: CodeIndexWorkerSelectionV1, /// The admitted process-wide plan. A persisted selection may differ until /// the daemon restarts, so unavailable is carried as `null` rather than @@ -431,7 +430,6 @@ pub async fn patch_user_settings( ¤t, profile_id, UserSettingsMutationV1 { - upload_enabled: admitted.upload_enabled, watcher_debounce_ms: admitted.watcher_debounce_ms, extraction_timeout_secs: admitted.extraction_timeout_secs, }, @@ -678,7 +676,6 @@ fn user_settings_payload( code_index_worker_configuration_revision_id: worker_configuration .configuration_revision_id .clone(), - upload_enabled: user.upload_enabled, code_index_workers: worker_configuration.code_index_workers, code_index_worker_status: tracedecay_code_index::parallelism::installed_worker_status(), watcher_debounce: user.watcher_debounce.clone(), diff --git a/crates/tracedecay-domain/src/configuration.rs b/crates/tracedecay-domain/src/configuration.rs index 8b771a20c6..304f2e80fd 100644 --- a/crates/tracedecay-domain/src/configuration.rs +++ b/crates/tracedecay-domain/src/configuration.rs @@ -61,13 +61,18 @@ pub const RETIRED_CORE_SETTING_KEYS_V1: &[&str] = &[ "index.git_ignore.v1", "semantic.runtime.v1", "sync.orphan_db_gc_days.v1", - USER_UPLOAD_ENABLED_SETTING_KEY, + "user.upload_enabled.v1", USER_WATCHER_DEBOUNCE_MS_SETTING_KEY, USER_EXTRACTION_TIMEOUT_SECS_SETTING_KEY, ]; +/// Profile setting keys that shipped and were then retired. A profile store +/// carrying one converges by dropping it on the next read; the value has no +/// registered owner anymore. `user.upload_enabled.v1` was the opt-in of the +/// worldwide token counter, removed without a final upload. +pub const RETIRED_PROFILE_SETTING_KEYS_V1: &[&str] = &["user.upload_enabled.v1"]; + /// Canonical user-profile settings. -pub const USER_UPLOAD_ENABLED_SETTING_KEY: &str = "user.upload_enabled.v1"; pub const USER_REPEATED_QUERY_HINTS_SETTING_KEY: &str = "user.repeated_query_hints.v1"; pub const USER_CODE_INDEX_WORKERS_SETTING_KEY: &str = "user.code_index_workers.v1"; pub const USER_WATCHER_DEBOUNCE_MS_SETTING_KEY: &str = "user.watcher_debounce_ms.v1"; @@ -80,7 +85,6 @@ pub const USER_WORK_EXPERTISE_CONSENT_SETTING_KEY: &str = "user.work_expertise_c /// Work read pins it together with the project's consent in one revision. pub const PROFILE_SETTING_KEYS_V1: &[&str] = &[ USER_CODE_INDEX_WORKERS_SETTING_KEY, - USER_UPLOAD_ENABLED_SETTING_KEY, USER_REPEATED_QUERY_HINTS_SETTING_KEY, USER_WATCHER_DEBOUNCE_MS_SETTING_KEY, USER_EXTRACTION_TIMEOUT_SECS_SETTING_KEY, @@ -129,7 +133,6 @@ pub const CONFIGURATION_SETTING_KEYS_V1: &[&str] = &[ AUTOMATION_SETTINGS_SETTING_KEY, LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, crate::feedback::PROXIMITY_RISK_THRESHOLD_SETTING_KEY_V1, - USER_UPLOAD_ENABLED_SETTING_KEY, USER_REPEATED_QUERY_HINTS_SETTING_KEY, USER_CODE_INDEX_WORKERS_SETTING_KEY, USER_WATCHER_DEBOUNCE_MS_SETTING_KEY, diff --git a/crates/tracedecay-domain/src/lib.rs b/crates/tracedecay-domain/src/lib.rs index 93fce9395a..3c73c73636 100644 --- a/crates/tracedecay-domain/src/lib.rs +++ b/crates/tracedecay-domain/src/lib.rs @@ -125,13 +125,13 @@ pub use configuration::{ TELEMETRY_TIMINGS_SETTING_KEY, TopologyConcurrencyPolicyV1, TopologyEscalationPolicyV1, TopologyGatePolicyV1, TopologyNotificationLevelV1, TopologyPolicyDigestV1, USER_CODE_INDEX_WORKERS_SETTING_KEY, USER_EXTRACTION_TIMEOUT_SECS_SETTING_KEY, - USER_REPEATED_QUERY_HINTS_SETTING_KEY, USER_UPLOAD_ENABLED_SETTING_KEY, - USER_WATCHER_DEBOUNCE_MS_SETTING_KEY, USER_WORK_EXPERTISE_CONSENT_SETTING_KEY, UserProfileId, - WORK_EXECUTABLE_BINDINGS_SETTING_KEY, WORK_TOPOLOGY_POLICY_SETTING_KEY, - WorkExecutableBindingV1, WorkExecutableCapabilityV1, WorkExpertiseCategoryV1, - WorkExpertiseConsentV1, WorkTopologyPolicyV1, WorktreeCleanlinessRequirementV1, - WorktreePlacementModeV1, WorktreePlacementRootId, WorktreeRetentionPolicyV1, - WorktreeRootPolicyV1, resolve_restrictive_capabilities, safe_work_topology_policy_v1, + USER_REPEATED_QUERY_HINTS_SETTING_KEY, USER_WATCHER_DEBOUNCE_MS_SETTING_KEY, + USER_WORK_EXPERTISE_CONSENT_SETTING_KEY, UserProfileId, WORK_EXECUTABLE_BINDINGS_SETTING_KEY, + WORK_TOPOLOGY_POLICY_SETTING_KEY, WorkExecutableBindingV1, WorkExecutableCapabilityV1, + WorkExpertiseCategoryV1, WorkExpertiseConsentV1, WorkTopologyPolicyV1, + WorktreeCleanlinessRequirementV1, WorktreePlacementModeV1, WorktreePlacementRootId, + WorktreeRetentionPolicyV1, WorktreeRootPolicyV1, resolve_restrictive_capabilities, + safe_work_topology_policy_v1, }; pub use cursor_binding::{ CURSOR_INVALID_CODE, CURSOR_PARAMETER_CHANGED_CODE, CursorBindingBuilderV1, diff --git a/crates/tracedecay-global-db/src/configuration/registry.rs b/crates/tracedecay-global-db/src/configuration/registry.rs index 6645a1aa37..e31e1ece16 100644 --- a/crates/tracedecay-global-db/src/configuration/registry.rs +++ b/crates/tracedecay-global-db/src/configuration/registry.rs @@ -23,10 +23,9 @@ use tracedecay_domain::configuration::{ SYNC_WATCH_MAX_DELAY_MS_SETTING_KEY, SYNC_WATCH_MAX_PROJECTS_SETTING_KEY, SettingDefinitionV1, SettingKey, SettingScopeV1, SettingSensitivityV1, TELEMETRY_TIMINGS_SETTING_KEY, USER_CODE_INDEX_WORKERS_SETTING_KEY, USER_EXTRACTION_TIMEOUT_SECS_SETTING_KEY, - USER_REPEATED_QUERY_HINTS_SETTING_KEY, USER_UPLOAD_ENABLED_SETTING_KEY, - USER_WATCHER_DEBOUNCE_MS_SETTING_KEY, USER_WORK_EXPERTISE_CONSENT_SETTING_KEY, - WORK_EXECUTABLE_BINDINGS_SETTING_KEY, WORK_TOPOLOGY_POLICY_SETTING_KEY, WorkExpertiseConsentV1, - safe_work_topology_policy_v1, + USER_REPEATED_QUERY_HINTS_SETTING_KEY, USER_WATCHER_DEBOUNCE_MS_SETTING_KEY, + USER_WORK_EXPERTISE_CONSENT_SETTING_KEY, WORK_EXECUTABLE_BINDINGS_SETTING_KEY, + WORK_TOPOLOGY_POLICY_SETTING_KEY, WorkExpertiseConsentV1, safe_work_topology_policy_v1, }; use tracedecay_domain::feedback::PROXIMITY_RISK_THRESHOLD_SETTING_KEY_V1; use tracedecay_domain::{DomainError, validate_index_path_patterns}; @@ -415,11 +414,6 @@ fn register_profile_scalar_settings( registry: &mut ConfigurationRegistry, ) -> Result<(), ConfigurationRegistryError> { for (key, default_value, restart_requirement) in [ - ( - USER_UPLOAD_ENABLED_SETTING_KEY, - ConfigurationValueV1::Boolean(false), - RestartRequirementV1::None, - ), ( USER_REPEATED_QUERY_HINTS_SETTING_KEY, ConfigurationValueV1::Boolean(false), @@ -862,11 +856,6 @@ mod user_profile_settings_tests { fn editable_profile_settings_are_registered_with_exact_scope_and_restart_semantics() { let registry = ConfigurationRegistry::profile().expect("registry"); for (raw_key, kind, restart) in [ - ( - USER_UPLOAD_ENABLED_SETTING_KEY, - ConfigurationValueKindV1::Boolean, - RestartRequirementV1::None, - ), ( USER_REPEATED_QUERY_HINTS_SETTING_KEY, ConfigurationValueKindV1::Boolean, diff --git a/crates/tracedecay-global-db/src/configuration/resolver.rs b/crates/tracedecay-global-db/src/configuration/resolver.rs index 233bb9b55d..2dffabd8ab 100644 --- a/crates/tracedecay-global-db/src/configuration/resolver.rs +++ b/crates/tracedecay-global-db/src/configuration/resolver.rs @@ -220,7 +220,7 @@ mod tests { }, revision_id: id("revision.project.1"), entries: BTreeMap::from([( - id("user.upload_enabled.v1"), + id("user.repeated_query_hints.v1"), ConfigurationValueV1::Boolean(true), )]), }; @@ -228,7 +228,7 @@ mod tests { resolve_configuration(®istry, &[layer]) .unwrap_err() .to_string(), - "setting user.upload_enabled.v1 cannot be placed in Project { project_id: ProjectId(\"project.fixture\") }" + "setting user.repeated_query_hints.v1 cannot be placed in Project { project_id: ProjectId(\"project.fixture\") }" ); let user_layer = ConfigurationLayerV1 { @@ -237,7 +237,7 @@ mod tests { }, revision_id: id("revision.profile.1"), entries: BTreeMap::from([( - id("user.upload_enabled.v1"), + id("user.repeated_query_hints.v1"), ConfigurationValueV1::Boolean(true), )]), }; diff --git a/crates/tracedecay-global-db/src/configuration/store.rs b/crates/tracedecay-global-db/src/configuration/store.rs index e8eed26376..b7a4948960 100644 --- a/crates/tracedecay-global-db/src/configuration/store.rs +++ b/crates/tracedecay-global-db/src/configuration/store.rs @@ -18,10 +18,10 @@ use tracedecay_domain::configuration::{ ConfigurationReceiptId, ConfigurationRevisionId, ConfigurationSnapshotV1, ConfigurationValueV1, INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, ProtectedChange, ProtectedChangePlan, ProtectedChangeSnapshotError, - RETIRED_CORE_SETTING_KEYS_V1, RedactedConfigurationChangeV1, RollbackModeV1, - SOURCE_BINDINGS_SETTING_KEY, SYNC_WATCH_LINKED_WORKTREES_SETTING_KEY, ScopeControlOperationV1, - SettingKey, USER_CODE_INDEX_WORKERS_SETTING_KEY, USER_EXTRACTION_TIMEOUT_SECS_SETTING_KEY, - USER_REPEATED_QUERY_HINTS_SETTING_KEY, USER_UPLOAD_ENABLED_SETTING_KEY, + RETIRED_CORE_SETTING_KEYS_V1, RETIRED_PROFILE_SETTING_KEYS_V1, RedactedConfigurationChangeV1, + RollbackModeV1, SOURCE_BINDINGS_SETTING_KEY, SYNC_WATCH_LINKED_WORKTREES_SETTING_KEY, + ScopeControlOperationV1, SettingKey, USER_CODE_INDEX_WORKERS_SETTING_KEY, + USER_EXTRACTION_TIMEOUT_SECS_SETTING_KEY, USER_REPEATED_QUERY_HINTS_SETTING_KEY, USER_WATCHER_DEBOUNCE_MS_SETTING_KEY, UserProfileId, WORK_TOPOLOGY_POLICY_SETTING_KEY, }; use tracedecay_domain::{AccessPolicyDigest, ActorId, ManifestDigest, UtcMicros, canonical_sha256}; @@ -666,7 +666,6 @@ pub struct ProfileCodeIndexWorkerCommitV1 { /// Profile settings registered after a profile store was first initialized. /// A store written before one existed converges by adding its default. const PROFILE_ADDITIVE_SETTING_KEYS: &[&str] = &[ - USER_UPLOAD_ENABLED_SETTING_KEY, USER_REPEATED_QUERY_HINTS_SETTING_KEY, USER_WATCHER_DEBOUNCE_MS_SETTING_KEY, USER_EXTRACTION_TIMEOUT_SECS_SETTING_KEY, @@ -725,12 +724,19 @@ impl<'db> ProfileConfigurationStore<'db> { } } let current = self.current_state().await?; - if registry.definitions().all(|definition| { - current - .snapshot - .effective_values - .contains_key(&definition.key) - }) { + let carries_retired_key = current.snapshot.effective_values.keys().any(|key| { + RETIRED_PROFILE_SETTING_KEYS_V1 + .iter() + .any(|retired| key.as_str() == *retired) + }); + if !carries_retired_key + && registry.definitions().all(|definition| { + current + .snapshot + .effective_values + .contains_key(&definition.key) + }) + { return Self::project(&self.profile_id, current, ®istry); } let converged = match self @@ -740,7 +746,7 @@ impl<'db> ProfileConfigurationStore<'db> { occurred_at, ®istry, PROFILE_ADDITIVE_SETTING_KEYS, - &[], + RETIRED_PROFILE_SETTING_KEYS_V1, ) .await { diff --git a/crates/tracedecay-global-db/src/configuration/store/tests/persistence.rs b/crates/tracedecay-global-db/src/configuration/store/tests/persistence.rs index 61ccf8c36a..a4913c6b31 100644 --- a/crates/tracedecay-global-db/src/configuration/store/tests/persistence.rs +++ b/crates/tracedecay-global-db/src/configuration/store/tests/persistence.rs @@ -18,8 +18,9 @@ use crate::configuration::registry::ConfigurationRegistry; use crate::configuration::resolver::resolve_configuration; use tracedecay_domain::configuration::{ CodeIndexWorkerSelectionV1, ConfigurationIdempotencyKey, ConfigurationLayerIdV1, - ConfigurationMutationOperationV1, SettingKey, USER_CODE_INDEX_WORKERS_SETTING_KEY, - USER_REPEATED_QUERY_HINTS_SETTING_KEY, USER_UPLOAD_ENABLED_SETTING_KEY, + ConfigurationMutationOperationV1, ConfigurationValueKindV1, DeprecationStateV1, + RestartRequirementV1, SettingDefinitionV1, SettingKey, SettingScopeV1, SettingSensitivityV1, + USER_CODE_INDEX_WORKERS_SETTING_KEY, USER_REPEATED_QUERY_HINTS_SETTING_KEY, }; #[asupersync::test] @@ -176,10 +177,6 @@ async fn worker_only_profile_store_converges_onto_the_profile_registry() { converged.code_index_workers().unwrap().selection, CodeIndexWorkerSelectionV1::Exact { workers: 8 } ); - assert_eq!( - converged.value(USER_UPLOAD_ENABLED_SETTING_KEY).unwrap(), - &ConfigurationValueV1::Boolean(false) - ); assert_eq!( converged .value(USER_REPEATED_QUERY_HINTS_SETTING_KEY) @@ -188,6 +185,122 @@ async fn worker_only_profile_store_converges_onto_the_profile_registry() { ); } +/// A profile store written while the worldwide counter shipped carries the +/// `user.upload_enabled.v1` opt-in, possibly enabled. It converges on the +/// first read: the retired key is dropped without any upload, the operator's +/// other values survive, and a second read finds nothing left to do. +#[asupersync::test] +async fn profile_store_carrying_the_retired_upload_setting_converges_and_keeps_other_values() { + let directory = tempfile::tempdir().unwrap(); + let profile_root = directory.path().join("profile"); + let runtime = crate::tests::harness::HostAdmissionTestRuntimeV1::profile(&profile_root) + .await + .unwrap(); + let db = runtime + .registered_database(HostAdmissionScope::Profile) + .unwrap(); + let profile_id = db.binding().shard_id.profile_id.clone(); + let upload_key = SettingKey::new("user.upload_enabled.v1").unwrap(); + let worker_key = SettingKey::new(USER_CODE_INDEX_WORKERS_SETTING_KEY).unwrap(); + assert!( + ConfigurationRegistry::profile() + .unwrap() + .definition(&upload_key) + .is_err(), + "the current profile registry must not register the retired key" + ); + let mut legacy_definitions = ConfigurationRegistry::profile() + .unwrap() + .definitions() + .cloned() + .collect::>(); + legacy_definitions.push(SettingDefinitionV1 { + key: upload_key.clone(), + schema_revision: crate::configuration::registry::CONFIGURATION_REGISTRY_SCHEMA_REVISION, + value_kind: ConfigurationValueKindV1::Boolean, + default_value: ConfigurationValueV1::Boolean(false), + sensitivity: SettingSensitivityV1::Public, + scope: SettingScopeV1::UserProfile, + restart_requirement: RestartRequirementV1::None, + deprecation: DeprecationStateV1::Active, + }); + let legacy = ConfigurationRegistry::from_definitions(legacy_definitions).unwrap(); + let initial_revision = id("configuration.profile.initial.v1"); + GlobalDbConfigurationControlStore::new_registered(db) + .initialize_canonical_with_registry( + &initial_revision, + &resolve_configuration(&legacy, &[]).unwrap(), + UtcMicros(1), + &legacy, + ) + .await + .unwrap(); + let layer = ConfigurationLayerIdV1::UserProfile { + profile_id: profile_id.clone(), + }; + let mut revision = initial_revision; + for (name, key, value) in [ + ( + "worker", + worker_key.clone(), + ConfigurationValueV1::CodeIndexWorkerSelection(CodeIndexWorkerSelectionV1::Exact { + workers: 8, + }), + ), + ( + "upload", + upload_key.clone(), + ConfigurationValueV1::Boolean(true), + ), + ] { + let authority = control_authority_with_key_for_layer( + ConfigurationMutationOperationV1::DirectMutation, + &revision, + Some( + ConfigurationIdempotencyKey::new(format!( + "configuration.idempotency.legacy-{name}" + )) + .unwrap(), + ), + layer.clone(), + ); + let transaction = db.begin_write_transaction().await.unwrap(); + let committed = super::super::mutation::commit_direct_in_transaction_with_registry( + &transaction, + &authority, + &DirectConfigurationMutation::Set { + layer: layer.clone(), + key, + value: Box::new(value), + }, + &revision, + &legacy, + ) + .await + .unwrap(); + transaction.commit().await.unwrap(); + revision = committed.current.revision_id; + } + + let store = super::super::ProfileConfigurationStore::new_registered(db, &profile_id).unwrap(); + let converged = store.read_or_initialize(UtcMicros(3)).await.unwrap(); + + assert_ne!(converged.revision_id, revision); + assert!( + !converged + .snapshot + .effective_values + .contains_key(&upload_key), + "the retired upload opt-in must be dropped, not carried" + ); + assert_eq!( + converged.code_index_workers().unwrap().selection, + CodeIndexWorkerSelectionV1::Exact { workers: 8 } + ); + let settled = store.read_or_initialize(UtcMicros(4)).await.unwrap(); + assert_eq!(settled.revision_id, converged.revision_id); +} + #[asupersync::test] async fn project_revision_store_rejects_profile_worker_snapshot() { let (_directory, runtime, root) = global_setup().await; diff --git a/crates/tracedecay-global-db/src/support.rs b/crates/tracedecay-global-db/src/support.rs index 6f35e1d369..7460e5a8ee 100644 --- a/crates/tracedecay-global-db/src/support.rs +++ b/crates/tracedecay-global-db/src/support.rs @@ -61,8 +61,8 @@ pub fn env_flag(name: &str) -> bool { std::env::var(name).is_ok_and(|value| env_value_truthy(&value)) } -/// Whether user-level global accounting (the cross-project `savings_ledger` -/// plus worldwide-counter flushes in the MCP server) is enabled. +/// Whether user-level global accounting (the cross-project `savings_ledger`) +/// is enabled. /// /// Enabled **by default**: every other writer of the user-level `global.db` /// (CLI sync, hooks, `tracedecay cost`, the dashboard) is ungated, and the diff --git a/crates/tracedecay-session-memory/src/user_config.rs b/crates/tracedecay-session-memory/src/user_config.rs index 82235f2b1b..9ca04d9d85 100644 --- a/crates/tracedecay-session-memory/src/user_config.rs +++ b/crates/tracedecay-session-memory/src/user_config.rs @@ -15,11 +15,16 @@ use tracedecay_domain::canonical_text::default_true; use tracedecay_automation::config::AutomationConfig; use tracedecay_runtime_core::storage::{append_lock_path, retry_transient_file_op}; -/// Keys canonical `user.*` configuration settings own, and caches of the -/// retired status worldwide/flag decoration. They are never read from this -/// file. -const RETIRED_KEYS: [&str; 7] = [ +/// Keys canonical `user.*` configuration settings own, and state of the +/// retired worldwide counter (its upload opt-in, pending amount and flush +/// timestamps) and status worldwide/flag decoration. They are never read from +/// this file; an older profile that still carries them loads normally and the +/// next write erases them. The pending amount is dropped without an upload. +const RETIRED_KEYS: [&str; 10] = [ "upload_enabled", + "pending_upload", + "last_upload_at", + "last_flush_attempt_at", "watcher_debounce", "extraction_timeout_secs", "last_worldwide_total", @@ -31,18 +36,6 @@ const RETIRED_KEYS: [&str; 7] = [ /// User-level tracedecay configuration. #[derive(Debug, Serialize, Deserialize)] pub struct UserConfig { - /// Tokens accumulated locally, not yet uploaded. - #[serde(default)] - pub pending_upload: u64, - - /// UNIX timestamp of last successful upload. - #[serde(default)] - pub last_upload_at: i64, - - /// UNIX timestamp of last flush attempt (success or failure). - #[serde(default)] - pub last_flush_attempt_at: i64, - /// Cached latest version from GitHub releases. #[serde(default)] pub cached_latest_version: String, @@ -102,9 +95,6 @@ pub struct UserConfig { impl Default for UserConfig { fn default() -> Self { Self { - pending_upload: 0, - last_upload_at: 0, - last_flush_attempt_at: 0, cached_latest_version: String::new(), last_version_check_at: 0, last_version_warning_at: 0, @@ -275,7 +265,7 @@ impl UserConfig { /// /// An unreadable or malformed file is an error, not defaults: those would /// silently turn a stored opt-out (dashboard, memory injection) back on - /// and drop the tracked hosts and pending counts. + /// and drop the tracked hosts. pub fn load(profile_root: &Path) -> std::result::Result { read_profile_config::(profile_root).map(Self::without_retired_keys) } @@ -441,7 +431,7 @@ mod tests { std::fs::create_dir_all(path.parent().unwrap()).unwrap(); // The torn write seen in the wild: a valid line followed by a bare // " true" orphan with no key. - let torn = "pending_upload = 0\n true"; + let torn = "last_version_check_at = 0\n true"; std::fs::write(&path, torn).unwrap(); let entries = || { let mut names = std::fs::read_dir(path.parent().unwrap()) @@ -452,7 +442,7 @@ mod tests { names }; let config = UserConfig { - pending_upload: 7, + last_version_check_at: 7, ..UserConfig::default() }; @@ -471,9 +461,9 @@ mod tests { assert_eq!(load_error.to_string(), expected); assert_eq!(entries(), vec!["config.toml".to_owned()]); - std::fs::write(&path, "pending_upload = 0\n").unwrap(); + std::fs::write(&path, "last_version_check_at = 0\n").unwrap(); config.save(profile).expect("a parseable config saves"); - assert_eq!(UserConfig::load(profile).unwrap().pending_upload, 7); + assert_eq!(UserConfig::load(profile).unwrap().last_version_check_at, 7); } #[test] @@ -529,13 +519,13 @@ mod tests { let path = config_path(profile); std::fs::create_dir_all(path.parent().unwrap()).unwrap(); - let handles: Vec<_> = (0..8u64) + let handles: Vec<_> = (0..8i64) .map(|thread_idx| { let profile = profile.to_path_buf(); std::thread::spawn(move || { - for i in 0..20u64 { + for i in 0..20i64 { let config = UserConfig { - pending_upload: thread_idx * 100 + i, + last_version_check_at: thread_idx * 100 + i, ..UserConfig::default() }; // Every write must succeed and leave a parseable file. @@ -554,10 +544,10 @@ mod tests { let parsed = toml::from_str::(&saved) .expect("file must be parseable after concurrent saves"); assert_eq!( - parsed.pending_upload % 100, + parsed.last_version_check_at % 100, 19, "the surviving file is one writer's final save, got {}", - parsed.pending_upload + parsed.last_version_check_at ); } @@ -583,9 +573,9 @@ mod tests { } }); - for i in 0..150u64 { + for i in 0..150i64 { let config = UserConfig { - pending_upload: i, + last_version_check_at: i, ..UserConfig::default() }; config.save(profile).expect("writer save should succeed"); @@ -595,7 +585,7 @@ mod tests { assert_eq!( toml::from_str::(&saved) .expect("final config parses") - .pending_upload, + .last_version_check_at, 149 ); } @@ -608,7 +598,9 @@ mod tests { std::fs::create_dir_all(path.parent().unwrap()).unwrap(); std::fs::write( &path, - "upload_enabled = true\nwatcher_debounce = \"9s\"\nextraction_timeout_secs = 5\n\ + "upload_enabled = true\npending_upload = 4200\nlast_upload_at = 1711375100\n\ + last_flush_attempt_at = 1711375150\n\ + watcher_debounce = \"9s\"\nextraction_timeout_secs = 5\n\ last_worldwide_total = 1000000\nlast_worldwide_fetch_at = 1711375200\n\ cached_country_flags = [\"🇺🇸\", \"🇩🇪\"]\nlast_flags_fetch_at = 1711375200\n\ future_key = \"keep-me\"\n[future_table]\nflag = true\n", @@ -616,7 +608,7 @@ mod tests { .unwrap(); let mut config = UserConfig::load(profile).unwrap(); - config.pending_upload = 3; + config.last_version_check_at = 3; config .save(profile) @@ -625,9 +617,12 @@ mod tests { assert!(saved.contains("future_key = \"keep-me\"")); assert!(saved.contains("[future_table]")); assert!(saved.contains("flag = true")); - assert!(saved.contains("pending_upload = 3")); + assert!(saved.contains("last_version_check_at = 3")); for retired in [ "upload_enabled", + "pending_upload", + "last_upload_at", + "last_flush_attempt_at", "watcher_debounce", "extraction_timeout_secs", "last_worldwide_total", diff --git a/crates/tracedecay/src/daemon/production_harness/configuration_idempotency_journey_test.rs b/crates/tracedecay/src/daemon/production_harness/configuration_idempotency_journey_test.rs index 73e5f96ac1..e35f722b0a 100644 --- a/crates/tracedecay/src/daemon/production_harness/configuration_idempotency_journey_test.rs +++ b/crates/tracedecay/src/daemon/production_harness/configuration_idempotency_journey_test.rs @@ -10,7 +10,7 @@ use tracedecay_contracts::{ use tracedecay_domain::configuration::{ ConfigurationIdempotencyKey, ConfigurationLayerIdV1, ConfigurationRevisionId, ConfigurationValueV1, SettingKey, TELEMETRY_TIMINGS_SETTING_KEY, - USER_UPLOAD_ENABLED_SETTING_KEY, UserProfileId, + USER_REPEATED_QUERY_HINTS_SETTING_KEY, UserProfileId, }; use tracedecay_sdk::client::{Client, ClientError, ConnectionMode}; use tracedecay_sdk::operations::{ApplicationConfigurationSet, TypedOperation}; @@ -330,7 +330,7 @@ async fn user_profile_configuration_batch_has_cli_dashboard_parity_after_restart layer: ConfigurationLayerIdV1::UserProfile { profile_id: profile_id.clone(), }, - key: SettingKey::new(USER_UPLOAD_ENABLED_SETTING_KEY).expect("user setting key"), + key: SettingKey::new(USER_REPEATED_QUERY_HINTS_SETTING_KEY).expect("user setting key"), value: Box::new(ConfigurationValueV1::Boolean(true)), }], expected_revision: expected_revision.clone(), @@ -390,7 +390,7 @@ async fn user_profile_configuration_batch_has_cli_dashboard_parity_after_restart let changed = ConfigurationBatchRequestV1 { mutations: vec![ConfigurationDirectMutationRequestV1::Set { layer: ConfigurationLayerIdV1::UserProfile { profile_id }, - key: SettingKey::new(USER_UPLOAD_ENABLED_SETTING_KEY).expect("user setting key"), + key: SettingKey::new(USER_REPEATED_QUERY_HINTS_SETTING_KEY).expect("user setting key"), value: Box::new(ConfigurationValueV1::Boolean(false)), }], ..request diff --git a/crates/tracedecay/src/doctor.rs b/crates/tracedecay/src/doctor.rs index 6b8fb5e20a..f7e5895b63 100644 --- a/crates/tracedecay/src/doctor.rs +++ b/crates/tracedecay/src/doctor.rs @@ -11,26 +11,13 @@ use tracedecay_contracts::project_open::{ use tracedecay_contracts::storage::{ SchemaConvergenceFindingV1, SchemaConvergenceProgressV1, SchemaConvergenceStateV1, }; -use tracedecay_contracts::{ApplicationOutcome, ResolvedSetting}; -use tracedecay_domain::configuration::{ - ConfigurationValueV1, SettingKey, USER_UPLOAD_ENABLED_SETTING_KEY, -}; use tracedecay_domain::errors::ProjectOpenFailureKind; -use tracedecay_tool_catalog::{ApplicationSurfaceOperation, BindingSurface}; use tracedecay_agent_hosts::agents::{self, DoctorCounters, HealthcheckContext}; use tracedecay_application::advisory::github_runtime; use tracedecay_automation_runtime::automation::effect_runtime::pending_automation_effect_resets_blocking; -use tracedecay_contracts::request_identity::{GlobalRequestSurface, mint_global_request_id}; -use tracedecay_contracts::{ConfigurationGetRequestV1, ConfigurationWireRequestV1}; -use tracedecay_daemon_protocol::ApplicationSurfaceRequest; -use tracedecay_daemon_protocol::RequestedOutputFormat; -use tracedecay_daemon_service::application_surface::{ - execute_application_surface, resolve_application_surface_dispatch, -}; #[cfg(unix)] use tracedecay_daemon_service::logging::recent_watcher_events; -use tracedecay_runtime_core::text::format_token_count; /// Opens an isolated daemon-registered profile database so Doctor tests can /// exercise the read-only session-temporal health adapter against the real @@ -96,7 +83,6 @@ impl DoctorTestRuntime { /// itself, the composition root cannot depend on the CLI crate. #[derive(Clone, Copy)] pub struct AdmittedDoctorNetworkProbes { - pub fetch_worldwide_total: fn() -> Option, pub fetch_latest_version: fn() -> Result, } @@ -173,14 +159,9 @@ pub async fn run_doctor( &mut pending_reset, )?; dc.replay_from(join_doctor_check(independent.watcher).await?); - let upload_enabled = join_doctor_check(independent.upload).await?; - check_user_config(&mut dc, profile.data_dir(), upload_enabled.as_ref()); - independent.network.admit_worldwide(upload_enabled.as_ref()); + check_user_config(&mut dc, profile.data_dir()); dc.replay_from(join_doctor_check(independent.hosts).await?); - independent - .network - .report(&mut dc, upload_enabled.as_ref()) - .await?; + independent.network.report(&mut dc).await?; print_summary(&dc); let completion = doctor_result(&dc, pending_reset); @@ -197,7 +178,6 @@ struct IndependentDoctorChecks { daemon_status: DoctorCheck>>>, watcher: DoctorCheck, - upload: DoctorCheck>, hosts: DoctorCheck, network: DoctorNetworkChecks, } @@ -220,7 +200,6 @@ fn launch_independent_doctor_checks( let project_for_hosts = project_path.to_path_buf(); let profile_for_status = profile.clone(); let project_for_status = project_path.to_path_buf(); - let profile_for_upload = profile.clone(); IndependentDoctorChecks { daemon_service: spawn_doctor_check(move || { @@ -248,15 +227,6 @@ fn launch_independent_doctor_checks( watcher: spawn_doctor_check(move || { collect_quiet(|dc| check_watcher(dc, &profile_for_watcher)) }), - upload: spawn_doctor_task(async move { - if daemon_listening { - configured_upload_enabled(&profile_for_upload) - .await - .map(UploadSetting::Resolved) - } else { - Ok(UploadSetting::DaemonUnavailable) - } - }), hosts: spawn_doctor_check(move || { collect_quiet(|dc| check_host_integrations(dc, &profile_for_hosts, &project_for_hosts)) }), @@ -463,7 +433,6 @@ struct ObservedDoctorFindingsV1 { } /// Typed state for a profile with no listening daemon. -const DAEMON_UNAVAILABLE: &str = "daemon_unavailable"; const DAEMON_UNAVAILABLE_STATEMENT: &str = "daemon_unavailable: no TraceDecay daemon is listening \ for this profile, so the daemon's canonical Doctor findings were not read and only \ binary-local checks ran. Pending operator action: start the daemon (`tracedecay daemon \ @@ -1216,135 +1185,24 @@ fn automation_effect_reset_findings( .collect()) } -#[tracing::instrument(name = "doctor.config.upload", level = "trace", skip_all)] -async fn configured_upload_enabled( - profile: &tracedecay_runtime_core::config::ProfileRoot, -) -> tracedecay_domain::errors::Result { - let setting = configured_setting(profile, USER_UPLOAD_ENABLED_SETTING_KEY).await?; - match setting.effective_value { - ConfigurationValueV1::Boolean(enabled) => Ok(enabled), - _ => Err(tracedecay_domain::errors::TraceDecayError::Config { - message: "worldwide counter upload setting is not boolean".to_owned(), - }), - } -} - -async fn configured_setting( - profile: &tracedecay_runtime_core::config::ProfileRoot, - key: &str, -) -> tracedecay_domain::errors::Result { - let operation = ApplicationSurfaceOperation::ConfigurationGet; - let key = SettingKey::new(key).map_err(|error| { - tracedecay_domain::errors::TraceDecayError::Config { - message: error.to_string(), - } - })?; - let request_id = - mint_global_request_id(GlobalRequestSurface::DaemonDoctor).map_err(|error| { - tracedecay_domain::errors::TraceDecayError::Config { - message: format!("could not create Doctor configuration request: {error}"), - } - })?; - // The setting belongs to the profile: the request names no project. - let handshake = crate::daemon::handshake_for_current_client(profile, None, None, false, false)?; - let client = crate::daemon::invocation_client_for_current(profile, handshake)?; - let dispatched = resolve_application_surface_dispatch( - BindingSurface::Cli, - operation, - request_id.clone(), - ApplicationSurfaceRequest::Configuration(ConfigurationWireRequestV1::Get( - ConfigurationGetRequestV1 { key: key.clone() }, - )), - RequestedOutputFormat::Json, - ) - .map_err(|error| tracedecay_domain::errors::TraceDecayError::Config { - message: error.to_string(), - })?; - let result = execute_application_surface(operation, dispatched, Some(&client)) - .await - .map_err(|error| tracedecay_domain::errors::TraceDecayError::Config { - message: error.to_string(), - })?; - let envelope = match result.result { - Ok(envelope) => envelope, - Err(problem) => { - return Err(tracedecay_domain::errors::TraceDecayError::Config { - message: problem.problem.summary(), - }); - } - }; - let ApplicationOutcome::Evidence(evidence) = envelope.outcome else { - return Err(tracedecay_domain::errors::TraceDecayError::Config { - message: "configuration get returned a non-evidence outcome".to_owned(), - }); - }; - let setting: ResolvedSetting = serde_json::from_value(evidence.payload.ok_or_else(|| { - tracedecay_domain::errors::TraceDecayError::Config { - message: "configuration get omitted its payload".to_owned(), - } - })?) - .map_err(|error| tracedecay_domain::errors::TraceDecayError::Config { - message: format!("configuration get returned an invalid setting: {error}"), - })?; - if setting.key != key { - return Err(tracedecay_domain::errors::TraceDecayError::Config { - message: "configuration get returned the wrong setting".to_owned(), - }); - } - Ok(setting) -} - -/// The worldwide-counter upload setting as the profile's configuration owner -/// answered it. -#[derive(Debug, PartialEq, Eq)] -enum UploadSetting { - Resolved(bool), - /// No daemon listens for this profile, so no configuration owner answered. - DaemonUnavailable, -} - -/// Check canonical user configuration and pending upload state. -fn check_user_config( - dc: &mut DoctorCounters, - profile_root: &Path, - upload_enabled: Result<&UploadSetting, &tracedecay_domain::errors::TraceDecayError>, -) { +/// Check the profile `config.toml` and the GitHub review sources it names. +fn check_user_config(dc: &mut DoctorCounters, profile_root: &Path) { dc.section("User config"); - match upload_enabled { - Ok(UploadSetting::Resolved(true)) => dc.pass("Worldwide counter upload enabled"), - Ok(UploadSetting::Resolved(false)) => { - dc.info("Worldwide counter upload disabled (default)"); - } - Ok(UploadSetting::DaemonUnavailable) => { - dc.info(&format!( - "Worldwide counter upload setting unread: {DAEMON_UNAVAILABLE}" - )); - } - Err(error) => dc.warn(&format!( - "Worldwide counter upload setting unavailable from canonical configuration: {error}" - )), - } match tracedecay_session_memory::user_config::UserConfig::load(profile_root) { - Ok(config) => { - if config.pending_upload > 0 { - dc.info(&format!("Pending upload: {} tokens", config.pending_upload)); - } - match github_runtime::check_configured_github_review_sources_v1(profile_root) { - Ok(findings) => { - for finding in findings { - dc.fail(&format!( - "GitHub review source {finding}, so it is not registered; fix or \ + Ok(_) => match github_runtime::check_configured_github_review_sources_v1(profile_root) { + Ok(findings) => { + for finding in findings { + dc.fail(&format!( + "GitHub review source {finding}, so it is not registered; fix or \ remove its github_review_sources entry in {}", - tracedecay_session_memory::user_config::config_path(profile_root) - .display() - )); - } + tracedecay_session_memory::user_config::config_path(profile_root).display() + )); } - Err(error) => dc.fail(&format!( - "GitHub review sources are unusable, none is registered: {error}" - )), } - } + Err(error) => dc.fail(&format!( + "GitHub review sources are unusable, none is registered: {error}" + )), + }, Err(error) => dc.fail(&format!("Profile config is unusable: {error}")), } } @@ -1464,72 +1322,28 @@ fn tracked_hosts(dc: &mut DoctorCounters, profile_root: &Path) -> Vec { struct DoctorNetworkChecks { latest: DoctorCheck>, - worldwide: Option>>, - fetch_worldwide_total: fn() -> Option, } impl DoctorNetworkChecks { fn start(network: AdmittedDoctorNetworkProbes) -> Self { Self { latest: spawn_doctor_check(move || (network.fetch_latest_version)()), - worldwide: None, - fetch_worldwide_total: network.fetch_worldwide_total, - } - } - - fn admit_worldwide( - &mut self, - upload_enabled: Result<&UploadSetting, &tracedecay_domain::errors::TraceDecayError>, - ) { - if matches!(upload_enabled, Ok(UploadSetting::Resolved(true))) { - self.worldwide = Some(spawn_doctor_check(self.fetch_worldwide_total)); } } #[tracing::instrument(name = "doctor.check.network", level = "trace", skip_all)] - async fn report( - self, - dc: &mut DoctorCounters, - upload_enabled: Result<&UploadSetting, &tracedecay_domain::errors::TraceDecayError>, - ) -> tracedecay_domain::errors::Result<()> { + async fn report(self, dc: &mut DoctorCounters) -> tracedecay_domain::errors::Result<()> { let latest = join_doctor_check(self.latest).await?; - let worldwide = match self.worldwide { - Some(handle) => Some(join_doctor_check(handle).await?), - None => None, - }; - render_network(dc, upload_enabled, worldwide, latest); + render_network(dc, latest); Ok(()) } } fn render_network( dc: &mut DoctorCounters, - upload_enabled: Result<&UploadSetting, &tracedecay_domain::errors::TraceDecayError>, - worldwide: Option>, latest: Result, ) { dc.section("Network"); - match upload_enabled { - Ok(UploadSetting::Resolved(true)) => { - if let Some(total) = worldwide.flatten() { - dc.pass(&format!( - "Worldwide counter reachable (total: {})", - format_token_count(total) - )); - } else { - dc.warn("Worldwide counter unreachable (offline or timeout)"); - } - } - Ok(UploadSetting::Resolved(false)) => { - dc.info("Worldwide counter skipped (upload disabled)"); - } - Ok(UploadSetting::DaemonUnavailable) => { - dc.info(&format!("Worldwide counter check skipped: {DAEMON_UNAVAILABLE}")); - } - Err(error) => dc.warn(&format!( - "Worldwide counter check skipped because canonical configuration is unavailable: {error}" - )), - } match latest { Ok(latest) => dc.pass(&format!("GitHub releases API reachable (latest v{latest})")), Err(error) => dc.warn(&format!("GitHub release lookup failed, {error}")), diff --git a/crates/tracedecay/src/doctor/tests.rs b/crates/tracedecay/src/doctor/tests.rs index 79c89692ff..a5a9cf4db3 100644 --- a/crates/tracedecay/src/doctor/tests.rs +++ b/crates/tracedecay/src/doctor/tests.rs @@ -746,50 +746,26 @@ async fn temporal_health_does_not_block_on_a_saturated_general_reader_lane() { drop(held); } -static NETWORK_PROBE_STARTED: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0); - -fn rendezvous_network_probe() { - use std::sync::atomic::Ordering; - - NETWORK_PROBE_STARTED.fetch_add(1, Ordering::SeqCst); - while NETWORK_PROBE_STARTED.load(Ordering::SeqCst) < 2 { - std::thread::yield_now(); - } -} - #[tokio::test] -async fn independent_network_probes_overlap() { - use std::sync::atomic::Ordering; - - NETWORK_PROBE_STARTED.store(0, Ordering::SeqCst); +async fn network_section_probes_only_the_github_release_lookup() { let mut counters = DoctorCounters::quiet(); - let mut network = DoctorNetworkChecks::start(AdmittedDoctorNetworkProbes { - fetch_worldwide_total: || { - rendezvous_network_probe(); - Some(7) - }, - fetch_latest_version: || { - rendezvous_network_probe(); - Ok("1.2.3".to_owned()) - }, + let network = DoctorNetworkChecks::start(AdmittedDoctorNetworkProbes { + fetch_latest_version: || Ok("1.2.3".to_owned()), }); - network.admit_worldwide(Ok(&UploadSetting::Resolved(true))); - network - .report(&mut counters, Ok(&UploadSetting::Resolved(true))) - .await - .unwrap(); + network.report(&mut counters).await.unwrap(); assert!( counters.checks.iter().any(|check| check .message - .contains("Worldwide counter reachable (total: 7)")), + .contains("GitHub releases API reachable (latest v1.2.3)")), "{:?}", counters.checks ); assert!( - counters.checks.iter().any(|check| check - .message - .contains("GitHub releases API reachable (latest v1.2.3)")), - "{:?}", + !counters + .checks + .iter() + .any(|check| check.message.to_lowercase().contains("worldwide")), + "doctor must not probe or mention the retired worldwide counter: {:?}", counters.checks ); } diff --git a/crates/tracedecay/src/mcp/server.rs b/crates/tracedecay/src/mcp/server.rs index 0fe82fc0f2..0e509a912a 100644 --- a/crates/tracedecay/src/mcp/server.rs +++ b/crates/tracedecay/src/mcp/server.rs @@ -303,9 +303,6 @@ pub struct McpServer { /// clone and swap in the freshly synced map on completion. file_token_map: Arc>>, tokens_saved: Option, - last_flushed_tokens: Option, - /// UNIX timestamp of last worldwide flush (`0` = never). - last_flush_at: AtomicI64, /// User-level database tracking all projects (best-effort). Wrapped in /// `Arc` so spawned savings-recording tasks can hold a cheap clone of /// the handle instead of opening a new connection per call. @@ -1006,8 +1003,6 @@ impl McpServer { diagnostics_lsp, file_token_map: Arc::new(std::sync::Mutex::new(file_token_map)), tokens_saved: persisted_tokens_saved.map(AtomicU64::new), - last_flushed_tokens: persisted_tokens_saved.map(AtomicU64::new), - last_flush_at: AtomicI64::new(0), global_db, accounting_db, profile, diff --git a/crates/tracedecay/src/mcp/server/connection.rs b/crates/tracedecay/src/mcp/server/connection.rs index 69fcf2f938..81f6da9bd1 100644 --- a/crates/tracedecay/src/mcp/server/connection.rs +++ b/crates/tracedecay/src/mcp/server/connection.rs @@ -224,8 +224,8 @@ impl McpServer { } impl McpServer { - /// Persists the tokens-saved counter, flushes pending tokens to the - /// worldwide counter, checkpoints the WAL, and logs a session summary. + /// Persists the tokens-saved counter, checkpoints the WAL, and logs a + /// session summary. /// /// Idempotent, safe to call multiple times. `run` invokes it once when /// its main loop exits; callers (e.g. `main.rs`, tests) may invoke it @@ -279,46 +279,6 @@ impl McpServer { } gdb.checkpoint().await; } - - // Flush remaining delta to worldwide counter (what periodic flushes missed). - if let Some(last_flushed_tokens) = self.last_flushed_tokens.as_ref() { - let last_flushed = last_flushed_tokens.load(Ordering::Relaxed); - if (self.accounting_db.is_some() || self.global_db.is_some()) - && tokens_saved > last_flushed - { - let delta = tokens_saved - last_flushed; - match (self.canonical_upload_enabled().await, self.owner_profile()) { - (Err(error), _) => failures.push(format!( - "worldwide counter upload configuration unavailable: {error}" - )), - (Ok(_), None) => failures.push( - "worldwide counter upload skipped: server has no owning profile" - .to_owned(), - ), - (Ok(upload_enabled), Some(profile)) => { - // The upload is a blocking HTTP call; it stays off - // the native worker that runs shutdown. - let profile_root = profile.data_dir().to_path_buf(); - let saved = tracedecay_runtime_core::native_runtime::spawn_blocking( - move || { - super::ledger::persist_worldwide_delta( - &profile_root, - delta, - upload_enabled, - ) - .map_err(|error| error.to_string()) - }, - ) - .await - .and_then(std::convert::identity); - if let Err(error) = saved { - failures - .push(format!("worldwide counter delta not saved: {error}")); - } - } - } - } - } } // Checkpoint WAL to merge it into the main database file diff --git a/crates/tracedecay/src/mcp/server/ledger.rs b/crates/tracedecay/src/mcp/server/ledger.rs index d7231f1000..31a0070a02 100644 --- a/crates/tracedecay/src/mcp/server/ledger.rs +++ b/crates/tracedecay/src/mcp/server/ledger.rs @@ -55,15 +55,9 @@ fn derive_hook_span_git_context( Some((worktree, branch)) } -fn configuration_authority_unavailable(detail: impl std::fmt::Display) -> TraceDecayError { - TraceDecayError::Config { - message: format!("configuration authority unavailable: {detail}"), - } -} - -// Global accounting (savings ledger + worldwide-counter flushes) is enabled -// by default; see `tracedecay_global_db::global_accounting_mode` for the env -// override precedence. +// Global accounting (the savings ledger) is enabled by default; see +// `tracedecay_global_db::global_accounting_mode` for the env override +// precedence. /// Where this server's savings-ledger and analytics writes land. /// @@ -111,37 +105,6 @@ impl McpServer { matches!(self.ledger_sink(), LedgerSink::Mounted(_)) } - /// Reads the upload policy from the owning profile's configuration store. - /// There is deliberately no `config.toml` fallback: without the canonical - /// authority, the upload decision is unavailable. - #[tracing::instrument(name = "mcp.ledger.read_upload_policy", level = "trace", skip_all)] - pub(super) async fn canonical_upload_enabled(&self) -> Result { - let (Some(database), Some(identity)) = - (self.profile_session_db.clone(), self.profile_identity()) - else { - return Err(configuration_authority_unavailable( - "server has no registered profile configuration store", - )); - }; - let profile = tracedecay_project::config::read_or_initialize_profile_configuration( - database, - identity.profile_id(), - ) - .await - .map_err(|error| { - configuration_authority_unavailable(format!("cannot read upload setting: {error}")) - })?; - match profile.value(tracedecay_domain::configuration::USER_UPLOAD_ENABLED_SETTING_KEY) { - Ok(tracedecay_domain::configuration::ConfigurationValueV1::Boolean(enabled)) => { - Ok(*enabled) - } - Ok(_) => Err(configuration_authority_unavailable( - "upload setting is not boolean", - )), - Err(error) => Err(configuration_authority_unavailable(error)), - } - } - /// Estimates the raw-file token cost ("before") for the given file /// paths from the cached file-token map (indexed file bytes / 4). /// Pure lookup, persists nothing. @@ -311,74 +274,6 @@ impl McpServer { crate::mcp::server::requests::recover_lock(&self.file_token_map).clone() } - /// Admits one background worldwide-counter flush at each 30-second boundary. - /// - /// The complete flush runs on the observed ledger-write path so responses - /// never await configuration or cloud I/O and shutdown still drains it. - #[tracing::instrument(name = "mcp.ledger.flush_worldwide", level = "trace", skip_all)] - pub(crate) fn maybe_flush_worldwide(self: &Arc) { - let now = tracedecay_runtime_core::tracedecay::current_timestamp(); - let last = self.last_flush_at.load(Ordering::Relaxed); - if now - last < 30 { - return; - } - if !claim_worldwide_flush(&self.last_flush_at, last, now) { - return; - } - - let (Some(tokens_saved), Some(last_flushed_tokens)) = - (&self.tokens_saved, &self.last_flushed_tokens) - else { - return; - }; - let current = tokens_saved.load(Ordering::Relaxed); - let last_flushed = last_flushed_tokens.load(Ordering::Relaxed); - if current <= last_flushed { - return; - } - let delta = current - last_flushed; - - if !self.ledger_sink_is_mounted() { - return; - } - let Some(profile_root) = self - .owner_profile() - .map(|profile| profile.data_dir().to_path_buf()) - else { - tracing::warn!("worldwide counter flush skipped: server has no owning profile"); - return; - }; - - let server = Arc::clone(self); - self.spawn_observed_ledger_write(async move { - let upload_enabled = match server.canonical_upload_enabled().await { - Ok(enabled) => enabled, - Err(error) => { - tracing::warn!( - error = %error, - "worldwide counter upload skipped because configuration authority is unavailable" - ); - return; - } - }; - let saved = tracedecay_runtime_core::native_runtime::spawn_blocking(move || { - persist_worldwide_delta(&profile_root, delta, upload_enabled) - .inspect_err(|error| tracing::warn!(%error, "could not save upload config")) - .is_ok() - }) - .await - .unwrap_or_else(|error| { - tracing::warn!(%error, "worldwide counter flush task failed"); - false - }); - if saved - && let Some(last_flushed_tokens) = server.last_flushed_tokens.as_ref() - { - last_flushed_tokens.store(current, Ordering::Release); - } - }); - } - #[tracing::instrument(name = "mcp.ledger.record_error_analytics", level = "trace", skip_all)] pub(crate) fn record_mcp_tool_error_analytics( &self, @@ -602,163 +497,58 @@ impl McpServer { } } -/// Adds `delta` to the profile's pending worldwide-counter count, uploading it -/// when enabled, and saves the profile config. -pub(super) fn persist_worldwide_delta( - profile_root: &Path, - delta: u64, - upload_enabled: bool, -) -> std::result::Result<(), tracedecay_session_memory::user_config::ConfigSaveError> { - let mut config = tracedecay_session_memory::user_config::UserConfig::load(profile_root)?; - config.pending_upload = config.pending_upload.saturating_add(delta); - if upload_enabled - && tracedecay_dashboard_api::cloud::flush_pending(config.pending_upload).is_some() - { - config.pending_upload = 0; - config.last_upload_at = tracedecay_runtime_core::tracedecay::current_timestamp(); - } - config.save(profile_root) -} - -fn claim_worldwide_flush(last_flush_at: &AtomicI64, expected: i64, now: i64) -> bool { - last_flush_at - .compare_exchange(expected, now, Ordering::AcqRel, Ordering::Acquire) - .is_ok() -} - #[cfg(test)] mod tests { - use std::sync::atomic::AtomicU64; + use serde_json::json; + use tracedecay_mcp::transport::JsonRpcRequest; use super::*; - #[test] - fn concurrent_boundary_calls_claim_exactly_one_worldwide_flush() { - let last_flush_at = Arc::new(AtomicI64::new(10)); - let barrier = Arc::new(std::sync::Barrier::new(16)); - let claimed = Arc::new(AtomicU64::new(0)); - let workers = (0..16) - .map(|_| { - let last_flush_at = Arc::clone(&last_flush_at); - let barrier = Arc::clone(&barrier); - let claimed = Arc::clone(&claimed); - std::thread::spawn(move || { - barrier.wait(); - if claim_worldwide_flush(&last_flush_at, 10, 40) { - claimed.fetch_add(1, Ordering::AcqRel); - } - }) - }) - .collect::>(); - for worker in workers { - worker.join().expect("flush claimant"); - } - - assert_eq!(claimed.load(Ordering::Acquire), 1); - assert_eq!(last_flush_at.load(Ordering::Acquire), 40); - } - - #[test] - fn disabled_upload_records_each_delta_once_after_durable_save() { - let profile = tempfile::tempdir().expect("profile"); - let profile = profile.path(); - let mut config = tracedecay_session_memory::user_config::UserConfig::load(profile).unwrap(); - config.pending_upload = 0; - config - .save(profile) - .expect("initialize isolated user config"); - let current = 37_u64; - let last_flushed = AtomicU64::new(0); - - for _ in 0..2 { - let previous = last_flushed.load(Ordering::Acquire); - if current > previous - && persist_worldwide_delta(profile, current - previous, false).is_ok() - { - last_flushed.store(current, Ordering::Release); - } - } - - let persisted = tracedecay_session_memory::user_config::UserConfig::load(profile).unwrap(); - assert_eq!(persisted.pending_upload, current); - assert_eq!(last_flushed.load(Ordering::Acquire), current); - } - + /// A completed tool response persists its tokens-saved accounting locally + /// and leaves the profile `config.toml` alone. The retired worldwide + /// counter used to write its pending-upload amount there after a response. #[tokio::test] - async fn concurrent_response_boundary_admits_one_observed_background_flush() { + async fn completed_response_persists_local_accounting_without_counter_state() { let (cg, _project, authority) = super::super::writer_test_support::init_indexed_repo().await; + let config_path = + tracedecay_session_memory::user_config::config_path(authority.profile_root()); + let config_before = std::fs::read(&config_path).ok(); + let project_root = cg.project_root().to_path_buf(); let context = super::super::writer_test_support::registered_context(cg, &authority); let server = super::super::McpServer::new_with_context(context).await; let tokens_saved = server.tokens_saved.as_ref().expect("fixture token counter"); - let last_flushed = server - .last_flushed_tokens - .as_ref() - .expect("fixture flushed counter"); - let baseline = last_flushed.load(Ordering::Acquire); + let baseline = tokens_saved.load(Ordering::Acquire); tokens_saved.store(baseline + 41, Ordering::Release); - server.last_flush_at.store(0, Ordering::Release); - let started_before = server.ledger_writes_started.load(Ordering::SeqCst); - let barrier = Arc::new(tokio::sync::Barrier::new(16)); - let callers = (0..16) - .map(|_| { - let server = Arc::clone(&server); - let barrier = Arc::clone(&barrier); - tokio::spawn(async move { - barrier.wait().await; - server.maybe_flush_worldwide(); - }) - }) - .collect::>(); - for caller in callers { - caller.await.expect("boundary caller"); - } + let request = JsonRpcRequest { + jsonrpc: "2.0".to_string(), + id: Some(json!(1)), + method: "tools/call".to_string(), + params: Some(json!({ + "name": "tracedecay_search", + "arguments": { "query": "main" } + })), + }; + server + .handle_request(&request) + .await + .expect("tool call should produce a response"); + server.ledger_writes_settled().await; assert_eq!( - server.ledger_writes_started.load(Ordering::SeqCst), - started_before + 1, - "concurrent completed responses must admit exactly one flush owner" + std::fs::read(&config_path).ok(), + config_before, + "a response must not write counter state into the profile config" ); + + let accounted = tokens_saved.load(Ordering::Acquire); + server.spawn_token_accounting_persist(&project_root, "tracedecay_search", 41, 100); server.ledger_writes_settled().await; assert_eq!( - last_flushed.load(Ordering::Acquire), - baseline + 41, - "shutdown-observed settlement must advance durable token accounting" + server.cg_snapshot().await.get_tokens_saved().await.unwrap(), + accounted + 41, + "local tokens-saved accounting must stay persisted" ); server.shutdown_background_tasks().await; } - - /// The observed ledger write runs where no Tokio runtime exists and still - /// advances durable accounting. - #[test] - fn worldwide_flush_settles_without_a_tokio_runtime() { - let hazards = crate::test_support::native_runtime_hazards(); - tracedecay_runtime_core::native_runtime::block_on(async { - assert!(tokio::runtime::Handle::try_current().is_err()); - let (cg, _project, authority) = - super::super::writer_test_support::init_indexed_repo().await; - let context = super::super::writer_test_support::registered_context(cg, &authority); - let server = super::super::McpServer::new_with_context(context).await; - let tokens_saved = server.tokens_saved.as_ref().expect("fixture token counter"); - let last_flushed = server - .last_flushed_tokens - .as_ref() - .expect("fixture flushed counter"); - let baseline = last_flushed.load(Ordering::Acquire); - tokens_saved.store(baseline + 7, Ordering::Release); - server.last_flush_at.store(0, Ordering::Release); - - server.maybe_flush_worldwide(); - - assert!( - server - .ledger_writes_settled_within(Duration::from_secs(30)) - .await - ); - assert_eq!(last_flushed.load(Ordering::Acquire), baseline + 7); - server.shutdown_background_tasks().await; - }) - .expect("native runtime"); - assert_eq!(crate::test_support::native_runtime_hazards(), hazards); - } } diff --git a/crates/tracedecay/src/mcp/server/requests.rs b/crates/tracedecay/src/mcp/server/requests.rs index d7b090349c..0dfcc44f9c 100644 --- a/crates/tracedecay/src/mcp/server/requests.rs +++ b/crates/tracedecay/src/mcp/server/requests.rs @@ -996,7 +996,6 @@ impl McpServer { accounting.net_saved_tokens(), accounting.raw_file_tokens, ); - self.maybe_flush_worldwide(); accounting } diff --git a/crates/tracedecay/tests/dashboard_api_test/settings.rs b/crates/tracedecay/tests/dashboard_api_test/settings.rs index e7957a8596..8b4fa7527d 100644 --- a/crates/tracedecay/tests/dashboard_api_test/settings.rs +++ b/crates/tracedecay/tests/dashboard_api_test/settings.rs @@ -86,7 +86,10 @@ fn settings_dashboard_api_aggregates_and_updates_config() { "settings must expose the profile worker snapshot: {settings}" ); - assert_eq!(settings["user"]["upload_enabled"], false); + assert!( + settings["user"].get("upload_enabled").is_none(), + "the retired worldwide-counter upload setting must not be served: {settings}" + ); assert_eq!( settings["user"]["code_index_workers"], json!({ "mode": "automatic" }), @@ -334,7 +337,7 @@ fn settings_dashboard_api_aggregates_and_updates_config() { &json!({ "expected_revision_id": user_revision, "idempotency_key": "configuration.idempotency.dashboard-user-unmounted", - "upload_enabled": false, + "extraction_timeout_secs": 90, "watcher_debounce": "15s" }), ); @@ -353,13 +356,31 @@ fn settings_dashboard_api_aggregates_and_updates_config() { &user_url, &json!({ "expected_revision_id": user_revision, - "upload_enabled": false + "extraction_timeout_secs": 90 }), ); assert_eq!( status, 400, "missing user idempotency must be rejected: {absent_user_idempotency}" ); + + let (status, retired_upload) = patch_json_body( + &agent, + &user_url, + &json!({ + "expected_revision_id": user_revision, + "idempotency_key": "configuration.idempotency.dashboard-user-retired-upload", + "upload_enabled": true + }), + ); + assert_eq!( + status, 400, + "the retired upload field must be refused, not silently accepted: {retired_upload}" + ); + assert_eq!( + retired_upload["validation_errors"][0]["field"], + "upload_enabled" + ); assert_eq!( absent_user_idempotency["validation_errors"][0]["field"], "idempotency_key" @@ -437,7 +458,7 @@ fn settings_dashboard_api_aggregates_and_updates_config() { assert_eq!(reloaded["project"]["config"]["max_file_size"], 1_048_576); assert_eq!(reloaded["project"]["configuration_revision_id"], revision); assert_eq!(reloaded["user"]["configuration_revision_id"], user_revision); - assert_eq!(reloaded["user"]["upload_enabled"], false); + assert!(reloaded["user"].get("upload_enabled").is_none()); assert_eq!(reloaded["user"]["watcher_debounce"], "2s"); }); } diff --git a/crates/tracedecay/tests/mcp_suite/mcp_cli_serve_test.rs b/crates/tracedecay/tests/mcp_suite/mcp_cli_serve_test.rs index a1a682ce91..9a2f7b770c 100644 --- a/crates/tracedecay/tests/mcp_suite/mcp_cli_serve_test.rs +++ b/crates/tracedecay/tests/mcp_suite/mcp_cli_serve_test.rs @@ -972,13 +972,6 @@ async fn projectless_session_serves_user_settings_and_refuses_project_keys() { let home = TempDir::new().unwrap(); let home = canonical_existing_path(home.path()); let _daemon = common::spawn_tracedecay_daemon(&home); - let enable = tracedecay_command_with_home(&home) - .current_dir(&home) - .arg("enable-upload-counter") - .stdin(Stdio::null()) - .output() - .expect("enable-upload-counter should run"); - assert!(enable.status.success(), "{enable:?}"); // The home directory is the ambient root, so the handshake names no project. let output = run_serve_requests( @@ -987,7 +980,7 @@ async fn projectless_session_serves_user_settings_and_refuses_project_keys() { None, &[ json!({ "jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {} }), - configuration_get_request(2, "user.upload_enabled.v1"), + configuration_get_request(2, "user.repeated_query_hints.v1"), json!({ "jsonrpc": "2.0", "id": 3, "method": "tools/list" }), configuration_get_request(4, "index.max_file_size.v1"), ], @@ -1001,8 +994,8 @@ async fn projectless_session_serves_user_settings_and_refuses_project_keys() { &setting["outcome"]["value"]["payload"]["effective_value"], ), ( - &json!("user.upload_enabled.v1"), - &json!({ "kind": "boolean", "value": true }), + &json!("user.repeated_query_hints.v1"), + &json!({ "kind": "boolean", "value": false }), ), "{setting}" ); @@ -1053,13 +1046,6 @@ async fn unenrolled_directory_session_serves_user_settings() { let home = canonical_existing_path(home.path()); let unenrolled = TempDir::new().unwrap(); let _daemon = common::spawn_tracedecay_daemon(&home); - let enable = tracedecay_command_with_home(&home) - .current_dir(unenrolled.path()) - .arg("enable-upload-counter") - .stdin(Stdio::null()) - .output() - .expect("enable-upload-counter should run"); - assert!(enable.status.success(), "{enable:?}"); let output = run_serve_requests( &home, @@ -1067,7 +1053,7 @@ async fn unenrolled_directory_session_serves_user_settings() { None, &[ json!({ "jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {} }), - configuration_get_request(2, "user.upload_enabled.v1"), + configuration_get_request(2, "user.repeated_query_hints.v1"), configuration_get_request(3, "index.max_file_size.v1"), ], ); @@ -1075,7 +1061,7 @@ async fn unenrolled_directory_session_serves_user_settings() { let setting = json_rpc_tool_payload(&output.stdout, 2); assert_eq!( setting["outcome"]["value"]["payload"]["effective_value"], - json!({ "kind": "boolean", "value": true }), + json!({ "kind": "boolean", "value": false }), "{setting}" ); let refusal = json_rpc_response(&output.stdout, 3); diff --git a/crates/tracedecay/tests/runtime_acceptance_suite/runtime_surface_acceptance.rs b/crates/tracedecay/tests/runtime_acceptance_suite/runtime_surface_acceptance.rs index 0062c9c373..23cdd92dfb 100644 --- a/crates/tracedecay/tests/runtime_acceptance_suite/runtime_surface_acceptance.rs +++ b/crates/tracedecay/tests/runtime_acceptance_suite/runtime_surface_acceptance.rs @@ -1163,13 +1163,13 @@ async fn dashboard_user_settings_replay_through_application_restart() { let request = serde_json::json!({ "expected_revision_id": initial_revision.clone(), "idempotency_key": idempotency_key, - "upload_enabled": true, + "extraction_timeout_secs": 90, "watcher_debounce": "15s" }); let (status, applied) = patch_dashboard_json(&agent, &user_url, &request); assert_eq!(status, 200, "profile mutation failed: {applied}"); assert_eq!(applied["payload"]["restart_recommended"], true); - assert_eq!(applied["payload"]["user"]["upload_enabled"], true); + assert_eq!(applied["payload"]["user"]["extraction_timeout_secs"], 90); assert_eq!(applied["payload"]["user"]["watcher_debounce"], "15s"); let applied_revision = applied["payload"]["user"]["configuration_revision_id"] .as_str() @@ -1222,7 +1222,7 @@ async fn dashboard_user_settings_replay_through_application_restart() { let conflicting_request = serde_json::json!({ "expected_revision_id": initial_revision.clone(), "idempotency_key": idempotency_key, - "upload_enabled": false, + "extraction_timeout_secs": 30, "watcher_debounce": "15s" }); let (status, conflict) = diff --git a/dashboard/src/workspaces/observatory/AnalyticsControls.dom.test.tsx b/dashboard/src/workspaces/observatory/AnalyticsControls.dom.test.tsx index 1c19e01ba7..a2fc69ec56 100644 --- a/dashboard/src/workspaces/observatory/AnalyticsControls.dom.test.tsx +++ b/dashboard/src/workspaces/observatory/AnalyticsControls.dom.test.tsx @@ -11,12 +11,11 @@ import type { ObservatoryAccountingReads } from './accountingReads.ts'; /** * The Plan 26 `analytics-controls` view. * - * Two real reads back parts of it, `/api/settings` for the profile upload - * setting and `/api/doctor/findings?family=storage` for the typed retention-backlog status, - * and they are read independently. The assertions pin the two reassuring - * falsehoods this surface could most easily tell: that an unpublished - * collection mode is `Off`, and that an absent exporter means zero egress - * failures. + * `/api/doctor/findings?family=storage` backs the typed retention-backlog + * status and is read independently of the observatory projection. The + * assertions pin the two reassuring falsehoods this surface could most easily + * tell: that an unpublished collection mode is `Off`, and that an absent + * exporter means zero egress failures. */ const NOW_MICROS = 1_753_003_600_000_000; @@ -26,30 +25,29 @@ afterEach(() => { }); describe('Observatory analytics controls', () => { - - it('keeps a failed settings read from blanking the retention evidence', async () => { - // Independent sources: one refusing must not take the other down with it. - renderControls({ settingsStatus: 503 }); + it('states retention evidence and no longer reads or shows an upload setting', async () => { + const fetchMock = renderControls(); await screen.findByText('retention sweep observed 4 entries'); - const upload = screen.getByRole('region', { name: 'profile upload setting' }); - expect(upload.textContent).toContain('the profile settings read did not resolve'); - expect(upload.textContent).not.toContain('disabled'); + expect(screen.queryByRole('region', { name: 'profile upload setting' })).toBeNull(); + expect( + fetchMock.mock.calls.some(([input]) => String(input).includes('/api/settings')), + ).toBe(false); }); - it('keeps a failed findings read from blanking the profile setting', async () => { + it('keeps a failed findings read from blanking the collection mode', async () => { renderControls({ findingsStatus: 503 }); - await screen.findByText('user.upload_enabled.v1'); - const retention = screen.getByRole('region', { name: 'retention and deletion' }); - expect(retention.textContent).toContain('could not be read'); + const retention = await screen.findByRole('region', { name: 'retention and deletion' }); + await screen.findByText('could not be read', { exact: false }); expect(retention.textContent).not.toContain('retention sweep observed'); + const mode = screen.getByRole('region', { name: 'collection mode' }); + expect(mode.textContent).not.toContain('could not be read'); }); }); function renderControls( options: { - settingsStatus?: number; findingsStatus?: number; retentionStatuses?: unknown[]; } = {}, @@ -62,31 +60,24 @@ function renderControls( reason: 'retention sweep observed 4 entries', }, ]; - vi.stubGlobal( - 'fetch', - vi.fn(async (input: RequestInfo | URL) => { - const url = String(input); - if (url.includes('/api/settings')) { - if (options.settingsStatus != null) { - return new Response('{}', { status: options.settingsStatus }); - } - return new Response(JSON.stringify(envelope(settingsPayload())), { status: 200 }); - } - if (url.includes('/api/doctor/findings?family=storage')) { - if (options.findingsStatus != null) { - return new Response('{}', { status: options.findingsStatus }); - } - return new Response(JSON.stringify(envelope(findingsPayload(statuses))), { status: 200 }); + const fetchMock = vi.fn(async (input: RequestInfo | URL) => { + const url = String(input); + if (url.includes('/api/doctor/findings?family=storage')) { + if (options.findingsStatus != null) { + return new Response('{}', { status: options.findingsStatus }); } - return new Response('{}', { status: 404 }); - }), - ); + return new Response(JSON.stringify(envelope(findingsPayload(statuses))), { status: 200 }); + } + return new Response('{}', { status: 404 }); + }); + vi.stubGlobal('fetch', fetchMock); const client = new QueryClient({ defaultOptions: { queries: { retry: false, gcTime: 0 } } }); render( , ); + return fetchMock; } function accountingReads(): ObservatoryAccountingReads { @@ -183,74 +174,6 @@ function findingsPayload(kindStatuses: unknown[]) { }; } -function settingsPayload() { - return { - automation: { - availability: { available: true, reason: null, required_authority: null }, - backend: null, - config_endpoint: '/api/plugins/holographic/curation/config', - enabled: false, - host_mode: null, - source_coverage: { effective: 'global', global: 'default', project: 'absent' }, - }, - environment: { - global_accounting_enabled: false, - global_accounting_mode: 'off', - pricing_offline: true, - variables: [], - }, - project: { - config: { - exclude: [], - extract_docstrings: true, - context_scout: false, - include: [], - max_file_size: 1_048_576, - sync: { auto_track_pr_branches: false, auto_track_pr_poll_secs: 300 }, - telemetry: { timings: false }, - track_call_sites: true, - }, - configuration_revision_id: 'revision-1', - configuration_snapshot_id: 'snapshot-1', - pr_autotrack: { tracked: [] }, - }, - restart_recommended: null, - resync_recommended: null, - storage: { - dashboard_root: '/store/dashboard', - graph_db: '/store/graph.db', - lcm_db: '/store/lcm.db', - lcm_scope: 'project', - memory_db: '/store/memory.db', - project_id: 'tracedecay', - project_root: '/repo', - savings_db: '/store/savings.db', - storage_mode: 'profile_sharded', - store_root: '/store', - }, - user: { - code_index_worker_status: { - available_logical_cpus: 4, - configured: { mode: 'automatic' }, - effective_workers: 4, - environment_override_workers: null, - limiting_reason: 'automatic_all_cores', - memory_safe_workers: 6, - }, - code_index_workers: { mode: 'automatic' }, - code_index_worker_configuration_revision_id: 'worker-revision-1', - code_index_worker_configuration_snapshot_id: 'worker-snapshot-1', - configuration_revision_id: 'revision-1', - configuration_snapshot_id: 'snapshot-1', - extraction_timeout_secs: 30, - installed_agents: ['claude'], - upload_enabled: false, - watcher_debounce: '500ms', - }, - version: { cached_latest_version: null, channel: 'stable', version: '0.0.0' }, - }; -} - function envelope(payload: unknown) { return { schema_revision: 1, diff --git a/dashboard/src/workspaces/observatory/AnalyticsControls.tsx b/dashboard/src/workspaces/observatory/AnalyticsControls.tsx index 0859ad798e..1324084eb2 100644 --- a/dashboard/src/workspaces/observatory/AnalyticsControls.tsx +++ b/dashboard/src/workspaces/observatory/AnalyticsControls.tsx @@ -3,19 +3,15 @@ * analytics: local mode, share staging age, retention/deletion, and egress * failures. * - * Two real reads back parts of this surface, `GET /api/settings` for the - * `user.upload_enabled.v1` profile setting, and `GET /api/doctor/findings?family=storage` for - * the typed `retention_backlog` status. They are read independently: a failed - * settings read must not blank the retention evidence, and neither may stand in - * for the analytics collection mode, which nothing publishes. + * `GET /api/doctor/findings?family=storage` backs the typed `retention_backlog` + * status. It is read independently of the observatory projection, and neither + * may stand in for the analytics collection mode, which nothing publishes. * * The rule this surface exists to hold is the one in `analyticsControls.ts`: an * unread mode is not `Off`, and an absent exporter is not zero egress failures. * Both would be reassuring, and both would be false. */ import type { ComponentProps, ReactNode } from 'react'; -import { SettingsPayloadV1Schema } from '../../contracts/generated.ts'; -import { useEnvelope } from '../../data/query/useEnvelope.ts'; import { useStorageFindings } from '../../data/query/storageFindings.ts'; import { envelopeReadState } from '../../ui/ReadSection.tsx'; import { StateChip } from '../../ui/StateChip'; @@ -26,20 +22,13 @@ import { egressFailureReading, retentionBacklogReading, shareStagingReading, - uploadSettingReading, type RetentionBacklogReading, - type UploadSettingReading, } from './analyticsControls.ts'; import type { ObservatoryAccountingReads } from './accountingReads.ts'; export function AnalyticsControls({ reads }: { reads: ObservatoryAccountingReads }) { - const settings = useEnvelope(['settings', 'analytics-controls'], '/api/settings', SettingsPayloadV1Schema); const findings = useStorageFindings(); - const settingsState = envelopeReadState(settings.isPending, settings.data, { - loading: 'requesting profile settings', - transport: 'profile settings could not be read', - }); const findingsState = envelopeReadState(findings.isPending, findings.data, { loading: 'requesting doctor storage findings', transport: 'doctor storage findings could not be read', @@ -175,26 +164,6 @@ export function AnalyticsControls({ reads }: { reads: ObservatoryAccountingReads : `${egress.failures.toLocaleString()} failures were recorded by the canonical projection.`}

- - - {settingsState.kind === 'ready' ? ( - - ) : ( -

- the profile settings read did not resolve, so the setting is not stated -

- )} - ); @@ -258,26 +227,3 @@ function RetentionBacklog({ reading }: { reading: RetentionBacklogReading }) { ); } - -function UploadSetting({ reading }: { reading: UploadSettingReading }) { - return ( - <> -
-
-
setting
-
{reading.settingKey}
-
-
-
value
-
- {reading.enabled == null ? 'not reported' : reading.enabled ? 'enabled' : 'disabled'} -
-
-
-

{reading.disclaimer}

- - ); -} diff --git a/dashboard/src/workspaces/observatory/analyticsControls.test.ts b/dashboard/src/workspaces/observatory/analyticsControls.test.ts index c90a60e70d..db09173a6b 100644 --- a/dashboard/src/workspaces/observatory/analyticsControls.test.ts +++ b/dashboard/src/workspaces/observatory/analyticsControls.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; import type { AnalyticsModeReadModelV1, MetricValueV1, StorageFindingKindStatusV1 } from '../../contracts/generated.ts'; -import { analyticsModeReading, egressFailureReading, retentionBacklogReading, shareStagingReading, uploadSettingReading } from './analyticsControls.ts'; +import { analyticsModeReading, egressFailureReading, retentionBacklogReading, shareStagingReading } from './analyticsControls.ts'; describe('analyticsModeReading', () => { it('reports an unavailable mode as unknown and explicitly not Off', () => { @@ -78,16 +78,6 @@ describe('retentionBacklogReading', () => { }); }); -describe('uploadSettingReading', () => { - - it('reports an unread setting as unknown rather than as disabled', () => { - const reading = uploadSettingReading(undefined); - expect(reading.enabled).toBeNull(); - expect(reading.enabled).not.toBe(false); - expect(reading.state).toBe('unknown'); - }); -}); - function status( kind: StorageFindingKindStatusV1['kind'], state: StorageFindingKindStatusV1['state'], diff --git a/dashboard/src/workspaces/observatory/analyticsControls.ts b/dashboard/src/workspaces/observatory/analyticsControls.ts index 79f30d6c29..f2df3bc539 100644 --- a/dashboard/src/workspaces/observatory/analyticsControls.ts +++ b/dashboard/src/workspaces/observatory/analyticsControls.ts @@ -1,9 +1,8 @@ -/** Plan 26 analytics controls over canonical settings and Observatory reads. */ +/** Plan 26 analytics controls over Observatory reads. */ import type { AnalyticsModeV1, AnalyticsModeReadModelV1, MetricValueV1, - SettingsPayloadV1, StorageFindingKindStatusV1, } from '../../contracts/generated.ts'; import type { DomainStateKind } from '../../ui/StateChip.tsx'; @@ -230,28 +229,3 @@ export function shareStagingReading(metrics: readonly MetricValueV1[]): ShareSta } return { ageSeconds: metric.value, state: 'ready', reason: '' }; } - -export interface UploadSettingReading { - /** The profile setting value, or `null` when the settings read carried none. */ - enabled: boolean | null; - settingKey: string; - state: DomainStateKind; - /** The one sentence that keeps this from being mistaken for the analytics - * collection mode. */ - disclaimer: string; -} - -/** The `user.upload_enabled.v1` profile setting from `/api/settings`. Real, - * wire-published, and explicitly not `AnalyticsModeV1`. */ -export function uploadSettingReading( - settings: SettingsPayloadV1 | undefined, -): UploadSettingReading { - const enabled = settings?.user.upload_enabled ?? null; - return { - enabled, - settingKey: 'user.upload_enabled.v1', - state: enabled == null ? 'unknown' : 'ready', - disclaimer: - 'This is the profile upload setting, not the Plan 26 analytics collection mode. It does not select Off, LocalOnly, or AggregateShare and does not govern adoption analytics.', - }; -} diff --git a/dashboard/src/workspaces/observatory/evidence.ts b/dashboard/src/workspaces/observatory/evidence.ts index 0239ee618f..c77adf2779 100644 --- a/dashboard/src/workspaces/observatory/evidence.ts +++ b/dashboard/src/workspaces/observatory/evidence.ts @@ -209,7 +209,7 @@ export const SOURCE_IDENTITY: Record = { id: 'analytics', title: 'Analytics controls', route: '/api/observatory', - authority: 'canonical analytics mode, retention lifecycle, egress and upload settings', + authority: 'canonical analytics mode, retention lifecycle and egress', }, telemetry: { id: 'telemetry', diff --git a/dashboard/src/workspaces/settings/SettingsPage.dom.test.tsx b/dashboard/src/workspaces/settings/SettingsPage.dom.test.tsx index 370012bae5..084c53beb7 100644 --- a/dashboard/src/workspaces/settings/SettingsPage.dom.test.tsx +++ b/dashboard/src/workspaces/settings/SettingsPage.dom.test.tsx @@ -51,7 +51,6 @@ const FIXTURE_SETTINGS: ReadonlyArray = [ 'user.configuration_snapshot_id', 'user.extraction_timeout_secs', 'user.installed_agents', - 'user.upload_enabled', 'user.watcher_debounce', ], ], diff --git a/dashboard/src/workspaces/settings/settingsEditorMachine.ts b/dashboard/src/workspaces/settings/settingsEditorMachine.ts index c893afde88..f48313af5b 100644 --- a/dashboard/src/workspaces/settings/settingsEditorMachine.ts +++ b/dashboard/src/workspaces/settings/settingsEditorMachine.ts @@ -728,7 +728,6 @@ function sameProjectValues( function sameUserValues(left: UserSettingsValues, right: UserSettingsValues): boolean { return ( - left.upload_enabled === right.upload_enabled && left.watcher_debounce === right.watcher_debounce && left.extraction_timeout_secs === right.extraction_timeout_secs ); diff --git a/dashboard/src/workspaces/settings/settingsModel.ts b/dashboard/src/workspaces/settings/settingsModel.ts index 19cc9f3337..752e2e47dc 100644 --- a/dashboard/src/workspaces/settings/settingsModel.ts +++ b/dashboard/src/workspaces/settings/settingsModel.ts @@ -12,7 +12,7 @@ * The daemon really does merge two layers there, and then ships only the * result. The winning layer is not on the wire. * - `user` and `project.config` do not even address the same keys - * (`upload_enabled`/`watcher_debounce` vs `include`/`exclude`/…), so they + * (`watcher_debounce`/`extraction_timeout_secs` vs `include`/`exclude`/…), so they * are not competing layers over a shared namespace. * * Therefore this model does NOT rank groups into a resolution stack and does @@ -150,7 +150,6 @@ export interface ProjectSettingsValues { } export interface UserSettingsValues { - readonly upload_enabled: boolean; readonly watcher_debounce: string; readonly extraction_timeout_secs: string; } @@ -205,7 +204,6 @@ export interface ProjectSettingsChangeSet { /** The user-scope counterpart of `ProjectSettingsChangeSet`. */ export interface UserSettingsChangeSet { - upload_enabled?: boolean; watcher_debounce?: string; extraction_timeout_secs?: number; } @@ -443,7 +441,6 @@ export function buildSettingsEditor(payload: SettingsPayloadV1): SettingsEditor auto_track_pr_poll_secs: pollSecs, }, user: { - upload_enabled: payload.user.upload_enabled, watcher_debounce: payload.user.watcher_debounce, extraction_timeout_secs: extractionTimeout, }, @@ -551,9 +548,6 @@ export function planUserChangeAgainst( if (errors.length > 0) return { outcome: 'invalid', errors }; const patch: UserSettingsChangeSet = {}; - if (values.upload_enabled !== current.user.upload_enabled) { - patch.upload_enabled = values.upload_enabled; - } if (values.watcher_debounce !== current.user.watcher_debounce) { patch.watcher_debounce = values.watcher_debounce; } diff --git a/dashboard/src/workspaces/settings/settingsRows.test.ts b/dashboard/src/workspaces/settings/settingsRows.test.ts index 3288aa2625..9a3b80e00c 100644 --- a/dashboard/src/workspaces/settings/settingsRows.test.ts +++ b/dashboard/src/workspaces/settings/settingsRows.test.ts @@ -162,7 +162,7 @@ describe('write bindings', () => { expect(applyRequirement(bindingFor('project.config.track_call_sites')!)).toEqual({ kind: 'reported_on_apply', }); - expect(applyRequirement(bindingFor('user.upload_enabled')!)).toEqual({ + expect(applyRequirement(bindingFor('user.watcher_debounce')!)).toEqual({ kind: 'reported_on_apply', }); }); diff --git a/dashboard/src/workspaces/settings/settingsRows.ts b/dashboard/src/workspaces/settings/settingsRows.ts index c96383879c..44d9336aaf 100644 --- a/dashboard/src/workspaces/settings/settingsRows.ts +++ b/dashboard/src/workspaces/settings/settingsRows.ts @@ -114,7 +114,6 @@ const BINDINGS: Readonly> = { field: 'auto_track_pr_poll_secs', input: 'integer', }, - 'user.upload_enabled': { scope: 'user', field: 'upload_enabled', input: 'boolean' }, 'user.watcher_debounce': { scope: 'user', field: 'watcher_debounce', input: 'duration' }, 'user.extraction_timeout_secs': { scope: 'user', @@ -360,8 +359,6 @@ function withUserField( value: unknown, ): UserSettingsValues { switch (field) { - case 'upload_enabled': - return typeof value === 'boolean' ? { ...values, [field]: value } : values; case 'watcher_debounce': case 'extraction_timeout_secs': return typeof value === 'string' ? { ...values, [field]: value } : values; diff --git a/dashboard/stories/fixtures/data.ts b/dashboard/stories/fixtures/data.ts index f0a00736f1..05390f0ddd 100644 --- a/dashboard/stories/fixtures/data.ts +++ b/dashboard/stories/fixtures/data.ts @@ -3242,7 +3242,6 @@ const settingsPayload: Record = { configuration_revision_id: 'user-rev-7', code_index_worker_configuration_snapshot_id: 'profile-worker-snap-7', code_index_worker_configuration_revision_id: 'profile-worker-rev-7', - upload_enabled: false, code_index_workers: { mode: 'automatic' }, code_index_worker_status: { configured: { mode: 'automatic' }, diff --git a/docs/REBRAND-COMPATIBILITY-POLICY.md b/docs/REBRAND-COMPATIBILITY-POLICY.md index c082d932b7..2c8c8e9e30 100644 --- a/docs/REBRAND-COMPATIBILITY-POLICY.md +++ b/docs/REBRAND-COMPATIBILITY-POLICY.md @@ -71,11 +71,12 @@ benchmark reports, and daemon-removal instructions should remain unless it is factually wrong. These references should be labeled as historical when helpful, not mechanically rewritten. -### A5. Worldwide counter endpoint +### A5. Worldwide counter endpoint (retired) -The existing counter endpoint name may remain `tracedecay-counter` while the service -continues to be best-effort and documented. A future endpoint rename is allowed only -when the replacement worker is deployed and continuity/failure behavior is preserved. +The worldwide counter was removed from the client (issue #3619); no TraceDecay +binary calls the `tracedecay-counter` Cloudflare Worker anymore. Shutting the +hosted Worker and its Upstash Redis store down is a maintainer action outside +this repository, see the worldwide token counter note in `docs/USER-GUIDE.md`. ## Category B, automatically migrated or cleaned up @@ -285,7 +286,7 @@ Every audited surface maps to exactly one policy category below. | Antigravity legacy CLI plugin path | B | Remove generated `tracedecay.json` on uninstall/reconcile. | | Claude legacy hook/config migration | B | Repair/remove known generated legacy hook and config shapes. | | `tracedecay-large-treesitters` package/dependency name | E | Keep exact upstream name until upstream rename or explicit maintained-fork policy. | -| Worldwide counter endpoint `tracedecay-counter` | A | Retain until replacement endpoint is deployed with preserved behavior. | +| Worldwide counter endpoint `tracedecay-counter` | A | Retired from the client; the hosted Worker is decommissioned by a maintainer. | | Legacy release asset names for explicit requested versions | B | Probe after current asset names for explicit old-version installs/upgrades. | | Latest-version filtering of legacy-only releases | D | Reject legacy-only/pre-reset releases as automatic latest candidates. | | Extracted binary name `tracedecay` in old archives | B | Accept only as fallback while extracting explicit old archives. | diff --git a/docs/USER-GUIDE.md b/docs/USER-GUIDE.md index a148361d5f..64dc8bf90c 100644 --- a/docs/USER-GUIDE.md +++ b/docs/USER-GUIDE.md @@ -825,11 +825,10 @@ It verifies: exact worktree/ref/commit/generation, freshness, coverage, and typed authority state - **Global registry**, daemon-owned project/profile enrollment and availability -- **User config**, `~/.tracedecay/config.toml` and upload settings +- **User config**, `~/.tracedecay/config.toml` and the GitHub review sources it names - **Agent integrations**. MCP server registration, hook installation, tool permissions, prompt rules -- **Network**, the configured worldwide counter and GitHub releases API; each - reports its own available or unavailable state, and a failed release lookup - names its outcome (see [Updating TraceDecay](#updating-tracedecay)) +- **Network**, the GitHub releases API; it reports its own available or + unavailable state, and a failed release lookup names its outcome (see [Updating TraceDecay](#updating-tracedecay)) If any tool permissions are missing after an upgrade, Doctor reports the missing capability and the supported install/update operation. Doctor only reports @@ -1123,33 +1122,25 @@ Network effects are separate from local indexing and retrieval. They can be disabled, unavailable, or denied without turning those states into successful local results. The available effects are described below. -### Worldwide token counter +### Worldwide token counter (retired) -TraceDecay tracks how many tokens it has saved locally. If you opt in, the -daemon's token-savings status path uploads that aggregate count to the worldwide -counter. Repository content, file names, and project names are not part of the -counter payload. The counter service still receives ordinary transport metadata -such as the source IP and may derive aggregate geography from it; submitting an -aggregate count is not an anonymity guarantee. +TraceDecay no longer has a worldwide token counter and makes no counter network +call. Token savings, costs, and usage stay local: `tracedecay current-counter`, +`tracedecay reset-counter`, `tracedecay gain`, `tracedecay cost`, and the +dashboard are unchanged. -`tracedecay status` does not display the counter. When you opt in, `tracedecay doctor` -reads the public total only to report whether the counter service is reachable. +`tracedecay enable-upload-counter` and `tracedecay disable-upload-counter` no +longer exist and fail as unknown subcommands. The `user.upload_enabled.v1` +setting and the dashboard's `upload_enabled` field are gone, so a configuration +read or patch that names them is refused. An older profile that still stores the +opt-in, a pending upload amount, or flush timestamps loads normally: the next +write erases them, and the pending amount is never uploaded. -**To opt in:** - -```bash -tracedecay enable-upload-counter -``` - -Disable it again at any time: - -```bash -tracedecay disable-upload-counter -``` - -The setting belongs to your profile, not to a project: it applies everywhere -and can be changed or read (`tracedecay tool configuration_get --args -'{"key":"user.upload_enabled.v1"}'`) from any directory. +The hosted counter service was a Cloudflare Worker named `tracedecay-counter`, +backed by Upstash Redis. Nothing in this repository deploys or operates it, so +removing the client does not shut it down. Decommissioning it is a separate +maintainer action: delete the Worker and its Redis store from the Cloudflare +and Upstash accounts that host them. Its status is tracked on issue #3619. ### Version check @@ -1370,7 +1361,7 @@ tracedecay memory status --path /path/to/project --json Created in your home directory. Contains: -- `config.toml`, user preferences (cached version info, pending upload count) +- `config.toml`, user preferences (cached version info, installed agents) - `global.db`, daemon-owned registry/usage metadata for enrolled projects; it is not a fact authority and clients never open it directly - `projects//`, daemon-owned project authority data when profile storage is enabled @@ -1378,13 +1369,11 @@ Created in your home directory. Contains: The `config.toml` is plain TOML and fully transparent: ```toml -pending_upload = 4823 # tokens waiting to be uploaded -last_upload_at = 1711375200 # last successful upload timestamp +cached_latest_version = "1.2.3" # latest release seen by the version check +last_version_check_at = 1711375200 # last version check timestamp +installed_agents = ["claude"] # hosts TraceDecay has integrated ``` -The upload opt-in is the `user.upload_enabled.v1` setting, not a key in this -file. - #### Mandatory structured sanitization of ingested transcripts Agent transcripts can contain credentials or other sensitive values. TraceDecay diff --git a/evals/agent_adoption/run.sh b/evals/agent_adoption/run.sh index 05f725f593..c7a074ed32 100755 --- a/evals/agent_adoption/run.sh +++ b/evals/agent_adoption/run.sh @@ -358,7 +358,6 @@ echo "query hints: $REPEATED_QUERY_HINTS" if [[ -n "$LEARNED_SKILL_WORKFLOW" ]]; then echo "learned: $LEARNED_SKILL_WORKFLOW variants=[${ACTIVE_VARIANTS# }]" fi -"$TD" disable-upload-counter >/dev/null 2>&1 || true # ---- build fixtures -------------------------------------------------------- # gitc() { git -C "$1" -c user.email=eval@tracedecay -c user.name=eval "${@:2}"; } diff --git a/scripts/mcp-conformance-smoke.sh b/scripts/mcp-conformance-smoke.sh index 22205c6de5..187bdc7ae4 100755 --- a/scripts/mcp-conformance-smoke.sh +++ b/scripts/mcp-conformance-smoke.sh @@ -116,7 +116,6 @@ NODE echo "error: tracedecay init failed" >&2 return 1 fi - "$TRACEDECAY_BIN" disable-upload-counter >/dev/null 2>&1 || true # Resolve and cache the inspector package before any timed call: on a cold # runner `npx` spends its first invocation downloading the package, and From cf66a51ac1a3d636b0f746b9f938df981a8d1fef Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 12:44:39 +0000 Subject: [PATCH 05/89] fix(projection): keep the stored cwd when a session names another --- .../src/observation_projection/apply.rs | 18 ++- .../src/observation_projection/state.rs | 33 ++-- .../src/session_project_rebind_tests.rs | 144 +++++++++++++++++- 3 files changed, 176 insertions(+), 19 deletions(-) diff --git a/crates/tracedecay-global-db/src/observation_projection/apply.rs b/crates/tracedecay-global-db/src/observation_projection/apply.rs index aefe8f89fa..55a0a41ba6 100644 --- a/crates/tracedecay-global-db/src/observation_projection/apply.rs +++ b/crates/tracedecay-global-db/src/observation_projection/apply.rs @@ -2559,7 +2559,7 @@ mod tests { use super::*; #[test] - fn projected_session_enrichment_is_commutative_and_rejects_real_path_collisions() { + fn projected_session_enrichment_is_commutative_and_rejects_real_transcript_collisions() { let sparse = SessionRecord { provider: "cursor".to_owned(), session_id: "session.fixture".to_owned(), @@ -2623,10 +2623,20 @@ mod tests { Some(r#"{"source":"runtime_preflight"}"#) ); - let conflicting = SessionRecord { + let other_cwd = SessionRecord { project_path: "/workspace/other".to_owned(), - ..rich + ..rich.clone() }; - assert!(reconcile_session_rows_detailed(&forward, &conflicting).is_err()); + let kept = reconcile_session_rows_detailed(&forward, &other_cwd).unwrap(); + assert_eq!(kept.project_path, "/workspace/project"); + + let transcript = |path: &str| SessionRecord { + transcript_path: Some(path.to_owned()), + ..rich.clone() + }; + let conflict = + reconcile_session_rows_detailed(&transcript("/a.jsonl"), &transcript("/b.jsonl")) + .unwrap_err(); + assert_eq!(conflict.field(), "transcript_path"); } } diff --git a/crates/tracedecay-global-db/src/observation_projection/state.rs b/crates/tracedecay-global-db/src/observation_projection/state.rs index 6104d62edd..c5884d47d5 100644 --- a/crates/tracedecay-global-db/src/observation_projection/state.rs +++ b/crates/tracedecay-global-db/src/observation_projection/state.rs @@ -1156,10 +1156,14 @@ pub(super) fn reconcile_session_rows_detailed( // A rebind whose paths are both project ids takes the current id. // A stored path-shaped key takes the incoming path. expected.project_path.clone() - } else if expected.project_path == expected.project_key { - actual.project_path.clone() } else { - return Err(SessionReconcileConflict("project_path")); + // Two concrete working directories name one session: it was read + // through another rollout, or a resumed rollout moved its cwd. The + // stored cwd stays, like the session's title and annotations, and each + // message row keeps its own source. Replaying the same records in the + // same order therefore reaches the same row, instead of dropping every + // later message of the session as a collision. + actual.project_path.clone() }; Ok(SessionRecord { provider: actual.provider.clone(), @@ -1569,20 +1573,23 @@ mod reconcile_tests { super::durable_project_path_key(&expected.project_path) ); + // Another directory under the same project key is the same session + // seen from another cwd: the stored path stays and nothing collides. let other = tmp.path().join("other-project"); std::fs::create_dir_all(&other).unwrap(); expected.project_path = other.to_string_lossy().into_owned(); let transaction = harness.registered.begin_write_transaction().await.unwrap(); - assert!(matches!( - super::super::apply::apply_session(&transaction, &expected).await, - Err( - tracedecay_store::ProjectionStoreError::SessionOutputCollision { - field: "project_path", - .. - } - ) - )); - transaction.rollback().await.unwrap(); + super::super::apply::apply_session(&transaction, &expected) + .await + .unwrap(); + transaction.commit().await.unwrap(); + let kept = harness + .registered + .get_session(&original.provider, &original.session_id) + .await + .unwrap() + .unwrap(); + assert_eq!(kept.project_path, persisted.project_path); } #[cfg(target_os = "macos")] diff --git a/crates/tracedecay-global-db/src/session_project_rebind_tests.rs b/crates/tracedecay-global-db/src/session_project_rebind_tests.rs index 066318e313..5f0b994967 100644 --- a/crates/tracedecay-global-db/src/session_project_rebind_tests.rs +++ b/crates/tracedecay-global-db/src/session_project_rebind_tests.rs @@ -77,11 +77,37 @@ fn observation( text: &str, receipt_id: &str, session: Option, +) -> DurableObservationV1 { + observation_from_source( + provider, session_id, None, scope, record_id, text, receipt_id, session, + ) +} + +/// `source_key` names the rollout file the record came from, so one session can +/// be read through several sources, each with its own cursor. +#[allow(clippy::too_many_arguments)] +fn observation_from_source( + provider: &str, + session_id: &str, + source_key: Option<&str>, + scope: ObservationScopeV1, + record_id: &str, + text: &str, + receipt_id: &str, + session: Option, ) -> DurableObservationV1 { let provider_id = ProviderId::new(provider).unwrap(); let session_id = SessionId::new(session_id).unwrap(); - let source = - ObservationSourceIdentityV1::for_provider(provider_id.clone(), session_id.clone()).unwrap(); + let source = match source_key { + Some(key) => ObservationSourceIdentityV1::for_provider_source( + provider_id.clone(), + session_id.clone(), + SessionId::new(key).unwrap(), + ) + .unwrap(), + None => ObservationSourceIdentityV1::for_provider(provider_id.clone(), session_id.clone()) + .unwrap(), + }; let range = ObservationSourceRangeV1::new(0, 1).unwrap(); let record = ObservationId::new(record_id).unwrap(); let mut facts = Vec::new(); @@ -681,3 +707,117 @@ async fn queue_last_error(database: &RegisteredGlobalDb, observation_id: &str) - let row = rows.next().await.unwrap().unwrap(); row.get(0).unwrap() } + +async fn queue_attempts( + database: &RegisteredGlobalDb, + observation_id: &str, +) -> Option<(i64, Option)> { + let snapshot = database.read_snapshot().await.unwrap(); + let mut rows = snapshot + .query( + "SELECT attempt_count, last_error FROM projection_queue WHERE observation_id = ?1", + params![observation_id], + ) + .await + .unwrap(); + let row = rows.next().await.unwrap()?; + Some((row.get(0).unwrap(), row.get(1).unwrap())) +} + +/// A Codex session whose later records name another working directory (a +/// rollout resumed from a linked worktree) is one session. Its messages must +/// project and the session row keeps the first cwd instead of colliding on +/// `project_path` for every later record. +#[asupersync::test] +async fn session_seen_from_another_cwd_projects_every_message() { + let tmp = TempDir::new().unwrap(); + let profile = tmp.path().join("profile"); + let first_cwd = tmp.path().join("repo"); + let second_cwd = tmp.path().join("repo-worktree"); + std::fs::create_dir_all(&first_cwd).unwrap(); + std::fs::create_dir_all(&second_cwd).unwrap(); + let runtime = HostAdmissionTestRuntimeV1::profile(&profile).await.unwrap(); + let database = runtime + .registered_database(HostAdmissionScope::Profile) + .unwrap(); + let store = runtime + .observation_store(HostAdmissionScope::Profile) + .unwrap(); + + let mut ids = Vec::new(); + for (record, text, cwd) in [ + ( + "record.codex.cwd-a", + "message from the first cwd", + &first_cwd, + ), + ( + "record.codex.cwd-b-1", + "message from the second cwd", + &second_cwd, + ), + ( + "record.codex.cwd-b-2", + "another message from the second cwd", + &second_cwd, + ), + ] { + let write = observation_from_source( + "codex", + CODEX_SESSION, + Some(&format!("rollout.{record}")), + ObservationScopeV1::Profile, + record, + text, + &format!("receipt.{record}"), + Some(session_fact(&cwd.to_string_lossy())), + ); + ids.push(write.observation_id().clone()); + persist(&store, write).await; + } + drain_projection_queue(&store).await; + + let texts = [ + "message from the first cwd", + "message from the second cwd", + "another message from the second cwd", + ]; + let expected_path = durable_project_path_key(&first_cwd.to_string_lossy()); + for id in &ids { + assert_eq!( + queue_attempts(database, id.as_str()).await, + None, + "a projected observation leaves the queue instead of parking on an error" + ); + } + for text in texts { + assert!( + projected_text_contains(database, "codex", CODEX_SESSION, text).await, + "{text:?} must project" + ); + } + let (_, project_path) = session_project_binding(database, "codex", CODEX_SESSION).await; + assert_eq!(project_path, expected_path); + + // A rebuild replays the same records through the staging path. + let frontier = store.projection_checkpoint().await.unwrap().last_sequence(); + let rebuilt = loop { + let outcome = store.rebuild_projection(frontier).await.unwrap(); + if outcome.is_complete() { + break outcome; + } + }; + assert_eq!( + rebuilt.skipped_observations(), + 0, + "no replayed record is skipped" + ); + for text in texts { + assert!( + projected_text_contains(database, "codex", CODEX_SESSION, text).await, + "{text:?} must survive the rebuild" + ); + } + let (_, project_path) = session_project_binding(database, "codex", CODEX_SESSION).await; + assert_eq!(project_path, expected_path); +} From b9475c69c5428a2f19f1c94b7b4056430c9692db Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 16:10:04 +0000 Subject: [PATCH 06/89] fix: drop unused mut and Debug-free parse assertion --- crates/tracedecay-cli/src/startup_tests.rs | 3 ++- crates/tracedecay/src/doctor.rs | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-cli/src/startup_tests.rs b/crates/tracedecay-cli/src/startup_tests.rs index 7d030ee651..e7ab1a9d38 100644 --- a/crates/tracedecay-cli/src/startup_tests.rs +++ b/crates/tracedecay-cli/src/startup_tests.rs @@ -698,7 +698,8 @@ fn post_update_full_reinstall_advances_both_version_markers() { fn retired_worldwide_counter_commands_are_refused_by_the_parser() { for retired in ["enable-upload-counter", "disable-upload-counter"] { let error = Cli::try_parse_from(["tracedecay", retired]) - .expect_err("a retired subcommand must not parse"); + .err() + .expect("a retired subcommand must not parse"); assert_eq!(error.kind(), clap::error::ErrorKind::InvalidSubcommand); } } diff --git a/crates/tracedecay/src/doctor.rs b/crates/tracedecay/src/doctor.rs index f7e5895b63..3b98ef2c49 100644 --- a/crates/tracedecay/src/doctor.rs +++ b/crates/tracedecay/src/doctor.rs @@ -136,7 +136,7 @@ pub async fn run_doctor( let project_path = std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")); let daemon_listening = tracedecay_daemon_control::daemon_socket_connectable(profile); - let mut independent = launch_independent_doctor_checks( + let independent = launch_independent_doctor_checks( profile, build_version, &project_path, From 642433a8b82980322c3845c4d3e2b354d9380d25 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 16:14:11 +0000 Subject: [PATCH 07/89] feat(global-db): add transact_or and JobExit for job bodies --- .../src/configuration/store.rs | 2 +- .../src/observability_rollup.rs | 8 ++- .../src/observability_rollup/retention.rs | 4 +- .../src/observation_projection/apply.rs | 1 + crates/tracedecay-global-db/src/registered.rs | 71 +++++++++++++------ .../src/registry_maintenance.rs | 2 +- .../src/runtime/workflow/workflow_index.rs | 2 +- .../runtime/workflow/workflow_index/port.rs | 6 -- 8 files changed, 61 insertions(+), 35 deletions(-) diff --git a/crates/tracedecay-global-db/src/configuration/store.rs b/crates/tracedecay-global-db/src/configuration/store.rs index e8eed26376..b5401b5075 100644 --- a/crates/tracedecay-global-db/src/configuration/store.rs +++ b/crates/tracedecay-global-db/src/configuration/store.rs @@ -75,7 +75,7 @@ where R: Send + 'static, E: Send + 'static, { - db.transact(body).await.map_err(unavailable)? + db.transact_or(unavailable, body).await } /// Concrete control-plane adapter over one already-open owned session store. diff --git a/crates/tracedecay-global-db/src/observability_rollup.rs b/crates/tracedecay-global-db/src/observability_rollup.rs index 06b8f0c0cc..dce17ddbd5 100644 --- a/crates/tracedecay-global-db/src/observability_rollup.rs +++ b/crates/tracedecay-global-db/src/observability_rollup.rs @@ -41,9 +41,11 @@ where R: Send + 'static, E: From + Send + 'static, { - db.transact(body) - .await - .map_err(|error| E::from(format!("failed to {context}: {error}")))? + db.transact_or( + |error| E::from(format!("failed to {context}: {error}")), + body, + ) + .await } /// A rebuild job that ends without publishing. A replayed receipt rolls the diff --git a/crates/tracedecay-global-db/src/observability_rollup/retention.rs b/crates/tracedecay-global-db/src/observability_rollup/retention.rs index 5d8880efb0..23a22bd8c0 100644 --- a/crates/tracedecay-global-db/src/observability_rollup/retention.rs +++ b/crates/tracedecay-global-db/src/observability_rollup/retention.rs @@ -1,4 +1,4 @@ -use crate::RegisteredGlobalDbWriteTransaction; +use tracedecay_runtime_core::db::engine::Executor; use super::{ OBSERVABILITY_ROLLUP_RETENTION_DAYS_V1, ObservabilityRollupRetentionReceiptV1, SECONDS_PER_DAY, @@ -11,7 +11,7 @@ use super::{ /// expired day as known-empty from already-pruned events. A query reaching /// past the horizon reports partial coverage. pub(crate) async fn expire_observability_rollups( - transaction: &RegisteredGlobalDbWriteTransaction<'_>, + transaction: &impl Executor, now_seconds: i64, ) -> Result { let cutoff = diff --git a/crates/tracedecay-global-db/src/observation_projection/apply.rs b/crates/tracedecay-global-db/src/observation_projection/apply.rs index aefe8f89fa..533ef92e4b 100644 --- a/crates/tracedecay-global-db/src/observation_projection/apply.rs +++ b/crates/tracedecay-global-db/src/observation_projection/apply.rs @@ -679,6 +679,7 @@ impl_send_projection_derive!(DatabaseWriteTransaction<'_>); impl_send_projection_derive!(DatabaseEngineReadConnection); impl_send_projection_derive!(DatabaseEngineReadSnapshot); impl_send_projection_derive!(crate::registered::RegisteredGlobalDbWriteTransaction<'_>); +impl_send_projection_derive!(crate::registered::RegisteredGlobalDbWriteTx<'_, '_>); /// Objective used for goal-state dedupe: prefer native `/objective` (Codex), /// else the already-extracted `content_text`. diff --git a/crates/tracedecay-global-db/src/registered.rs b/crates/tracedecay-global-db/src/registered.rs index e3e52c44f6..23b9cbec73 100644 --- a/crates/tracedecay-global-db/src/registered.rs +++ b/crates/tracedecay-global-db/src/registered.rs @@ -634,6 +634,42 @@ impl RegisteredGlobalDb { .await } + /// [`Self::transact`] for a body whose error type is the caller's own: a + /// refused or failed job becomes `wrap(cause)`, and the body's error is + /// returned as is. + pub(crate) async fn transact_or( + &self, + wrap: impl FnOnce(TraceDecayError) -> E, + body: impl AsyncFn(RegisteredGlobalDbWriteTx<'_, '_>) -> Result + Send + Sync + 'static, + ) -> Result + where + R: Send + 'static, + E: Send + 'static, + { + self.transact(body).await.map_err(wrap)? + } + + /// [`Self::transact_or`] for a body that can also end early with a value + /// the caller wants as success. The job rolls back either way, so an + /// early exit leaves no commit behind. + pub(crate) async fn transact_or_rollback( + &self, + wrap: impl FnOnce(TraceDecayError) -> E, + body: impl AsyncFn(RegisteredGlobalDbWriteTx<'_, '_>) -> Result> + + Send + + Sync + + 'static, + ) -> Result + where + R: Send + 'static, + E: Send + 'static, + { + match self.transact(body).await.map_err(wrap)? { + Ok(value) | Err(JobExit::Rollback(value)) => Ok(value), + Err(JobExit::Fail(error)) => Err(error), + } + } + pub fn binding(&self) -> &tracedecay_store::StoreRuntimeBindingV1 { self.database.registered_binding() } @@ -866,27 +902,6 @@ impl tracedecay_sessions::runtime::git_correlation::GitCorrelationWriteTxn } } -impl tracedecay_sessions::runtime::workflow_index::WorkflowIngestWriteTxn - for RegisteredGlobalDbWriteTransaction<'_> -{ - #[allow(clippy::manual_async_fn)] - fn commit( - self, - ) -> impl Future< - Output = Result<(), tracedecay_sessions::runtime::workflow_index::WorkflowIndexError>, - > + Send { - async move { - RegisteredGlobalDbWriteTransaction::commit(self) - .await - .map_err(|error| { - tracedecay_sessions::runtime::workflow_index::WorkflowIndexError::Db( - error.to_string(), - ) - }) - } - } -} - impl RegisteredGlobalDbWriteTransaction<'_> { pub async fn execute

( &self, @@ -955,6 +970,20 @@ impl RegisteredGlobalDbWriteTransaction<'_> { } } +/// The error side of a job body that can end without committing. `Rollback` +/// carries a value for the caller, so a replay can be reported as a success +/// without leaving a commit behind; `Fail` is an ordinary failure. +pub(crate) enum JobExit { + Rollback(R), + Fail(E), +} + +impl From for JobExit { + fn from(error: E) -> Self { + Self::Fail(error) + } +} + /// The transaction a [`RegisteredGlobalDb::transact`] body runs in. It is a /// borrowed handle, so the body takes it by value and lends `&transaction` to /// helpers written against the executor traits. diff --git a/crates/tracedecay-global-db/src/registry_maintenance.rs b/crates/tracedecay-global-db/src/registry_maintenance.rs index fb474b5a1e..a8f82c7e9a 100644 --- a/crates/tracedecay-global-db/src/registry_maintenance.rs +++ b/crates/tracedecay-global-db/src/registry_maintenance.rs @@ -6,7 +6,7 @@ use serde::Serialize; use crate::{ CodeProjectRecord, GraphScopeUpsert, ProjectRegistryContext, RegisteredGlobalDb, - RegisteredGlobalDbWriteTransaction, StoreArtifactUpsert, StoreInstanceUpsert, + StoreArtifactUpsert, StoreInstanceUpsert, }; use tracedecay_runtime_core::branch_meta; use tracedecay_runtime_core::storage::{ diff --git a/crates/tracedecay-sessions/src/runtime/workflow/workflow_index.rs b/crates/tracedecay-sessions/src/runtime/workflow/workflow_index.rs index 17cfda8d2a..0ca3ddc53f 100644 --- a/crates/tracedecay-sessions/src/runtime/workflow/workflow_index.rs +++ b/crates/tracedecay-sessions/src/runtime/workflow/workflow_index.rs @@ -698,7 +698,7 @@ async fn runs_for_git_scope( } mod port; -pub use port::{WorkflowIngestSink, WorkflowIngestWriteTxn}; +pub use port::WorkflowIngestSink; #[cfg(test)] #[allow(clippy::unwrap_used, clippy::expect_used)] diff --git a/crates/tracedecay-sessions/src/runtime/workflow/workflow_index/port.rs b/crates/tracedecay-sessions/src/runtime/workflow/workflow_index/port.rs index 782d744a1e..8dd90aee1c 100644 --- a/crates/tracedecay-sessions/src/runtime/workflow/workflow_index/port.rs +++ b/crates/tracedecay-sessions/src/runtime/workflow/workflow_index/port.rs @@ -6,15 +6,9 @@ use std::future::Future; -use tracedecay_runtime_core::db::engine::{Executor, QueryExecutor}; - use super::WorkflowIndexError; use crate::{WorkflowAgent, WorkflowRun}; -pub trait WorkflowIngestWriteTxn: QueryExecutor + Executor + Sized + Send { - fn commit(self) -> impl Future> + Send; -} - /// Authoritative sink one workflow-ingest sweep upserts through. /// /// The sweep needs an authority check, a watermark it can advance, and a From 156af12380ec8be06ca8169884d1ad478c91a782 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 16:14:12 +0000 Subject: [PATCH 08/89] refactor(global-db): run accounting and workflow writes as jobs --- .../src/registered_accounting.rs | 72 ++++++++++--------- .../src/workflow_adapter.rs | 60 ++++++++-------- 2 files changed, 68 insertions(+), 64 deletions(-) diff --git a/crates/tracedecay-global-db/src/registered_accounting.rs b/crates/tracedecay-global-db/src/registered_accounting.rs index 9d7895488a..4b056d7147 100644 --- a/crates/tracedecay-global-db/src/registered_accounting.rs +++ b/crates/tracedecay-global-db/src/registered_accounting.rs @@ -1,6 +1,6 @@ use std::path::Path; -use tracedecay_runtime_core::db::engine::Value; +use tracedecay_runtime_core::db::engine::{Executor as _, Value}; use super::{ RegisteredGlobalDb, SavingsDay, SavingsTotal, global_db_operation_error, @@ -19,20 +19,22 @@ impl RegisteredGlobalDb { tokens_saved: u64, ) -> tracedecay_domain::errors::Result<()> { let path = super::project_path_alias_key(project_path); - let transaction = self.begin_write_transaction().await?; - transaction - .execute( - "INSERT INTO projects (path, tokens_saved) VALUES (?1, ?2) - ON CONFLICT(path) DO UPDATE SET - tokens_saved = MAX(tokens_saved, excluded.tokens_saved)", - tracedecay_runtime_core::db::engine::params![path, tokens_saved as i64], - ) - .await - .map_err(|error| global_db_operation_error("update project token total", error))?; - transaction - .commit() - .await - .map_err(|error| global_db_operation_error("commit project token total", error)) + self.transact(async move |transaction| { + transaction + .execute( + "INSERT INTO projects (path, tokens_saved) VALUES (?1, ?2) + ON CONFLICT(path) DO UPDATE SET + tokens_saved = MAX(tokens_saved, excluded.tokens_saved)", + tracedecay_runtime_core::db::engine::params![ + path.as_str(), + tokens_saved as i64 + ], + ) + .await + .map_err(|error| global_db_operation_error("update project token total", error))?; + Ok(()) + }) + .await? } /// A project with no registry row has genuinely saved nothing, so an @@ -87,26 +89,26 @@ impl RegisteredGlobalDb { timestamp: i64, ) -> tracedecay_domain::errors::Result<()> { let project_path = RegisteredGlobalDb::canonical_project_key(Path::new(project_path)); - let transaction = self.begin_write_transaction().await?; - transaction - .execute( - "INSERT INTO savings_ledger - (ts, project_path, tool_name, before_tokens, after_tokens) - VALUES (?1, ?2, ?3, ?4, ?5)", - tracedecay_runtime_core::db::engine::params![ - timestamp, - project_path, - tool_name, - before_tokens as i64, - after_tokens as i64 - ], - ) - .await - .map_err(|error| global_db_operation_error("append savings ledger entry", error))?; - transaction - .commit() - .await - .map_err(|error| global_db_operation_error("commit savings ledger entry", error)) + let tool_name = tool_name.to_owned(); + self.transact(async move |transaction| { + transaction + .execute( + "INSERT INTO savings_ledger + (ts, project_path, tool_name, before_tokens, after_tokens) + VALUES (?1, ?2, ?3, ?4, ?5)", + tracedecay_runtime_core::db::engine::params![ + timestamp, + project_path.as_str(), + tool_name.as_str(), + before_tokens as i64, + after_tokens as i64 + ], + ) + .await + .map_err(|error| global_db_operation_error("append savings ledger entry", error))?; + Ok(()) + }) + .await? } /// Sums settled savings-ledger rows for an optional project path. diff --git a/crates/tracedecay-global-db/src/workflow_adapter.rs b/crates/tracedecay-global-db/src/workflow_adapter.rs index 0dc375dbc5..bd14d14813 100644 --- a/crates/tracedecay-global-db/src/workflow_adapter.rs +++ b/crates/tracedecay-global-db/src/workflow_adapter.rs @@ -7,14 +7,13 @@ use std::path::Path; use tracedecay_domain::ProjectId; use tracedecay_store::StoreShardScopeV1; -use crate::{RegisteredGlobalDb, RegisteredGlobalDbWriteTransaction}; -use tracedecay_runtime_core::db::engine::params; +use crate::RegisteredGlobalDb; +use tracedecay_runtime_core::db::engine::{Executor as _, params}; #[cfg(any(test, feature = "test-helpers"))] use tracedecay_sessions::runtime::source::TranscriptIngestResult; use tracedecay_sessions::runtime::workflow_index::{ INGEST_WATERMARK_KEY, RegisteredWorkflowIndexSnapshot, WorkflowAgent, WorkflowIndexError, - WorkflowIngestSink, WorkflowIngestWriteTxn, WorkflowRun, read_ingest_watermark, upsert_agent, - upsert_run, + WorkflowIngestSink, WorkflowRun, read_ingest_watermark, upsert_agent, upsert_run, }; #[cfg(any(test, feature = "test-helpers"))] use tracedecay_sessions::runtime::workflow_ingest::{ @@ -118,26 +117,27 @@ where #[tracing::instrument(name = "global_db.workflow.bump_watermark", level = "trace", skip_all)] async fn bump_ingest_watermark(&self, value: i64) { - let Ok(transaction) = self.db().begin_write_transaction().await else { - tracing::debug!("workflow ingest writer unavailable"); - return; - }; - let write = async { - transaction - .execute( - "INSERT INTO workflow_index_meta(key, value) + let write = self + .db() + .transact(async move |transaction| { + transaction + .execute( + "INSERT INTO workflow_index_meta(key, value) VALUES (?1, ?2) ON CONFLICT(key) DO UPDATE SET value = MAX(value, excluded.value), updated_at = unixepoch()", - params![INGEST_WATERMARK_KEY, value], - ) - .await?; - RegisteredGlobalDbWriteTransaction::commit(transaction).await - } - .await; - if let Err(err) = write { - tracing::debug!(error = %err, "workflow ingest watermark not advanced"); + params![INGEST_WATERMARK_KEY, value], + ) + .await + }) + .await; + match write { + Ok(Ok(_)) => {} + Ok(Err(err)) => { + tracing::debug!(error = %err, "workflow ingest watermark not advanced"); + } + Err(err) => tracing::debug!(error = %err, "workflow ingest writer unavailable"), } } @@ -147,15 +147,17 @@ where run: &WorkflowRun, agents: &[WorkflowAgent], ) -> Result<(), WorkflowIndexError> { - let transaction = self - .db() - .begin_write_transaction() + let run = run.clone(); + let agents = agents.to_vec(); + self.db() + .transact(async move |transaction| { + upsert_run(&transaction, &run).await?; + for agent in &agents { + upsert_agent(&transaction, agent).await?; + } + Ok(()) + }) .await - .map_err(|error| WorkflowIndexError::Db(error.to_string()))?; - upsert_run(&transaction, run).await?; - for agent in agents { - upsert_agent(&transaction, agent).await?; - } - WorkflowIngestWriteTxn::commit(transaction).await + .map_err(|error| WorkflowIndexError::Db(error.to_string()))? } } From 9111a8fb3f587ca2bf45d547a60d6203947fec96 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 16:14:13 +0000 Subject: [PATCH 09/89] refactor(global-db): run topology anchor and discovery writes as jobs --- .../src/discovery_queue.rs | 143 +++++++++--------- .../src/git_topology_anchor.rs | 83 +++++----- 2 files changed, 109 insertions(+), 117 deletions(-) diff --git a/crates/tracedecay-global-db/src/discovery_queue.rs b/crates/tracedecay-global-db/src/discovery_queue.rs index abe0d86ace..c8bc387903 100644 --- a/crates/tracedecay-global-db/src/discovery_queue.rs +++ b/crates/tracedecay-global-db/src/discovery_queue.rs @@ -7,7 +7,7 @@ use std::path::{Path, PathBuf}; use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; -use tracedecay_runtime_core::db::engine::params; +use tracedecay_runtime_core::db::engine::{Executor as _, QueryExecutor as _, params}; use crate::RegisteredGlobalDb; @@ -133,78 +133,77 @@ impl RegisteredGlobalDb { let Some(last_path) = paths.last().cloned() else { return Ok(None); }; - let transaction = self - .begin_write_transaction() - .await - .map_err(|error| format!("begin host discovery queue transaction: {error}"))?; let (range_start, range_end) = exclusive_prefix_range(&prefix)?; - let mut sequence_rows = transaction - .query( - "SELECT COALESCE(MAX(byte_offset), 0) - FROM parse_offsets - WHERE file_path >= ?1 AND file_path < ?2 AND file_id = 1", - params![range_start, range_end], - ) - .await - .map_err(|error| format!("read host discovery queue tail: {error}"))?; - let sequence_row = sequence_rows - .next() - .await - .map_err(|error| format!("read host discovery queue tail: {error}"))? - .ok_or_else(|| "host discovery queue tail is unavailable".to_owned())?; - let mut sequence = sequence_row - .get::(0) - .map_err(|error| format!("decode host discovery queue tail: {error}"))?; - if sequence < 0 { - return Err("host discovery queue tail is negative".to_owned()); - } - drop(sequence_rows); - for path in paths { - let key = queue_key(&prefix, &path); - let proposed = sequence - .checked_add(1) - .ok_or_else(|| "host discovery queue sequence is exhausted".to_owned())?; - let inserted = transaction - .execute( - "INSERT INTO parse_offsets (file_path, byte_offset, mtime, file_id) - VALUES (?1, ?2, 0, 1) - ON CONFLICT(file_path) DO NOTHING", - params![key, proposed], - ) - .await - .map_err(|error| format!("enqueue host discovery path: {error}"))?; - if inserted > 0 { - sequence = proposed; - } - } - let last_key = queue_key(&prefix, &last_path); - let mut rows = transaction - .query( - "SELECT byte_offset, file_path - FROM parse_offsets - WHERE file_path = ?1 AND file_id = 1", - params![last_key], - ) - .await - .map_err(|error| format!("resolve host discovery queue identity: {error}"))?; - let row = rows - .next() - .await - .map_err(|error| format!("read host discovery queue identity: {error}"))? - .ok_or_else(|| "enqueued host discovery path is unavailable".to_owned())?; - let entry = decode_entry( - &prefix, - row.get::(0) - .map_err(|error| format!("decode host discovery queue sequence: {error}"))?, - row.get::(1) - .map_err(|error| format!("decode host discovery queue key: {error}"))?, - )?; - drop(rows); - transaction - .commit() - .await - .map_err(|error| format!("commit host discovery queue: {error}"))?; - Ok(Some(entry)) + self.transact_or( + |error| format!("run host discovery queue transaction: {error}"), + async move |transaction| { + let mut sequence_rows = transaction + .query( + "SELECT COALESCE(MAX(byte_offset), 0) + FROM parse_offsets + WHERE file_path >= ?1 AND file_path < ?2 AND file_id = 1", + params![range_start.as_str(), range_end.as_str()], + ) + .await + .map_err(|error| format!("read host discovery queue tail: {error}"))?; + let sequence_row = sequence_rows + .next() + .await + .map_err(|error| format!("read host discovery queue tail: {error}"))? + .ok_or_else(|| "host discovery queue tail is unavailable".to_owned())?; + let mut sequence = sequence_row + .get::(0) + .map_err(|error| format!("decode host discovery queue tail: {error}"))?; + if sequence < 0 { + return Err("host discovery queue tail is negative".to_owned()); + } + drop(sequence_rows); + for path in &paths { + let key = queue_key(&prefix, path); + let proposed = sequence + .checked_add(1) + .ok_or_else(|| "host discovery queue sequence is exhausted".to_owned())?; + let inserted = transaction + .execute( + "INSERT INTO parse_offsets (file_path, byte_offset, mtime, file_id) + VALUES (?1, ?2, 0, 1) + ON CONFLICT(file_path) DO NOTHING", + params![key, proposed], + ) + .await + .map_err(|error| format!("enqueue host discovery path: {error}"))?; + if inserted > 0 { + sequence = proposed; + } + } + let last_key = queue_key(&prefix, &last_path); + let mut rows = transaction + .query( + "SELECT byte_offset, file_path + FROM parse_offsets + WHERE file_path = ?1 AND file_id = 1", + params![last_key], + ) + .await + .map_err(|error| format!("resolve host discovery queue identity: {error}"))?; + let row = rows + .next() + .await + .map_err(|error| format!("read host discovery queue identity: {error}"))? + .ok_or_else(|| "enqueued host discovery path is unavailable".to_owned())?; + let entry = decode_entry( + &prefix, + row.get::(0).map_err(|error| { + format!("decode host discovery queue sequence: {error}") + })?, + row.get::(1) + .map_err(|error| format!("decode host discovery queue key: {error}"))?, + )?; + drop(rows); + Ok(Some(entry)) + }, + ) + .await } #[tracing::instrument(name = "global_db.discovery_queue.query", level = "trace", skip_all)] diff --git a/crates/tracedecay-global-db/src/git_topology_anchor.rs b/crates/tracedecay-global-db/src/git_topology_anchor.rs index 9875b44b8c..88b5b79060 100644 --- a/crates/tracedecay-global-db/src/git_topology_anchor.rs +++ b/crates/tracedecay-global-db/src/git_topology_anchor.rs @@ -8,7 +8,7 @@ use tracedecay_contracts::retrieval::{ GitTopologyAnchorResolutionOutcome, }; use tracedecay_domain::{ObservationScopeV1, RetrievalAnchorRecord, RetrievalAnchorTarget}; -use tracedecay_runtime_core::db::engine::{params, params_from_iter}; +use tracedecay_runtime_core::db::engine::{Executor as _, params, params_from_iter}; use tracedecay_store::StoreShardScopeV1; use crate::{RegisteredGlobalDb, RegisteredGlobalDbLeaseV1}; @@ -30,58 +30,51 @@ impl RegisteredGitTopologyAnchorAuthority { if !binding_matches_owner(&self.database, publication.owner()) { return Err(GitTopologyAnchorAuthorityError::Unavailable); } - let transaction = self - .database - .begin_write_transaction() - .await - .map_err(map_database_error)?; let records = publication.into_records(); let anchor_ids = records .iter() .map(|record| record.anchor_id().as_str().to_owned()) .collect::>(); - let existing = read_records(&transaction, &anchor_ids).await?; - let mut published = false; - for candidate in records { - let existing = existing.get(candidate.anchor_id().as_str()); - match existing { - Some(existing) if existing.is_semantic_replay_of(&candidate) => continue, - Some(_) => { + self.database + .transact(async move |transaction| { + let existing = read_records(&transaction, &anchor_ids).await?; + let mut published = false; + for candidate in &records { + let existing = existing.get(candidate.anchor_id().as_str()); + match existing { + Some(existing) if existing.is_semantic_replay_of(candidate) => continue, + Some(_) => return Err(GitTopologyAnchorAuthorityError::Conflict), + None => {} + } + let anchor_json = serde_json::to_string(candidate) + .map_err(|_| GitTopologyAnchorAuthorityError::Conflict)?; + let owner_json = candidate + .owner_column_json() + .map_err(|_| GitTopologyAnchorAuthorityError::Conflict)?; transaction - .rollback() + .execute( + "INSERT INTO retrieval_anchors ( + anchor_id, anchor_json, owner_json, projection_generation + ) VALUES (?1, ?2, ?3, ?4)", + params![ + candidate.anchor_id().as_str(), + anchor_json, + owner_json, + candidate.projection_generation().as_str(), + ], + ) .await - .map_err(|_| GitTopologyAnchorAuthorityError::Unavailable)?; - return Err(GitTopologyAnchorAuthorityError::Conflict); + .map_err(map_engine_error)?; + published = true; } - None => {} - } - let anchor_json = serde_json::to_string(&candidate) - .map_err(|_| GitTopologyAnchorAuthorityError::Conflict)?; - let owner_json = candidate - .owner_column_json() - .map_err(|_| GitTopologyAnchorAuthorityError::Conflict)?; - transaction - .execute( - "INSERT INTO retrieval_anchors ( - anchor_id, anchor_json, owner_json, projection_generation - ) VALUES (?1, ?2, ?3, ?4)", - params![ - candidate.anchor_id().as_str(), - anchor_json, - owner_json, - candidate.projection_generation().as_str(), - ], - ) - .await - .map_err(map_engine_error)?; - published = true; - } - transaction.commit().await.map_err(map_engine_error)?; - Ok(if published { - GitTopologyAnchorPublicationOutcome::Published - } else { - GitTopologyAnchorPublicationOutcome::Replayed - }) + Ok(if published { + GitTopologyAnchorPublicationOutcome::Published + } else { + GitTopologyAnchorPublicationOutcome::Replayed + }) + }) + .await + .map_err(map_database_error)? } async fn resolve_record( From cca8a2f5bf4059c90c883a2f0eb20e40a9212df7 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 16:14:14 +0000 Subject: [PATCH 10/89] refactor(global-db): run stack delivery writes as jobs --- .../src/registered/delivery_settlement.rs | 239 +++++----- .../src/stack_delivery.rs | 448 +++++++++--------- 2 files changed, 352 insertions(+), 335 deletions(-) diff --git a/crates/tracedecay-global-db/src/registered/delivery_settlement.rs b/crates/tracedecay-global-db/src/registered/delivery_settlement.rs index 3eec135196..1ac55c2bcd 100644 --- a/crates/tracedecay-global-db/src/registered/delivery_settlement.rs +++ b/crates/tracedecay-global-db/src/registered/delivery_settlement.rs @@ -1,9 +1,10 @@ -use super::{RegisteredGlobalDb, RegisteredGlobalDbWriteTransaction}; +use super::RegisteredGlobalDb; use tracedecay_domain::{ CoverageStateV1, DeliveryDropReasonV1, DeliveryEventClassV1, DeliverySettlementAttemptV1, DeliverySettlementCensusV1, DeliverySettlementOutcomeV1, DeliverySettlementV1, DeliverySurfaceFamilyV1, UtcMicros, WorkAttemptIdentityV1, canonical_sha256, }; +use tracedecay_runtime_core::db::engine::{Executor, QueryExecutor}; /// One Work attempt can have at most this many separately addressed delivery /// fan-outs. The recipient bound is enforced per fan-out by the domain; this @@ -151,59 +152,62 @@ impl RegisteredGlobalDb { source_receipt_ref: Option<&str>, ) -> Result { validate_project_and_attempt(project_id, attempt)?; - let transaction = self - .begin_write_transaction() - .await - .map_err(|error| format!("failed to begin delivery attempt transaction: {error}"))?; - ensure_fanout_identity(&transaction, project_id, attempt).await?; + let project_id = project_id.to_owned(); + let attempt = attempt.clone(); + let source_receipt_ref = source_receipt_ref.map(str::to_owned); + self.transact_or( + |error| format!("failed to run delivery attempt transaction: {error}"), + async move |transaction| { + let project_id = project_id.as_str(); + let attempt = &attempt; + ensure_fanout_identity(&transaction, project_id, attempt).await?; - if let Some(stored) = read_delivery_attempt(&transaction, project_id, attempt).await? { - if stored.attempt != *attempt { - return Err("delivery attempt identity conflict".to_owned()); - } - let outcome = if stored.outcome.is_some() { - DeliveryAttemptClaimV1::AlreadySettled(stored.into_settlement()?) - } else { - DeliveryAttemptClaimV1::ReplayedAttempt - }; - if let Some(source_receipt_ref) = source_receipt_ref { - bind_source_receipt(&transaction, project_id, attempt, source_receipt_ref).await?; - } - transaction - .commit() - .await - .map_err(|error| format!("failed to close delivery attempt replay: {error}"))?; - return Ok(outcome); - } + if let Some(stored) = + read_delivery_attempt(&transaction, project_id, attempt).await? + { + if stored.attempt != *attempt { + return Err("delivery attempt identity conflict".to_owned()); + } + let outcome = if stored.outcome.is_some() { + DeliveryAttemptClaimV1::AlreadySettled(stored.into_settlement()?) + } else { + DeliveryAttemptClaimV1::ReplayedAttempt + }; + if let Some(source_receipt_ref) = source_receipt_ref.as_deref() { + bind_source_receipt(&transaction, project_id, attempt, source_receipt_ref) + .await?; + } + return Ok(outcome); + } - let existing = count_attempts(&transaction, project_id, attempt).await?; - if existing >= u64::from(attempt.eligible) { - return Err("delivery attempt exceeds eligible recipients".to_owned()); - } - transaction - .execute( - "INSERT INTO delivery_settlements - (project_id, owner_event_id, surface, channel_ref, attempted_at_micros, - outcome, settled_at_micros, drop_reason, census_json) - VALUES (?1, ?2, ?3, ?4, ?5, NULL, NULL, NULL, NULL)", - tracedecay_runtime_core::db::engine::params![ - project_id, - attempt.owner_event_id.as_str(), - surface_name(attempt.channel.surface), - attempt.channel.channel_ref.as_str(), - attempt.attempted_at.0, - ], - ) - .await - .map_err(|error| format!("failed to record delivery attempt: {error}"))?; - if let Some(source_receipt_ref) = source_receipt_ref { - bind_source_receipt(&transaction, project_id, attempt, source_receipt_ref).await?; - } - transaction - .commit() - .await - .map_err(|error| format!("failed to commit delivery attempt: {error}"))?; - Ok(DeliveryAttemptClaimV1::Claimed) + let existing = count_attempts(&transaction, project_id, attempt).await?; + if existing >= u64::from(attempt.eligible) { + return Err("delivery attempt exceeds eligible recipients".to_owned()); + } + transaction + .execute( + "INSERT INTO delivery_settlements + (project_id, owner_event_id, surface, channel_ref, attempted_at_micros, + outcome, settled_at_micros, drop_reason, census_json) + VALUES (?1, ?2, ?3, ?4, ?5, NULL, NULL, NULL, NULL)", + tracedecay_runtime_core::db::engine::params![ + project_id, + attempt.owner_event_id.as_str(), + surface_name(attempt.channel.surface), + attempt.channel.channel_ref.as_str(), + attempt.attempted_at.0, + ], + ) + .await + .map_err(|error| format!("failed to record delivery attempt: {error}"))?; + if let Some(source_receipt_ref) = source_receipt_ref.as_deref() { + bind_source_receipt(&transaction, project_id, attempt, source_receipt_ref) + .await?; + } + Ok(DeliveryAttemptClaimV1::Claimed) + }, + ) + .await } /// Resolves a project-scoped source acknowledgement token to the exact @@ -359,67 +363,68 @@ impl RegisteredGlobalDb { settlement .validate() .map_err(|error| format!("invalid delivery settlement: {error}"))?; - let transaction = self - .begin_write_transaction() - .await - .map_err(|error| format!("failed to begin delivery settlement transaction: {error}"))?; - ensure_fanout_identity(&transaction, project_id, &settlement.attempt).await?; - let stored = read_delivery_attempt(&transaction, project_id, &settlement.attempt) - .await? - .ok_or_else(|| "delivery attempt is unavailable".to_owned())?; - if stored.attempt != settlement.attempt { - return Err("delivery attempt identity conflict".to_owned()); - } - let (replayed, census) = if stored.outcome.is_some() { - let census = stored - .census - .clone() - .ok_or_else(|| "delivery settlement census is missing".to_owned())?; - if stored.into_settlement()? != *settlement { - return Err("delivery settlement conflict".to_owned()); - } - (true, census) - } else { - let census = read_pending_delivery_census( - &transaction, - project_id, - &settlement.attempt, - settlement, - ) - .await?; - let census_json = serde_json::to_string(&census) - .map_err(|error| format!("failed to serialize delivery census: {error}"))?; - transaction - .execute( - "UPDATE delivery_settlements - SET outcome = ?5, settled_at_micros = ?6, - drop_reason = ?7, census_json = ?8 - WHERE project_id = ?1 AND owner_event_id = ?2 AND surface = ?3 - AND channel_ref = ?4 AND outcome IS NULL", - tracedecay_runtime_core::db::engine::params![ + let project_id = project_id.to_owned(); + let settlement = settlement.clone(); + self.transact_or( + |error| format!("failed to run delivery settlement transaction: {error}"), + async move |transaction| { + let project_id = project_id.as_str(); + ensure_fanout_identity(&transaction, project_id, &settlement.attempt).await?; + let stored = read_delivery_attempt(&transaction, project_id, &settlement.attempt) + .await? + .ok_or_else(|| "delivery attempt is unavailable".to_owned())?; + if stored.attempt != settlement.attempt { + return Err("delivery attempt identity conflict".to_owned()); + } + let (replayed, census) = if stored.outcome.is_some() { + let census = stored + .census + .clone() + .ok_or_else(|| "delivery settlement census is missing".to_owned())?; + if stored.into_settlement()? != settlement { + return Err("delivery settlement conflict".to_owned()); + } + (true, census) + } else { + let census = read_pending_delivery_census( + &transaction, project_id, - settlement.attempt.owner_event_id.as_str(), - surface_name(settlement.attempt.channel.surface), - settlement.attempt.channel.channel_ref.as_str(), - outcome_name(settlement.outcome), - settlement.settled_at.0, - settlement.drop_reason.map(drop_reason_name), - census_json, - ], - ) - .await - .map_err(|error| format!("failed to settle delivery attempt: {error}"))?; - (false, census) - }; - transaction - .commit() - .await - .map_err(|error| format!("failed to commit delivery settlement: {error}"))?; - Ok(DurableDeliverySettlementReceiptV1 { - settlement: settlement.clone(), - census, - replayed, - }) + &settlement.attempt, + &settlement, + ) + .await?; + let census_json = serde_json::to_string(&census) + .map_err(|error| format!("failed to serialize delivery census: {error}"))?; + transaction + .execute( + "UPDATE delivery_settlements + SET outcome = ?5, settled_at_micros = ?6, + drop_reason = ?7, census_json = ?8 + WHERE project_id = ?1 AND owner_event_id = ?2 AND surface = ?3 + AND channel_ref = ?4 AND outcome IS NULL", + tracedecay_runtime_core::db::engine::params![ + project_id, + settlement.attempt.owner_event_id.as_str(), + surface_name(settlement.attempt.channel.surface), + settlement.attempt.channel.channel_ref.as_str(), + outcome_name(settlement.outcome), + settlement.settled_at.0, + settlement.drop_reason.map(drop_reason_name), + census_json, + ], + ) + .await + .map_err(|error| format!("failed to settle delivery attempt: {error}"))?; + (false, census) + }; + Ok(DurableDeliverySettlementReceiptV1 { + settlement: settlement.clone(), + census, + replayed, + }) + }, + ) + .await } } @@ -497,7 +502,7 @@ fn validate_source_receipt_ref(source_receipt_ref: &str) -> Result<(), String> { } async fn bind_source_receipt( - transaction: &RegisteredGlobalDbWriteTransaction<'_>, + transaction: &impl Executor, project_id: &str, attempt: &DeliverySettlementAttemptV1, source_receipt_ref: &str, @@ -587,7 +592,7 @@ async fn bind_source_receipt( } async fn ensure_fanout_identity( - transaction: &RegisteredGlobalDbWriteTransaction<'_>, + transaction: &impl Executor, project_id: &str, attempt: &DeliverySettlementAttemptV1, ) -> Result<(), String> { @@ -697,7 +702,7 @@ impl StoredDeliveryAttempt { } async fn read_delivery_attempt( - transaction: &RegisteredGlobalDbWriteTransaction<'_>, + transaction: &impl QueryExecutor, project_id: &str, attempt: &DeliverySettlementAttemptV1, ) -> Result, String> { @@ -756,7 +761,7 @@ async fn read_delivery_attempt( } async fn count_attempts( - transaction: &RegisteredGlobalDbWriteTransaction<'_>, + transaction: &impl QueryExecutor, project_id: &str, attempt: &DeliverySettlementAttemptV1, ) -> Result { @@ -785,7 +790,7 @@ async fn count_attempts( } async fn read_pending_delivery_census( - transaction: &RegisteredGlobalDbWriteTransaction<'_>, + transaction: &impl QueryExecutor, project_id: &str, attempt: &DeliverySettlementAttemptV1, settlement: &DeliverySettlementV1, diff --git a/crates/tracedecay-global-db/src/stack_delivery.rs b/crates/tracedecay-global-db/src/stack_delivery.rs index 585fc67f37..0e2ff3e286 100644 --- a/crates/tracedecay-global-db/src/stack_delivery.rs +++ b/crates/tracedecay-global-db/src/stack_delivery.rs @@ -10,7 +10,7 @@ use std::collections::{BTreeMap, BTreeSet}; use serde::{Deserialize, Serialize}; use tracedecay_runtime_core::db::engine::{Executor, QueryExecutor, params, params_from_iter}; -use crate::RegisteredGlobalDb; +use crate::{RegisteredGlobalDb, registered::JobExit}; /// Maximum number of rows in the `pending` state for one project. pub const MAX_GITHUB_STACK_ACTIVE_PENDING_V1: usize = 256; @@ -540,93 +540,88 @@ impl RegisteredGlobalDb { MAX_GITHUB_STACK_DELIVERY_BATCH_V1 )); } - let transaction = self - .begin_write_transaction() - .await - .map_err(|error| format!("begin GitHub stack signal append: {error}"))?; - if let Some(existing) = - lookup_signal(&transaction, &record.project_id, &record.signal_id).await? - { - if existing != record { + self.transact_or_rollback( + |error| format!("GitHub stack signal append: {error}"), + async move |transaction| { + if let Some(existing) = + lookup_signal(&transaction, &record.project_id, &record.signal_id).await? + { + if existing != record { + return Err("GitHub stack signal identity conflict".to_owned().into()); + } + let (pending_count, deferred_count) = + counts(&transaction, &record.project_id).await?; + return Err(JobExit::Rollback( + GitHubStackSignalAppendOutcomeV1::Replayed { + pending_count, + deferred_count, + }, + )); + } + transaction - .rollback() + .execute( + "INSERT INTO github_stack_delivery_signals + (project_id, signal_id, scope_digest, repository_id, + watermark_id, observed_at_micros, signal_json) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", + params![ + record.project_id.as_str(), + record.signal_id.as_str(), + record.scope_digest.as_str(), + record.repository_id.as_str(), + record.watermark_id.as_str(), + record.observed_at_micros, + record.signal_json.as_str() + ], + ) .await - .map_err(|error| format!("close conflicting GitHub stack append: {error}"))?; - return Err("GitHub stack signal identity conflict".to_owned()); - } - let (pending_count, deferred_count) = counts(&transaction, &record.project_id).await?; - transaction - .rollback() - .await - .map_err(|error| format!("close replayed GitHub stack append: {error}"))?; - return Ok(GitHubStackSignalAppendOutcomeV1::Replayed { - pending_count, - deferred_count, - }); - } - - transaction - .execute( - "INSERT INTO github_stack_delivery_signals - (project_id, signal_id, scope_digest, repository_id, - watermark_id, observed_at_micros, signal_json) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", - params![ - record.project_id.as_str(), - record.signal_id.as_str(), - record.scope_digest.as_str(), - record.repository_id.as_str(), - record.watermark_id.as_str(), - record.observed_at_micros, - record.signal_json.as_str() - ], - ) - .await - .map_err(|error| format!("append GitHub stack signal: {error}"))?; - // Repair a deferred queue left by an interrupted older writer before - // assigning this append's capacity. All callers hold the immediate - // write transaction, so the quota decision is serialized. - promote_deferred(&transaction, &record.project_id).await?; - let (pending_before, _) = counts(&transaction, &record.project_id).await?; - let capacity = MAX_GITHUB_STACK_ACTIVE_PENDING_V1.saturating_sub(pending_before); - for (index, recipient) in recipient_set.into_iter().enumerate() { - let state = if index < capacity { - GitHubStackDeliveryStateV1::Pending - } else { - GitHubStackDeliveryStateV1::Deferred - }; - transaction - .execute( - "INSERT INTO github_stack_delivery_recipients - (project_id, signal_id, recipient, state) - VALUES (?1, ?2, ?3, ?4)", - params![ - record.project_id.as_str(), - record.signal_id.as_str(), - recipient, - state.as_str() - ], - ) - .await - .map_err(|error| format!("append GitHub stack delivery recipient: {error}"))?; - } - let (pending_count, deferred_count) = counts(&transaction, &record.project_id).await?; - let saturated = deferred_count > 0; - transaction - .commit() - .await - .map_err(|error| format!("commit GitHub stack signal append: {error}"))?; - Ok(if saturated { - GitHubStackSignalAppendOutcomeV1::Saturated { - pending_count, - deferred_count, - } - } else { - GitHubStackSignalAppendOutcomeV1::Appended { - pending_count, - deferred_count, - } - }) + .map_err(|error| format!("append GitHub stack signal: {error}"))?; + // Repair a deferred queue left by an interrupted older writer before + // assigning this append's capacity. The write job is serialized, so + // the quota decision is too. + promote_deferred(&transaction, &record.project_id).await?; + let (pending_before, _) = counts(&transaction, &record.project_id).await?; + let capacity = MAX_GITHUB_STACK_ACTIVE_PENDING_V1.saturating_sub(pending_before); + for (index, recipient) in recipient_set.iter().enumerate() { + let state = if index < capacity { + GitHubStackDeliveryStateV1::Pending + } else { + GitHubStackDeliveryStateV1::Deferred + }; + transaction + .execute( + "INSERT INTO github_stack_delivery_recipients + (project_id, signal_id, recipient, state) + VALUES (?1, ?2, ?3, ?4)", + params![ + record.project_id.as_str(), + record.signal_id.as_str(), + recipient.as_str(), + state.as_str() + ], + ) + .await + .map_err(|error| { + format!("append GitHub stack delivery recipient: {error}") + })?; + } + let (pending_count, deferred_count) = + counts(&transaction, &record.project_id).await?; + Ok(if deferred_count > 0 { + GitHubStackSignalAppendOutcomeV1::Saturated { + pending_count, + deferred_count, + } + } else { + GitHubStackSignalAppendOutcomeV1::Appended { + pending_count, + deferred_count, + } + }) + }, + ) + .await } /// Returns a deterministic page of coordinator-pending deliveries. @@ -742,46 +737,47 @@ impl RegisteredGlobalDb { validate_text(&key.signal_id, "signal id")?; validate_recipient(&key.recipient)?; } - let transaction = self - .begin_write_transaction() - .await - .map_err(|error| format!("begin GitHub stack host publication: {error}"))?; - let batch = read_delivery_batch(&transaction, project_id, &keys).await?; - let mut pending = Vec::with_capacity(keys.len()); - for key in &keys { - let row = batch - .get(key) - .ok_or_else(|| "GitHub stack delivery batch is incomplete".to_owned())?; - let signal_watermark = row - .watermark_id - .as_deref() - .ok_or_else(|| "GitHub stack delivery signal is unavailable".to_owned())?; - if signal_watermark != watermark_id { - return Err("GitHub stack delivery watermark mismatch".to_owned()); - } - let state = row - .state - .ok_or_else(|| "GitHub stack recipient binding is unavailable".to_owned())?; - match state { - GitHubStackDeliveryStateV1::Pending => { - pending.push(key.clone()); + let project_id = project_id.to_owned(); + let watermark_id = watermark_id.to_owned(); + self.transact_or( + |error| format!("GitHub stack host publication: {error}"), + async move |transaction| { + let batch = read_delivery_batch(&transaction, &project_id, &keys).await?; + let mut pending = Vec::with_capacity(keys.len()); + for key in &keys { + let row = batch + .get(key) + .ok_or_else(|| "GitHub stack delivery batch is incomplete".to_owned())?; + let signal_watermark = row + .watermark_id + .as_deref() + .ok_or_else(|| "GitHub stack delivery signal is unavailable".to_owned())?; + if signal_watermark != watermark_id { + return Err("GitHub stack delivery watermark mismatch".to_owned()); + } + let state = row.state.ok_or_else(|| { + "GitHub stack recipient binding is unavailable".to_owned() + })?; + match state { + GitHubStackDeliveryStateV1::Pending => { + pending.push(key.clone()); + } + GitHubStackDeliveryStateV1::HostPending => {} + GitHubStackDeliveryStateV1::Deferred => { + return Err("GitHub stack delivery is deferred".to_owned()); + } + GitHubStackDeliveryStateV1::Settled => {} + GitHubStackDeliveryStateV1::AuthorizationLost => { + return Err("GitHub stack delivery authorization was lost".to_owned()); + } + } } - GitHubStackDeliveryStateV1::HostPending => {} - GitHubStackDeliveryStateV1::Deferred => { - return Err("GitHub stack delivery is deferred".to_owned()); - } - GitHubStackDeliveryStateV1::Settled => {} - GitHubStackDeliveryStateV1::AuthorizationLost => { - return Err("GitHub stack delivery authorization was lost".to_owned()); - } - } - } - transition_pending_batch(&transaction, project_id, &pending).await?; - promote_deferred(&transaction, project_id).await?; - transaction - .commit() - .await - .map_err(|error| format!("commit GitHub stack host publication: {error}")) + transition_pending_batch(&transaction, &project_id, &pending).await?; + promote_deferred(&transaction, &project_id).await?; + Ok(()) + }, + ) + .await } /// Final host receipt. Replaying a receipt after settlement is harmless; @@ -800,58 +796,70 @@ impl RegisteredGlobalDb { validate_text(project_id, "project id")?; validate_text(signal_id, "signal id")?; validate_recipient(recipient)?; - let transaction = self - .begin_write_transaction() - .await - .map_err(|error| format!("begin GitHub stack host acknowledgement: {error}"))?; - let mut rows = transaction - .query( - "SELECT state FROM github_stack_delivery_recipients - WHERE project_id = ?1 AND signal_id = ?2 AND recipient = ?3", - params![project_id, signal_id, recipient], - ) - .await - .map_err(|error| format!("query GitHub stack host acknowledgement: {error}"))?; - let Some(row) = rows - .next() - .await - .map_err(|error| format!("read GitHub stack host acknowledgement: {error}"))? - else { - return Err("GitHub stack recipient binding is unavailable".to_owned()); - }; - let state = GitHubStackDeliveryStateV1::parse( - row.get::(0) - .map_err(|error| format!("decode GitHub stack host state: {error}"))?, - )?; - match state { - GitHubStackDeliveryStateV1::HostPending => { - transaction - .execute( - "UPDATE github_stack_delivery_recipients - SET state = 'settled' - WHERE project_id = ?1 AND signal_id = ?2 AND recipient = ?3 - AND state = 'host_pending'", - params![project_id, signal_id, recipient], + let (project_id, signal_id, recipient) = ( + project_id.to_owned(), + signal_id.to_owned(), + recipient.to_owned(), + ); + self.transact_or( + |error| format!("GitHub stack host acknowledgement: {error}"), + async move |transaction| { + let mut rows = transaction + .query( + "SELECT state FROM github_stack_delivery_recipients + WHERE project_id = ?1 AND signal_id = ?2 AND recipient = ?3", + params![project_id.as_str(), signal_id.as_str(), recipient.as_str()], ) .await - .map_err(|error| format!("settle GitHub stack host delivery: {error}"))?; - } - GitHubStackDeliveryStateV1::Settled => {} - GitHubStackDeliveryStateV1::Pending => { - return Err("GitHub stack delivery is still pending host publication".to_owned()); - } - GitHubStackDeliveryStateV1::Deferred => { - return Err("GitHub stack delivery is deferred".to_owned()); - } - GitHubStackDeliveryStateV1::AuthorizationLost => { - return Err("GitHub stack delivery authorization was lost".to_owned()); - } - } - promote_deferred(&transaction, project_id).await?; - transaction - .commit() - .await - .map_err(|error| format!("commit GitHub stack host acknowledgement: {error}")) + .map_err(|error| format!("query GitHub stack host acknowledgement: {error}"))?; + let Some(row) = rows + .next() + .await + .map_err(|error| format!("read GitHub stack host acknowledgement: {error}"))? + else { + return Err("GitHub stack recipient binding is unavailable".to_owned()); + }; + let state = GitHubStackDeliveryStateV1::parse( + row.get::(0) + .map_err(|error| format!("decode GitHub stack host state: {error}"))?, + )?; + match state { + GitHubStackDeliveryStateV1::HostPending => { + transaction + .execute( + "UPDATE github_stack_delivery_recipients + SET state = 'settled' + WHERE project_id = ?1 AND signal_id = ?2 AND recipient = ?3 + AND state = 'host_pending'", + params![ + project_id.as_str(), + signal_id.as_str(), + recipient.as_str() + ], + ) + .await + .map_err(|error| { + format!("settle GitHub stack host delivery: {error}") + })?; + } + GitHubStackDeliveryStateV1::Settled => {} + GitHubStackDeliveryStateV1::Pending => { + return Err( + "GitHub stack delivery is still pending host publication".to_owned() + ); + } + GitHubStackDeliveryStateV1::Deferred => { + return Err("GitHub stack delivery is deferred".to_owned()); + } + GitHubStackDeliveryStateV1::AuthorizationLost => { + return Err("GitHub stack delivery authorization was lost".to_owned()); + } + } + promote_deferred(&transaction, &project_id).await?; + Ok(()) + }, + ) + .await } /// Marks a pending/deferred recipient as permanently unauthorized. The @@ -866,49 +874,53 @@ impl RegisteredGlobalDb { validate_text(project_id, "project id")?; validate_text(signal_id, "signal id")?; validate_recipient(recipient)?; - let transaction = self - .begin_write_transaction() - .await - .map_err(|error| format!("begin GitHub stack authorization loss: {error}"))?; - let changed = transaction - .execute( - "UPDATE github_stack_delivery_recipients - SET state = 'authorization_lost' - WHERE project_id = ?1 AND signal_id = ?2 AND recipient = ?3 - AND state IN ('pending', 'deferred')", - params![project_id, signal_id, recipient], - ) - .await - .map_err(|error| format!("record GitHub stack authorization loss: {error}"))?; - if changed == 0 { - let mut rows = transaction - .query( - "SELECT state FROM github_stack_delivery_recipients - WHERE project_id = ?1 AND signal_id = ?2 AND recipient = ?3", - params![project_id, signal_id, recipient], - ) - .await - .map_err(|error| format!("query GitHub stack authorization state: {error}"))?; - let Some(row) = rows - .next() - .await - .map_err(|error| format!("read GitHub stack authorization state: {error}"))? - else { - return Err("GitHub stack recipient binding is unavailable".to_owned()); - }; - let state = - GitHubStackDeliveryStateV1::parse(row.get::(0).map_err(|error| { - format!("decode GitHub stack authorization state: {error}") - })?)?; - if state != GitHubStackDeliveryStateV1::AuthorizationLost { - return Err("GitHub stack delivery is no longer authorizable".to_owned()); - } - } - promote_deferred(&transaction, project_id).await?; - transaction - .commit() - .await - .map_err(|error| format!("commit GitHub stack authorization loss: {error}")) + let (project_id, signal_id, recipient) = ( + project_id.to_owned(), + signal_id.to_owned(), + recipient.to_owned(), + ); + self.transact_or( + |error| format!("GitHub stack authorization loss: {error}"), + async move |transaction| { + let changed = transaction + .execute( + "UPDATE github_stack_delivery_recipients + SET state = 'authorization_lost' + WHERE project_id = ?1 AND signal_id = ?2 AND recipient = ?3 + AND state IN ('pending', 'deferred')", + params![project_id.as_str(), signal_id.as_str(), recipient.as_str()], + ) + .await + .map_err(|error| format!("record GitHub stack authorization loss: {error}"))?; + if changed == 0 { + let mut rows = transaction + .query( + "SELECT state FROM github_stack_delivery_recipients + WHERE project_id = ?1 AND signal_id = ?2 AND recipient = ?3", + params![project_id.as_str(), signal_id.as_str(), recipient.as_str()], + ) + .await + .map_err(|error| { + format!("query GitHub stack authorization state: {error}") + })?; + let Some(row) = rows.next().await.map_err(|error| { + format!("read GitHub stack authorization state: {error}") + })? + else { + return Err("GitHub stack recipient binding is unavailable".to_owned()); + }; + let state = GitHubStackDeliveryStateV1::parse(row.get::(0).map_err( + |error| format!("decode GitHub stack authorization state: {error}"), + )?)?; + if state != GitHubStackDeliveryStateV1::AuthorizationLost { + return Err("GitHub stack delivery is no longer authorizable".to_owned()); + } + } + promote_deferred(&transaction, &project_id).await?; + Ok(()) + }, + ) + .await } /// Looks up the immutable signal payload by its exact registered-project From 45baa936c45fa216684c440099ee063e27c3fbe9 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 16:14:15 +0000 Subject: [PATCH 11/89] refactor(global-db): run source rewrite writes as jobs --- .../src/registered_source_rewrite.rs | 297 +++++++++--------- 1 file changed, 152 insertions(+), 145 deletions(-) diff --git a/crates/tracedecay-global-db/src/registered_source_rewrite.rs b/crates/tracedecay-global-db/src/registered_source_rewrite.rs index 48cc673b79..e36a2eab5b 100644 --- a/crates/tracedecay-global-db/src/registered_source_rewrite.rs +++ b/crates/tracedecay-global-db/src/registered_source_rewrite.rs @@ -12,11 +12,12 @@ use tracedecay_domain::{ CanonicalObservationIdV1, ObservationScopeV1, ObservationSourceGenerationV1, ObservationSourceIdentityV1, }; -use tracedecay_runtime_core::db::engine::{QueryExecutor, params}; +use tracedecay_runtime_core::db::engine::{Executor, QueryExecutor, params}; use tracedecay_store::{ProjectionStoreError, ProjectionStoreResult}; use super::RegisteredGlobalDb; use super::observation_projection::retire_source_record; +use super::registered::JobExit; /// Unoffered-record reads are keyset-paginated at this page size. The exact- /// SQL engine refuses a query that materializes more rows than its hard cap, @@ -87,33 +88,42 @@ impl RegisteredGlobalDb { return Ok(()); } let key = source_key(source, scope)?; - let transaction = self - .begin_write_transaction() - .await - .map_err(|error| storage("begin source presence transaction", error))?; - for record in records { - transaction - .execute( - "INSERT INTO observation_source_presence ( - source_key, observation_id, generation, start_offset - ) VALUES (?1, ?2, ?3, ?4) - ON CONFLICT(source_key, observation_id) DO UPDATE SET - generation = excluded.generation, - start_offset = excluded.start_offset", - params![ - key.as_str(), - record.observation_id.as_str(), - record.generation.generation_id().to_string(), - offset(record.start_offset)?, - ], - ) - .await - .map_err(|error| storage("record source presence", error))?; - } - transaction - .commit() - .await - .map_err(|error| storage("commit source presence", error)) + let rows = records + .iter() + .map(|record| { + Ok(( + record.observation_id.as_str().to_owned(), + record.generation.generation_id().to_string(), + offset(record.start_offset)?, + )) + }) + .collect::>>()?; + self.transact_or( + |error| storage("run source presence transaction", error), + async move |transaction| { + for (observation_id, generation, start_offset) in &rows { + transaction + .execute( + "INSERT INTO observation_source_presence ( + source_key, observation_id, generation, start_offset + ) VALUES (?1, ?2, ?3, ?4) + ON CONFLICT(source_key, observation_id) DO UPDATE SET + generation = excluded.generation, + start_offset = excluded.start_offset", + params![ + key.as_str(), + observation_id.as_str(), + generation.as_str(), + *start_offset, + ], + ) + .await + .map_err(|error| storage("record source presence", error))?; + } + Ok(()) + }, + ) + .await } /// Starts a rewrite of the source from `previous` to `generation`. The @@ -138,44 +148,45 @@ impl RegisteredGlobalDb { } let key = source_key(source, scope)?; let generation = generation.generation_id().to_string(); - let transaction = self - .begin_write_transaction() - .await - .map_err(|error| storage("begin source rewrite transaction", error))?; - // A rewrite the previous generation never finished still names its - // own layout: its retained prefix was relabeled to it. - let unfinished = pending_rewrite(&transaction, &key).await?; - transaction - .execute( - "UPDATE observation_source_presence SET generation = ?2 - WHERE source_key = ?1 AND start_offset < ?3 - AND generation IN (?4, ?5)", - params![ - key.as_str(), - generation.as_str(), - offset(retained_through)?, - previous.generation_id().to_string(), - unfinished, - ], - ) - .await - .map_err(|error| storage("retain source prefix", error))?; - transaction - .execute( - "INSERT INTO observation_source_rewrites ( - source_key, generation, retained_through - ) VALUES (?1, ?2, ?3) - ON CONFLICT(source_key) DO UPDATE SET - generation = excluded.generation, - retained_through = excluded.retained_through", - params![key.as_str(), generation.as_str(), offset(retained_through)?], - ) - .await - .map_err(|error| storage("record source rewrite", error))?; - transaction - .commit() - .await - .map_err(|error| storage("commit source rewrite", error)) + let retained_through = offset(retained_through)?; + let previous = previous.generation_id().to_string(); + self.transact_or( + |error| storage("run source rewrite transaction", error), + async move |transaction| { + // A rewrite the previous generation never finished still names its + // own layout: its retained prefix was relabeled to it. + let unfinished = pending_rewrite(&transaction, &key).await?; + transaction + .execute( + "UPDATE observation_source_presence SET generation = ?2 + WHERE source_key = ?1 AND start_offset < ?3 + AND generation IN (?4, ?5)", + params![ + key.as_str(), + generation.as_str(), + retained_through, + previous.as_str(), + unfinished.as_deref(), + ], + ) + .await + .map_err(|error| storage("retain source prefix", error))?; + transaction + .execute( + "INSERT INTO observation_source_rewrites ( + source_key, generation, retained_through + ) VALUES (?1, ?2, ?3) + ON CONFLICT(source_key) DO UPDATE SET + generation = excluded.generation, + retained_through = excluded.retained_through", + params![key.as_str(), generation.as_str(), retained_through], + ) + .await + .map_err(|error| storage("record source rewrite", error))?; + Ok(()) + }, + ) + .await } /// Completes a pending rewrite once `generation` was read to the end of @@ -202,85 +213,81 @@ impl RegisteredGlobalDb { return Ok(0); } drop(snapshot); - let transaction = self - .begin_write_transaction() - .await - .map_err(|error| storage("begin source rewrite completion", error))?; - if pending_rewrite(&transaction, &key).await?.as_deref() != Some(generation.as_str()) { - transaction - .rollback() - .await - .map_err(|error| storage("release source rewrite completion", error))?; - return Ok(0); - } - let mut retired = 0_u64; - // Keyset-paginate the unoffered ids instead of materializing the whole - // set in one statement: a single rewritten source can hold more - // unoffered records than the engine's per-query materialization cap, - // and one oversized read wedged every completion attempt forever. - // `retire_source_record` never touches `observation_source_presence`, - // so the ordered id cursor stays stable across pages; the settle - // deletes below remain the authoritative cleanup. - let mut after_observation_id = String::new(); - loop { - let mut rows = transaction - .query( - "SELECT observation_id FROM observation_source_presence - WHERE source_key = ?1 AND generation <> ?2 - AND observation_id > ?3 - ORDER BY observation_id - LIMIT ?4", - params![ - key.as_str(), - generation.as_str(), - after_observation_id, - UNOFFERED_RECORD_PAGE_SIZE - ], - ) - .await - .map_err(|error| storage("read unoffered source records", error))?; - let mut page = Vec::new(); - while let Some(row) = rows - .next() - .await - .map_err(|error| storage("read unoffered source records", error))? - { - page.push( - row.get::(0) - .map_err(|error| storage("read unoffered source records", error))?, - ); - } - drop(rows); - if page.is_empty() { - break; - } - let page_len = page.len(); - after_observation_id = page[page_len - 1].clone(); - for observation_id in &page { - let retired_one: Pin< - Box> + Send + '_>, - > = Box::pin(retire_source_record(&transaction, observation_id)); - if retired_one.await? { - retired += 1; + self.transact_or_rollback( + |error| storage("run source rewrite completion", error), + async move |transaction| { + if pending_rewrite(&transaction, &key).await?.as_deref() + != Some(generation.as_str()) + { + return Err(JobExit::Rollback(0)); } - } - if (page_len as i64) < UNOFFERED_RECORD_PAGE_SIZE { - break; - } - } - for sql in [ - "DELETE FROM observation_source_presence WHERE source_key = ?1 AND generation <> ?2", - "DELETE FROM observation_source_rewrites WHERE source_key = ?1 AND generation = ?2", - ] { - transaction - .execute(sql, params![key.as_str(), generation.as_str()]) - .await - .map_err(|error| storage("settle source rewrite", error))?; - } - transaction - .commit() - .await - .map_err(|error| storage("commit source rewrite completion", error))?; - Ok(retired) + let mut retired = 0_u64; + // Keyset-paginate the unoffered ids instead of materializing the whole + // set in one statement: a single rewritten source can hold more + // unoffered records than the engine's per-query materialization cap, + // and one oversized read wedged every completion attempt forever. + // `retire_source_record` never touches `observation_source_presence`, + // so the ordered id cursor stays stable across pages; the settle + // deletes below remain the authoritative cleanup. + let mut after_observation_id = String::new(); + loop { + let mut rows = transaction + .query( + "SELECT observation_id FROM observation_source_presence + WHERE source_key = ?1 AND generation <> ?2 + AND observation_id > ?3 + ORDER BY observation_id + LIMIT ?4", + params![ + key.as_str(), + generation.as_str(), + after_observation_id.as_str(), + UNOFFERED_RECORD_PAGE_SIZE + ], + ) + .await + .map_err(|error| storage("read unoffered source records", error))?; + let mut page = Vec::new(); + while let Some(row) = rows + .next() + .await + .map_err(|error| storage("read unoffered source records", error))? + { + page.push( + row.get::(0) + .map_err(|error| storage("read unoffered source records", error))?, + ); + } + drop(rows); + if page.is_empty() { + break; + } + let page_len = page.len(); + after_observation_id = page[page_len - 1].clone(); + for observation_id in &page { + let retired_one: Pin< + Box> + Send + '_>, + > = Box::pin(retire_source_record(&transaction, observation_id)); + if retired_one.await? { + retired += 1; + } + } + if (page_len as i64) < UNOFFERED_RECORD_PAGE_SIZE { + break; + } + } + for sql in [ + "DELETE FROM observation_source_presence WHERE source_key = ?1 AND generation <> ?2", + "DELETE FROM observation_source_rewrites WHERE source_key = ?1 AND generation = ?2", + ] { + transaction + .execute(sql, params![key.as_str(), generation.as_str()]) + .await + .map_err(|error| storage("settle source rewrite", error))?; + } + Ok(retired) + }, + ) + .await } } From 6d7ab42a99e981c5389589d9ad16b0e9fcc2d5db Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 16:14:16 +0000 Subject: [PATCH 12/89] refactor(global-db): run registry and remote deletion writes as jobs --- .../src/registry_maintenance/lifecycle.rs | 41 ++-- .../src/remote_deletion.rs | 221 +++++++++--------- 2 files changed, 131 insertions(+), 131 deletions(-) diff --git a/crates/tracedecay-global-db/src/registry_maintenance/lifecycle.rs b/crates/tracedecay-global-db/src/registry_maintenance/lifecycle.rs index 09ca393e33..2c7ee003c4 100644 --- a/crates/tracedecay-global-db/src/registry_maintenance/lifecycle.rs +++ b/crates/tracedecay-global-db/src/registry_maintenance/lifecycle.rs @@ -209,7 +209,7 @@ pub async fn apply_registry_gc( } pub(super) async fn delete_registry_gc_candidates_in_transaction( - transaction: &RegisteredGlobalDbWriteTransaction<'_>, + transaction: &impl tracedecay_runtime_core::db::engine::Executor, project_ids: &[String], project_paths: &[PathBuf], ) -> tracedecay_domain::errors::Result<(usize, usize)> { @@ -269,19 +269,22 @@ pub async fn forget_registry_project( project_id: &str, project_paths: &[PathBuf], ) -> tracedecay_domain::errors::Result { - let transaction = db.begin_write_transaction().await?; - let (code_projects_deleted, path_ledger_rows_deleted) = - delete_registry_gc_candidates_in_transaction( - &transaction, - &[project_id.to_string()], - project_paths, - ) - .await?; - transaction.commit().await?; - Ok(ForgetRegistryProjectRows { - code_projects_deleted, - path_ledger_rows_deleted, + let project_ids = [project_id.to_string()]; + let project_paths = project_paths.to_vec(); + db.transact(async move |transaction| { + let (code_projects_deleted, path_ledger_rows_deleted) = + delete_registry_gc_candidates_in_transaction( + &transaction, + &project_ids, + &project_paths, + ) + .await?; + Ok::<_, tracedecay_domain::errors::TraceDecayError>(ForgetRegistryProjectRows { + code_projects_deleted, + path_ledger_rows_deleted, + }) }) + .await? } /// Retires exact registry rows for project roots already removed by an @@ -290,9 +293,11 @@ pub async fn retire_registry_project_paths( db: &RegisteredGlobalDb, project_paths: &[PathBuf], ) -> tracedecay_domain::errors::Result { - let transaction = db.begin_write_transaction().await?; - let (_, retired) = - delete_registry_gc_candidates_in_transaction(&transaction, &[], project_paths).await?; - transaction.commit().await?; - Ok(retired) + let project_paths = project_paths.to_vec(); + db.transact(async move |transaction| { + let (_, retired) = + delete_registry_gc_candidates_in_transaction(&transaction, &[], &project_paths).await?; + Ok::<_, tracedecay_domain::errors::TraceDecayError>(retired) + }) + .await? } diff --git a/crates/tracedecay-global-db/src/remote_deletion.rs b/crates/tracedecay-global-db/src/remote_deletion.rs index d85975e412..8a4fd115c4 100644 --- a/crates/tracedecay-global-db/src/remote_deletion.rs +++ b/crates/tracedecay-global-db/src/remote_deletion.rs @@ -2,7 +2,7 @@ use serde::{Deserialize, Serialize}; use tracedecay_domain::errors::TraceDecayError; -use tracedecay_runtime_core::db::engine::{QueryExecutor, params}; +use tracedecay_runtime_core::db::engine::{Executor, QueryExecutor, params}; use crate::RegisteredGlobalDb; @@ -323,45 +323,44 @@ impl RegisteredGlobalDb { tombstone: RemoteDeletionTombstone, ) -> Result { tombstone.validate_new()?; - let transaction = self.begin_write_transaction().await?; - if let Some(existing) = read_tombstone( - &transaction, - &tombstone.profile_id, - tombstone.target, - tombstone.project_key(), - ) - .await? - { - transaction.commit().await.map_err(|error| { - remote_deletion_error("commit remote deletion tombstone replay", error) - })?; - return if existing.tombstone_id == tombstone.tombstone_id { - Ok(RemoteDeletionTombstoneRecordOutcome::Replayed(existing)) - } else { - Ok(RemoteDeletionTombstoneRecordOutcome::Conflict { existing }) - }; - } - transaction - .execute( - "INSERT INTO remote_deletion_tombstones - (profile_id, target_kind, project_id, tombstone_id, recorded_at_micros, - cleanup_status, failure_code, failure_phase, retryable) - VALUES (?1, ?2, ?3, ?4, ?5, 'pending', NULL, NULL, NULL)", - params![ - tombstone.profile_id.as_str(), - tombstone.target.as_str(), - tombstone.project_key(), - tombstone.tombstone_id.as_str(), - tombstone.recorded_at_micros, - ], + self.transact(async move |transaction| { + if let Some(existing) = read_tombstone( + &transaction, + &tombstone.profile_id, + tombstone.target, + tombstone.project_key(), ) - .await - .map_err(|error| remote_deletion_error("record remote deletion tombstone", error))?; - transaction - .commit() - .await - .map_err(|error| remote_deletion_error("commit remote deletion tombstone", error))?; - Ok(RemoteDeletionTombstoneRecordOutcome::Recorded(tombstone)) + .await? + { + return Ok(if existing.tombstone_id == tombstone.tombstone_id { + RemoteDeletionTombstoneRecordOutcome::Replayed(existing) + } else { + RemoteDeletionTombstoneRecordOutcome::Conflict { existing } + }); + } + transaction + .execute( + "INSERT INTO remote_deletion_tombstones + (profile_id, target_kind, project_id, tombstone_id, recorded_at_micros, + cleanup_status, failure_code, failure_phase, retryable) + VALUES (?1, ?2, ?3, ?4, ?5, 'pending', NULL, NULL, NULL)", + params![ + tombstone.profile_id.as_str(), + tombstone.target.as_str(), + tombstone.project_key(), + tombstone.tombstone_id.as_str(), + tombstone.recorded_at_micros, + ], + ) + .await + .map_err(|error| { + remote_deletion_error("record remote deletion tombstone", error) + })?; + Ok(RemoteDeletionTombstoneRecordOutcome::Recorded( + tombstone.clone(), + )) + }) + .await? } #[tracing::instrument( @@ -376,70 +375,65 @@ impl RegisteredGlobalDb { next: RemoteDeletionCleanupState, ) -> Result { validate_transition(&next)?; - let transaction = self.begin_write_transaction().await?; - let existing = read_tombstone( - &transaction, - &tombstone.profile_id, - tombstone.target, - tombstone.project_key(), - ) - .await? - .ok_or_else(|| { - remote_deletion_error( - "transition remote deletion tombstone", - "tombstone is unavailable", - ) - })?; - if existing.tombstone_id != tombstone.tombstone_id { - transaction.commit().await.map_err(|error| { - remote_deletion_error("commit remote deletion conflict read", error) - })?; - return Ok(RemoteDeletionTombstoneTransitionOutcome::Conflict { existing }); - } - if existing.cleanup != expected { - transaction.commit().await.map_err(|error| { - remote_deletion_error("commit remote deletion state read", error) - })?; - return Ok(RemoteDeletionTombstoneTransitionOutcome::StateChanged { existing }); - } - let (failure_code, failure_phase, retryable) = - next.failure() - .map_or((None, None, None), |(code, phase, retryable)| { - ( - Some(code.as_str()), - Some(phase.as_str()), - Some(i64::from(retryable)), - ) - }); - transaction - .execute( - "UPDATE remote_deletion_tombstones - SET cleanup_status = ?1, failure_code = ?2, failure_phase = ?3, retryable = ?4 - WHERE profile_id = ?5 AND target_kind = ?6 AND project_id = ?7 - AND tombstone_id = ?8", - params![ - next.status(), - failure_code, - failure_phase, - retryable, - tombstone.profile_id.as_str(), - tombstone.target.as_str(), - tombstone.project_key(), - tombstone.tombstone_id.as_str(), - ], + let tombstone = tombstone.clone(); + let expected = expected.clone(); + self.transact(async move |transaction| { + let existing = read_tombstone( + &transaction, + &tombstone.profile_id, + tombstone.target, + tombstone.project_key(), ) - .await - .map_err(|error| { - remote_deletion_error("transition remote deletion tombstone", error) + .await? + .ok_or_else(|| { + remote_deletion_error( + "transition remote deletion tombstone", + "tombstone is unavailable", + ) })?; - transaction.commit().await.map_err(|error| { - remote_deletion_error("commit remote deletion tombstone transition", error) - })?; - let updated = RemoteDeletionTombstone { - cleanup: next, - ..existing - }; - Ok(RemoteDeletionTombstoneTransitionOutcome::Updated(updated)) + if existing.tombstone_id != tombstone.tombstone_id { + return Ok(RemoteDeletionTombstoneTransitionOutcome::Conflict { existing }); + } + if existing.cleanup != expected { + return Ok(RemoteDeletionTombstoneTransitionOutcome::StateChanged { existing }); + } + let (failure_code, failure_phase, retryable) = + next.failure() + .map_or((None, None, None), |(code, phase, retryable)| { + ( + Some(code.as_str()), + Some(phase.as_str()), + Some(i64::from(retryable)), + ) + }); + transaction + .execute( + "UPDATE remote_deletion_tombstones + SET cleanup_status = ?1, failure_code = ?2, failure_phase = ?3, retryable = ?4 + WHERE profile_id = ?5 AND target_kind = ?6 AND project_id = ?7 + AND tombstone_id = ?8", + params![ + next.status(), + failure_code, + failure_phase, + retryable, + tombstone.profile_id.as_str(), + tombstone.target.as_str(), + tombstone.project_key(), + tombstone.tombstone_id.as_str(), + ], + ) + .await + .map_err(|error| { + remote_deletion_error("transition remote deletion tombstone", error) + })?; + let updated = RemoteDeletionTombstone { + cleanup: next.clone(), + ..existing + }; + Ok(RemoteDeletionTombstoneTransitionOutcome::Updated(updated)) + }) + .await? } pub async fn remote_deletion_tombstone_for_project( @@ -475,19 +469,20 @@ impl RegisteredGlobalDb { pub async fn delete_remote_deleted_project_registry_row(&self, project_id: &str) -> Result<()> { validate_identifier("remote deletion project id", project_id)?; - let transaction = self.begin_write_transaction().await?; - transaction - .execute( - "DELETE FROM code_projects WHERE project_id = ?1", - params![project_id], - ) - .await - .map_err(|error| { - remote_deletion_error("remove remote-deleted project registry row", error) - })?; - transaction.commit().await.map_err(|error| { - remote_deletion_error("commit remote-deleted project registry row", error) + let project_id = project_id.to_owned(); + self.transact(async move |transaction| { + transaction + .execute( + "DELETE FROM code_projects WHERE project_id = ?1", + params![project_id.as_str()], + ) + .await + .map_err(|error| { + remote_deletion_error("remove remote-deleted project registry row", error) + })?; + Ok(()) }) + .await? } } From 68af603fdbd6e396ddad1fba3105e1d8b8836455 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 16:14:18 +0000 Subject: [PATCH 13/89] refactor(global-db): run analytics writes as jobs --- .../src/registered_analytics.rs | 454 +++++++++--------- 1 file changed, 231 insertions(+), 223 deletions(-) diff --git a/crates/tracedecay-global-db/src/registered_analytics.rs b/crates/tracedecay-global-db/src/registered_analytics.rs index 609e51ecf1..468eef0bc4 100644 --- a/crates/tracedecay-global-db/src/registered_analytics.rs +++ b/crates/tracedecay-global-db/src/registered_analytics.rs @@ -3,7 +3,7 @@ use std::fmt::Write as _; use std::future::Future; use std::pin::Pin; -use tracedecay_runtime_core::db::engine::{Value, opt_text}; +use tracedecay_runtime_core::db::engine::{Executor, Value, opt_text}; use super::{ AnalyticsEventInsert, AnalyticsEventQuery, AnalyticsEventRecord, AnalyticsHintCounts, @@ -236,23 +236,19 @@ impl RegisteredGlobalDb { if events.is_empty() { return Ok(Vec::new()); } - let transaction = self - .begin_write_transaction() - .await - .map_err(|error| format!("failed to begin analytics append transaction: {error}"))?; - let ids = match kind { - AnalyticsAppendKind::General => { - append_analytics_events_in_existing_tx(&transaction, events).await? - } - AnalyticsAppendKind::Observability => { - append_observability_events_in_existing_tx(&transaction, events).await? - } - }; - transaction - .commit() - .await - .map_err(|error| format!("failed to commit analytics append transaction: {error}"))?; - Ok(ids) + let events = events.to_vec(); + self.transact_or( + |error| format!("failed to run analytics append transaction: {error}"), + async move |transaction| match kind { + AnalyticsAppendKind::General => { + append_analytics_events_in_existing_tx(&transaction, &events).await + } + AnalyticsAppendKind::Observability => { + append_observability_events_in_existing_tx(&transaction, &events).await + } + }, + ) + .await } #[tracing::instrument( @@ -323,52 +319,55 @@ impl RegisteredGlobalDb { owner_fact_json, delivery_envelope_json, )?; - let transaction = self - .begin_write_transaction() - .await - .map_err(|error| format!("failed to begin observability outbox claim: {error}"))?; - if let Some(stored) = read_outbox_record(&transaction, project_id, owner_event_id).await? { - if stored.owner_fact_json != owner_fact_json { - return Err("observability owner fact conflict".to_owned()); - } - let outcome = if let Some(analytics_event_id) = stored.analytics_event_id { - ObservabilityEmissionClaimV1::Settled { - delivery_envelope_json: stored.delivery_envelope_json, - analytics_event_id, - } - } else { - ObservabilityEmissionClaimV1::Pending { - delivery_envelope_json: stored.delivery_envelope_json, + let project_id = project_id.to_owned(); + let owner_event_id = owner_event_id.to_owned(); + let owner_fact_json = owner_fact_json.to_owned(); + let delivery_envelope_json = delivery_envelope_json.to_owned(); + self.transact_or( + |error| format!("failed to run observability outbox claim: {error}"), + async move |transaction| { + if let Some(stored) = + read_outbox_record(&transaction, &project_id, &owner_event_id).await? + { + if stored.owner_fact_json != owner_fact_json { + return Err("observability owner fact conflict".to_owned()); + } + return Ok( + if let Some(analytics_event_id) = stored.analytics_event_id { + ObservabilityEmissionClaimV1::Settled { + delivery_envelope_json: stored.delivery_envelope_json, + analytics_event_id, + } + } else { + ObservabilityEmissionClaimV1::Pending { + delivery_envelope_json: stored.delivery_envelope_json, + } + }, + ); } - }; - transaction - .commit() - .await - .map_err(|error| format!("failed to close observability outbox replay: {error}"))?; - return Ok(outcome); - } - transaction - .execute( - "INSERT INTO observability_emission_outbox - (project_id, owner_event_id, owner_fact_json, - delivery_envelope_json, state, analytics_event_id) - VALUES (?1, ?2, ?3, ?4, 'pending', NULL)", - tracedecay_runtime_core::db::engine::params![ - project_id, - owner_event_id, - owner_fact_json, - delivery_envelope_json - ], - ) - .await - .map_err(|error| format!("failed to claim observability outbox event: {error}"))?; - transaction - .commit() - .await - .map_err(|error| format!("failed to commit observability outbox claim: {error}"))?; - Ok(ObservabilityEmissionClaimV1::Claimed { - delivery_envelope_json: delivery_envelope_json.to_owned(), - }) + transaction + .execute( + "INSERT INTO observability_emission_outbox + (project_id, owner_event_id, owner_fact_json, + delivery_envelope_json, state, analytics_event_id) + VALUES (?1, ?2, ?3, ?4, 'pending', NULL)", + tracedecay_runtime_core::db::engine::params![ + project_id.as_str(), + owner_event_id.as_str(), + owner_fact_json.as_str(), + delivery_envelope_json.as_str() + ], + ) + .await + .map_err(|error| { + format!("failed to claim observability outbox event: {error}") + })?; + Ok(ObservabilityEmissionClaimV1::Claimed { + delivery_envelope_json: delivery_envelope_json.clone(), + }) + }, + ) + .await } /// CAS a pending delivery to its delayed-coverage representation. @@ -391,59 +390,56 @@ impl RegisteredGlobalDb { owner_fact_json, delayed_delivery_envelope_json, )?; - let transaction = self - .begin_write_transaction() - .await - .map_err(|error| format!("failed to begin observability outbox delay: {error}"))?; - let stored = read_outbox_record(&transaction, project_id, owner_event_id) - .await? - .ok_or_else(|| "observability outbox event is unavailable".to_owned())?; - if stored.owner_fact_json != owner_fact_json { - return Err("observability owner fact conflict".to_owned()); - } - if let Some(analytics_event_id) = stored.analytics_event_id { - transaction - .commit() - .await - .map_err(|error| format!("failed to close settled outbox delay: {error}"))?; - return Ok(ObservabilityEmissionClaimV1::Settled { - delivery_envelope_json: stored.delivery_envelope_json, - analytics_event_id, - }); - } - if stored.delivery_envelope_json != expected_delivery_envelope_json { - transaction - .commit() - .await - .map_err(|error| format!("failed to close replayed outbox delay: {error}"))?; - return Ok(ObservabilityEmissionClaimV1::Pending { - delivery_envelope_json: stored.delivery_envelope_json, - }); - } - transaction - .execute( - "UPDATE observability_emission_outbox - SET delivery_envelope_json = ?4 - WHERE project_id = ?1 AND owner_event_id = ?2 - AND owner_fact_json = ?3 AND state = 'pending' - AND delivery_envelope_json = ?5", - tracedecay_runtime_core::db::engine::params![ - project_id, - owner_event_id, - owner_fact_json, - delayed_delivery_envelope_json, - expected_delivery_envelope_json - ], - ) - .await - .map_err(|error| format!("failed to mark observability delivery delayed: {error}"))?; - transaction - .commit() - .await - .map_err(|error| format!("failed to commit observability outbox delay: {error}"))?; - Ok(ObservabilityEmissionClaimV1::Pending { - delivery_envelope_json: delayed_delivery_envelope_json.to_owned(), - }) + let project_id = project_id.to_owned(); + let owner_event_id = owner_event_id.to_owned(); + let owner_fact_json = owner_fact_json.to_owned(); + let expected_delivery_envelope_json = expected_delivery_envelope_json.to_owned(); + let delayed_delivery_envelope_json = delayed_delivery_envelope_json.to_owned(); + self.transact_or( + |error| format!("failed to run observability outbox delay: {error}"), + async move |transaction| { + let stored = read_outbox_record(&transaction, &project_id, &owner_event_id) + .await? + .ok_or_else(|| "observability outbox event is unavailable".to_owned())?; + if stored.owner_fact_json != owner_fact_json { + return Err("observability owner fact conflict".to_owned()); + } + if let Some(analytics_event_id) = stored.analytics_event_id { + return Ok(ObservabilityEmissionClaimV1::Settled { + delivery_envelope_json: stored.delivery_envelope_json, + analytics_event_id, + }); + } + if stored.delivery_envelope_json != expected_delivery_envelope_json { + return Ok(ObservabilityEmissionClaimV1::Pending { + delivery_envelope_json: stored.delivery_envelope_json, + }); + } + transaction + .execute( + "UPDATE observability_emission_outbox + SET delivery_envelope_json = ?4 + WHERE project_id = ?1 AND owner_event_id = ?2 + AND owner_fact_json = ?3 AND state = 'pending' + AND delivery_envelope_json = ?5", + tracedecay_runtime_core::db::engine::params![ + project_id.as_str(), + owner_event_id.as_str(), + owner_fact_json.as_str(), + delayed_delivery_envelope_json.as_str(), + expected_delivery_envelope_json.as_str() + ], + ) + .await + .map_err(|error| { + format!("failed to mark observability delivery delayed: {error}") + })?; + Ok(ObservabilityEmissionClaimV1::Pending { + delivery_envelope_json: delayed_delivery_envelope_json.clone(), + }) + }, + ) + .await } /// Append the exact delivery and settle its outbox row in one transaction. @@ -472,48 +468,54 @@ impl RegisteredGlobalDb { { return Err("observability outbox delivery binding conflict".to_owned()); } - let transaction = self - .begin_write_transaction() - .await - .map_err(|error| format!("failed to begin observability outbox settlement: {error}"))?; - let stored = read_outbox_record(&transaction, project_id, owner_event_id) - .await? - .ok_or_else(|| "observability outbox event is unavailable".to_owned())?; - if stored.owner_fact_json != owner_fact_json - || stored.delivery_envelope_json != delivery_envelope_json - { - return Err("observability outbox settlement conflict".to_owned()); - } - let id = append_observability_event_in_existing_tx(&transaction, event) - .await - .map_err(|error| error.to_string())?; - if let Some(settled_id) = stored.analytics_event_id { - if settled_id != id { - return Err("observability outbox receipt conflict".to_owned()); - } - } else { - transaction - .execute( - "UPDATE observability_emission_outbox - SET state = 'settled', analytics_event_id = ?5 - WHERE project_id = ?1 AND owner_event_id = ?2 - AND owner_fact_json = ?3 AND delivery_envelope_json = ?4 - AND state = 'pending'", - tracedecay_runtime_core::db::engine::params![ - project_id, - owner_event_id, - owner_fact_json, - delivery_envelope_json, - id - ], - ) - .await - .map_err(|error| format!("failed to settle observability outbox event: {error}"))?; - } - transaction.commit().await.map_err(|error| { - format!("failed to commit observability outbox settlement: {error}") - })?; - Ok(id) + let project_id = project_id.to_owned(); + let owner_event_id = owner_event_id.to_owned(); + let owner_fact_json = owner_fact_json.to_owned(); + let delivery_envelope_json = delivery_envelope_json.to_owned(); + let event = event.clone(); + self.transact_or( + |error| format!("failed to run observability outbox settlement: {error}"), + async move |transaction| { + let stored = read_outbox_record(&transaction, &project_id, &owner_event_id) + .await? + .ok_or_else(|| "observability outbox event is unavailable".to_owned())?; + if stored.owner_fact_json != owner_fact_json + || stored.delivery_envelope_json != delivery_envelope_json + { + return Err("observability outbox settlement conflict".to_owned()); + } + let id = append_observability_event_in_existing_tx(&transaction, &event) + .await + .map_err(|error| error.to_string())?; + if let Some(settled_id) = stored.analytics_event_id { + if settled_id != id { + return Err("observability outbox receipt conflict".to_owned()); + } + } else { + transaction + .execute( + "UPDATE observability_emission_outbox + SET state = 'settled', analytics_event_id = ?5 + WHERE project_id = ?1 AND owner_event_id = ?2 + AND owner_fact_json = ?3 AND delivery_envelope_json = ?4 + AND state = 'pending'", + tracedecay_runtime_core::db::engine::params![ + project_id.as_str(), + owner_event_id.as_str(), + owner_fact_json.as_str(), + delivery_envelope_json.as_str(), + id + ], + ) + .await + .map_err(|error| { + format!("failed to settle observability outbox event: {error}") + })?; + } + Ok(id) + }, + ) + .await } /// Claim and settle one run of worker-owned owner facts in a single write. @@ -630,43 +632,43 @@ impl RegisteredGlobalDb { if now_seconds < 0 { return Err("invalid observability retention time".to_owned()); } - let transaction = self - .begin_write_transaction() - .await - .map_err(|error| format!("failed to begin observability retention: {error}"))?; - // Expired dirty-day markers go first: they protect their source rows - // from the analytics pass below, and a day past the horizon has no - // generation left to rebuild. - let expired_rollups = expire_observability_rollups(&transaction, now_seconds).await?; - let (expired_detail_outbox, expired_detail) = prune_observability_retention_class( - &transaction, - now_seconds.saturating_sub(OBSERVABILITY_DETAIL_RETENTION_SECONDS), - "optional_local_detail30d", - ) - .await?; - let (expired_rollup_outbox, expired_rollup) = prune_observability_retention_class( - &transaction, - now_seconds.saturating_sub(OBSERVABILITY_ROLLUP_RETENTION_SECONDS), - "local_rollup395d", - ) - .await?; - let has_more = observability_retention_has_more( - &transaction, - now_seconds.saturating_sub(OBSERVABILITY_DETAIL_RETENTION_SECONDS), - now_seconds.saturating_sub(OBSERVABILITY_ROLLUP_RETENTION_SECONDS), + self.transact_or( + |error| format!("failed to run observability retention: {error}"), + async move |transaction| { + // Expired dirty-day markers go first: they protect their source rows + // from the analytics pass below, and a day past the horizon has no + // generation left to rebuild. + let expired_rollups = + expire_observability_rollups(&transaction, now_seconds).await?; + let (expired_detail_outbox, expired_detail) = prune_observability_retention_class( + &transaction, + now_seconds.saturating_sub(OBSERVABILITY_DETAIL_RETENTION_SECONDS), + "optional_local_detail30d", + ) + .await?; + let (expired_rollup_outbox, expired_rollup) = prune_observability_retention_class( + &transaction, + now_seconds.saturating_sub(OBSERVABILITY_ROLLUP_RETENTION_SECONDS), + "local_rollup395d", + ) + .await?; + let has_more = observability_retention_has_more( + &transaction, + now_seconds.saturating_sub(OBSERVABILITY_DETAIL_RETENTION_SECONDS), + now_seconds.saturating_sub(OBSERVABILITY_ROLLUP_RETENTION_SECONDS), + ) + .await?; + Ok(ObservabilityRetentionReceiptV1 { + expired_detail, + expired_rollup, + expired_settled_outbox: expired_detail_outbox + .saturating_add(expired_rollup_outbox), + expired_rollups, + has_more, + }) + }, ) - .await?; - transaction - .commit() - .await - .map_err(|error| format!("failed to commit observability retention: {error}"))?; - Ok(ObservabilityRetentionReceiptV1 { - expired_detail, - expired_rollup, - expired_settled_outbox: expired_detail_outbox.saturating_add(expired_rollup_outbox), - expired_rollups, - has_more, - }) + .await } #[tracing::instrument( @@ -701,22 +703,28 @@ impl RegisteredGlobalDb { expected_cursor: super::ParseOffset, cursor: super::ParseOffset, ) -> Result, String> { - let transaction = self - .begin_write_transaction() - .await - .map_err(|error| format!("failed to begin analytics import transaction: {error}"))?; - super::transcript::require_expected_offset(&transaction, cursor_path, expected_cursor) - .await - .map_err(|error| format!("failed to claim analytics import cursor: {error}"))?; - let ids = append_analytics_events_in_existing_tx(&transaction, events).await?; - super::transcript::set_parse_offset(&transaction, cursor_path, cursor) - .await - .map_err(|error| format!("failed to persist analytics import cursor: {error}"))?; - transaction - .commit() - .await - .map_err(|error| format!("failed to commit analytics import transaction: {error}"))?; - Ok(ids) + let events = events.to_vec(); + let cursor_path = cursor_path.to_owned(); + self.transact_or( + |error| format!("failed to run analytics import transaction: {error}"), + async move |transaction| { + super::transcript::require_expected_offset( + &transaction, + &cursor_path, + expected_cursor, + ) + .await + .map_err(|error| format!("failed to claim analytics import cursor: {error}"))?; + let ids = append_analytics_events_in_existing_tx(&transaction, &events).await?; + super::transcript::set_parse_offset(&transaction, &cursor_path, cursor) + .await + .map_err(|error| { + format!("failed to persist analytics import cursor: {error}") + })?; + Ok(ids) + }, + ) + .await } #[tracing::instrument( @@ -950,7 +958,7 @@ impl RegisteredGlobalDb { } async fn append_analytics_events_in_existing_tx( - transaction: &super::RegisteredGlobalDbWriteTransaction<'_>, + transaction: &impl Executor, events: &[AnalyticsEventInsert], ) -> Result, String> { let mut ids = Vec::with_capacity(events.len()); @@ -1016,7 +1024,7 @@ async fn append_analytics_events_in_existing_tx( } async fn append_analytics_event_in_existing_tx( - transaction: &super::RegisteredGlobalDbWriteTransaction<'_>, + transaction: &impl Executor, event: &AnalyticsEventInsert, ) -> Result { let mut rows = transaction @@ -1098,7 +1106,7 @@ fn validate_prepared_owner_emission( } async fn claim_and_settle_owner_facts_in_tx( - transaction: &RegisteredGlobalDbWriteTransaction<'_>, + transaction: &impl Executor, emissions: &[ObservabilityOwnerEmissionWriteV1], prepare_new: F, ) -> Result, String> @@ -1131,7 +1139,7 @@ where /// Resolves every still-open fact against the outbox and prepares a delivery /// only for a fact no row owns yet. async fn prepare_unclaimed_owner_facts( - transaction: &RegisteredGlobalDbWriteTransaction<'_>, + transaction: &impl Executor, emissions: &[ObservabilityOwnerEmissionWriteV1], outcomes: &mut [Option], mut prepare_new: F, @@ -1183,7 +1191,7 @@ where /// Adopts an exact stored event, appends the rest in one statement, and /// settles their outbox rows in one statement. async fn settle_prepared_owner_facts( - transaction: &RegisteredGlobalDbWriteTransaction<'_>, + transaction: &impl Executor, emissions: &[ObservabilityOwnerEmissionWriteV1], outcomes: &mut [Option], prepared: Vec<(usize, PreparedObservabilityEmissionV1)>, @@ -1263,7 +1271,7 @@ fn bind_key_pairs(mut sql: String, keys: Vec<(String, String)>) -> (String, Vec< /// Stored owner facts for `keys`, by `(project_id, owner_event_id)`. async fn stored_owner_facts( - transaction: &RegisteredGlobalDbWriteTransaction<'_>, + transaction: &impl Executor, keys: Vec<(String, String)>, ) -> Result, String> { let mut stored = BTreeMap::new(); @@ -1298,7 +1306,7 @@ async fn stored_owner_facts( /// Stored canonical observability events for `keys`, by /// `(project_id, hint_id)`. async fn stored_observability_events( - transaction: &RegisteredGlobalDbWriteTransaction<'_>, + transaction: &impl Executor, keys: Vec<(String, String)>, ) -> Result, String> { let mut stored = BTreeMap::new(); @@ -1336,7 +1344,7 @@ async fn stored_observability_events( } async fn insert_settled_outbox_rows( - transaction: &RegisteredGlobalDbWriteTransaction<'_>, + transaction: &impl Executor, rows: &[(&ObservabilityOwnerEmissionWriteV1, String, i64)], ) -> Result<(), String> { for chunk in rows.chunks(ANALYTICS_INSERT_ROWS_PER_STATEMENT) { @@ -1407,7 +1415,7 @@ fn validate_outbox_input( } async fn read_outbox_record( - transaction: &RegisteredGlobalDbWriteTransaction<'_>, + transaction: &impl Executor, project_id: &str, owner_event_id: &str, ) -> Result, String> { @@ -1454,7 +1462,7 @@ enum ObservabilityAppendError { /// All runs share the caller's transaction, so a later conflict rolls back /// earlier appends just as the single-event path does. async fn append_observability_events_in_existing_tx( - transaction: &RegisteredGlobalDbWriteTransaction<'_>, + transaction: &impl Executor, events: &[AnalyticsEventInsert], ) -> Result, String> { let mut ids = Vec::with_capacity(events.len()); @@ -1514,7 +1522,7 @@ async fn append_observability_events_in_existing_tx( } async fn append_observability_event_in_existing_tx( - transaction: &RegisteredGlobalDbWriteTransaction<'_>, + transaction: &impl Executor, event: &AnalyticsEventInsert, ) -> Result { validate_observability_event(event).map_err(ObservabilityAppendError::Failed)?; @@ -1583,7 +1591,7 @@ fn analytics_record_matches_insert( } async fn prune_observability_retention_class( - transaction: &RegisteredGlobalDbWriteTransaction<'_>, + transaction: &impl Executor, cutoff_seconds: i64, retention_class: &str, ) -> Result<(u64, u64), String> { @@ -1625,7 +1633,7 @@ async fn prune_observability_retention_class( } async fn observability_retention_has_more( - transaction: &RegisteredGlobalDbWriteTransaction<'_>, + transaction: &impl Executor, detail_cutoff_seconds: i64, rollup_cutoff_seconds: i64, ) -> Result { From f2f81a2598fe438307613a0fd53890f8c7125a2a Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 16:14:19 +0000 Subject: [PATCH 14/89] test(global-db): prove job exits and scope loss roll back --- crates/tracedecay-global-db/src/tests.rs | 86 ++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/crates/tracedecay-global-db/src/tests.rs b/crates/tracedecay-global-db/src/tests.rs index 9e471a6fdd..8629cda561 100644 --- a/crates/tracedecay-global-db/src/tests.rs +++ b/crates/tracedecay-global-db/src/tests.rs @@ -1483,6 +1483,92 @@ async fn transact_rolls_back_when_scope_ends_before_commit() { ); } +#[asupersync::test] +async fn transact_or_rollback_keeps_an_early_value_but_never_a_commit() { + use crate::registered::JobExit; + use tracedecay_runtime_core::db::engine::Executor as _; + let harness = RegisteredGlobalDbHarness::open("transact-or-rollback-exits").await; + let db = harness.registered.clone(); + + let replayed = db + .transact_or_rollback( + |error| error.to_string(), + async move |transaction| { + transaction + .execute("CREATE TABLE job_exit_rolled_back (value INTEGER)", ()) + .await + .map_err(|error| error.to_string())?; + Err::(JobExit::Rollback(7)) + }, + ) + .await; + assert_eq!(replayed, Ok(7)); + assert!( + !table_exists(&db, "job_exit_rolled_back").await, + "an early exit left its writes behind" + ); + + let failed = db + .transact_or_rollback( + |error| error.to_string(), + async move |transaction| { + transaction + .execute("CREATE TABLE job_exit_failed (value INTEGER)", ()) + .await + .map_err(|error| error.to_string())?; + Err::(JobExit::Fail("body failed".to_owned())) + }, + ) + .await; + assert_eq!(failed, Err("body failed".to_owned())); + assert!(!table_exists(&db, "job_exit_failed").await); +} + +#[asupersync::test] +async fn transact_or_wraps_a_scope_that_ended_mid_body_and_rolls_back() { + use tracedecay_runtime_core::db::engine::Executor as _; + let mut harness = RegisteredGlobalDbHarness::open("transact-or-scope-end").await; + let db = harness.registered.clone(); + let scope = std::sync::Arc::new(std::sync::Mutex::new(harness.take_scope())); + + let error = db + .transact_or( + |error| format!("wrapped: {error}"), + async move |transaction| { + transaction + .execute("CREATE TABLE transact_or_staged (value INTEGER)", ()) + .await + .map_err(|error| error.to_string())?; + drop(scope.lock().unwrap().take()); + Ok::<_, String>(()) + }, + ) + .await + .expect_err("transact_or committed after the scope ended"); + + assert!(error.starts_with("wrapped: "), "{error}"); + assert!(error.contains("active daemon"), "{error}"); + assert!( + !table_exists(&db, "transact_or_staged").await, + "scope loss did not roll the job back" + ); +} + +#[asupersync::test] +async fn converted_write_is_refused_without_a_write_scope() { + let mut harness = RegisteredGlobalDbHarness::open("converted-write-no-scope").await; + let db = harness.registered.clone(); + harness.revoke(); + + let error = db + .try_record_savings("/project", "tool", 10, 4, 1) + .await + .expect_err("a converted write ran without a write scope"); + + assert!(error.to_string().contains("active daemon"), "{error}"); + assert_eq!(row_count(&db, "savings_ledger").await, 0); +} + #[asupersync::test] async fn begun_registered_transaction_can_roll_back_after_scope_drop() { let mut harness = RegisteredGlobalDbHarness::open("registered-rollback").await; From 9afc4aa0a30a8c4960d40bba6f3c6471ab99025d Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 16:14:39 +0000 Subject: [PATCH 15/89] docs(storage): record step 5i-4 global-db job writes --- docs/plans/storage/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/plans/storage/README.md b/docs/plans/storage/README.md index 021f5d860f..67e644e9a0 100644 --- a/docs/plans/storage/README.md +++ b/docs/plans/storage/README.md @@ -773,6 +773,24 @@ changes a file that holds memory. clones what it owns into the closure. A rollup rebuild that finds its stored receipt still ends by rolling back, so replay does not advance the transaction generation. +- Step 5i-4, global-db job writes (2026-10-18, branch + `refactor/storage-global-db-jobs-2`). The accounting, workflow-index, topology + anchor, discovery queue, GitHub stack delivery, delivery settlement, source + rewrite, registry forget and retire, remote deletion and analytics outbox writes + now run as `RegisteredGlobalDb::transact` jobs. `transact_or` maps a refused job + to the site's own error type. `transact_or_rollback` lets a body end early with + a value the caller wants as success, as a replayed append does, and the job + rolls back so a replay leaves no commit. Helpers under these sites take + `&impl Executor`, so one body serves the job and the remaining interactive + transactions. `WorkflowIngestWriteTxn` had no caller left and is deleted. Still + interactive, for later slices: `apply_registry_gc` (it reads outside the + transaction while holding it), `relink_orphan_store_instance` (a manifest file + effect sits between the SQL and the commit), `claim_and_settle_observability_emissions` + (its `FnMut` allocates producer sequences, so a re-run would allocate twice), + `append_analytics_event` (a writer connection, not an interactive transaction), + and the project registry, observation projection, LCM, provider usage, + dashboard, session, schema-stage and invariant, git-index and observation + retention and canonical-body groups. - #4138 (2026-10-16). Shutdown store close waits up to 3 s for leases to drain before deciding a store's WAL cannot be truncated. Under load, owners joined before store close could still be unwinding the tasks that held their leases, From 6ce5353a6038c908f4daf40d64b5e18361eed87a Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 16:17:58 +0000 Subject: [PATCH 16/89] chore: regenerate dashboard contracts without upload_enabled --- .../codegen/schemas/dashboard-contracts.schema.json | 10 ---------- dashboard/src/contracts/decoders.ts | 4 ---- dashboard/src/contracts/types.ts | 2 -- 3 files changed, 16 deletions(-) diff --git a/dashboard/codegen/schemas/dashboard-contracts.schema.json b/dashboard/codegen/schemas/dashboard-contracts.schema.json index e65724f847..ee47a131a9 100644 --- a/dashboard/codegen/schemas/dashboard-contracts.schema.json +++ b/dashboard/codegen/schemas/dashboard-contracts.schema.json @@ -26581,12 +26581,6 @@ "idempotency_key": { "type": "string" }, - "upload_enabled": { - "type": [ - "boolean", - "null" - ] - }, "watcher_debounce": { "type": [ "string", @@ -26640,9 +26634,6 @@ }, "type": "array" }, - "upload_enabled": { - "type": "boolean" - }, "watcher_debounce": { "type": "string" } @@ -26652,7 +26643,6 @@ "configuration_revision_id", "code_index_worker_configuration_snapshot_id", "code_index_worker_configuration_revision_id", - "upload_enabled", "code_index_workers", "code_index_worker_status", "watcher_debounce", diff --git a/dashboard/src/contracts/decoders.ts b/dashboard/src/contracts/decoders.ts index 67b2c136d4..38737d6a67 100644 --- a/dashboard/src/contracts/decoders.ts +++ b/dashboard/src/contracts/decoders.ts @@ -9455,13 +9455,11 @@ export const UserSettingsPatchSchema: z.ZodObject<{ expected_revision_id: z.ZodType; extraction_timeout_secs: z.ZodOptional>; idempotency_key: z.ZodType; - upload_enabled: z.ZodOptional>; watcher_debounce: z.ZodOptional>; }, z.core.$strict> = z.object({ expected_revision_id: z.string(), extraction_timeout_secs: z.number().int().min(0).nullable().optional(), idempotency_key: z.string(), - upload_enabled: z.boolean().nullable().optional(), watcher_debounce: z.string().nullable().optional(), }).strict(); @@ -9474,7 +9472,6 @@ export const UserSettingsPayloadV1Schema: z.ZodObject<{ configuration_snapshot_id: z.ZodType; extraction_timeout_secs: z.ZodType; installed_agents: z.ZodType, unknown>; - upload_enabled: z.ZodType; watcher_debounce: z.ZodType; }> = z.object({ code_index_worker_configuration_revision_id: z.string(), @@ -9485,7 +9482,6 @@ export const UserSettingsPayloadV1Schema: z.ZodObject<{ configuration_snapshot_id: z.string(), extraction_timeout_secs: z.number().int().min(0), installed_agents: z.array(z.string()), - upload_enabled: z.boolean(), watcher_debounce: z.string(), }); diff --git a/dashboard/src/contracts/types.ts b/dashboard/src/contracts/types.ts index 7eceeb6733..b9e3fcaa73 100644 --- a/dashboard/src/contracts/types.ts +++ b/dashboard/src/contracts/types.ts @@ -5423,7 +5423,6 @@ export type UserSettingsPatch = { expected_revision_id: string; extraction_timeout_secs?: number | null | undefined; idempotency_key: string; - upload_enabled?: boolean | null | undefined; watcher_debounce?: string | null | undefined; }; @@ -5436,7 +5435,6 @@ export type UserSettingsPayloadV1 = { configuration_snapshot_id: string; extraction_timeout_secs: number; installed_agents: Array; - upload_enabled: boolean; watcher_debounce: string; }; From 95381c8ae9b7a722cefc97c7c0b4184a32b840bb Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 16:31:16 +0000 Subject: [PATCH 17/89] build(evals): resolve eval binary through the shipping Bazel target --- docs/memory-evals.md | 4 +- evals/agent_adoption/README.md | 17 +++-- evals/agent_adoption/run.sh | 41 +++--------- evals/hermetic/README.md | 28 +++++--- evals/hermetic/run.sh | 47 ++----------- evals/memory/run_real_model.py | 32 ++++++--- evals/memory/test_run_real_model.py | 27 ++++++++ scripts/BUILD.bazel | 8 +++ scripts/build-eval-binary.sh | 58 ++++++++++++++++ scripts/test-build-eval-binary.py | 100 ++++++++++++++++++++++++++++ 10 files changed, 262 insertions(+), 100 deletions(-) create mode 100755 scripts/build-eval-binary.sh create mode 100644 scripts/test-build-eval-binary.py diff --git a/docs/memory-evals.md b/docs/memory-evals.md index 67bdc14d70..ff0502f977 100644 --- a/docs/memory-evals.md +++ b/docs/memory-evals.md @@ -117,7 +117,7 @@ actually determines whether durable memory helps a user. That is the ### How it works -`evals/hermetic/run.sh` builds the dev binary, stages it at a non-cargo path, +`evals/hermetic/run.sh` builds the shipping Bazel binary, stages a copy under the env dir, installs the plugin into a throwaway `CLAUDE_CONFIG_DIR`/`CODEX_HOME`/ `TRACEDECAY_DATA_DIR`, indexes a project, then drives a **real** agent (`claude -p` or `codex exec --json`) at each corpus prompt and records the @@ -128,7 +128,7 @@ each session used. The adoption corpus is per bucket. ```bash -ENV=$(evals/hermetic/run.sh setup --agent codex --debug) # subscription; no API +ENV=$(evals/hermetic/run.sh setup --agent codex) # subscription; no API evals/hermetic/run.sh index --env-dir "$ENV" --project evals/hermetic/run.sh run --agent codex --env-dir "$ENV" \ --corpus evals/hermetic/corpora/fact-store-adoption.jsonl diff --git a/evals/agent_adoption/README.md b/evals/agent_adoption/README.md index 6844d7cf17..cd1697c3fa 100644 --- a/evals/agent_adoption/README.md +++ b/evals/agent_adoption/README.md @@ -88,7 +88,8 @@ paths, so do **not** commit run artifacts. | `PARALLEL` | `1` | concurrent live agent runs; scenarios are read-only against shared fixtures and each `claude` process spawns its own MCP server, so `4` is a safe default for high-rep cells | | `REPEATED_QUERY_HINTS` | `off` | `on` turns on the profile setting `user.repeated_query_hints.v1` for the eval's isolated profile before any agent launches, and suffixes transcripts with `__qhints`; see "Repeated-query hints on vs off" | | `REAL_CLAUDE_DIR` | `$CLAUDE_CONFIG_DIR` or `~/.claude` | where the Claude credential comes from: its `.claude_code_oauth_token` and `settings.json` `env`/`model` keys; see "Store isolation & auth" | -| `TRACEDECAY_BIN` | candidate branch (live) | explicit tracedecay binary override | +| `TRACEDECAY_BIN` | Bazel build of this checkout | explicit prebuilt tracedecay binary; skips the build and must exist | +| `TRACEDECAY_EVAL_BAZEL_CONFIG` | `release` | Bazel config for the build; `dist` builds the exact release bytes | | `EVAL_OUT` | unset | dir to copy scoreboard + report into | | `LEARNED_SKILL_WORKFLOW` | unset | run only that workflow's held-out scenarios, once per variant, on `full` | | `LEARNED_SKILL_VARIANTS` | `none generated manual` | learned-skill variants to compare | @@ -128,11 +129,15 @@ fails, and Codex drops the graph server without any transcript error. Name the m model that actually answered; an alias resolved through `ANTHROPIC_DEFAULT_*_MODEL` would be mislabelled. -Live runs build a debug `tracedecay` binary from the checked-out candidate -branch with ordinary Cargo and use it for fixture setup, MCP, hooks, and the -throwaway Codex plugin install. -`TRACEDECAY_BIN` pins a previously built candidate or release binary and avoids -the runner build path. The daemon also receives a disposable HOME, XDG paths, +Every run, dry or live, builds the shipping target +`//crates/tracedecay-cli:tracedecay` from the checked-out candidate branch with +`bazel build --config=release` (`scripts/build-eval-binary.sh`) and uses it for +fixture setup, MCP, hooks, and the throwaway Codex plugin install. The runner +prints the binary path, version, and sha256. A failed build stops the run; no +older or PATH binary is substituted. Set `TRACEDECAY_EVAL_BAZEL_CONFIG=dist` to +build the exact release bytes (thin LTO, stripped). +`TRACEDECAY_BIN` pins a previously built candidate or release binary, which must +exist, and skips the build; its identity line is printed the same way. The daemon also receives a disposable HOME, XDG paths, host-config paths, temporary directory, and working directory. Only explicit read-only authentication sources are carried across that boundary. Every Claude condition, including `full`, loads the candidate `plugin/` via diff --git a/evals/agent_adoption/run.sh b/evals/agent_adoption/run.sh index c85245e7db..21d4d96309 100755 --- a/evals/agent_adoption/run.sh +++ b/evals/agent_adoption/run.sh @@ -43,7 +43,11 @@ # read-only against shared fixtures, so this is safe # REPEATED_QUERY_HINTS on|off: the eval profile's repeated-query hint setting # (default: off); on suffixes transcripts with __qhints -# TRACEDECAY_BIN tracedecay binary (default: resolve from PATH) +# TRACEDECAY_BIN prebuilt tracedecay binary to run instead of the Bazel +# build (default: bazel build --config=release +# //crates/tracedecay-cli:tracedecay) +# TRACEDECAY_EVAL_BAZEL_CONFIG Bazel config for that build (default: release; +# dist builds the exact release bytes) # EVAL_OUT directory to also copy scoreboard.json + report.md into # LEARNED_SKILL_WORKFLOW run only learned_skills/'s held-out scenarios, # once per skill variant, on the full condition @@ -111,36 +115,11 @@ STEER_TEXT="This repository has indexed code-relationship evidence available. Ch live=0 if [[ "${TRACEDECAY_AGENT_EVALS:-}" == "1" ]]; then live=1; fi -# Live runs default to the candidate branch binary. TRACEDECAY_BIN remains an -# explicit override for release-binary comparisons; dry runs reuse a built -# candidate when present and otherwise fall back to PATH. -if [[ -n "${TRACEDECAY_BIN:-}" ]]; then - TD="$TRACEDECAY_BIN" -elif [[ "$live" == "1" ]]; then - echo "building candidate tracedecay binary..." - if [[ -n "${CARGO_TARGET_DIR:-}" ]]; then - eval_target="$CARGO_TARGET_DIR" - elif [[ -d /fast/cargo-target && -w /fast/cargo-target ]]; then - eval_target="/fast/cargo-target/tracedecay-agent-adoption-evals" - else - eval_target="$repo_root/target" - fi - ( - cd "$repo_root" - CARGO_TARGET_DIR="$eval_target" \ - cargo build --quiet --package tracedecay-cli --bin tracedecay - ) - TD="$eval_target/debug/tracedecay" -elif [[ -x "$repo_root/target/debug/tracedecay" ]]; then - TD="$repo_root/target/debug/tracedecay" -else - TD="$(command -v tracedecay || true)" -fi -if [[ -z "$TD" || ! -x "$TD" ]]; then - echo "error: tracedecay binary not found; set TRACEDECAY_BIN" >&2 - exit 2 -fi -TD="$(cd "$(dirname "$TD")" && pwd)/$(basename "$TD")" +# Live and dry runs both use the shipping Bazel binary built from this checkout +# (scripts/build-eval-binary.sh). TRACEDECAY_BIN selects an already-built +# artifact, such as a release binary, and must exist. A failed build exits; +# no PATH or older-build fallback exists. +TD="$("$repo_root/scripts/build-eval-binary.sh")" || exit 2 # Neutrality lint (USER DOCTRINE): fail fast, before building fixtures or # spending a single token, if any scenario prompt names tracedecay/MCP/a diff --git a/evals/hermetic/README.md b/evals/hermetic/README.md index e8bb5a58d0..61cb95ce0c 100644 --- a/evals/hermetic/README.md +++ b/evals/hermetic/README.md @@ -8,11 +8,11 @@ sessions depend on the real `~/.claude`, `~/.tracedecay`, and ```bash # One-shot: build + isolate + install + index + one trivial scenario. -evals/hermetic/run.sh smoke --agent claude --model sonnet --debug -evals/hermetic/run.sh smoke --agent codex --debug +evals/hermetic/run.sh smoke --agent claude --model sonnet +evals/hermetic/run.sh smoke --agent codex # Full corpus against a reusable env: -ENV=$(evals/hermetic/run.sh setup --agent claude --debug) +ENV=$(evals/hermetic/run.sh setup --agent claude) evals/hermetic/run.sh index --env-dir "$ENV" --project /path/to/repo evals/hermetic/run.sh run --agent claude --env-dir "$ENV" --corpus my-corpus.jsonl --model sonnet evals/hermetic/run.sh teardown --env-dir "$ENV" @@ -22,6 +22,14 @@ Without `--project`, the harness indexes the checkout containing `run.sh`, not the caller's working directory or a machine-specific primary checkout. Pass `--project` explicitly to evaluate another repository. +`setup` and `smoke` build the shipping target `//crates/tracedecay-cli:tracedecay` +through `scripts/build-eval-binary.sh` (`bazel build --config=release`; set +`TRACEDECAY_EVAL_BAZEL_CONFIG=dist` for the exact release bytes). The script +prints the binary's version and sha256. A failed build aborts the run; there is +no stale-binary fallback. `TRACEDECAY_BIN=` stages an already-built +binary instead (for example a release binary to compare against) and the same +identity line records which one ran. + Offline harness path-selection checks (no build, agent, or daemon required): ```bash @@ -147,12 +155,12 @@ Each scored row in `results.jsonl` also records derived fields: **`verify_pass`* The args-ergonomics corpus compares MCP-first behavior with CLI fallback: ```bash -ENV=$(evals/hermetic/run.sh setup --agent claude --debug) +ENV=$(evals/hermetic/run.sh setup --agent claude) evals/hermetic/run.sh index --env-dir "$ENV" --project /path/to/tracedecay-worktree evals/hermetic/run.sh run --agent claude --env-dir "$ENV" \ --corpus evals/hermetic/corpora/tool-args-ergonomics.jsonl --model sonnet -ENV=$(evals/hermetic/run.sh setup --agent codex --debug) +ENV=$(evals/hermetic/run.sh setup --agent codex) evals/hermetic/run.sh index --env-dir "$ENV" --project /path/to/tracedecay-worktree evals/hermetic/run.sh run --agent codex --env-dir "$ENV" \ --corpus evals/hermetic/corpora/tool-args-ergonomics.jsonl @@ -175,17 +183,15 @@ scenario) and `summary.md` (pass count + per-scenario table). - **Codex auth.** Codex mode copies only `~/.codex/auth.json` into the isolated `CODEX_HOME`. If Codex changes auth storage, `codex exec` will fail closed in the eval env instead of mutating the user's real config. -- **Global cargo caches** (`~/.cargo/registry`) are shared, only the *output* - (`CARGO_TARGET_DIR=/target`) is worktree-scoped. The build cannot - corrupt the user's install because it never writes to `~/.cargo/bin`. +- **The build cache** is Bazel's, shared across worktrees. The build cannot + corrupt the user's install because it only stages a copy under the env dir. ## Post-merge re-eval protocol After the plugin-suite changes merge, re-run to compare against the baseline facts recorded in tracedecay memory: -1. **Rebuild** from the merged checkout: `run.sh setup` (drop `--debug` for a - representative release build). +1. **Rebuild** from the merged checkout: `run.sh setup`. 2. **Re-index** the same target project into the fresh env. 3. **Rerun the same corpus** at the same `--model`. 4. **Compare** the new `summary.md` pass rate and per-scenario tracedecay-vs- @@ -212,7 +218,7 @@ how close the agent is to the intended loop. Run recipe (adapted from the reusable-env examples above): ```bash -ENV=$(evals/hermetic/run.sh setup --agent claude --debug) +ENV=$(evals/hermetic/run.sh setup --agent claude) evals/hermetic/run.sh index --env-dir "$ENV" --project /path/to/repo evals/hermetic/run.sh run --agent claude --env-dir "$ENV" \ --corpus evals/hermetic/corpora/fact-store-adoption.jsonl --model sonnet diff --git a/evals/hermetic/run.sh b/evals/hermetic/run.sh index fe41caf708..b91391fcd2 100755 --- a/evals/hermetic/run.sh +++ b/evals/hermetic/run.sh @@ -9,7 +9,7 @@ # See ./README.md for the design, isolation guarantees, and limitations. # # Subcommands: -# build Build (or reuse) the dev binary and stage it at a stable path. +# build Build the shipping Bazel binary and stage it at a stable path. # setup Build + create an isolated env dir + install the dev plugin into it. # index Index a target project with the dev binary (into the isolated home). # run Execute a corpus JSONL against the isolated env and score it. @@ -51,42 +51,11 @@ die() { printf '[hermetic] ERROR: %s\n' "$*" >&2; exit 1; } # * current_exe (the staged copy) is baked into hook commands, and # * the MCP server command "tracedecay" resolves to the staged copy via PATH. -# Build the dev binary from THIS worktree. Echoes the built artifact path. +# Resolve the dev binary: the shipping Bazel target built from THIS worktree, +# or an explicit TRACEDECAY_BIN. A failed build aborts; an older artifact is +# never staged in its place. Echoes the binary path. build_binary() { - local profile_dir="release" - local -a cargo_args=(build --release --package tracedecay-cli --bin tracedecay) - if [[ "${BUILD_DEBUG:-0}" == "1" ]]; then - profile_dir="debug" - cargo_args=(build --package tracedecay-cli --bin tracedecay) - fi - - # Isolate build artifacts from the user's normal target dir so a concurrent - # `cargo` in this worktree is not disturbed and so is_cargo_target_binary can - # recognise the location deterministically. - local target_dir="${WORKTREE_ROOT}/target" - log "building dev binary (${profile_dir}) from ${WORKTREE_ROOT}" - # CRITICAL: a cargo failure must abort the whole run. Otherwise a stale - # artifact from a previous build would be silently staged and the eval would - # run against the WRONG binary -- exactly the failure this harness prevents. - # We check cargo's exit status explicitly (a bare subshell in a command-sub - # is not always fatal under set -e), and unless HERMETIC_ALLOW_STALE=1 we - # refuse to proceed on a build failure even if an old artifact exists. - local rc=0 - ( - cd "${WORKTREE_ROOT}" - CARGO_TARGET_DIR="${target_dir}" cargo "${cargo_args[@]}" >&2 - ) || rc=$? - if [[ "${rc}" -ne 0 ]]; then - if [[ "${HERMETIC_ALLOW_STALE:-0}" == "1" ]]; then - log "WARNING: cargo build failed (rc=${rc}); HERMETIC_ALLOW_STALE=1 set, reusing prior artifact" - else - die "cargo build failed (rc=${rc}); refusing to run against a possibly stale binary. Set HERMETIC_ALLOW_STALE=1 only if you understand the artifact may be old." - fi - fi - - local built="${target_dir}/${profile_dir}/tracedecay" - [[ -x "${built}" ]] || die "expected built binary not found at ${built}" - printf '%s\n' "${built}" + "${WORKTREE_ROOT}/scripts/build-eval-binary.sh" } # Copy the built binary to /bin/tracedecay (a stable non-cargo path). @@ -594,11 +563,10 @@ Common options: --corpus Corpus file for `run`. --model Model override (default: sonnet for claude; gpt-5.6-sol for codex). --reps Re-run the corpus N times (default: 1; `run` only). - --debug Reuse/produce a debug build instead of release (faster). Examples: - run.sh smoke --agent claude --debug - run.sh setup --agent codex --debug + run.sh smoke --agent claude + run.sh setup --agent codex run.sh run --agent claude --env-dir /tmp/eval-env-... --corpus my-corpus.jsonl --model sonnet EOF } @@ -617,7 +585,6 @@ main() { --corpus) corpus="$2"; shift 2 ;; --model) model="$2"; shift 2 ;; --reps) reps="$2"; shift 2 ;; - --debug) export BUILD_DEBUG=1; shift ;; -h|--help) usage; exit 0 ;; *) die "unknown option: $1" ;; esac diff --git a/evals/memory/run_real_model.py b/evals/memory/run_real_model.py index b4aa1d363e..f966448a81 100755 --- a/evals/memory/run_real_model.py +++ b/evals/memory/run_real_model.py @@ -12,7 +12,7 @@ Without both flags the runner records a blocked report and exits with code 2. The deterministic no-LLM layer lives in `tests/memory_eval_test.rs` and runs -as part of the normal cargo test suite. CI never calls this script. +as part of the normal Bazel test suite. CI never calls this script. Example: python3 evals/memory/run_real_model.py --scenario memory-no-pollution \ @@ -104,7 +104,10 @@ def parse_args(argv): parser.add_argument( "--tracedecay-bin", type=Path, - help="tracedecay binary (default: target/debug/tracedecay if built, else PATH).", + help=( + "prebuilt tracedecay binary (default: bazel build --config=release " + "//crates/tracedecay-cli:tracedecay)." + ), ) args = parser.parse_args(argv) if args.model is None: @@ -160,15 +163,24 @@ def create_eval_environment(scenario_id): def resolve_tracedecay_bin(explicit): + """The shipping Bazel binary, built fresh, or the explicit one if it exists. + + scripts/build-eval-binary.sh owns both paths; it prints the binary's version + and sha256 on stderr and fails rather than reuse an older build. + """ + env = dict(os.environ) if explicit: - return str(explicit) - debug_bin = REPO_ROOT / "target/debug/tracedecay" - if debug_bin.exists(): - return str(debug_bin) - found = shutil.which("tracedecay") - if not found: - sys.exit("no tracedecay binary: build target/debug or install one on PATH") - return found + env["TRACEDECAY_BIN"] = str(explicit) + result = subprocess.run( + [str(REPO_ROOT / "scripts/build-eval-binary.sh")], + env=env, + stdout=subprocess.PIPE, + text=True, + check=False, + ) + if result.returncode != 0: + sys.exit(f"no tracedecay binary: scripts/build-eval-binary.sh exited {result.returncode}") + return result.stdout.strip() def load_scenarios(ids): diff --git a/evals/memory/test_run_real_model.py b/evals/memory/test_run_real_model.py index 03d7a87d07..a171e039b4 100644 --- a/evals/memory/test_run_real_model.py +++ b/evals/memory/test_run_real_model.py @@ -71,6 +71,33 @@ def test_install_uses_the_isolated_user_hermes_home(self): self.assertEqual(run_mock.call_args.kwargs["env"], env.env) +class BinaryResolutionTest(unittest.TestCase): + def test_explicit_binary_is_used_and_must_exist(self): + with tempfile.TemporaryDirectory() as tmp: + binary = Path(tmp) / "tracedecay" + binary.write_text("#!/bin/sh\necho tracedecay 0.0.0\n") + binary.chmod(0o755) + with contextlib.redirect_stderr(io.StringIO()): + self.assertEqual( + run_real_model.resolve_tracedecay_bin(binary), str(binary) + ) + with self.assertRaises(SystemExit): + run_real_model.resolve_tracedecay_bin(Path(tmp) / "missing") + + def test_default_never_falls_back_to_a_prebuilt_or_path_binary(self): + failed = subprocess.CompletedProcess([], 2, stdout="") + with tempfile.TemporaryDirectory() as tmp: + path_binary = Path(tmp) / "tracedecay" + path_binary.write_text("#!/bin/sh\n") + path_binary.chmod(0o755) + with ( + mock.patch.dict("os.environ", {"PATH": tmp}), + mock.patch.object(run_real_model.subprocess, "run", return_value=failed), + self.assertRaises(SystemExit), + ): + run_real_model.resolve_tracedecay_bin(None) + + class CursorComposerSelectionTest(unittest.TestCase): def test_cursor_driver_loads_the_installed_plugin_in_composer(self): env = run_real_model.create_eval_environment("cursor-composer") diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index 9c43205848..67aaa9ed2e 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -43,6 +43,14 @@ py_test( main = "test-check-sqlite-plan-stability.py", ) +py_test( + name = "build_eval_binary_test", + size = "small", + srcs = ["test-build-eval-binary.py"], + data = ["build-eval-binary.sh"], + main = "test-build-eval-binary.py", +) + py_test( name = "ambient_tokio_check_test", size = "small", diff --git a/scripts/build-eval-binary.sh b/scripts/build-eval-binary.sh new file mode 100755 index 0000000000..cdca2ebbda --- /dev/null +++ b/scripts/build-eval-binary.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Resolve the tracedecay binary an eval runs against and print its absolute path +# on stdout. Every eval harness (evals/agent_adoption, evals/hermetic, +# evals/memory) calls this, so they all execute the shipping target +# //crates/tracedecay-cli:tracedecay with the same features and dependency +# patches the release builds. +# +# Default: build that target with Bazel. A failed build is an error; no older +# binary is ever substituted. TRACEDECAY_BIN selects an already-built artifact +# (for example a release binary to compare against); it must exist and be +# executable. +# +# Env: +# TRACEDECAY_BIN explicit binary; skips the build +# TRACEDECAY_EVAL_BAZEL_CONFIG Bazel config to build with (default: release; +# `dist` adds thin LTO and symbol stripping to +# build the exact release bytes) +# +# The selected artifact's version and sha256 go to stderr. +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +target="//crates/tracedecay-cli:tracedecay" +config="${TRACEDECAY_EVAL_BAZEL_CONFIG:-release}" +build_cmd="bazel build --config=$config $target" + +fail() { + echo "error: $*" >&2 + exit 2 +} + +if [[ -n "${TRACEDECAY_BIN:-}" ]]; then + [[ -f "$TRACEDECAY_BIN" && -x "$TRACEDECAY_BIN" ]] || + fail "TRACEDECAY_BIN=$TRACEDECAY_BIN is not an executable file; unset it to build the default with: $build_cmd" + bin="$(cd "$(dirname "$TRACEDECAY_BIN")" && pwd)/$(basename "$TRACEDECAY_BIN")" + origin="explicit TRACEDECAY_BIN" +else + command -v bazel >/dev/null 2>&1 || + fail "bazel is required to build the eval binary (run: $build_cmd) or set TRACEDECAY_BIN" + echo "building eval binary: $build_cmd" >&2 + (cd "$repo_root" && bazel build --config="$config" "$target") >&2 || + fail "eval binary build failed; not reusing any older binary. Reproduce with: $build_cmd" + bazel_bin="$(cd "$repo_root" && bazel info --config="$config" bazel-bin)" || + fail "could not resolve the Bazel output directory (bazel info --config=$config bazel-bin)" + bin="$bazel_bin/crates/tracedecay-cli/tracedecay" + [[ -f "$bin" && -x "$bin" ]] || + fail "expected a binary at $bin after: $build_cmd" + origin="bazel --config=$config $target" +fi + +if command -v sha256sum >/dev/null 2>&1; then + digest="$(sha256sum "$bin")" +else + digest="$(shasum -a 256 "$bin")" +fi +version="$("$bin" --version 2>/dev/null | head -1)" || version=unknown +echo "eval binary: $bin ($origin; ${version:-unknown}; sha256 ${digest%% *})" >&2 +printf '%s\n' "$bin" diff --git a/scripts/test-build-eval-binary.py b/scripts/test-build-eval-binary.py new file mode 100644 index 0000000000..22de025f37 --- /dev/null +++ b/scripts/test-build-eval-binary.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""build-eval-binary.sh builds the shipping target and never reuses a stale binary.""" + +import os +import shutil +import stat +import subprocess +import tempfile +import unittest +from pathlib import Path + +SCRIPT = Path(__file__).with_name("build-eval-binary.sh") + +# Fake bazel: records its argv; `build` exits $FAKE_BUILD_RC, `info` prints $FAKE_BAZEL_BIN. +FAKE_BAZEL = """#!/bin/sh +echo "$@" >> "$FAKE_BAZEL_LOG" +case "$1" in + build) exit "${FAKE_BUILD_RC:-0}" ;; + info) echo "$FAKE_BAZEL_BIN" ;; +esac +""" + + +class BuildEvalBinaryTest(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + root = Path(tmp.name) + self.repo = root / "repo" + (self.repo / "scripts").mkdir(parents=True) + shutil.copy(SCRIPT, self.repo / "scripts" / SCRIPT.name) + self.bazel_bin = root / "bazel-bin" + cli = self.bazel_bin / "crates" / "tracedecay-cli" + cli.mkdir(parents=True) + # A binary from an earlier build, present before the build runs. + self.built = cli / "tracedecay" + self.built.write_text("#!/bin/sh\necho tracedecay 1.2.3\n") + self.built.chmod(0o755) + self.log = root / "bazel.log" + fakes = root / "fakes" + fakes.mkdir() + bazel = fakes / "bazel" + bazel.write_text(FAKE_BAZEL) + bazel.chmod(bazel.stat().st_mode | stat.S_IXUSR) + self.env = { + "PATH": f"{fakes}:{os.environ['PATH']}", + "FAKE_BAZEL_LOG": str(self.log), + "FAKE_BAZEL_BIN": str(self.bazel_bin), + } + + def run_script(self, **env): + return subprocess.run( + ["bash", str(self.repo / "scripts" / SCRIPT.name)], + env={**self.env, **env}, + text=True, + capture_output=True, + check=False, + ) + + def test_builds_the_shipping_target_and_prints_its_identity(self): + result = self.run_script() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), str(self.built)) + self.assertIn( + "build --config=release //crates/tracedecay-cli:tracedecay", + self.log.read_text(), + ) + self.assertIn("tracedecay 1.2.3", result.stderr) + self.assertIn("sha256 ", result.stderr) + + def test_config_override_reaches_bazel(self): + result = self.run_script(TRACEDECAY_EVAL_BAZEL_CONFIG="dist") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("build --config=dist //crates", self.log.read_text()) + + def test_failed_build_does_not_fall_back_to_the_stale_binary(self): + result = self.run_script(FAKE_BUILD_RC="1") + self.assertEqual(result.returncode, 2) + self.assertEqual(result.stdout, "") + self.assertIn("bazel build --config=release //crates/tracedecay-cli:tracedecay", result.stderr) + + def test_explicit_binary_skips_the_build(self): + explicit = self.built.with_name("candidate") + explicit.write_text("#!/bin/sh\necho tracedecay 9.9.9\n") + explicit.chmod(0o755) + result = self.run_script(TRACEDECAY_BIN=str(explicit)) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), str(explicit)) + self.assertIn("tracedecay 9.9.9", result.stderr) + self.assertFalse(self.log.exists()) + + def test_missing_explicit_binary_names_the_build_command(self): + result = self.run_script(TRACEDECAY_BIN=str(self.built.with_name("absent"))) + self.assertEqual(result.returncode, 2) + self.assertEqual(result.stdout, "") + self.assertIn("bazel build --config=release //crates/tracedecay-cli:tracedecay", result.stderr) + + +if __name__ == "__main__": + unittest.main() From e253929260b7542538aa81fd17ae15fe4a0b35a1 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 16:34:07 +0000 Subject: [PATCH 18/89] build(dist): apply Bazel asupersync patches to the extracted-crate build --- BUILD.bazel | 1 + scripts/BUILD.bazel | 11 +++++ scripts/check-distribution-acceptance.sh | 51 +++++++++++++++++++++++- scripts/test-bazel-dependency-patches.py | 48 ++++++++++++++++++++++ 4 files changed, 110 insertions(+), 1 deletion(-) create mode 100644 scripts/test-bazel-dependency-patches.py diff --git a/BUILD.bazel b/BUILD.bazel index 0e6f37c990..54139007a8 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -12,6 +12,7 @@ npm_link_all_packages(name = "node_modules") exports_files( [ ".github/release-targets.json", + "MODULE.bazel", "version.txt", "Cargo.toml", "clippy.toml", diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index 67aaa9ed2e..c55e526b23 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -51,6 +51,17 @@ py_test( main = "test-build-eval-binary.py", ) +py_test( + name = "bazel_dependency_patches_test", + size = "small", + srcs = ["test-bazel-dependency-patches.py"], + data = [ + "check-distribution-acceptance.sh", + "//:MODULE.bazel", + ] + glob(["bazel/patches/*.patch"]), + main = "test-bazel-dependency-patches.py", +) + py_test( name = "ambient_tokio_check_test", size = "small", diff --git a/scripts/check-distribution-acceptance.sh b/scripts/check-distribution-acceptance.sh index 9a2b686071..c85f68f40e 100755 --- a/scripts/check-distribution-acceptance.sh +++ b/scripts/check-distribution-acceptance.sh @@ -592,8 +592,46 @@ assert_required_assets "$root_package" "$cli_package" [[ ! -e "$package_root/.git" && ! -L "$package_root/.git" ]] || die "extracted package workspace unexpectedly contains .git metadata: $package_root/.git" +# The Bazel release build compiles asupersync with the patches listed in +# MODULE.bazel, which live in scripts/bazel/patches/asupersync-*.patch. Cargo +# has no patch-file mechanism, so the extracted-crate build compiles a copy of +# the same pinned source with those same files applied, in the sorted order +# crate_universe applies them. scripts/test-bazel-dependency-patches.py keeps +# the file set and the MODULE.bazel annotation identical. +asupersync_source=$( + cargo metadata \ + --manifest-path "$staged/Cargo.toml" \ + --format-version 1 | + python3 -c ' +import json +import os +import sys + +manifests = [ + package["manifest_path"] + for package in json.load(sys.stdin)["packages"] + if package["name"] == "asupersync" +] +if len(manifests) != 1: + sys.exit(f"expected exactly one asupersync package, found {len(manifests)}") +print(os.path.dirname(manifests[0])) +' +) || die "could not locate the asupersync source" +patched_asupersync="$work/patched-deps/asupersync" +mkdir -p -- "$patched_asupersync" +tar -C "$asupersync_source" --exclude=.git -cf - . | tar -C "$patched_asupersync" -xf - +shopt -s nullglob +asupersync_patches=("$repo"/scripts/bazel/patches/asupersync-*.patch) +shopt -u nullglob +[[ ${#asupersync_patches[@]} -gt 0 ]] || + die "no asupersync patches found under scripts/bazel/patches" +for asupersync_patch in "${asupersync_patches[@]}"; do + (cd -- "$patched_asupersync" && git apply -p1 "$asupersync_patch") || + die "could not apply $asupersync_patch to asupersync" +done + patch_config="$work/packaged-crates.toml" -python3 - "$metadata" "$packages" "$staged/Cargo.toml" >"$patch_config" <<'PY' +python3 - "$metadata" "$packages" "$staged/Cargo.toml" "$patched_asupersync" >"$patch_config" <<'PY' import json import pathlib import sys @@ -634,6 +672,17 @@ for name, spec in workspace_manifest.get("patch", {}).get("crates-io", {}).items f"distribution acceptance: workspace path patch {name} is not staged at {patched}" ) print(f'{json.dumps(name)} = {{ path = {json.dumps(str(patched))} }}') +# The asupersync copy patched above replaces the pinned git source, so the +# runtime the extracted build links is the one Bazel builds. +asupersync_spec = workspace_manifest["workspace"]["dependencies"]["asupersync"] +if "git" not in asupersync_spec: + raise SystemExit( + "distribution acceptance: asupersync is no longer a git dependency; " + "patch it through [patch.crates-io] here or drop its Bazel patches" + ) +print() +print(f'[patch.{json.dumps(asupersync_spec["git"])}]') +print(f'asupersync = {{ path = {json.dumps(sys.argv[4])} }}') # Extracted packages do not see workspace Cargo.toml profiles. Copy # [profile.dist] from that manifest so `--profile dist` matches shipping. dist_profile = workspace_manifest.get("profile", {}).get("dist") diff --git a/scripts/test-bazel-dependency-patches.py b/scripts/test-bazel-dependency-patches.py new file mode 100644 index 0000000000..459c2c8431 --- /dev/null +++ b/scripts/test-bazel-dependency-patches.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""The Bazel dependency patches and the extracted-crate Cargo build stay in step. + +check-distribution-acceptance.sh applies every scripts/bazel/patches/asupersync-*.patch +to the asupersync source it compiles. MODULE.bazel lists the patches Bazel applies. +A patch present in only one of them means the two builds run different runtimes. +""" + +import re +import unittest +from pathlib import Path + +ROOT = Path(__file__).absolute().parent.parent +PATCHES = ROOT / "scripts" / "bazel" / "patches" +LABEL = re.compile(r'"//scripts:bazel/patches/([^"]+\.patch)"') + + +def annotation_patches(module: str, crate: str) -> list[str]: + for block in re.findall(r"crate\.annotation\((.*?)\n\)", module, re.DOTALL): + if f'crate = "{crate}"' in block: + return LABEL.findall(block) + raise AssertionError(f"MODULE.bazel has no crate.annotation for {crate}") + + +class DependencyPatchParityTest(unittest.TestCase): + def setUp(self): + self.module = (ROOT / "MODULE.bazel").read_text() + + def test_asupersync_patches_match_the_distribution_build(self): + applied_by_distribution = sorted(p.name for p in PATCHES.glob("asupersync-*.patch")) + self.assertTrue(applied_by_distribution) + # crate_universe applies an annotation's patches in sorted label order. + self.assertEqual( + annotation_patches(self.module, "asupersync"), applied_by_distribution + ) + + def test_distribution_build_applies_that_glob(self): + script = (ROOT / "scripts" / "check-distribution-acceptance.sh").read_text() + self.assertIn("scripts/bazel/patches/asupersync-*.patch", script) + + def test_no_patch_file_is_unreferenced(self): + referenced = set(LABEL.findall(self.module)) + unreferenced = {p.name for p in PATCHES.glob("*.patch")} - referenced + self.assertEqual(unreferenced, set()) + + +if __name__ == "__main__": + unittest.main() From 0ec2ce53693e25560e3fa9306409296c28f23303 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 16:34:53 +0000 Subject: [PATCH 19/89] ci: mark the Windows cargo check as a type-check-only exception Also runs the eval binary script test in the hermetic eval workflow. --- .github/workflows/ci.yml | 17 +++++++++++------ .github/workflows/hermetic-evals.yml | 4 ++++ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6278fb284b..17d047cdf0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -591,12 +591,17 @@ jobs: - name: Run blocking Clippy policy run: bazel build --config=ci --config=clippy //... - # Master pushes run no Windows lane, so `cfg(windows)` code otherwise - # first compiles in the release build. Bazel has no MSVC toolchain on - # Linux, so this cross-target type check stays on Cargo: `cargo check` - # needs no linker, and mingw supplies the C compiler the native build - # scripts run. macOS has no equivalent: its build scripts need the Apple - # SDK headers. + # The one deliberate Cargo step left in CI, and only because Bazel cannot + # replace it yet. Master pushes and ordinary PRs run no Windows lane + # (`windows-test` is dispatch-only), so `cfg(windows)` code otherwise + # first compiles in the release build. The workspace registers no + # Windows target toolchain for Linux, so this cross-target type check + # stays on Cargo: `cargo check` needs no linker, and mingw supplies the + # C compiler the native build scripts run. It type-checks only and + # produces no executable, so no eval, test, or release consumes its + # output and it cannot diverge a shipped runtime. Remove it when Bazel + # grows a Linux-hosted Windows target. macOS has no equivalent: its + # build scripts need the Apple SDK headers. - name: Install pinned toolchain and Windows check target run: | rustup toolchain install diff --git a/.github/workflows/hermetic-evals.yml b/.github/workflows/hermetic-evals.yml index 75d634062d..17a95296d6 100644 --- a/.github/workflows/hermetic-evals.yml +++ b/.github/workflows/hermetic-evals.yml @@ -5,12 +5,14 @@ on: paths: - "evals/hermetic/**" - "evals/agent_adoption/**" + - "scripts/build-eval-binary.sh" - ".github/workflows/hermetic-evals.yml" push: branches: [master] paths: - "evals/hermetic/**" - "evals/agent_adoption/**" + - "scripts/build-eval-binary.sh" - ".github/workflows/hermetic-evals.yml" permissions: @@ -34,6 +36,7 @@ jobs: run: | bash -n evals/hermetic/run.sh bash -n evals/agent_adoption/run.sh + bash -n scripts/build-eval-binary.sh - name: Run scorer and shell-runner regressions run: >- bazel test @@ -41,4 +44,5 @@ jobs: //evals/hermetic:run_test //evals/hermetic:scorecard_test //evals/agent_adoption:selftest + //scripts:build_eval_binary_test --test_output=errors From d1539a103dfcad1d85abc41c3e8348388a35b858 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 17:48:54 +0000 Subject: [PATCH 20/89] fix(projection): settle passes parked on terminal session collisions --- .../src/observation_projection/rebuild.rs | 7 +- .../src/session_project_rebind_tests.rs | 9 ++ .../tests/transcript_ingest_suite/codex.rs | 111 ++++++++++++++++++ .../restart_atomicity.rs | 22 +++- 4 files changed, 143 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay-global-db/src/observation_projection/rebuild.rs b/crates/tracedecay-global-db/src/observation_projection/rebuild.rs index 6332fdf643..1234a622ea 100644 --- a/crates/tracedecay-global-db/src/observation_projection/rebuild.rs +++ b/crates/tracedecay-global-db/src/observation_projection/rebuild.rs @@ -298,9 +298,14 @@ async fn next_ready_projection_head( )) } +/// Whether a later drain still has work. A row parked on a terminal collision +/// is never drained again, so it must not keep every pass reporting more work. async fn projection_queue_has_items(conn: &impl QueryExecutor) -> ProjectionStoreResult { let mut rows = conn - .query("SELECT 1 FROM projection_queue LIMIT 1", ()) + .query( + "SELECT 1 FROM projection_queue WHERE next_retry_at_micros < ?1 LIMIT 1", + params![PROJECTION_TERMINAL_RETRY_MICROS], + ) .await .map_err(|error| storage("probe projection queue", error))?; Ok(rows diff --git a/crates/tracedecay-global-db/src/session_project_rebind_tests.rs b/crates/tracedecay-global-db/src/session_project_rebind_tests.rs index 5f0b994967..6ee48632b2 100644 --- a/crates/tracedecay-global-db/src/session_project_rebind_tests.rs +++ b/crates/tracedecay-global-db/src/session_project_rebind_tests.rs @@ -652,6 +652,15 @@ async fn genuine_session_collision_is_recorded_and_later_sessions_project() { store.next_queued_observation().await.unwrap().is_none(), "the terminal collision must not stay at the head of the queue" ); + let window = store + .project_queued_observations(8) + .await + .unwrap() + .expect("the store drains in windows"); + assert!( + window.items.is_empty() && !window.has_more, + "a parked terminal collision is not pending work, or every pass reports backpressure" + ); } fn session_fact_with_transcript(cwd: &str, transcript_path: &str) -> CanonicalObservationFactV1 { diff --git a/crates/tracedecay/tests/transcript_ingest_suite/codex.rs b/crates/tracedecay/tests/transcript_ingest_suite/codex.rs index 6f709b53c6..51fa735ec6 100644 --- a/crates/tracedecay/tests/transcript_ingest_suite/codex.rs +++ b/crates/tracedecay/tests/transcript_ingest_suite/codex.rs @@ -4,9 +4,11 @@ use tempfile::TempDir; use tracedecay_project::test_support::host_admission::HostAdmissionTestRuntimeV1; use tracedecay_sessions::admission::HostAdmissionScope; use tracedecay_sessions::runtime::SessionProvider; +use tracedecay_sessions::runtime::shared::durable_project_path_key; use crate::restart_atomicity::{ ingest_global_sources_for_provider, ingest_user_provider, open_project_session_db, + run_user_provider_pass, }; pub(crate) fn write_jsonl(path: &std::path::Path, lines: &[serde_json::Value]) { @@ -497,3 +499,112 @@ pub(crate) fn write_codex_rollout_with_structured_events( ); path } + +/// Seeds a `moved-session` row under another working directory, naming +/// `stored_transcript` (or the rollout itself), then runs user-scope passes +/// over the session's rollout until one reports no failure. Returns the +/// runtime and the number of passes that took. +async fn ingest_rollout_over_stored_session( + tmp: &TempDir, + stored_transcript: Option<&str>, +) -> (HostAdmissionTestRuntimeV1, usize) { + let home = tmp.path().join("home"); + let general = tmp.path().join("general-chat"); + let stored_cwd = tmp.path().join("earlier-cwd"); + std::fs::create_dir_all(&general).unwrap(); + std::fs::create_dir_all(&stored_cwd).unwrap(); + let rollout = write_codex_rollout(&home, &general, "moved-session"); + let runtime = HostAdmissionTestRuntimeV1::profile(tmp.path().join("profile")) + .await + .unwrap(); + let stored = tracedecay_sessions::runtime::SessionRecord { + provider: "codex".to_owned(), + session_id: "moved-session".to_owned(), + project_key: "user".to_owned(), + project_path: durable_project_path_key(&stored_cwd.to_string_lossy()), + title: None, + started_at: Some(1), + ended_at: None, + transcript_path: Some( + stored_transcript.map_or_else(|| rollout.to_string_lossy().into_owned(), str::to_owned), + ), + metadata_json: None, + parent_session_id: None, + is_subagent: false, + agent_id: None, + parent_tool_use_id: None, + }; + assert!( + runtime + .upsert_session_for_test(HostAdmissionScope::Profile, &stored) + .await + .unwrap() + ); + + let mut passes = 0; + loop { + passes += 1; + let outcome = + run_user_provider_pass(&runtime, &home, SessionProvider::Codex, Vec::new()).await; + if outcome.failures.is_empty() { + break; + } + assert!(passes < 4, "still retrying: {:?}", outcome.failures); + } + // Settled: another pass has nothing to retry. + ingest_user_provider(&runtime, &home, SessionProvider::Codex, Vec::new()).await; + (runtime, passes) +} + +async fn billing_messages(runtime: &HostAdmissionTestRuntimeV1) -> usize { + runtime + .search_session_messages_for_test( + HostAdmissionScope::Profile, + "codex", + None, + "billing pipeline", + 10, + ) + .await + .unwrap() + .len() +} + +/// A session row written earlier under another working directory (a hook +/// payload, LCM, or an older build) still names the same transcript. The +/// rollout's messages must project and the stored directory stays. Before the +/// fix each message was parked as a `project_path` session collision and every +/// pass reported `ingest_pass_backpressured` again. +#[tokio::test] +async fn user_scope_projects_a_codex_rollout_whose_cwd_differs_from_the_stored_session() { + let tmp = TempDir::new().unwrap(); + let (runtime, passes) = ingest_rollout_over_stored_session(&tmp, None).await; + + assert_eq!(passes, 1, "one pass must settle, not retry"); + assert_eq!(billing_messages(&runtime).await, 2, "both messages project"); + let session = runtime + .session_for_test(HostAdmissionScope::Profile, "codex", "moved-session") + .await + .unwrap() + .unwrap(); + assert_eq!( + session.project_path, + durable_project_path_key(&tmp.path().join("earlier-cwd").to_string_lossy()) + ); +} + +/// A stored session that names a different transcript is a genuine duplicate: +/// its messages are still refused as a collision. The parked collision must +/// not keep every later pass reporting `ingest_pass_backpressured`. +#[tokio::test] +async fn user_scope_settles_after_a_genuine_codex_session_collision() { + let tmp = TempDir::new().unwrap(); + let (runtime, _) = + ingest_rollout_over_stored_session(&tmp, Some("/elsewhere/other-rollout.jsonl")).await; + + assert_eq!( + billing_messages(&runtime).await, + 0, + "a conflicting transcript identity stays a collision" + ); +} diff --git a/crates/tracedecay/tests/transcript_ingest_suite/restart_atomicity.rs b/crates/tracedecay/tests/transcript_ingest_suite/restart_atomicity.rs index e305f80af4..a30ab18bd8 100644 --- a/crates/tracedecay/tests/transcript_ingest_suite/restart_atomicity.rs +++ b/crates/tracedecay/tests/transcript_ingest_suite/restart_atomicity.rs @@ -217,6 +217,23 @@ pub(super) async fn ingest_user_provider( home: &Path, provider: SessionProvider, registered_roots: Vec, +) -> IngestPassOutcome { + let outcome = run_user_provider_pass(runtime, home, provider, registered_roots).await; + assert!( + outcome.failures.is_empty(), + "user {provider:?} pass failed: {:?}", + outcome.failures + ); + outcome +} + +/// One production user-scope pass, whatever it reports. A bounded pass may +/// report backpressure while discovery still has ground to cover. +pub(super) async fn run_user_provider_pass( + runtime: &HostAdmissionTestRuntimeV1, + home: &Path, + provider: SessionProvider, + registered_roots: Vec, ) -> IngestPassOutcome { let database = runtime .registered_database(HostAdmissionScope::Profile) @@ -236,11 +253,6 @@ pub(super) async fn ingest_user_provider( ), ) .await; - assert!( - outcome.failures.is_empty(), - "user {provider:?} pass failed: {:?}", - outcome.failures - ); outcome } From 5dc09fa586d02ba7653e00dd297d40d837cc73bf Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 18:02:41 +0000 Subject: [PATCH 21/89] test(session-runtime): cut refresh groups without a wall-clock deadline --- .../worker.rs | 39 ++++++++++++++----- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker.rs index 489de73c34..5427dd0fa8 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker.rs @@ -1915,13 +1915,12 @@ mod tests { use tracedecay_runtime_core::db::engine::params; use tracedecay_sessions::runtime::{SessionMessageRecord, SessionRecord}; - /// Wide enough that one item's commit lands inside it on a loaded host. - const COMMIT_DEADLINE: Duration = Duration::from_secs(2); - - /// Commits `item_count` items. A group of several items never commits - /// before the deadline, and neither does a `slow` item alone; every other - /// item alone commits at once. Returns how many commits the deadline cut - /// short. + /// Commits `item_count` items. A group of several items never commits, and + /// neither does a `slow` item alone: its transaction ends in the + /// `DeadlineExceeded` that storage reports when its own deadline passes. + /// Every other item alone commits. Real writes race no wall-clock deadline + /// (`Duration::MAX` starts no timer), so host load cannot cut an item that + /// the test means to commit. Returns how many commits were cut short. async fn timed_refresh_group_progress(item_count: usize, slow: &[usize]) -> usize { let harness = RegisteredGlobalDbHarness::open("refresh-group-progress").await; let database = &harness.registered; @@ -1937,7 +1936,7 @@ mod tests { let cut_short = std::sync::atomic::AtomicUsize::new(0); let cut_short = &cut_short; - let outcomes = commit_refresh_groups(item_count, COMMIT_DEADLINE, |range| async move { + let outcomes = commit_refresh_groups(item_count, Duration::MAX, |range| async move { let transaction = database .begin_write_transaction() .await @@ -1953,7 +1952,7 @@ mod tests { } if range.len() > 1 || slow.contains(&range.start) { cut_short.fetch_add(1, Ordering::SeqCst); - std::future::pending::<()>().await; + return Err(SessionStoreError::DeadlineExceeded); } transaction .commit() @@ -2010,6 +2009,28 @@ mod tests { ); } + /// The deadline itself ends a group that never commits. No I/O races it: + /// a commit that is ready at its first poll wins even past the deadline, + /// so only the pending group is ever cut, however loaded the host. + #[tokio::test] + async fn a_refresh_group_that_outlives_its_deadline_is_retried_item_by_item() { + let outcomes = commit_refresh_groups(3, Duration::from_millis(10), |range| async move { + if range.len() > 1 { + std::future::pending::<()>().await; + } + Ok(range.map(Ok).collect()) + }) + .await + .unwrap(); + assert_eq!( + outcomes + .into_iter() + .map(|outcome| outcome.unwrap()) + .collect::>(), + [0, 1, 2] + ); + } + #[tokio::test] async fn a_timed_out_completion_stays_claimable_by_the_next_pass() { let harness = RegisteredGlobalDbHarness::open("refresh-timed-out-completion").await; From 8fdd7f9aa9eff6bf3033ff662eadc312516325d8 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 18:10:47 +0000 Subject: [PATCH 22/89] test(sessions): return the user pass outcome directly --- .../tests/transcript_ingest_suite/restart_atomicity.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay/tests/transcript_ingest_suite/restart_atomicity.rs b/crates/tracedecay/tests/transcript_ingest_suite/restart_atomicity.rs index a30ab18bd8..7714cf7af4 100644 --- a/crates/tracedecay/tests/transcript_ingest_suite/restart_atomicity.rs +++ b/crates/tracedecay/tests/transcript_ingest_suite/restart_atomicity.rs @@ -241,7 +241,7 @@ pub(super) async fn run_user_provider_pass( let authority = GlobalDbSessionIngestAuthority::new(database).with_background_cpu(runtime.background_cpu()); let shard = &database.binding().shard_id; - let outcome = with_transcript_source_profile( + with_transcript_source_profile( ProfileRoot::new(runtime.profile_root_for_test()).with_home(home), ingest_user_global_sources_for_provider_with_roots_bounded( (&shard.brain_id, &shard.profile_id, &authority), @@ -252,8 +252,7 @@ pub(super) async fn run_user_provider_pass( &ObservationCancellation::default(), ), ) - .await; - outcome + .await } /// Runs one Claude source through the production observation pipeline against From 73e6ccfd20d682b82ad4d49752bddccb3490a08b Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 19:33:29 +0000 Subject: [PATCH 23/89] fix(cli): read glibc counters through mallinfo on the 2.31 baseline --- crates/tracedecay-cli/tests/c_library_allocator.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay-cli/tests/c_library_allocator.rs b/crates/tracedecay-cli/tests/c_library_allocator.rs index c434a4fd49..df4f9ef0ad 100644 --- a/crates/tracedecay-cli/tests/c_library_allocator.rs +++ b/crates/tracedecay-cli/tests/c_library_allocator.rs @@ -19,10 +19,15 @@ static MIMALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc; const MIB: usize = 1024 * 1024; /// Bytes glibc's arenas and its mmapped chunks hold for live allocations. +/// +/// The hermetic toolchain links against glibc 2.31, which predates +/// `mallinfo2`. `mallinfo` reports the same counters as `int`, which this +/// test's few-MiB working set stays far below. fn glibc_in_use() -> usize { - // SAFETY: `mallinfo2` only reads allocator statistics. - let info = unsafe { libc::mallinfo2() }; - info.uordblks + info.hblkhd + // SAFETY: `mallinfo` only reads allocator statistics. + let info = unsafe { libc::mallinfo() }; + usize::try_from(info.uordblks).expect("in-use arena bytes fit in int") + + usize::try_from(info.hblkhd).expect("mmapped bytes fit in int") } #[test] From ee650a8c6b1955731d26beb8e7e8d8449694d5a9 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 20:17:10 +0000 Subject: [PATCH 24/89] fix(ci): expose packaged LSP smoke imports to Bazel --- scripts/BUILD.bazel | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index c5ff54479a..11df536bae 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -61,9 +61,24 @@ py_binary( "lib/portable_process.py", "lib/process_control.py", ], + imports = ["."], main = "check-packaged-lsp-bridge.py", ) +py_test( + name = "packaged_lsp_bridge_test", + size = "small", + srcs = [ + "check-packaged-lsp-bridge.py", + "test-check-packaged-lsp-bridge.py", + "lib/__init__.py", + "lib/portable_process.py", + "lib/process_control.py", + ], + imports = ["."], + main = "test-check-packaged-lsp-bridge.py", +) + exports_files( [ "bazel/rust_test.bzl", From 9fdec8ede126ef250d048463dcf33b3a76743788 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 20:18:08 +0000 Subject: [PATCH 25/89] test(query): write historical fixtures without runfile permissions --- .../tests/search_quality_suite/candidate_producers.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs b/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs index 1535b0fc07..8542d1c80e 100644 --- a/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs +++ b/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs @@ -2685,12 +2685,13 @@ fn reader_refuses_historical_v10_writer_artifact_as_incompatible() { let directory = tempfile::tempdir().expect("private v10 reopen dir"); let artifact_path = directory.path().join("lexical-artifact-v10.sqlite"); - std::fs::copy(checked_in, &artifact_path).expect("copy historical v10 fixture"); + let bytes = std::fs::read(checked_in).expect("read historical v10 fixture"); + // Bazel runfiles are read-only; create an owned file without copying their mode. + std::fs::write(&artifact_path, &bytes).expect("write historical v10 fixture"); drop( tracedecay_private_fs::make_private_file(&artifact_path) .expect("restore private-file protection for content-addressed open"), ); - let bytes = std::fs::read(&artifact_path).expect("read historical v10 fixture"); let file_size_bytes = u64::try_from(bytes.len()).expect("v10 fixture length"); let digest = ManifestDigest::new(format!("sha256:{}", hex::encode(Sha256::digest(&bytes)))) .expect("v10 fixture digest"); From 64e8b5e8c2c16be5dfad0ac16fd620e5f9f0d800 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 20:18:11 +0000 Subject: [PATCH 26/89] fix(evals): honor consent and platform executable paths --- benchmark_data/BUILD.bazel | 2 ++ evals/hermetic/BUILD.bazel | 2 ++ evals/memory/BUILD.bazel | 15 +++++++++++++++ evals/memory/run_real_model.py | 6 ++++-- evals/memory/test_run_real_model.py | 26 ++++++++++++++++++++++++++ scripts/BUILD.bazel | 1 + scripts/build-eval-binary.sh | 6 +++++- scripts/test-build-eval-binary.py | 26 +++++++++++++++++++++++++- 8 files changed, 80 insertions(+), 4 deletions(-) diff --git a/benchmark_data/BUILD.bazel b/benchmark_data/BUILD.bazel index c510e6d63a..f2c7affbe9 100644 --- a/benchmark_data/BUILD.bazel +++ b/benchmark_data/BUILD.bazel @@ -82,3 +82,5 @@ exports_files(["session-temporal/workload-v1.json"], visibility = ["//visibility:public"], ) # END generated by scripts/bazel/gen_builds.py + +exports_files(["run_benchmarks.py"], visibility = ["//evals/memory:__pkg__"]) diff --git a/evals/hermetic/BUILD.bazel b/evals/hermetic/BUILD.bazel index b16f0187d2..8761819a7d 100644 --- a/evals/hermetic/BUILD.bazel +++ b/evals/hermetic/BUILD.bazel @@ -28,3 +28,5 @@ py_test( imports = ["."], legacy_create_init = 0, ) + +exports_files(["score.py"], visibility = ["//evals/memory:__pkg__"]) diff --git a/evals/memory/BUILD.bazel b/evals/memory/BUILD.bazel index af098d6a49..2b78708c10 100644 --- a/evals/memory/BUILD.bazel +++ b/evals/memory/BUILD.bazel @@ -1,3 +1,18 @@ +load("@rules_python//python:defs.bzl", "py_test") + +py_test( + name = "real_model_runner_test", + srcs = ["run_real_model.py", "test_run_real_model.py"], + data = [ + "//benchmark_data:run_benchmarks.py", + "//evals/hermetic:score.py", + "//scripts:build-eval-binary.sh", + ], + imports = ["."], + legacy_create_init = 0, + main = "test_run_real_model.py", +) + filegroup( name = "scenarios", srcs = glob(["scenarios/*.json"]), diff --git a/evals/memory/run_real_model.py b/evals/memory/run_real_model.py index f966448a81..0136f31486 100755 --- a/evals/memory/run_real_model.py +++ b/evals/memory/run_real_model.py @@ -699,7 +699,6 @@ def evaluate_assertions(scenario, tracedecay_bin, fixture, env): def main(argv): args = parse_args(argv) - args.tracedecay_bin = resolve_tracedecay_bin(args.tracedecay_bin) scenarios = load_scenarios(args.scenario) if not scenarios: sys.exit("no runnable scenarios selected") @@ -712,7 +711,7 @@ def main(argv): "timestamp": timestamp, "driver": args.driver, "model": args.model, - "tracedecay_bin": str(args.tracedecay_bin), + "tracedecay_bin": None, "scenarios": [], } @@ -729,6 +728,9 @@ def main(argv): print(f"\nblocked report written to {report_path}", file=sys.stderr) return 2 + args.tracedecay_bin = resolve_tracedecay_bin(args.tracedecay_bin) + report["tracedecay_bin"] = args.tracedecay_bin + overall_ok = True for scenario in scenarios: eval_env = create_eval_environment(scenario["id"]) diff --git a/evals/memory/test_run_real_model.py b/evals/memory/test_run_real_model.py index a171e039b4..179e2383fb 100644 --- a/evals/memory/test_run_real_model.py +++ b/evals/memory/test_run_real_model.py @@ -26,6 +26,14 @@ SCORE_SPEC.loader.exec_module(hermetic_score) +def setUpModule(): + runs = tempfile.TemporaryDirectory() + unittest.addModuleCleanup(runs.cleanup) + patch = mock.patch.object(run_real_model, "RUNS_DIR", Path(runs.name)) + patch.start() + unittest.addModuleCleanup(patch.stop) + + class HermesHomeSelectionTest(unittest.TestCase): def test_driver_defaults_resolve_before_launch_and_reporting(self): self.assertEqual(run_real_model.parse_args([]).model, "gpt-5.4-mini") @@ -72,6 +80,24 @@ def test_install_uses_the_isolated_user_hermes_home(self): class BinaryResolutionTest(unittest.TestCase): + def test_blocked_run_writes_report_without_resolving_a_binary(self): + for flags in ([], ["--agent-turn"], ["--i-understand-model-cost"]): + with self.subTest(flags=flags), tempfile.TemporaryDirectory() as tmp: + with ( + mock.patch.object(run_real_model, "RUNS_DIR", Path(tmp)), + mock.patch.object(run_real_model, "resolve_tracedecay_bin") as resolve, + mock.patch.object(run_real_model, "load_scenarios", return_value=[{"id": "test"}]), + contextlib.redirect_stdout(io.StringIO()), + contextlib.redirect_stderr(io.StringIO()), + ): + self.assertEqual(run_real_model.main(flags), 2) + resolve.assert_not_called() + reports = list(Path(tmp).glob("*/report.json")) + self.assertEqual(len(reports), 1) + report = json.loads(reports[0].read_text()) + self.assertEqual(report["status"], "blocked") + self.assertIsNone(report["tracedecay_bin"]) + def test_explicit_binary_is_used_and_must_exist(self): with tempfile.TemporaryDirectory() as tmp: binary = Path(tmp) / "tracedecay" diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index 85dd02f7d6..5048ad182f 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -85,6 +85,7 @@ py_binary( exports_files( [ "bazel/rust_test.bzl", + "build-eval-binary.sh", "test-rust-cache-lineage.py", "test-check-packaged-lsp-bridge.py", "test-check-sdk-publish-workflow.py", diff --git a/scripts/build-eval-binary.sh b/scripts/build-eval-binary.sh index cdca2ebbda..2b2e82a804 100755 --- a/scripts/build-eval-binary.sh +++ b/scripts/build-eval-binary.sh @@ -42,7 +42,11 @@ else fail "eval binary build failed; not reusing any older binary. Reproduce with: $build_cmd" bazel_bin="$(cd "$repo_root" && bazel info --config="$config" bazel-bin)" || fail "could not resolve the Bazel output directory (bazel info --config=$config bazel-bin)" - bin="$bazel_bin/crates/tracedecay-cli/tracedecay" + exe="" + case "$(uname -s)" in + MINGW*|MSYS*|CYGWIN*) exe=".exe" ;; + esac + bin="$bazel_bin/crates/tracedecay-cli/tracedecay${exe}" [[ -f "$bin" && -x "$bin" ]] || fail "expected a binary at $bin after: $build_cmd" origin="bazel --config=$config $target" diff --git a/scripts/test-build-eval-binary.py b/scripts/test-build-eval-binary.py index 22de025f37..9f6751f422 100644 --- a/scripts/test-build-eval-binary.py +++ b/scripts/test-build-eval-binary.py @@ -37,7 +37,8 @@ def setUp(self): self.built.write_text("#!/bin/sh\necho tracedecay 1.2.3\n") self.built.chmod(0o755) self.log = root / "bazel.log" - fakes = root / "fakes" + self.fakes = root / "fakes" + fakes = self.fakes fakes.mkdir() bazel = fakes / "bazel" bazel.write_text(FAKE_BAZEL) @@ -68,6 +69,16 @@ def test_builds_the_shipping_target_and_prints_its_identity(self): self.assertIn("tracedecay 1.2.3", result.stderr) self.assertIn("sha256 ", result.stderr) + def test_windows_build_selects_the_executable_suffix(self): + windows_binary = self.built.with_suffix(".exe") + self.built.rename(windows_binary) + uname = self.fakes / "uname" + uname.write_text("#!/bin/sh\necho MINGW64_NT-10.0\n") + uname.chmod(0o755) + result = self.run_script() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), str(windows_binary)) + def test_config_override_reaches_bazel(self): result = self.run_script(TRACEDECAY_EVAL_BAZEL_CONFIG="dist") self.assertEqual(result.returncode, 0, result.stderr) @@ -89,6 +100,19 @@ def test_explicit_binary_skips_the_build(self): self.assertIn("tracedecay 9.9.9", result.stderr) self.assertFalse(self.log.exists()) + def test_isolated_reentry_reuses_the_selected_binary_without_another_build(self): + outer = self.run_script() + self.assertEqual(outer.returncode, 0, outer.stderr) + with tempfile.TemporaryDirectory() as isolated_home: + child = self.run_script( + TRACEDECAY_BIN=outer.stdout.strip(), + HOME=isolated_home, + XDG_CACHE_HOME=str(Path(isolated_home) / ".cache"), + ) + self.assertEqual(child.returncode, 0, child.stderr) + self.assertEqual(child.stdout, outer.stdout) + self.assertEqual(sum(line.startswith("build ") for line in self.log.read_text().splitlines()), 1) + def test_missing_explicit_binary_names_the_build_command(self): result = self.run_script(TRACEDECAY_BIN=str(self.built.with_name("absent"))) self.assertEqual(result.returncode, 2) From 451721797b8ae53bc539072fc19994df9cb66d34 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 20:27:12 +0000 Subject: [PATCH 27/89] fix(dashboard): decode canonical settings settlements --- .../workflows/dashboard-contract-export.yml | 74 -- .../schemas/dashboard-contracts.schema.json | 1105 +++++++++++++++++ dashboard/codegen/src/generate.ts | 2 +- dashboard/src/contracts/decoders.ts | 401 ++++++ dashboard/src/contracts/types.ts | 440 +++++-- .../src/test/fixtures/settings-patch.json | 639 ++++++++++ .../settings/SettingsPage.dom.test.tsx | 40 +- .../settings/SettingsReviewPanel.tsx | 5 +- .../settings/settingsEditorMachine.ts | 4 +- .../settings/settingsMutation.test.ts | 89 ++ .../workspaces/settings/settingsMutation.ts | 155 +-- 11 files changed, 2673 insertions(+), 281 deletions(-) delete mode 100644 .github/workflows/dashboard-contract-export.yml create mode 100644 dashboard/src/test/fixtures/settings-patch.json create mode 100644 dashboard/src/workspaces/settings/settingsMutation.test.ts diff --git a/.github/workflows/dashboard-contract-export.yml b/.github/workflows/dashboard-contract-export.yml deleted file mode 100644 index c9aeab7cda..0000000000 --- a/.github/workflows/dashboard-contract-export.yml +++ /dev/null @@ -1,74 +0,0 @@ -# Temporary build route: remove after importing the canonical generated JSON. -name: Export dashboard contract artifacts -run-name: Export dashboard contracts for ${{ github.event.pull_request.head.sha }} - -on: - pull_request: - types: [opened, synchronize, reopened] - -permissions: - contents: read - -concurrency: - group: dashboard-contract-export-${{ github.event.pull_request.number }} - cancel-in-progress: true - -jobs: - export: - if: github.event.pull_request.head.repo.full_name == github.repository - runs-on: ubuntu-latest - timeout-minutes: 90 - env: - CARGO_BAZEL_TIMEOUT: "3600" - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ github.event.pull_request.head.sha }} - persist-credentials: false - - - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 - with: - node-version: "26" - - - uses: ./.github/actions/setup-pnpm - - - uses: ./.github/actions/setup-bazel - with: - cache-key: ci-linux-test-v2 - - - name: Export the canonical dashboard schema - run: | - set -euo pipefail - EXPORT_DIRECTORY="$RUNNER_TEMP/dashboard-contract-export" - mkdir -p "$EXPORT_DIRECTORY" - bazel run --config=ci --jobs=1 //sdks/codegen:dashboard_schema -- "$EXPORT_DIRECTORY/dashboard-contracts.schema.json" - test -s "$EXPORT_DIRECTORY/dashboard-contracts.schema.json" - - # Preserve the schema before the independent fixture build starts. - - name: Upload the canonical dashboard schema - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: dashboard-schema-${{ github.event.pull_request.head.sha }} - path: ${{ runner.temp }}/dashboard-contract-export/dashboard-contracts.schema.json - if-no-files-found: error - retention-days: 1 - - - name: Serialize the real settings response fixture - run: | - set -euo pipefail - EXPORT_DIRECTORY="$RUNNER_TEMP/dashboard-contract-export" - bazel test --config=ci --jobs=1 //crates/tracedecay-dashboard-api:unit_test \ - --test_arg=settings_api::tests::wire::project_patch_serialization_preserves_settlement_and_refusals \ - --test_arg=--exact --test_output=all --cache_test_results=no \ - --nozip_undeclared_test_outputs - fixture=bazel-testlogs/crates/tracedecay-dashboard-api/unit_test/test.outputs/settings-patch.json - test -s "$fixture" - cp -- "$fixture" "$EXPORT_DIRECTORY/settings-patch.json" - - - name: Upload the canonical settings response fixture - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: settings-patch-fixture-${{ github.event.pull_request.head.sha }} - path: ${{ runner.temp }}/dashboard-contract-export/settings-patch.json - if-no-files-found: error - retention-days: 1 diff --git a/dashboard/codegen/schemas/dashboard-contracts.schema.json b/dashboard/codegen/schemas/dashboard-contracts.schema.json index e65724f847..76951811fd 100644 --- a/dashboard/codegen/schemas/dashboard-contracts.schema.json +++ b/dashboard/codegen/schemas/dashboard-contracts.schema.json @@ -1343,6 +1343,74 @@ ], "type": "string" }, + "ApplicationOutcome": { + "description": "Canonical outcome family for an admitted application operation.", + "oneOf": [ + { + "properties": { + "outcome": { + "const": "evidence", + "type": "string" + }, + "value": { + "$ref": "#/$defs/EvidencePacket" + } + }, + "required": [ + "outcome", + "value" + ], + "type": "object" + }, + { + "properties": { + "outcome": { + "const": "preview", + "type": "string" + }, + "value": { + "$ref": "#/$defs/PreviewResult" + } + }, + "required": [ + "outcome", + "value" + ], + "type": "object" + }, + { + "properties": { + "outcome": { + "const": "effect", + "type": "string" + }, + "value": { + "$ref": "#/$defs/EffectResult" + } + }, + "required": [ + "outcome", + "value" + ], + "type": "object" + }, + { + "description": "An operation-owned typed result that carries its own receipts, such as\na source-edit result with its durable effect, or a rendered document.", + "properties": { + "outcome": { + "const": "result", + "type": "string" + }, + "value": true + }, + "required": [ + "outcome", + "value" + ], + "type": "object" + } + ] + }, "ApplicationProblemDetailV1": { "description": "The structured facts behind a problem. Adapters read these fields; the\nproblem's `message` is only their one human rendering.", "oneOf": [ @@ -1828,6 +1896,45 @@ "description": "Strongly typed canonical identity: `AttemptId`.", "type": "string" }, + "AuthorityReceipt": { + "additionalProperties": false, + "description": "Proof that this request crossed the current authorization boundary.", + "properties": { + "authorized_scope_digest": { + "$ref": "#/$defs/ManifestDigest" + }, + "disclosure": { + "$ref": "#/$defs/DisclosureClass" + }, + "grant_digest": { + "$ref": "#/$defs/ManifestDigest" + }, + "grant_id": { + "type": "string" + }, + "grant_revision": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "policy": { + "$ref": "#/$defs/PolicyDecisionRef" + }, + "revalidated_at": { + "$ref": "#/$defs/UtcMicros" + } + }, + "required": [ + "grant_id", + "grant_revision", + "grant_digest", + "authorized_scope_digest", + "disclosure", + "policy", + "revalidated_at" + ], + "type": "object" + }, "AuthorizedRoot": { "additionalProperties": false, "description": "One exact application scope paired with its registered physical locator.\n\nThe scope remains the identity authority. The locator is retained only so\na later read or restart can reopen the exact registered root without an\nactive-graph or CWD fallback.", @@ -3345,6 +3452,15 @@ ], "type": "object" }, + "BudgetClass": { + "description": "Bounded elapsed-work classification supplied by a retriever.", + "enum": [ + "within_budget", + "approaching_limit", + "exhausted" + ], + "type": "string" + }, "CallChainMeasurementV1": { "properties": { "complete": { @@ -3452,6 +3568,22 @@ ], "type": "object" }, + "CancellationObservation": { + "additionalProperties": false, + "properties": { + "observed_at": { + "$ref": "#/$defs/UtcMicros" + }, + "stage": { + "$ref": "#/$defs/CancellationStage" + } + }, + "required": [ + "stage", + "observed_at" + ], + "type": "object" + }, "CancellationStage": { "description": "Exact stage at which cancellation or deadline state was observed.", "enum": [ @@ -3813,6 +3945,43 @@ "description": "Strongly typed canonical identity: `CodeGenerationId`.", "type": "string" }, + "CodeGraphReadFreshnessV1": { + "description": "Freshness of the generation a verified graph read serves.", + "oneOf": [ + { + "properties": { + "state": { + "const": "current", + "type": "string" + } + }, + "required": [ + "state" + ], + "type": "object" + }, + { + "properties": { + "rebuild_in_flight": { + "type": "boolean" + }, + "sealed_at": { + "$ref": "#/$defs/UtcMicros" + }, + "state": { + "const": "last_complete_stale", + "type": "string" + } + }, + "required": [ + "state", + "sealed_at", + "rebuild_in_flight" + ], + "type": "object" + } + ] + }, "CodeGraphServingReadinessV1": { "description": "Interactive graph-serving state for the latest sealed generation.\n\nA sealed generation can expose truthful census statistics before its graph\nprojection is ready to serve queries, so readiness is reported separately.", "oneOf": [ @@ -4752,6 +4921,70 @@ "description": "Strongly typed canonical identity: `ComponentVersion`.", "type": "string" }, + "ConfigurationRefusalV1": { + "description": "Configuration refusals carrying the same stable codes as the HTTP routes.", + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "actual_revision_id": { + "type": "string" + }, + "code": { + "const": "configuration_revision_conflict", + "type": "string" + }, + "detail": { + "type": "string" + }, + "expected_revision_id": { + "type": "string" + } + }, + "required": [ + "code", + "detail", + "expected_revision_id", + "actual_revision_id" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "code": { + "const": "configuration_authority_unavailable", + "type": "string" + }, + "detail": { + "type": "string" + } + }, + "required": [ + "code", + "detail" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "code": { + "const": "application_contract_violation", + "type": "string" + }, + "detail": { + "type": "string" + } + }, + "required": [ + "code", + "detail" + ], + "type": "object" + } + ] + }, "ConfigurationRevisionId": { "description": "Strongly typed canonical identity: `ConfigurationRevisionId`.", "type": "string" @@ -5167,6 +5400,36 @@ } ] }, + "DashboardConfigurationRefusalV1": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "detail": { + "type": "string" + }, + "validation_errors": { + "items": { + "$ref": "#/$defs/SettingsValidationIssueV1" + }, + "type": "array" + } + }, + "required": [ + "detail", + "validation_errors" + ], + "type": "object" + }, + { + "$ref": "#/$defs/ConfigurationRefusalV1" + }, + { + "$ref": "#/$defs/ApplicationProblemEnvelope" + } + ], + "description": "Every refusal body produced by the dashboard configuration adapter. The\nuntagged wrapper preserves the existing validation, CAS, and application\nproblem wire formats while giving producers and consumers one authority." + }, "DashboardCoverageCompletenessV1": { "description": "Coverage completeness axis. `Unsupported` distinguishes \"the source that\nwould establish coverage is not wired\" from `Unknown` (\"coverage could not\nbe determined\").", "enum": [ @@ -6917,6 +7180,70 @@ ], "type": "object" }, + "DashboardEnvelopeV130": { + "description": "The normative read-model envelope. Every V2 read-model route returns exactly\nthis shape; only `payload` varies by route.", + "properties": { + "authorization": { + "$ref": "#/$defs/DashboardAuthorizationV1" + }, + "coverage": { + "$ref": "#/$defs/DashboardCoverageV1" + }, + "domain_state": { + "$ref": "#/$defs/DashboardDomainStateV1" + }, + "freshness": { + "$ref": "#/$defs/DashboardFreshnessV1" + }, + "legal_actions": { + "items": { + "$ref": "#/$defs/DashboardLegalActionRefV1" + }, + "type": "array" + }, + "payload": { + "$ref": "#/$defs/SettingsPayloadV1" + }, + "schema_revision": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "scope": { + "$ref": "#/$defs/DashboardScopeV1" + }, + "source_watermark": { + "anyOf": [ + { + "$ref": "#/$defs/DashboardWatermarkV1" + }, + { + "type": "null" + } + ] + }, + "time": { + "$ref": "#/$defs/DashboardTimeV1" + }, + "version": { + "$ref": "#/$defs/DashboardVersionV1" + } + }, + "required": [ + "schema_revision", + "scope", + "version", + "time", + "source_watermark", + "authorization", + "coverage", + "freshness", + "domain_state", + "legal_actions", + "payload" + ], + "type": "object" + }, "DashboardEnvelopeV14": { "description": "The normative read-model envelope. Every V2 read-model route returns exactly\nthis shape; only `payload` varies by route.", "properties": { @@ -7487,6 +7814,19 @@ ], "type": "object" }, + "Deadline": { + "additionalProperties": false, + "description": "One immutable deadline supplied by the caller or upstream admission layer.", + "properties": { + "expires_at": { + "$ref": "#/$defs/UtcMicros" + } + }, + "required": [ + "expires_at" + ], + "type": "object" + }, "DecideWorkProposalRequestV1": { "additionalProperties": false, "properties": { @@ -11160,6 +11500,15 @@ ], "type": "object" }, + "DisclosureClass": { + "description": "Disclosure ceiling carried by an immutable grant and revalidated at sinks.", + "enum": [ + "metadata", + "evidence", + "sensitive" + ], + "type": "string" + }, "DoctorCoverageCompletenessV1": { "description": "Whether Doctor observed all of a family's evidence sources.", "oneOf": [ @@ -11729,6 +12078,49 @@ ], "type": "object" }, + "EffectResult": { + "additionalProperties": false, + "description": "Result of an admitted effect. `EffectUnknown` remains a receipt state and\ncannot be remapped into a pre-admission problem.", + "properties": { + "authority": { + "$ref": "#/$defs/AuthorityReceipt" + }, + "effect_class": { + "$ref": "#/$defs/EffectClass" + }, + "effect_id": { + "type": "string" + }, + "execution": { + "$ref": "#/$defs/OperationReceipt" + }, + "expected_state": { + "$ref": "#/$defs/ManifestDigest" + }, + "idempotency_key": { + "type": "string" + }, + "payload": true, + "receipt": { + "$ref": "#/$defs/EffectReceipt" + }, + "reconciliation": { + "$ref": "#/$defs/ReconciliationState" + } + }, + "required": [ + "effect_id", + "effect_class", + "idempotency_key", + "authority", + "expected_state", + "execution", + "reconciliation", + "receipt", + "payload" + ], + "type": "object" + }, "EffectTermination": { "description": "Durable effect receipt terminal state.", "enum": [ @@ -11793,6 +12185,46 @@ ], "type": "object" }, + "EvidenceAuthority": { + "additionalProperties": false, + "description": "Evidence authority is separate from caller authority and cannot grant access.", + "properties": { + "evidence_id": { + "type": "string" + }, + "horizon": { + "anyOf": [ + { + "$ref": "#/$defs/UtcMicros" + }, + { + "type": "null" + } + ] + }, + "producer": { + "type": "string" + }, + "revision": { + "$ref": "#/$defs/ComponentVersion" + }, + "scope": { + "$ref": "#/$defs/ResolvedScope" + }, + "source_kind": { + "type": "string" + } + }, + "required": [ + "evidence_id", + "source_kind", + "producer", + "scope", + "revision", + "horizon" + ], + "type": "object" + }, "EvidenceCoverage": { "additionalProperties": false, "description": "Deterministic coverage fold input for an evidence packet.", @@ -11858,6 +12290,184 @@ ], "type": "string" }, + "EvidencePacket": { + "additionalProperties": false, + "description": "Immutable evidence packet consumed by adapters and later planner work.", + "properties": { + "authority": { + "$ref": "#/$defs/AuthorityReceipt" + }, + "contributions": { + "items": { + "$ref": "#/$defs/RetrieverContribution" + }, + "type": "array" + }, + "coverage": { + "$ref": "#/$defs/EvidenceCoverage" + }, + "evidence_authorities": { + "items": { + "$ref": "#/$defs/EvidenceAuthority" + }, + "type": "array" + }, + "execution": { + "$ref": "#/$defs/OperationReceipt" + }, + "omissions": { + "items": { + "$ref": "#/$defs/Omission" + }, + "type": "array" + }, + "page": { + "$ref": "#/$defs/PageState" + }, + "payload": true, + "scores": { + "items": { + "$ref": "#/$defs/EvidenceScore" + }, + "type": "array" + }, + "temporal": { + "$ref": "#/$defs/TemporalState" + } + }, + "required": [ + "temporal", + "authority", + "evidence_authorities", + "coverage", + "omissions", + "scores", + "contributions", + "page", + "execution", + "payload" + ], + "type": "object" + }, + "EvidenceScore": { + "additionalProperties": false, + "properties": { + "calibration_revision": { + "anyOf": [ + { + "$ref": "#/$defs/ComponentVersion" + }, + { + "type": "null" + } + ] + }, + "calibration_valid": { + "type": [ + "boolean", + "null" + ] + }, + "deterministic_components": { + "items": { + "type": "string" + }, + "type": "array" + }, + "kind": { + "$ref": "#/$defs/EvidenceScoreKind" + }, + "score_id": { + "type": "string" + }, + "value": { + "$ref": "#/$defs/EvidenceScoreValue" + } + }, + "required": [ + "score_id", + "kind", + "value", + "calibration_revision", + "calibration_valid", + "deterministic_components" + ], + "type": "object" + }, + "EvidenceScoreKind": { + "description": "Evidence score semantics. Scores are metadata, never authority.", + "enum": [ + "ordinal_rank", + "heuristic_score", + "calibrated_probability", + "calibrated_interval" + ], + "type": "string" + }, + "EvidenceScoreValue": { + "oneOf": [ + { + "properties": { + "kind": { + "const": "ordinal", + "type": "string" + }, + "rank": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind", + "rank" + ], + "type": "object" + }, + { + "properties": { + "kind": { + "const": "fixed_point", + "type": "string" + }, + "micros": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind", + "micros" + ], + "type": "object" + }, + { + "properties": { + "kind": { + "const": "interval", + "type": "string" + }, + "lower_micros": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "upper_micros": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind", + "lower_micros", + "upper_micros" + ], + "type": "object" + } + ] + }, "ExecutionBlockedCauseV1": { "description": "Cause a work item was blocked for.", "enum": [ @@ -14461,6 +15071,15 @@ "description": "Strongly typed canonical identity: `FileOccurrenceId`.", "type": "string" }, + "FreshnessState": { + "description": "Application-level freshness. Missing or partial truth never becomes current.", + "enum": [ + "current", + "stale", + "unknown" + ], + "type": "string" + }, "GenerateProposalRequest": { "additionalProperties": false, "description": "Read-only proposal generation over one exact current product graph.", @@ -20495,6 +21114,225 @@ ], "type": "object" }, + "Omission": { + "additionalProperties": false, + "properties": { + "count": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "domain": { + "$ref": "#/$defs/EvidenceDomain" + }, + "reason": { + "$ref": "#/$defs/OmissionReason" + } + }, + "required": [ + "domain", + "count", + "reason" + ], + "type": "object" + }, + "OmissionReason": { + "description": "Safe reason why authorized requested evidence was omitted.", + "oneOf": [ + { + "enum": [ + "budget", + "redacted", + "unavailable", + "unsupported", + "stale", + "failed", + "cancelled", + "timed_out", + "conflict" + ], + "type": "string" + }, + { + "const": "macro_body_unparsed", + "description": "Evidence came from a macro body that was not expanded, so what the\nexpansion defines or calls is not covered.", + "type": "string" + }, + { + "const": "import_unmodeled", + "description": "A caller sits under a `use` whose binding the index could not model\n(a glob in a block or inline module), so whether it calls the target\nis not covered.", + "type": "string" + }, + { + "const": "cursor_expired", + "description": "The continuation cursor can no longer be redeemed (its lifetime ended\nor the snapshot it pages is gone); restart the request without it.", + "type": "string" + }, + { + "const": "cursor_foreign", + "description": "The continuation cursor was issued for another project, worktree, or\nref; redeem it where it was issued.", + "type": "string" + } + ] + }, + "OperationBudgetUsage": { + "additionalProperties": false, + "description": "Bounded work accounting supplied by an owning port or transaction.", + "properties": { + "bytes_consumed": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "elapsed_micros": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "units_consumed": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "units_consumed", + "bytes_consumed", + "elapsed_micros" + ], + "type": "object" + }, + "OperationReceipt": { + "additionalProperties": false, + "description": "Canonical operation evidence. An admitted failure remains represented here\nrather than being replaced by a transport exception.", + "properties": { + "budget": { + "$ref": "#/$defs/OperationBudgetUsage" + }, + "cancellation": { + "anyOf": [ + { + "$ref": "#/$defs/CancellationObservation" + }, + { + "type": "null" + } + ] + }, + "effective_deadline": { + "$ref": "#/$defs/Deadline" + }, + "ended_at": { + "$ref": "#/$defs/UtcMicros" + }, + "started_at": { + "$ref": "#/$defs/UtcMicros" + }, + "termination": { + "$ref": "#/$defs/OperationTermination" + } + }, + "required": [ + "started_at", + "ended_at", + "effective_deadline", + "cancellation", + "budget", + "termination" + ], + "type": "object" + }, + "OperationTermination": { + "description": "Terminal state after an operation has been admitted.", + "enum": [ + "completed", + "cancelled", + "timed_out", + "failed", + "unavailable", + "partial", + "effect_unknown" + ], + "type": "string" + }, + "PageCursor": { + "description": "Exact continuation authority for the enclosing evidence page: an opaque\ncursor bound to the operation and request that minted it.", + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "cursor": { + "type": "string" + }, + "kind": { + "const": "opaque", + "type": "string" + } + }, + "required": [ + "kind", + "cursor" + ], + "type": "object" + } + ] + }, + "PageState": { + "additionalProperties": false, + "description": "Stable page state. General cursor bytes stay opaque until authorization is\nrevalidated; structural fact cursors retain their canonical ordering type.", + "properties": { + "cursor": { + "anyOf": [ + { + "$ref": "#/$defs/PageCursor" + }, + { + "type": "null" + } + ] + }, + "expires_at": { + "anyOf": [ + { + "$ref": "#/$defs/UtcMicros" + }, + { + "type": "null" + } + ] + }, + "returned": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "sort_contract_id": { + "$ref": "#/$defs/SortContractId" + }, + "sort_revision": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "total": { + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "sort_contract_id", + "sort_revision", + "total", + "returned", + "cursor", + "expires_at" + ], + "type": "object" + }, "PauseWorkRunCommand": { "additionalProperties": false, "properties": { @@ -20675,6 +21513,33 @@ ], "type": "object" }, + "PolicyDecisionRef": { + "additionalProperties": false, + "description": "A policy decision pinned into a receipt or provider identity.", + "properties": { + "decision_id": { + "type": "string" + }, + "digest": { + "$ref": "#/$defs/ManifestDigest" + }, + "evaluator_revision": { + "$ref": "#/$defs/ComponentVersion" + }, + "revision": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "decision_id", + "revision", + "digest", + "evaluator_revision" + ], + "type": "object" + }, "PolicyIdentifierV1": { "description": "A bounded, canonical identifier owned by the policy input schema.\n\nIt represents immutable references only; it is never a path, display\nlabel, provider account, branch name, or native object identifier.", "type": "string" @@ -20774,6 +21639,41 @@ ], "type": "object" }, + "PreviewResult": { + "additionalProperties": false, + "description": "Read-only preview of a future typed effect.", + "properties": { + "authority": { + "$ref": "#/$defs/AuthorityReceipt" + }, + "effect_class": { + "$ref": "#/$defs/EffectClass" + }, + "execution": { + "$ref": "#/$defs/OperationReceipt" + }, + "expected_state": { + "$ref": "#/$defs/ManifestDigest" + }, + "payload": true, + "preview_digest": { + "$ref": "#/$defs/ManifestDigest" + }, + "preview_id": { + "type": "string" + } + }, + "required": [ + "preview_id", + "preview_digest", + "effect_class", + "authority", + "expected_state", + "execution", + "payload" + ], + "type": "object" + }, "PrimitiveFreshnessStateV1": { "description": "Whether the served code generation is known current at serve time.", "enum": [ @@ -21300,6 +22200,29 @@ ], "type": "object" }, + "ProjectSettingsPatchResponseV1": { + "properties": { + "application_outcome": { + "anyOf": [ + { + "$ref": "#/$defs/ApplicationOutcome" + }, + { + "type": "null" + } + ], + "description": "Exact application settlement returned by the configuration binding.\n`None` means the submitted patch was already the current state." + }, + "current": { + "$ref": "#/$defs/DashboardEnvelopeV130" + } + }, + "required": [ + "application_outcome", + "current" + ], + "type": "object" + }, "ProjectSettingsPayloadV1": { "properties": { "config": { @@ -21768,6 +22691,15 @@ ], "type": "string" }, + "ReconciliationState": { + "description": "Reconciliation state retained after an admitted effect.", + "enum": [ + "pending", + "reconciled", + "failed" + ], + "type": "string" + }, "RecordWorkHandoffRequestV1": { "additionalProperties": false, "properties": { @@ -22514,6 +23446,94 @@ "description": "Strongly typed canonical identity: `RetrievalAnchorId`.", "type": "string" }, + "RetrieverContribution": { + "additionalProperties": false, + "description": "One source-owned contribution to the packet fold.", + "properties": { + "contract": { + "$ref": "#/$defs/ResultContractRef" + }, + "coverage": { + "$ref": "#/$defs/EvidenceCoverage" + }, + "domain": { + "$ref": "#/$defs/EvidenceDomain" + }, + "elapsed_budget_class": { + "$ref": "#/$defs/BudgetClass" + }, + "evidence_authorities": { + "items": { + "type": "string" + }, + "type": "array" + }, + "omitted_count": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "producer_revision": { + "$ref": "#/$defs/ComponentVersion" + }, + "provenance_anchors": { + "items": { + "$ref": "#/$defs/RetrievalAnchorId" + }, + "type": "array" + }, + "retriever_id": { + "$ref": "#/$defs/RetrieverId" + }, + "returned_count": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "score_ids": { + "items": { + "type": "string" + }, + "type": "array" + }, + "state": { + "$ref": "#/$defs/RetrieverContributionState" + } + }, + "required": [ + "retriever_id", + "contract", + "producer_revision", + "domain", + "state", + "coverage", + "returned_count", + "omitted_count", + "score_ids", + "provenance_anchors", + "evidence_authorities", + "elapsed_budget_class" + ], + "type": "object" + }, + "RetrieverContributionState": { + "description": "Terminal state reported by one retriever contribution.", + "enum": [ + "completed", + "partial", + "unavailable", + "unsupported", + "stale", + "failed", + "cancelled", + "timed_out" + ], + "type": "string" + }, + "RetrieverId": { + "description": "Stable, canonical catalog identity for `RetrieverId`.", + "type": "string" + }, "RetryDirective": { "description": "Safe adapter-independent retry instruction. Adapters preserve it verbatim.", "enum": [ @@ -24500,6 +25520,22 @@ ], "type": "object" }, + "SettingsValidationIssueV1": { + "description": "One field-level settings validation failure.", + "properties": { + "field": { + "type": "string" + }, + "message": { + "type": "string" + } + }, + "required": [ + "field", + "message" + ], + "type": "object" + }, "SharedProfileStoreLocatorV1": { "additionalProperties": false, "description": "Logical and verified physical identity of the shared Profile shard.\n\nBrain and profile IDs select the shard; the verified store locator binds its\nphysical store. Lease incarnations and authority epochs remain runtime fences.", @@ -24856,6 +25892,10 @@ ], "type": "string" }, + "SortContractId": { + "description": "Stable, canonical catalog identity for `SortContractId`.", + "type": "string" + }, "SourceSpan": { "additionalProperties": false, "description": "Byte range inside one sanitized source file. Mutable line numbers are\nnever part of identity.", @@ -26364,6 +27404,63 @@ } ] }, + "TemporalState": { + "additionalProperties": false, + "description": "Temporal provenance for a packet.", + "properties": { + "code_graph_freshness": { + "anyOf": [ + { + "$ref": "#/$defs/CodeGraphReadFreshnessV1" + }, + { + "type": "null" + } + ] + }, + "freshness": { + "$ref": "#/$defs/FreshnessState" + }, + "requested_at": { + "$ref": "#/$defs/UtcMicros" + }, + "requested_mode": { + "$ref": "#/$defs/TemporalModeV1" + }, + "resolved_at": { + "$ref": "#/$defs/UtcMicros" + }, + "source_generation": { + "anyOf": [ + { + "$ref": "#/$defs/CodeGenerationId" + }, + { + "type": "null" + } + ] + }, + "watermark_digest": { + "anyOf": [ + { + "$ref": "#/$defs/ManifestDigest" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "requested_mode", + "requested_at", + "resolved_at", + "source_generation", + "watermark_digest", + "freshness" + ], + "type": "object" + }, "TestMapMeasurementV1": { "properties": { "algorithm": { @@ -35946,6 +37043,12 @@ "settings_project_patch": { "$ref": "#/$defs/ProjectSettingsPatch" }, + "settings_project_patch_response": { + "$ref": "#/$defs/ProjectSettingsPatchResponseV1" + }, + "settings_refusal": { + "$ref": "#/$defs/DashboardConfigurationRefusalV1" + }, "settings_user_patch": { "$ref": "#/$defs/UserSettingsPatch" }, @@ -36198,6 +37301,8 @@ "code_index_freshness", "settings", "settings_project_patch", + "settings_project_patch_response", + "settings_refusal", "settings_user_patch", "settings_code_index_worker_patch", "observatory", diff --git a/dashboard/codegen/src/generate.ts b/dashboard/codegen/src/generate.ts index 1c3d4fc890..9524e0ca05 100644 --- a/dashboard/codegen/src/generate.ts +++ b/dashboard/codegen/src/generate.ts @@ -449,7 +449,7 @@ function emitNamedDef( function emitAlias(desc: string, name: string, ann: string, zod: string, ts: string): EmittedDef { return { typeName: name, - types: `export type ${name} = ${ts};`, + types: `export type ${name} =${ts.startsWith("\n") ? "" : " "}${ts};`, decoders: `${desc}export const ${name}Schema: ${ann} = ${zod};`, }; } diff --git a/dashboard/src/contracts/decoders.ts b/dashboard/src/contracts/decoders.ts index 67b2c136d4..a9d127776f 100644 --- a/dashboard/src/contracts/decoders.ts +++ b/dashboard/src/contracts/decoders.ts @@ -54,6 +54,7 @@ import type { AnalyticsUsageCategoryV1, AnalyticsUsageSummaryV1, ApplicationExecutionFailureClassV1, + ApplicationOutcome, ApplicationProblemDetailV1, ApplicationProblemEnvelope, ApplicationProblemKind, @@ -61,6 +62,7 @@ import type { ApplicationUnavailableClassV1, ApplyWorkRelationReplanRequestV1, AttemptId, + AuthorityReceipt, AuthorizedRoot, AuthorizedScopeSet, AuthorizedWorkProductScopeV1, @@ -104,8 +106,10 @@ import type { BranchNamingPolicyV1, BranchTopologyKindV1, BranchTopologyPolicyV1, + BudgetClass, CallChainMeasurementV1, CallChainStepV1, + CancellationObservation, CancellationStage, CancelWorkAttemptCommand, CanonicalGitRefNameV1, @@ -118,6 +122,7 @@ import type { CodeCloneIndexResourcesV1, CodeCloneIndexStatusV1, CodeGenerationId, + CodeGraphReadFreshnessV1, CodeGraphServingReadinessV1, CodeIndexBuildBlockedReasonV1, CodeIndexBuildPhaseV1, @@ -143,6 +148,7 @@ import type { ComparisonDispositionV1, ComplexityAnalysisV1, ComponentVersion, + ConfigurationRefusalV1, ConfigurationRevisionId, ConfigurationSnapshotId, ContentDigest, @@ -160,6 +166,7 @@ import type { CrossMergePolicyV1, DaemonServiceUnitObservationV1, DashboardAuthorizationV1, + DashboardConfigurationRefusalV1, DashboardCoverageCompletenessV1, DashboardCoverageV1, DashboardDomainStateV1, @@ -172,6 +179,7 @@ import type { DashboardTimeV1, DashboardVersionV1, DashboardWatermarkV1, + Deadline, DecideWorkProposalRequestV1, DecideWorkRelationReplanRequestV1, DecisionEngineSettingsV1, @@ -244,6 +252,7 @@ import type { DeliverySharedCodeRefStateV1, DeliverySharedCodeRefV1, DiagnosticsSearchedTsconfigV1, + DisclosureClass, DoctorCoverageCompletenessV1, DoctorCoverageStatementV1, DoctorEvidenceReferenceV1, @@ -262,11 +271,17 @@ import type { DuplicateEffortKindV1, EffectClass, EffectReceipt, + EffectResult, EffectTermination, EnvironmentSettingsPayloadV1, EnvironmentVariableV1, + EvidenceAuthority, EvidenceCoverage, EvidenceDomain, + EvidencePacket, + EvidenceScore, + EvidenceScoreKind, + EvidenceScoreValue, ExecutionBlockedCauseV1, ExecutionConcurrencyPhaseV1, ExecutionConflictKindV1, @@ -352,6 +367,7 @@ import type { FeedbackSystemMetricV1, FeedbackSystemQualityReadModelV1, FileOccurrenceId, + FreshnessState, GeneratedWorkProposal, GenerateProposalRequest, GitHubStackedPullRequestPolicyV1, @@ -516,16 +532,25 @@ import type { ObservabilityHorizonV1, ObservationSourceIdentityV1, ObservatoryReadModelV1, + Omission, + OmissionReason, + OperationBudgetUsage, + OperationReceipt, + OperationTermination, + PageCursor, + PageState, PauseWorkRunCommand, PayloadAccessState, PayloadDigestV1, PayloadReferenceV1, PerformanceComparisonReadModelV1, + PolicyDecisionRef, PolicyIdentifierV1, PrAutoTrackEntryV1, PrAutoTrackPayloadV1, PrepareWorkDuplicateAdjudicationRequestV1, PrepareWorkProductMutationRequestV1, + PreviewResult, PrimitiveFreshnessStateV1, PrimitiveIndexingStateV1, PrimitiveSearchFreshnessV1, @@ -543,6 +568,7 @@ import type { ProjectRegistrySummary, ProjectRepoGroup, ProjectSettingsPatch, + ProjectSettingsPatchResponseV1, ProjectSettingsPayloadV1, ProjectsPayloadV1, ProposalId, @@ -560,6 +586,7 @@ import type { ProximityWarningClassV1, PublicCodeProject, QuantityEvidenceClassV1, + ReconciliationState, RecordWorkHandoffRequestV1, RefId, RegisteredRootLocatorV1, @@ -592,6 +619,9 @@ import type { ResumeWorkAttemptsCommand, ResumeWorkRunCommand, RetrievalAnchorId, + RetrieverContribution, + RetrieverContributionState, + RetrieverId, RetryDirective, RetryScope, RetryWorkAttemptCommandV1, @@ -654,6 +684,7 @@ import type { SessionRetrievalStructuralRefusalV1, SettingsAvailabilityV1, SettingsPayloadV1, + SettingsValidationIssueV1, SharedProfileStoreLocatorV1, SignificantTableGrowthSampleV1, SimilarCoverageV1, @@ -664,6 +695,7 @@ import type { SkillInstallTarget, SkillOutcomeRecord, SkillOutcomeVerdict, + SortContractId, SourceSpan, SourceStoreId, StartWorkAttemptCommand, @@ -701,6 +733,7 @@ import type { TelemetrySettingsPatch, TelemetrySettingsV1, TemporalModeV1, + TemporalState, TestMapMeasurementV1, TokenActualV1, TokenPairV1, @@ -1506,6 +1539,21 @@ export const AnalyticsUsageSummaryV1Schema: z.ZodObject<{ /** Stable non-retryable class for an admitted execution failure. */ export const ApplicationExecutionFailureClassV1Schema: z.ZodEnum<{ "denied": "denied"; "malformed_output": "malformed_output"; "permanent": "permanent" }> = z.enum(["denied", "malformed_output", "permanent"]); +/** Canonical outcome family for an admitted application operation. */ +export const ApplicationOutcomeSchema: z.ZodType = z.discriminatedUnion("outcome", [z.object({ + outcome: z.literal("effect"), + value: z.lazy(() => EffectResultSchema), +}), z.object({ + outcome: z.literal("evidence"), + value: z.lazy(() => EvidencePacketSchema), +}), z.object({ + outcome: z.literal("preview"), + value: z.lazy(() => PreviewResultSchema), +}), z.object({ + outcome: z.literal("result"), + value: z.unknown(), +})]); + /** The structured facts behind a problem. Adapters read these fields; the problem's `message` is only their one human rendering. */ export const ApplicationProblemDetailV1Schema: z.ZodType = z.discriminatedUnion("kind", [z.object({ @@ -1635,6 +1683,25 @@ export const ApplyWorkRelationReplanRequestV1Schema: z.ZodObject<{ /** Strongly typed canonical identity: `AttemptId`. */ export const AttemptIdSchema: z.ZodType = z.string(); +/** Proof that this request crossed the current authorization boundary. */ +export const AuthorityReceiptSchema: z.ZodObject<{ + authorized_scope_digest: z.ZodType; + disclosure: z.ZodType; + grant_digest: z.ZodType; + grant_id: z.ZodType; + grant_revision: z.ZodType; + policy: z.ZodType; + revalidated_at: z.ZodType; +}, z.core.$strict> = z.object({ + authorized_scope_digest: z.lazy(() => ManifestDigestSchema), + disclosure: z.lazy(() => DisclosureClassSchema), + grant_digest: z.lazy(() => ManifestDigestSchema), + grant_id: z.string(), + grant_revision: z.number().int().min(0), + policy: z.lazy(() => PolicyDecisionRefSchema), + revalidated_at: z.lazy(() => UtcMicrosSchema), +}).strict(); + /** One exact application scope paired with its registered physical locator. The scope remains the identity authority. The locator is retained only so @@ -2201,6 +2268,9 @@ export const BranchTopologyPolicyV1Schema: z.ZodObject<{ allowed: z.array(z.lazy(() => BranchTopologyKindV1Schema)), }).strict(); +/** Bounded elapsed-work classification supplied by a retriever. */ +export const BudgetClassSchema: z.ZodEnum<{ "approaching_limit": "approaching_limit"; "exhausted": "exhausted"; "within_budget": "within_budget" }> = z.enum(["approaching_limit", "exhausted", "within_budget"]); + export const CallChainMeasurementV1Schema: z.ZodObject<{ complete: z.ZodType; directed: z.ZodType; @@ -2233,6 +2303,14 @@ export const CallChainStepV1Schema: z.ZodObject<{ node: z.lazy(() => NodeRefV1Schema), }); +export const CancellationObservationSchema: z.ZodObject<{ + observed_at: z.ZodType; + stage: z.ZodType; +}, z.core.$strict> = z.object({ + observed_at: z.lazy(() => UtcMicrosSchema), + stage: z.lazy(() => CancellationStageSchema), +}).strict(); + /** Exact stage at which cancellation or deadline state was observed. */ export const CancellationStageSchema: z.ZodEnum<{ "after_commit": "after_commit"; "before_admission": "before_admission"; "before_effect": "before_effect"; "before_read": "before_read"; "during_read": "during_read"; "effect_in_flight": "effect_in_flight"; "reconciling": "reconciling" }> = z.enum(["after_commit", "before_admission", "before_effect", "before_read", "during_read", "effect_in_flight", "reconciling"]); @@ -2358,6 +2436,15 @@ export const CodeCloneIndexStatusV1Schema: z.ZodType = z.string(); +/** Freshness of the generation a verified graph read serves. */ +export const CodeGraphReadFreshnessV1Schema: z.ZodType = z.discriminatedUnion("state", [z.object({ + state: z.literal("current"), +}), z.object({ + rebuild_in_flight: z.boolean(), + sealed_at: z.lazy(() => UtcMicrosSchema), + state: z.literal("last_complete_stale"), +})]); + /** Interactive graph-serving state for the latest sealed generation. A sealed generation can expose truthful census statistics before its graph @@ -2677,6 +2764,20 @@ export const ComplexityAnalysisV1Schema: z.ZodUnion<[z.ZodLiteral<"complete">, z /** Strongly typed canonical identity: `ComponentVersion`. */ export const ComponentVersionSchema: z.ZodType = z.string(); +/** Configuration refusals carrying the same stable codes as the HTTP routes. */ +export const ConfigurationRefusalV1Schema: z.ZodType = z.discriminatedUnion("code", [z.object({ + code: z.literal("application_contract_violation"), + detail: z.string(), +}).strict(), z.object({ + code: z.literal("configuration_authority_unavailable"), + detail: z.string(), +}).strict(), z.object({ + actual_revision_id: z.string(), + code: z.literal("configuration_revision_conflict"), + detail: z.string(), + expected_revision_id: z.string(), +}).strict()]); + /** Strongly typed canonical identity: `ConfigurationRevisionId`. */ export const ConfigurationRevisionIdSchema: z.ZodType = z.string(); @@ -2830,6 +2931,17 @@ export const DashboardAuthorizationV1Schema: z.ZodType; +} | ConfigurationRefusalV1 | ApplicationProblemEnvelope, unknown> = z.union([z.object({ + detail: z.string(), + validation_errors: z.array(z.lazy(() => SettingsValidationIssueV1Schema)), +}).strict(), z.lazy(() => ConfigurationRefusalV1Schema), z.lazy(() => ApplicationProblemEnvelopeSchema)]); + /** Coverage completeness axis. `Unsupported` distinguishes "the source that would establish coverage is not wired" from `Unknown` ("coverage could not be determined"). */ @@ -2968,6 +3080,13 @@ export const DashboardWatermarkV1Schema: z.ZodObject<{ watermark: z.string(), }); +/** One immutable deadline supplied by the caller or upstream admission layer. */ +export const DeadlineSchema: z.ZodObject<{ + expires_at: z.ZodType; +}, z.core.$strict> = z.object({ + expires_at: z.lazy(() => UtcMicrosSchema), +}).strict(); + export const DecideWorkProposalRequestV1Schema: z.ZodObject<{ disposition: z.ZodType; mutation: z.ZodType; @@ -3969,6 +4088,9 @@ export const DiagnosticsSearchedTsconfigV1Schema: z.ZodObject<{ present: z.boolean(), }).strict(); +/** Disclosure ceiling carried by an immutable grant and revalidated at sinks. */ +export const DisclosureClassSchema: z.ZodEnum<{ "evidence": "evidence"; "metadata": "metadata"; "sensitive": "sensitive" }> = z.enum(["evidence", "metadata", "sensitive"]); + /** Whether Doctor observed all of a family's evidence sources. */ export const DoctorCoverageCompletenessV1Schema: z.ZodUnion<[z.ZodLiteral<"complete">, z.ZodLiteral<"partial">, z.ZodLiteral<"unknown">]> = z.union([z.literal("complete"), z.literal("partial"), z.literal("unknown")]); @@ -4153,6 +4275,30 @@ export const EffectReceiptSchema: z.ZodObject<{ scope: z.lazy(() => ResolvedScopeSchema), }).strict(); +/** Result of an admitted effect. `EffectUnknown` remains a receipt state and +cannot be remapped into a pre-admission problem. */ +export const EffectResultSchema: z.ZodObject<{ + authority: z.ZodType; + effect_class: z.ZodType; + effect_id: z.ZodType; + execution: z.ZodType; + expected_state: z.ZodType; + idempotency_key: z.ZodType; + payload: z.ZodType; + receipt: z.ZodType; + reconciliation: z.ZodType; +}, z.core.$strict> = z.object({ + authority: z.lazy(() => AuthorityReceiptSchema), + effect_class: z.lazy(() => EffectClassSchema), + effect_id: z.string(), + execution: z.lazy(() => OperationReceiptSchema), + expected_state: z.lazy(() => ManifestDigestSchema), + idempotency_key: z.string(), + payload: z.unknown(), + receipt: z.lazy(() => EffectReceiptSchema), + reconciliation: z.lazy(() => ReconciliationStateSchema), +}).strict(); + /** Durable effect receipt terminal state. */ export const EffectTerminationSchema: z.ZodEnum<{ "cancelled": "cancelled"; "completed": "completed"; "effect_unknown": "effect_unknown"; "failed": "failed"; "partial": "partial"; "timed_out": "timed_out" }> = z.enum(["cancelled", "completed", "effect_unknown", "failed", "partial", "timed_out"]); @@ -4180,6 +4326,23 @@ export const EnvironmentVariableV1Schema: z.ZodObject<{ value: z.string().nullable(), }); +/** Evidence authority is separate from caller authority and cannot grant access. */ +export const EvidenceAuthoritySchema: z.ZodObject<{ + evidence_id: z.ZodType; + horizon: z.ZodType; + producer: z.ZodType; + revision: z.ZodType; + scope: z.ZodType; + source_kind: z.ZodType; +}, z.core.$strict> = z.object({ + evidence_id: z.string(), + horizon: z.union([z.lazy(() => UtcMicrosSchema), z.null()]), + producer: z.string(), + revision: z.lazy(() => ComponentVersionSchema), + scope: z.lazy(() => ResolvedScopeSchema), + source_kind: z.string(), +}).strict(); + /** Deterministic coverage fold input for an evidence packet. */ export const EvidenceCoverageSchema: z.ZodObject<{ completeness: z.ZodType; @@ -4200,6 +4363,62 @@ export const EvidenceCoverageSchema: z.ZodObject<{ /** Requested evidence domain for bounded coverage and omissions. */ export const EvidenceDomainSchema: z.ZodEnum<{ "anchor": "anchor"; "diagnostic": "diagnostic"; "graph": "graph"; "operational": "operational"; "source": "source"; "symbol": "symbol"; "temporal": "temporal"; "test": "test" }> = z.enum(["anchor", "diagnostic", "graph", "operational", "source", "symbol", "temporal", "test"]); +/** Immutable evidence packet consumed by adapters and later planner work. */ +export const EvidencePacketSchema: z.ZodObject<{ + authority: z.ZodType; + contributions: z.ZodType, unknown>; + coverage: z.ZodType; + evidence_authorities: z.ZodType, unknown>; + execution: z.ZodType; + omissions: z.ZodType, unknown>; + page: z.ZodType; + payload: z.ZodType; + scores: z.ZodType, unknown>; + temporal: z.ZodType; +}, z.core.$strict> = z.object({ + authority: z.lazy(() => AuthorityReceiptSchema), + contributions: z.array(z.lazy(() => RetrieverContributionSchema)), + coverage: z.lazy(() => EvidenceCoverageSchema), + evidence_authorities: z.array(z.lazy(() => EvidenceAuthoritySchema)), + execution: z.lazy(() => OperationReceiptSchema), + omissions: z.array(z.lazy(() => OmissionSchema)), + page: z.lazy(() => PageStateSchema), + payload: z.unknown(), + scores: z.array(z.lazy(() => EvidenceScoreSchema)), + temporal: z.lazy(() => TemporalStateSchema), +}).strict(); + +export const EvidenceScoreSchema: z.ZodObject<{ + calibration_revision: z.ZodType; + calibration_valid: z.ZodType; + deterministic_components: z.ZodType, unknown>; + kind: z.ZodType; + score_id: z.ZodType; + value: z.ZodType; +}, z.core.$strict> = z.object({ + calibration_revision: z.union([z.lazy(() => ComponentVersionSchema), z.null()]), + calibration_valid: z.boolean().nullable(), + deterministic_components: z.array(z.string()), + kind: z.lazy(() => EvidenceScoreKindSchema), + score_id: z.string(), + value: z.lazy(() => EvidenceScoreValueSchema), +}).strict(); + +/** Evidence score semantics. Scores are metadata, never authority. */ +export const EvidenceScoreKindSchema: z.ZodEnum<{ "calibrated_interval": "calibrated_interval"; "calibrated_probability": "calibrated_probability"; "heuristic_score": "heuristic_score"; "ordinal_rank": "ordinal_rank" }> = z.enum(["calibrated_interval", "calibrated_probability", "heuristic_score", "ordinal_rank"]); + +export const EvidenceScoreValueSchema: z.ZodType = z.discriminatedUnion("kind", [z.object({ + kind: z.literal("fixed_point"), + micros: z.number().int().min(0), +}), z.object({ + kind: z.literal("interval"), + lower_micros: z.number().int().min(0), + upper_micros: z.number().int().min(0), +}), z.object({ + kind: z.literal("ordinal"), + rank: z.number().int().min(0), +})]); + /** Cause a work item was blocked for. */ export const ExecutionBlockedCauseV1Schema: z.ZodEnum<{ "backpressure": "backpressure"; "capability": "capability"; "ci": "ci"; "conflict": "conflict"; "dependency": "dependency"; "effect_unknown": "effect_unknown"; "lease": "lease"; "needs_input": "needs_input"; "other": "other"; "policy": "policy"; "review": "review"; "scope": "scope"; "test": "test"; "unknown": "unknown" }> = z.enum(["backpressure", "capability", "ci", "conflict", "dependency", "effect_unknown", "lease", "needs_input", "other", "policy", "review", "scope", "test", "unknown"]); @@ -5053,6 +5272,9 @@ export const FeedbackSystemQualityReadModelV1Schema: z.ZodObject<{ /** Strongly typed canonical identity: `FileOccurrenceId`. */ export const FileOccurrenceIdSchema: z.ZodType = z.string(); +/** Application-level freshness. Missing or partial truth never becomes current. */ +export const FreshnessStateSchema: z.ZodEnum<{ "current": "current"; "stale": "stale"; "unknown": "unknown" }> = z.enum(["current", "stale", "unknown"]); + /** A canonical product proposal and the exact verified graph that licensed it. `proposal` can be moved directly into a `DecideWorkProposalRequestV1`; @@ -7273,6 +7495,76 @@ export const ObservatoryReadModelV1Schema: z.ZodObject<{ watermark: z.string(), }); +export const OmissionSchema: z.ZodObject<{ + count: z.ZodType; + domain: z.ZodType; + reason: z.ZodType; +}, z.core.$strict> = z.object({ + count: z.number().int().min(0), + domain: z.lazy(() => EvidenceDomainSchema), + reason: z.lazy(() => OmissionReasonSchema), +}).strict(); + +/** Safe reason why authorized requested evidence was omitted. */ +export const OmissionReasonSchema: z.ZodType<"budget" | "cancelled" | "conflict" | "failed" | "redacted" | "stale" | "timed_out" | "unavailable" | "unsupported" | "macro_body_unparsed" | "import_unmodeled" | "cursor_expired" | "cursor_foreign", unknown> = z.union([z.enum(["budget", "cancelled", "conflict", "failed", "redacted", "stale", "timed_out", "unavailable", "unsupported"]), z.literal("macro_body_unparsed"), z.literal("import_unmodeled"), z.literal("cursor_expired"), z.literal("cursor_foreign")]); + +/** Bounded work accounting supplied by an owning port or transaction. */ +export const OperationBudgetUsageSchema: z.ZodObject<{ + bytes_consumed: z.ZodType; + elapsed_micros: z.ZodType; + units_consumed: z.ZodType; +}, z.core.$strict> = z.object({ + bytes_consumed: z.number().int().min(0), + elapsed_micros: z.number().int().min(0), + units_consumed: z.number().int().min(0), +}).strict(); + +/** Canonical operation evidence. An admitted failure remains represented here +rather than being replaced by a transport exception. */ +export const OperationReceiptSchema: z.ZodObject<{ + budget: z.ZodType; + cancellation: z.ZodType; + effective_deadline: z.ZodType; + ended_at: z.ZodType; + started_at: z.ZodType; + termination: z.ZodType; +}, z.core.$strict> = z.object({ + budget: z.lazy(() => OperationBudgetUsageSchema), + cancellation: z.union([z.lazy(() => CancellationObservationSchema), z.null()]), + effective_deadline: z.lazy(() => DeadlineSchema), + ended_at: z.lazy(() => UtcMicrosSchema), + started_at: z.lazy(() => UtcMicrosSchema), + termination: z.lazy(() => OperationTerminationSchema), +}).strict(); + +/** Terminal state after an operation has been admitted. */ +export const OperationTerminationSchema: z.ZodEnum<{ "cancelled": "cancelled"; "completed": "completed"; "effect_unknown": "effect_unknown"; "failed": "failed"; "partial": "partial"; "timed_out": "timed_out"; "unavailable": "unavailable" }> = z.enum(["cancelled", "completed", "effect_unknown", "failed", "partial", "timed_out", "unavailable"]); + +/** Exact continuation authority for the enclosing evidence page: an opaque +cursor bound to the operation and request that minted it. */ +export const PageCursorSchema: z.ZodType = z.discriminatedUnion("kind", [z.object({ + cursor: z.string(), + kind: z.literal("opaque"), +}).strict()]); + +/** Stable page state. General cursor bytes stay opaque until authorization is +revalidated; structural fact cursors retain their canonical ordering type. */ +export const PageStateSchema: z.ZodObject<{ + cursor: z.ZodType; + expires_at: z.ZodType; + returned: z.ZodType; + sort_contract_id: z.ZodType; + sort_revision: z.ZodType; + total: z.ZodType; +}, z.core.$strict> = z.object({ + cursor: z.union([z.lazy(() => PageCursorSchema), z.null()]), + expires_at: z.union([z.lazy(() => UtcMicrosSchema), z.null()]), + returned: z.number().int().min(0), + sort_contract_id: z.lazy(() => SortContractIdSchema), + sort_revision: z.number().refine(Number.isInteger).min(0), + total: z.number().int().min(0).nullable(), +}).strict(); + export const PauseWorkRunCommandSchema: z.ZodObject<{ expected_authority_version: z.ZodType; occurred_at: z.ZodType; @@ -7334,6 +7626,19 @@ export const PerformanceComparisonReadModelV1Schema: z.ZodObject<{ workload: z.string().nullable(), }); +/** A policy decision pinned into a receipt or provider identity. */ +export const PolicyDecisionRefSchema: z.ZodObject<{ + decision_id: z.ZodType; + digest: z.ZodType; + evaluator_revision: z.ZodType; + revision: z.ZodType; +}, z.core.$strict> = z.object({ + decision_id: z.string(), + digest: z.lazy(() => ManifestDigestSchema), + evaluator_revision: z.lazy(() => ComponentVersionSchema), + revision: z.number().int().min(0), +}).strict(); + /** A bounded, canonical identifier owned by the policy input schema. It represents immutable references only; it is never a path, display @@ -7386,6 +7691,25 @@ export const PrepareWorkProductMutationRequestV1Schema: z.ZodObject<{ selection: z.lazy(() => WorkProductSelectionScopeV1Schema), }).strict(); +/** Read-only preview of a future typed effect. */ +export const PreviewResultSchema: z.ZodObject<{ + authority: z.ZodType; + effect_class: z.ZodType; + execution: z.ZodType; + expected_state: z.ZodType; + payload: z.ZodType; + preview_digest: z.ZodType; + preview_id: z.ZodType; +}, z.core.$strict> = z.object({ + authority: z.lazy(() => AuthorityReceiptSchema), + effect_class: z.lazy(() => EffectClassSchema), + execution: z.lazy(() => OperationReceiptSchema), + expected_state: z.lazy(() => ManifestDigestSchema), + payload: z.unknown(), + preview_digest: z.lazy(() => ManifestDigestSchema), + preview_id: z.string(), +}).strict(); + /** Whether the served code generation is known current at serve time. */ export const PrimitiveFreshnessStateV1Schema: z.ZodEnum<{ "fresh": "fresh"; "possibly_stale": "possibly_stale" }> = z.enum(["fresh", "possibly_stale"]); @@ -7591,6 +7915,14 @@ export const ProjectSettingsPatchSchema: z.ZodObject<{ track_call_sites: z.boolean().nullable().optional(), }).strict(); +export const ProjectSettingsPatchResponseV1Schema: z.ZodObject<{ + application_outcome: z.ZodType; + current: z.ZodType, unknown>; +}> = z.object({ + application_outcome: z.union([z.lazy(() => ApplicationOutcomeSchema), z.null()]), + current: z.lazy(() => DashboardEnvelopeV1Schema(z.lazy(() => SettingsPayloadV1Schema))), +}); + export const ProjectSettingsPayloadV1Schema: z.ZodObject<{ config: z.ZodType; configuration_revision_id: z.ZodType; @@ -7761,6 +8093,9 @@ export const PublicCodeProjectSchema: z.ZodObject<{ export const QuantityEvidenceClassV1Schema: z.ZodEnum<{ "estimated": "estimated"; "locally_measured": "locally_measured"; "owner_receipt": "owner_receipt"; "unknown": "unknown" }> = z.enum(["estimated", "locally_measured", "owner_receipt", "unknown"]); +/** Reconciliation state retained after an admitted effect. */ +export const ReconciliationStateSchema: z.ZodEnum<{ "failed": "failed"; "pending": "pending"; "reconciled": "reconciled" }> = z.enum(["failed", "pending", "reconciled"]); + export const RecordWorkHandoffRequestV1Schema: z.ZodObject<{ handoff: z.ZodType; mutation: z.ZodType; @@ -8041,6 +8376,41 @@ export const ResumeWorkRunCommandSchema: z.ZodObject<{ /** Strongly typed canonical identity: `RetrievalAnchorId`. */ export const RetrievalAnchorIdSchema: z.ZodType = z.string(); +/** One source-owned contribution to the packet fold. */ +export const RetrieverContributionSchema: z.ZodObject<{ + contract: z.ZodType; + coverage: z.ZodType; + domain: z.ZodType; + elapsed_budget_class: z.ZodType; + evidence_authorities: z.ZodType, unknown>; + omitted_count: z.ZodType; + producer_revision: z.ZodType; + provenance_anchors: z.ZodType, unknown>; + retriever_id: z.ZodType; + returned_count: z.ZodType; + score_ids: z.ZodType, unknown>; + state: z.ZodType; +}, z.core.$strict> = z.object({ + contract: z.lazy(() => ResultContractRefSchema), + coverage: z.lazy(() => EvidenceCoverageSchema), + domain: z.lazy(() => EvidenceDomainSchema), + elapsed_budget_class: z.lazy(() => BudgetClassSchema), + evidence_authorities: z.array(z.string()), + omitted_count: z.number().int().min(0), + producer_revision: z.lazy(() => ComponentVersionSchema), + provenance_anchors: z.array(z.lazy(() => RetrievalAnchorIdSchema)), + retriever_id: z.lazy(() => RetrieverIdSchema), + returned_count: z.number().int().min(0), + score_ids: z.array(z.string()), + state: z.lazy(() => RetrieverContributionStateSchema), +}).strict(); + +/** Terminal state reported by one retriever contribution. */ +export const RetrieverContributionStateSchema: z.ZodEnum<{ "cancelled": "cancelled"; "completed": "completed"; "failed": "failed"; "partial": "partial"; "stale": "stale"; "timed_out": "timed_out"; "unavailable": "unavailable"; "unsupported": "unsupported" }> = z.enum(["cancelled", "completed", "failed", "partial", "stale", "timed_out", "unavailable", "unsupported"]); + +/** Stable, canonical catalog identity for `RetrieverId`. */ +export const RetrieverIdSchema: z.ZodType = z.string(); + /** Safe adapter-independent retry instruction. Adapters preserve it verbatim. */ export const RetryDirectiveSchema: z.ZodEnum<{ "after_delay": "after_delay"; "after_reconcile": "after_reconcile"; "after_revalidate": "after_revalidate"; "never": "never"; "same_request": "same_request" }> = z.enum(["after_delay", "after_reconcile", "after_revalidate", "never", "same_request"]); @@ -8761,6 +9131,15 @@ export const SettingsPayloadV1Schema: z.ZodObject<{ version: z.lazy(() => VersionSettingsPayloadV1Schema), }); +/** One field-level settings validation failure. */ +export const SettingsValidationIssueV1Schema: z.ZodObject<{ + field: z.ZodType; + message: z.ZodType; +}> = z.object({ + field: z.string(), + message: z.string(), +}); + /** Logical and verified physical identity of the shared Profile shard. Brain and profile IDs select the shard; the verified store locator binds its @@ -8890,6 +9269,9 @@ exists, so it is never a failure. Snapshots written before that distinction stored `unknown` as `ignored`. */ export const SkillOutcomeVerdictSchema: z.ZodEnum<{ "adopted": "adopted"; "too_early": "too_early"; "unknown": "unknown" }> = z.enum(["adopted", "too_early", "unknown"]); +/** Stable, canonical catalog identity for `SortContractId`. */ +export const SortContractIdSchema: z.ZodType = z.string(); + /** Byte range inside one sanitized source file. Mutable line numbers are never part of identity. */ export const SourceSpanSchema: z.ZodObject<{ @@ -9374,6 +9756,25 @@ export const TemporalModeV1Schema: z.ZodType = z.discri kind: z.literal("forensic"), }).strict()]); +/** Temporal provenance for a packet. */ +export const TemporalStateSchema: z.ZodObject<{ + code_graph_freshness: z.ZodOptional>; + freshness: z.ZodType; + requested_at: z.ZodType; + requested_mode: z.ZodType; + resolved_at: z.ZodType; + source_generation: z.ZodType; + watermark_digest: z.ZodType; +}, z.core.$strict> = z.object({ + code_graph_freshness: z.union([z.lazy(() => CodeGraphReadFreshnessV1Schema), z.null()]).optional(), + freshness: z.lazy(() => FreshnessStateSchema), + requested_at: z.lazy(() => UtcMicrosSchema), + requested_mode: z.lazy(() => TemporalModeV1Schema), + resolved_at: z.lazy(() => UtcMicrosSchema), + source_generation: z.union([z.lazy(() => CodeGenerationIdSchema), z.null()]), + watermark_digest: z.union([z.lazy(() => ManifestDigestSchema), z.null()]), +}).strict(); + export const TestMapMeasurementV1Schema: z.ZodObject<{ algorithm: z.ZodType; applicable: z.ZodType; diff --git a/dashboard/src/contracts/types.ts b/dashboard/src/contracts/types.ts index 7eceeb6733..5f38d42963 100644 --- a/dashboard/src/contracts/types.ts +++ b/dashboard/src/contracts/types.ts @@ -320,7 +320,25 @@ export type AnalyticsUsageSummaryV1 = { export type ApplicationExecutionFailureClassV1 = "denied" | "malformed_output" | "permanent"; -export type ApplicationProblemDetailV1 = +export type ApplicationOutcome = + | { + outcome: "effect"; + value: EffectResult; + } + | { + outcome: "evidence"; + value: EvidencePacket; + } + | { + outcome: "preview"; + value: PreviewResult; + } + | { + outcome: "result"; + value: unknown; + }; + +export type ApplicationProblemDetailV1 = | { kind: "code_graph_rewarming"; retry_after_millis: number; @@ -420,6 +438,16 @@ export type ApplyWorkRelationReplanRequestV1 = { export type AttemptId = string; +export type AuthorityReceipt = { + authorized_scope_digest: ManifestDigest; + disclosure: DisclosureClass; + grant_digest: ManifestDigest; + grant_id: string; + grant_revision: number; + policy: PolicyDecisionRef; + revalidated_at: UtcMicros; +}; + export type AuthorizedRoot = { locator: RegisteredRootLocatorV1 | null; scope: ResolvedScope; @@ -461,7 +489,7 @@ export type AutomaticFactReceiptsPayloadV1 = { export type AutomaticFactState = "applied" | "quarantined"; -export type AutomaticWorktreeGcV1 = +export type AutomaticWorktreeGcV1 = | { kind: "disabled"; } @@ -471,7 +499,7 @@ export type AutomaticWorktreeGcV1 = minimum_idle_seconds: number; }; -export type AutomationCommittedReceiptV1 = +export type AutomationCommittedReceiptV1 = | { kind: "automatic_fact"; receipt: MemoryAutomationFactReceiptV1; @@ -661,7 +689,7 @@ export type AutomationRunSummaryV1 = { skipped_count: number; }; -export type AutomationRunTerminalV1 = +export type AutomationRunTerminalV1 = | { status: "completed"; summary: AutomationRunSummaryV1; @@ -715,7 +743,7 @@ export type AutomationTrigger = "application" | "dashboard" | "host_receipt" | " export type BrainId = string; -export type BranchCollisionPolicyV1 = +export type BranchCollisionPolicyV1 = | { kind: "append_monotonic_ordinal"; maximum_attempts: number; @@ -724,7 +752,7 @@ export type BranchCollisionPolicyV1 = kind: "reject"; }; -export type BranchNameComponentV1 = +export type BranchNameComponentV1 = | { kind: "monotonic_collision_ordinal"; } @@ -755,6 +783,8 @@ export type BranchTopologyPolicyV1 = { allowed: Array; }; +export type BudgetClass = "approaching_limit" | "exhausted" | "within_budget"; + export type CallChainMeasurementV1 = { complete: boolean; directed: boolean; @@ -773,6 +803,11 @@ export type CallChainStepV1 = { node: NodeRefV1; }; +export type CancellationObservation = { + observed_at: UtcMicros; + stage: CancellationStage; +}; + export type CancellationStage = "after_commit" | "before_admission" | "before_effect" | "before_read" | "during_read" | "effect_in_flight" | "reconciling"; export type CancelWorkAttemptCommand = { @@ -828,7 +863,7 @@ export type CodeCloneIndexResourcesV1 = { stale_invalidations: number | null; }; -export type CodeCloneIndexStatusV1 = +export type CodeCloneIndexStatusV1 = | { observation: CodeCloneIndexObservationV1; omission_reasons: Array; @@ -850,7 +885,17 @@ export type CodeCloneIndexStatusV1 = export type CodeGenerationId = string; -export type CodeGraphServingReadinessV1 = +export type CodeGraphReadFreshnessV1 = + | { + state: "current"; + } + | { + rebuild_in_flight: boolean; + sealed_at: UtcMicros; + state: "last_complete_stale"; + }; + +export type CodeGraphServingReadinessV1 = | { state: "pending"; } @@ -955,7 +1000,7 @@ export type CodeIndexRouteRefusalV1 = { remediation: string; }; -export type CodeIndexSourceOmissionReasonV1 = +export type CodeIndexSourceOmissionReasonV1 = | { detail: string; kind: "privacy_withheld"; @@ -968,7 +1013,7 @@ export type CodeIndexStalenessStateV1 = "fresh" | "indexing" | "parked" | "refre export type CodeIndexWorkerLimitingReasonV1 = "automatic_all_cores" | "automatic_half_cores" | "configured_exact" | "environment_override" | "resident_memory"; -export type CodeIndexWorkerSelectionV1 = +export type CodeIndexWorkerSelectionV1 = | { mode: "automatic"; } @@ -1023,6 +1068,22 @@ export type ComplexityAnalysisV1 = "complete" | "traversal_budget_exhausted"; export type ComponentVersion = string; +export type ConfigurationRefusalV1 = + | { + code: "application_contract_violation"; + detail: string; + } + | { + code: "configuration_authority_unavailable"; + detail: string; + } + | { + actual_revision_id: string; + code: "configuration_revision_conflict"; + detail: string; + expected_revision_id: string; + }; + export type ConfigurationRevisionId = string; export type ConfigurationSnapshotId = string; @@ -1096,7 +1157,7 @@ export type CrossMergePolicyV1 = { default_mode: CrossMergeModeV1; }; -export type DaemonServiceUnitObservationV1 = +export type DaemonServiceUnitObservationV1 = | { path: string; serves: string; @@ -1110,7 +1171,7 @@ export type DaemonServiceUnitObservationV1 = state: "unobservable"; }; -export type DashboardAuthorizationV1 = +export type DashboardAuthorizationV1 = | { outcome: "authorized"; } @@ -1124,6 +1185,11 @@ export type DashboardAuthorizationV1 = outcome: "unauthorized"; }; +export type DashboardConfigurationRefusalV1 = { + detail: string; + validation_errors: Array; +} | ConfigurationRefusalV1 | ApplicationProblemEnvelope; + export type DashboardCoverageCompletenessV1 = "complete" | "partial" | "unknown" | "unsupported"; export type DashboardCoverageV1 = { @@ -1193,6 +1259,10 @@ export type DashboardWatermarkV1 = { watermark: string; }; +export type Deadline = { + expires_at: UtcMicros; +}; + export type DecideWorkProposalRequestV1 = { disposition: WorkProposalDispositionV12; mutation: WorkProductMutationIdentityV1; @@ -1233,7 +1303,7 @@ export type DeliveryAgentUsageV1 = { usage_coverage: ProviderUsageCoverageV1; }; -export type DeliveryAttentionEvidenceV1 = +export type DeliveryAttentionEvidenceV1 = | { failure_anchor: string; kind: "ci_failure"; @@ -1358,7 +1428,7 @@ export type DeliveryGenerationFreshnessV1 = { indexed_commit: string; }; -export type DeliveryGitHeadV1 = +export type DeliveryGitHeadV1 = | { branch: string; commit: string; @@ -1449,7 +1519,7 @@ export type DeliveryInboxV1 = { registry_state: DeliveryRegistryStateV1; }; -export type DeliveryMembershipBasisV1 = +export type DeliveryMembershipBasisV1 = | { branch_ref: string; head_commit_id: string; @@ -1492,7 +1562,7 @@ export type DeliveryOverviewV1 = { review_comments: DeliveryProjectionV14; }; -export type DeliveryProjectionV1 = +export type DeliveryProjectionV1 = | { state: "denied"; value: DeliveryGitStatusV1 | null; @@ -1535,7 +1605,7 @@ export type DeliveryProjectionV1 = value: DeliveryGitStatusV1 | null; }; -export type DeliveryProjectionV12 = +export type DeliveryProjectionV12 = | { state: "denied"; value: DeliveryCommitTimelineV1 | null; @@ -1578,7 +1648,7 @@ export type DeliveryProjectionV12 = value: DeliveryCommitTimelineV1 | null; }; -export type DeliveryProjectionV13 = +export type DeliveryProjectionV13 = | { state: "denied"; value: DeliveryPullRequestTimelineV1 | null; @@ -1621,7 +1691,7 @@ export type DeliveryProjectionV13 = value: DeliveryPullRequestTimelineV1 | null; }; -export type DeliveryProjectionV14 = +export type DeliveryProjectionV14 = | { state: "denied"; value: DeliveryReviewTimelineV1 | null; @@ -1664,7 +1734,7 @@ export type DeliveryProjectionV14 = value: DeliveryReviewTimelineV1 | null; }; -export type DeliveryProjectionV15 = +export type DeliveryProjectionV15 = | { state: "denied"; value: DeliveryCiTimelineV1 | null; @@ -1707,7 +1777,7 @@ export type DeliveryProjectionV15 = value: DeliveryCiTimelineV1 | null; }; -export type DeliveryProjectionV16 = +export type DeliveryProjectionV16 = | { state: "denied"; value: DeliveryFailureLocalizationTimelineV1 | null; @@ -1750,7 +1820,7 @@ export type DeliveryProjectionV16 = value: DeliveryFailureLocalizationTimelineV1 | null; }; -export type DeliveryProjectionV17 = +export type DeliveryProjectionV17 = | { state: "denied"; value: DeliveryReleaseTimelineV1 | null; @@ -1793,7 +1863,7 @@ export type DeliveryProjectionV17 = value: DeliveryReleaseTimelineV1 | null; }; -export type DeliveryProjectionV18 = +export type DeliveryProjectionV18 = | { state: "denied"; value: DeliveryGenerationFreshnessV1 | null; @@ -1836,7 +1906,7 @@ export type DeliveryProjectionV18 = value: DeliveryGenerationFreshnessV1 | null; }; -export type DeliveryProjectionV19 = +export type DeliveryProjectionV19 = | { state: "denied"; value: DeliveryAgentUsageV1 | null; @@ -2024,6 +2094,8 @@ export type DiagnosticsSearchedTsconfigV1 = { present: boolean; }; +export type DisclosureClass = "evidence" | "metadata" | "sensitive"; + export type DoctorCoverageCompletenessV1 = "complete" | "partial" | "unknown"; export type DoctorCoverageStatementV1 = { @@ -2040,7 +2112,7 @@ export type DoctorEvidenceRefV1 = { export type DoctorEvidenceStateV1 = "unsupported" | "absent" | "stale" | "degraded" | "partial" | "unknown" | "denied" | "healthy_complete_coverage"; -export type DoctorFamilyConsultationV1 = +export type DoctorFamilyConsultationV1 = | { status: "consulted"; } @@ -2112,6 +2184,18 @@ export type EffectReceipt = { scope: ResolvedScope; }; +export type EffectResult = { + authority: AuthorityReceipt; + effect_class: EffectClass; + effect_id: string; + execution: OperationReceipt; + expected_state: ManifestDigest; + idempotency_key: string; + payload: unknown; + receipt: EffectReceipt; + reconciliation: ReconciliationState; +}; + export type EffectTermination = "cancelled" | "completed" | "effect_unknown" | "failed" | "partial" | "timed_out"; export type EnvironmentSettingsPayloadV1 = { @@ -2128,6 +2212,15 @@ export type EnvironmentVariableV1 = { value: string | null; }; +export type EvidenceAuthority = { + evidence_id: string; + horizon: UtcMicros | null; + producer: string; + revision: ComponentVersion; + scope: ResolvedScope; + source_kind: string; +}; + export type EvidenceCoverage = { completeness: CoverageCompleteness; domains: Array; @@ -2139,6 +2232,45 @@ export type EvidenceCoverage = { export type EvidenceDomain = "anchor" | "diagnostic" | "graph" | "operational" | "source" | "symbol" | "temporal" | "test"; +export type EvidencePacket = { + authority: AuthorityReceipt; + contributions: Array; + coverage: EvidenceCoverage; + evidence_authorities: Array; + execution: OperationReceipt; + omissions: Array; + page: PageState; + payload: unknown; + scores: Array; + temporal: TemporalState; +}; + +export type EvidenceScore = { + calibration_revision: ComponentVersion | null; + calibration_valid: boolean | null; + deterministic_components: Array; + kind: EvidenceScoreKind; + score_id: string; + value: EvidenceScoreValue; +}; + +export type EvidenceScoreKind = "calibrated_interval" | "calibrated_probability" | "heuristic_score" | "ordinal_rank"; + +export type EvidenceScoreValue = + | { + kind: "fixed_point"; + micros: number; + } + | { + kind: "interval"; + lower_micros: number; + upper_micros: number; + } + | { + kind: "ordinal"; + rank: number; + }; + export type ExecutionBlockedCauseV1 = "backpressure" | "capability" | "ci" | "conflict" | "dependency" | "effect_unknown" | "lease" | "needs_input" | "other" | "policy" | "review" | "scope" | "test" | "unknown"; export type ExecutionConcurrencyPhaseV1 = "accepted" | "active" | "admitted" | "requested" | "useful"; @@ -2189,7 +2321,7 @@ export type ExecutionStackDriftKindV1 = "base_advanced" | "head_advanced" | "mer export type ExecutionSurfaceFamilyV1 = "cli" | "dashboard" | "hook" | "lsp" | "mcp" | "other"; -export type ExecutionTopologyDimensionV1 = +export type ExecutionTopologyDimensionV1 = | { dimension: "blocked_cause"; value: ExecutionBlockedCauseV1; @@ -2306,7 +2438,7 @@ export type ExecutionTopologyMetricsV1 = { watermark: string; }; -export type ExecutionTopologyViewV1 = +export type ExecutionTopologyViewV1 = | { state: "absent"; } @@ -2421,7 +2553,7 @@ export type FactCategoryV1 = "code_area" | "decision" | "general" | "project" | export type FactCommitDispositionV1 = "committed" | "idempotent_replay"; -export type FactCommitOwnerV1 = +export type FactCommitOwnerV1 = | { kind: "profile"; } @@ -2482,7 +2614,7 @@ export type FactOutcomeRecord = { export type FactOutcomeVerdict = "deleted" | "never_recalled" | "quarantined" | "recalled" | "recalled_and_helpful" | "unavailable"; -export type FactSearchGraphCoverageV1 = +export type FactSearchGraphCoverageV1 = | { expanded_fact_count: number; kind: "complete"; @@ -2613,7 +2745,7 @@ export type FeedbackProximityReadRequestV1 = { observed_at: UtcMicros; }; -export type FeedbackProximityReadResultV1 = +export type FeedbackProximityReadResultV1 = | { page: FeedbackProximityReadPageV1; state: "complete"; @@ -2641,7 +2773,7 @@ export type FeedbackProximityReadResultV1 = state: "unavailable"; }; -export type FeedbackProximityRelationV1 = +export type FeedbackProximityRelationV1 = | { relation_kind: "code_neighborhood_candidate"; warning_class: ProximityWarningClassV1; @@ -2702,6 +2834,8 @@ export type FeedbackSystemQualityReadModelV1 = { export type FileOccurrenceId = string; +export type FreshnessState = "current" | "stale" | "unknown"; + export type GeneratedWorkProposal = { calibration: WorkCalibrationEvidenceV1; decision: WorkProposalDecisionV1; @@ -2855,7 +2989,7 @@ export type GraphTotalsV1 = { export type HandoffOpenKindV1 = "investigation" | "task"; -export type HandoffOpenTargetV1 = +export type HandoffOpenTargetV1 = | { finding_id: FeedbackFindingId; kind: "investigation"; @@ -2892,7 +3026,7 @@ export type IntentWindowSettingsV1 = { max_user_turns: number; }; -export type JobDelivery = +export type JobDelivery = | { mode: "file"; path?: string | null | undefined; @@ -3174,7 +3308,7 @@ export type LoomEditedFileV1 = { session_id: string; }; -export type LoomEventV1 = +export type LoomEventV1 = | { kind: "pull_request"; label?: string | null | undefined; @@ -3346,7 +3480,7 @@ export type MemoryAutomationCurationMergeV1 = { winner_fact_id: FactId; }; -export type MemoryAutomationCurationOperationEffectV1 = +export type MemoryAutomationCurationOperationEffectV1 = | { closest_fact_id: FactId | null; commit: FactCommitReceiptV1 | null; @@ -3439,7 +3573,7 @@ export type MemoryAutomationFactDedupeValidationV1 = { export type MemoryAutomationFactDispositionV1 = "already_applied" | "applied" | "quarantined"; -export type MemoryAutomationFactEffectV1 = +export type MemoryAutomationFactEffectV1 = | { assertion_id: FactAssertionId; event_id: FactEventId; @@ -3602,7 +3736,7 @@ export type MemoryGraphEdgeV1 = { target: string; }; -export type MemoryGraphNodeV1 = +export type MemoryGraphNodeV1 = | { assertion_id: FactAssertionId; fact_id: FactId; @@ -3909,7 +4043,7 @@ export type MetricValueV1 = { value: number | null; }; -export type MultiRootCapabilityV1 = +export type MultiRootCapabilityV1 = | { revision: ScopeSetRevision; root_count: number; @@ -3941,7 +4075,7 @@ export type MultiRootExecuteRequestV1 = { scope_set_revision: ScopeSetRevision; }; -export type MultiRootOperationV1 = +export type MultiRootOperationV1 = | { kind: "feedback"; request: unknown; @@ -4031,6 +4165,46 @@ export type ObservatoryReadModelV1 = { watermark: string; }; +export type Omission = { + count: number; + domain: EvidenceDomain; + reason: OmissionReason; +}; + +export type OmissionReason = "budget" | "cancelled" | "conflict" | "failed" | "redacted" | "stale" | "timed_out" | "unavailable" | "unsupported" | "macro_body_unparsed" | "import_unmodeled" | "cursor_expired" | "cursor_foreign"; + +export type OperationBudgetUsage = { + bytes_consumed: number; + elapsed_micros: number; + units_consumed: number; +}; + +export type OperationReceipt = { + budget: OperationBudgetUsage; + cancellation: CancellationObservation | null; + effective_deadline: Deadline; + ended_at: UtcMicros; + started_at: UtcMicros; + termination: OperationTermination; +}; + +export type OperationTermination = "cancelled" | "completed" | "effect_unknown" | "failed" | "partial" | "timed_out" | "unavailable"; + +export type PageCursor = + | { + cursor: string; + kind: "opaque"; + }; + +export type PageState = { + cursor: PageCursor | null; + expires_at: UtcMicros | null; + returned: number; + sort_contract_id: SortContractId; + sort_revision: number; + total: number | null; +}; + export type PauseWorkRunCommand = { expected_authority_version: number | null; occurred_at: UtcMicros; @@ -4066,6 +4240,13 @@ export type PerformanceComparisonReadModelV1 = { workload: string | null; }; +export type PolicyDecisionRef = { + decision_id: string; + digest: ManifestDigest; + evaluator_revision: ComponentVersion; + revision: number; +}; + export type PolicyIdentifierV1 = string; export type PrAutoTrackEntryV1 = { @@ -4093,6 +4274,16 @@ export type PrepareWorkProductMutationRequestV1 = { selection: WorkProductSelectionScopeV1; }; +export type PreviewResult = { + authority: AuthorityReceipt; + effect_class: EffectClass; + execution: OperationReceipt; + expected_state: ManifestDigest; + payload: unknown; + preview_digest: ManifestDigest; + preview_id: string; +}; + export type PrimitiveFreshnessStateV1 = "fresh" | "possibly_stale"; export type PrimitiveIndexingStateV1 = { @@ -4202,6 +4393,11 @@ export type ProjectSettingsPatch = { track_call_sites?: boolean | null | undefined; }; +export type ProjectSettingsPatchResponseV1 = { + application_outcome: ApplicationOutcome | null; + current: DashboardEnvelopeV1; +}; + export type ProjectSettingsPayloadV1 = { config: ProjectEditableSettingsV1; configuration_revision_id: string; @@ -4230,7 +4426,7 @@ export type ProtectedRefRuleV1 = { selector: ProtectedRefSelectorV1; }; -export type ProtectedRefSelectorV1 = +export type ProtectedRefSelectorV1 = | { kind: "exact"; value: CanonicalGitRefNameV1; @@ -4304,6 +4500,8 @@ export type PublicCodeProject = { export type QuantityEvidenceClassV1 = "estimated" | "locally_measured" | "owner_receipt" | "unknown"; +export type ReconciliationState = "failed" | "pending" | "reconciled"; + export type RecordWorkHandoffRequestV1 = { handoff: WorkHandoffV1; mutation: WorkProductMutationIdentityV1; @@ -4359,7 +4557,7 @@ export type ReleaseWorkPlacementCommand = { export type RemoteAuthorityMissingReasonV1 = "authority_authentication_failed" | "authority_unreachable" | "caller_authentication_failed" | "enrollment_expired" | "enrollment_revoked" | "fence_unverified" | "insufficient_capability" | "placement_unknown" | "protocol_incompatible" | "registry_unavailable" | "scope_mismatch" | "writer_authority_unpublished"; -export type RemoteAuthoritySummaryV1 = +export type RemoteAuthoritySummaryV1 = | { fence: RemoteFenceSummaryV1; state: "available"; @@ -4385,7 +4583,7 @@ export type RemoteFenceSummaryV1 = { export type RemoteListenerKindV1 = "degraded" | "disabled" | "serving"; -export type RemoteOperationalStatusPayloadV1 = +export type RemoteOperationalStatusPayloadV1 = | { authority: RemoteAuthoritySummaryV1; coverage: DoctorCoverageCompletenessV1; @@ -4417,7 +4615,7 @@ export type RemoteSpoolSummaryV1 = { export type RepositoryId = string; -export type RepositoryPlacementScopeV1 = +export type RepositoryPlacementScopeV1 = | { kind: "all_authorized"; } @@ -4480,6 +4678,25 @@ export type ResumeWorkRunCommand = { export type RetrievalAnchorId = string; +export type RetrieverContribution = { + contract: ResultContractRef; + coverage: EvidenceCoverage; + domain: EvidenceDomain; + elapsed_budget_class: BudgetClass; + evidence_authorities: Array; + omitted_count: number; + producer_revision: ComponentVersion; + provenance_anchors: Array; + retriever_id: RetrieverId; + returned_count: number; + score_ids: Array; + state: RetrieverContributionState; +}; + +export type RetrieverContributionState = "cancelled" | "completed" | "failed" | "partial" | "stale" | "timed_out" | "unavailable" | "unsupported"; + +export type RetrieverId = string; + export type RetryDirective = "after_delay" | "after_reconcile" | "after_revalidate" | "never" | "same_request"; export type RetryScope = "fresh_request" | "same_operation" | "same_request"; @@ -4491,7 +4708,7 @@ export type RetryWorkAttemptCommandV1 = { original_attempt: WorkAttemptIdentityV1; }; -export type ReviewRequirementV1 = +export type ReviewRequirementV1 = | { kind: "code_owner_and_independent_review"; } @@ -4777,7 +4994,7 @@ export type SchemaConvergenceFindingV1 = { store: string; }; -export type SchemaConvergenceProgressV1 = +export type SchemaConvergenceProgressV1 = | { done: number; remaining: number; @@ -4795,7 +5012,7 @@ export type SchemaConvergenceStateV1 = "completed" | "degraded" | "pending_schem export type SchemaId = string; -export type ScopeOutcome_for_Array_of_AnyValue = +export type ScopeOutcome_for_Array_of_AnyValue = | { outcome: "denied"; } @@ -4817,7 +5034,7 @@ export type ScopeOutcome_for_Array_of_AnyValue = }; }; -export type ScopeOutcome_for_Nullable_RootGenerationV1 = +export type ScopeOutcome_for_Nullable_RootGenerationV1 = | { outcome: "denied"; } @@ -4839,7 +5056,7 @@ export type ScopeOutcome_for_Nullable_RootGenerationV1 = }; }; -export type ScopeOutcome_for_Nullable_string = +export type ScopeOutcome_for_Nullable_string = | { outcome: "denied"; } @@ -4883,7 +5100,7 @@ export type SessionRetrievalBudgetAccountingV1 = { observed: SessionRetrievalBudgetObservationV1; }; -export type SessionRetrievalBudgetObservationV1 = +export type SessionRetrievalBudgetObservationV1 = | { observation: "consumed_with_more_available"; units: number; @@ -4895,7 +5112,7 @@ export type SessionRetrievalBudgetObservationV1 = export type SessionRetrievalBudgetStageV1 = "candidate_read_exhausted" | "context_bytes" | "context_tokens" | "cursor_manifest_limit" | "estimator_version_mismatch" | "execution_work_exhausted" | "hydration_bytes" | "kernel_result_limit" | "participant_manifest_canonical_bytes" | "participant_manifest_participants" | "record_read_exhausted" | "request_candidate_bytes" | "request_context_bytes" | "request_hydration_bytes" | "request_hydration_limit" | "request_record_bytes" | "request_result_limit"; -export type SessionRetrievalStructuralRefusalV1 = +export type SessionRetrievalStructuralRefusalV1 = | { accounting: SessionRetrievalBudgetAccountingV1 | null; refusal: "budget_exhausted"; @@ -4925,6 +5142,11 @@ export type SettingsPayloadV1 = { version: VersionSettingsPayloadV1; }; +export type SettingsValidationIssueV1 = { + field: string; + message: string; +}; + export type SharedProfileStoreLocatorV1 = { brain_id: BrainId; profile_id: UserProfileId; @@ -4940,7 +5162,7 @@ export type SignificantTableGrowthSampleV1 = { table: string; }; -export type SimilarCoverageV1 = +export type SimilarCoverageV1 = | { status: "complete"; } @@ -5006,6 +5228,8 @@ export type SkillOutcomeRecord = { export type SkillOutcomeVerdict = "adopted" | "too_early" | "unknown"; +export type SortContractId = string; + export type SourceSpan = { end_byte: number; start_byte: number; @@ -5059,7 +5283,7 @@ export type StorageTelemetryPayloadV1 = { table_growth_threshold: TableGrowthThresholdV1; }; -export type StorageTelemetryReadV1 = +export type StorageTelemetryReadV1 = | { kind: "denied"; store: StoreKeyV1; @@ -5083,7 +5307,7 @@ export type StorageTelemetryReadV1 = store: StoreKeyV1; }; -export type StoreBudgetDimensionV1 = +export type StoreBudgetDimensionV1 = | { evaluation: StoreBudgetEvaluationV1; reason: string; @@ -5100,7 +5324,7 @@ export type StoreBudgetDimensionV1 = state: "unset"; }; -export type StoreBudgetEvaluationV1 = +export type StoreBudgetEvaluationV1 = | { observed: StorageByteSizeV1; overage: StorageByteSizeV1; @@ -5113,7 +5337,7 @@ export type StoreBudgetEvaluationV1 = state: "within_budget"; }; -export type StoreGrowthDimensionV1 = +export type StoreGrowthDimensionV1 = | { reason: string; state: "unknown"; @@ -5184,7 +5408,7 @@ export type StrataScanV1 = { files_examined: number; }; -export type StructureReadV1 = +export type StructureReadV1 = | { code: string; detail: string; @@ -5201,7 +5425,7 @@ export type StructureReadV1 = status: "unmeasured"; }; -export type StructureReadV12 = +export type StructureReadV12 = | { code: string; detail: string; @@ -5218,7 +5442,7 @@ export type StructureReadV12 = status: "unmeasured"; }; -export type StructureReadV13 = +export type StructureReadV13 = | { code: string; detail: string; @@ -5235,7 +5459,7 @@ export type StructureReadV13 = status: "unmeasured"; }; -export type StructureReadV14 = +export type StructureReadV14 = | { code: string; detail: string; @@ -5252,7 +5476,7 @@ export type StructureReadV14 = status: "unmeasured"; }; -export type StructureReadV15 = +export type StructureReadV15 = | { code: string; detail: string; @@ -5281,7 +5505,7 @@ export type SyncSettingsV1 = { auto_track_pr_poll_secs: number; }; -export type TableGrowthDimensionV1 = +export type TableGrowthDimensionV1 = | { coverage: DashboardCoverageV1; observed_at: number; @@ -5312,7 +5536,7 @@ export type TableGrowthDimensionV1 = state: "unsupported"; }; -export type TableGrowthOmissionV1 = +export type TableGrowthOmissionV1 = | { current_bytes: number; kind: "baseline_pending"; @@ -5358,7 +5582,7 @@ export type TelemetrySettingsV1 = { timings: boolean; }; -export type TemporalModeV1 = +export type TemporalModeV1 = | { cutoff: UtcMicros; kind: "as_of"; @@ -5373,6 +5597,16 @@ export type TemporalModeV1 = kind: "forensic"; }; +export type TemporalState = { + code_graph_freshness?: CodeGraphReadFreshnessV1 | null | undefined; + freshness: FreshnessState; + requested_at: UtcMicros; + requested_mode: TemporalModeV1; + resolved_at: UtcMicros; + source_generation: CodeGenerationId | null; + watermark_digest: ManifestDigest | null; +}; + export type TestMapMeasurementV1 = { algorithm: string; applicable: boolean; @@ -5505,7 +5739,7 @@ export type WorkArtifactHydrationRequestV1 = { page_size: number; }; -export type WorkArtifactHydrationV1 = +export type WorkArtifactHydrationV1 = | { state: "absent"; } @@ -5542,7 +5776,7 @@ export type WorkAttemptEvidenceRecordV1 = { stdout: WorkAttemptStreamSummaryV1 | null; }; -export type WorkAttemptEvidenceStateV1 = +export type WorkAttemptEvidenceStateV1 = | { state: "pending"; } @@ -5557,7 +5791,7 @@ export type WorkAttemptIdentityV1 = { task_id: TaskId; }; -export type WorkAttemptListCoverageV1 = +export type WorkAttemptListCoverageV1 = | { coverage: "capped"; remaining: number; @@ -5576,7 +5810,7 @@ export type WorkAttemptListRequestV1 = { page_size: number; }; -export type WorkAttemptListV1 = +export type WorkAttemptListV1 = | { state: "absent"; } @@ -5600,7 +5834,7 @@ export type WorkAttemptProjectionBindingV1 = { source_watermark: Record; }; -export type WorkAttemptProviderOutcomeV1 = +export type WorkAttemptProviderOutcomeV1 = | { outcome: "cancelled"; } @@ -5735,7 +5969,7 @@ export type WorkCancellationRequestV1 = { requested_at: UtcMicros; }; -export type WorkCancellationStateV1 = +export type WorkCancellationStateV1 = | { state: "acknowledged"; value: WorkCancellationAcknowledgementV1; @@ -5781,7 +6015,7 @@ export type WorkDecompositionProposalV1 = { rationale: Array; }; -export type WorkDuplicateAdjudicationAppendOutcomeV1 = +export type WorkDuplicateAdjudicationAppendOutcomeV1 = | { outcome: "appended"; receipt: WorkDuplicateAdjudicationReceiptV1; @@ -5831,7 +6065,7 @@ export type WorkEffectStateV1 = "compound_non_repeatable" | "intercepted" | "obs export type WorkEgressPolicy = "allowlisted" | "deny"; -export type WorkEvidenceContinuationV1 = +export type WorkEvidenceContinuationV1 = | { cursor: string; kind: "anchor"; @@ -5851,7 +6085,7 @@ export type WorkEvidenceCoverageV1 = { state: WorkEvidenceCoverageStateV1; }; -export type WorkEvidenceExpansionSelectorV1 = +export type WorkEvidenceExpansionSelectorV1 = | { kind: "anchor"; link_id: string; @@ -5903,7 +6137,7 @@ export type WorkEvidenceRetrieveRequestV1 = { verified_version: VerifiedWorkGraphVersionV1; }; -export type WorkEvidenceSourceV1 = +export type WorkEvidenceSourceV1 = | { hydration: WorkAnchorHydrationV1; kind: "anchor"; @@ -5940,7 +6174,7 @@ export type WorkExecutionEnvelopeV1 = { worktree_root: string; }; -export type WorkExecutionHistoryV1 = +export type WorkExecutionHistoryV1 = | { state: "absent"; } @@ -5999,7 +6233,7 @@ export type WorkExecutionSpanV1 = { wall_micros: number | null; }; -export type WorkExecutionTimingCoverageV1 = +export type WorkExecutionTimingCoverageV1 = | { coverage: "complete"; } @@ -6018,7 +6252,7 @@ export type WorkExperienceCandidateV1 = { item: WorkItemV1; }; -export type WorkExperienceCoverageV1 = +export type WorkExperienceCoverageV1 = | { applicable: number; coverage: "complete"; @@ -6056,7 +6290,7 @@ export type WorkExperienceV1 = { verified_version: VerifiedWorkGraphVersionV1; }; -export type WorkExpertiseAuthorizationV1 = +export type WorkExpertiseAuthorizationV1 = | { availability: "available"; categories: Array; @@ -6086,7 +6320,7 @@ export type WorkExpertiseLegalActionV1 = "allow_requested_categories" | "grant_p export type WorkExpertiseUnavailableReasonV1 = "project_consent_disabled" | "project_consent_expired" | "project_consent_not_yet_effective" | "requested_category_not_allowed" | "user_consent_disabled" | "user_consent_expired" | "user_consent_not_yet_effective"; -export type WorkFallbackTopology = +export type WorkFallbackTopology = | { executable: WorkExecutableReference; kind: "codex_cli"; @@ -6169,7 +6403,7 @@ export type WorkflowFanOutChildPlanV1 = { task_id: TaskId; }; -export type WorkflowFanOutFailurePolicyV1 = +export type WorkflowFanOutFailurePolicyV1 = | { policy: "collect"; } @@ -6232,7 +6466,7 @@ export type WorkflowRunEvent = { sequence: number; }; -export type WorkflowRunEventKind = +export type WorkflowRunEventKind = | { definition: WorkflowDefinition; fan_out_plans: Array; @@ -6353,7 +6587,7 @@ export type WorkflowSynthesisDraft = { export type WorkFrontierComparisonV1 = "agree" | "disagree" | "incomparable"; -export type WorkGraphChangeV1 = +export type WorkGraphChangeV1 = | { based_on_version: number; identity: WorkAttemptIdentityV1; @@ -6416,7 +6650,7 @@ export type WorkGraphChangeV1 = proposal_id: ProposalId; }; -export type WorkGraphReadModeV1 = +export type WorkGraphReadModeV1 = | { mode: "as_of"; valid_at: UtcMicros; @@ -6442,7 +6676,7 @@ export type WorkGraphReadRequestV1 = { selection: WorkProductSelectionScopeV1; }; -export type WorkGraphReadV1 = +export type WorkGraphReadV1 = | { authorized_scope: AuthorizedWorkProductScopeV1; mode: "absent"; @@ -6473,7 +6707,7 @@ export type WorkGraphReadV1 = timeline: WorkGraphTimelineV1; }; -export type WorkGraphSelectionCoverageV1 = +export type WorkGraphSelectionCoverageV1 = | { coverage: "complete"; covered_events: number; @@ -6485,7 +6719,7 @@ export type WorkGraphSelectionCoverageV1 = first_excluded_sequence: number; }; -export type WorkGraphTimelineCoverageV1 = +export type WorkGraphTimelineCoverageV1 = | { coverage: "complete"; returned: number; @@ -6573,7 +6807,7 @@ export type WorkKanbanProjectionV1 = { graph_version: number; }; -export type WorkLeakAdjudicationOutcomeV1 = +export type WorkLeakAdjudicationOutcomeV1 = | { outcome: "appended"; receipt: WorkLeakAdjudicationReceiptV1; @@ -6653,7 +6887,7 @@ export type WorkPlacementPreflightV1 = { target: WorkPlacementTargetV1; }; -export type WorkPlacementReadingV1 = +export type WorkPlacementReadingV1 = | { state: "absent"; } @@ -6704,7 +6938,7 @@ export type WorkPriorOutcomeV1 = { export type WorkPriorTerminalV1 = "cancelled" | "failed" | "succeeded" | "timed_out"; -export type WorkProductAuthorizedRelationScopeV1 = +export type WorkProductAuthorizedRelationScopeV1 = | { kind: "project"; project_id: ProjectId; @@ -6715,7 +6949,7 @@ export type WorkProductAuthorizedRelationScopeV1 = repository_id: RepositoryId; }; -export type WorkProductChangeDraftV1 = +export type WorkProductChangeDraftV1 = | { change: "accept_task"; evidence_by_criterion: Record; @@ -6766,7 +7000,7 @@ export type WorkProductEventEvidenceV1 = { source_store_id: SourceStoreId; }; -export type WorkProductEventPayloadV1 = +export type WorkProductEventPayloadV1 = | { change: WorkGraphChangeV1; kind: "changed"; @@ -6796,7 +7030,7 @@ export type WorkProductEventV1 = { source_watermark: Record; }; -export type WorkProductExpectedAuthorityV1 = +export type WorkProductExpectedAuthorityV1 = | { authority: "no_prior_graph"; } @@ -6831,7 +7065,7 @@ export type WorkProductMutationReceiptV1 = { verified_graph_version: VerifiedWorkGraphVersionV1; }; -export type WorkProductMutationRequestV1 = +export type WorkProductMutationRequestV1 = | { mutation: "accept_task"; request: AcceptWorkTaskRequestV1; @@ -6889,7 +7123,7 @@ export type WorkProductProjectionBundleV1 = { workload: WorkWorkloadProjectionV1; }; -export type WorkProductRelationV1 = +export type WorkProductRelationV1 = | { identity: WorkAttemptIdentityV1; kind: "accepted_attempt"; @@ -6947,7 +7181,7 @@ export type WorkProductRevisionPinsV1 = { policy_revision_id: string; }; -export type WorkProductSelectionScopeV1 = +export type WorkProductSelectionScopeV1 = | { selection: "profile_owned_no_git"; } @@ -7070,7 +7304,7 @@ export type WorkRankedRouteV1 = { sensitive_data_fitness: WorkOrdinalBandV1; }; -export type WorkRecoveryStateV1 = +export type WorkRecoveryStateV1 = | { state: "fresh"; } @@ -7109,7 +7343,7 @@ export type WorkRelationReplanProposalV1 = { export type WorkRestartReasonV1 = "failure_observed" | "lease_lost" | "process_lost" | "provider_unavailable"; -export type WorkRetryAttemptOutcomeV1 = +export type WorkRetryAttemptOutcomeV1 = | { attempt: WorkAttemptV1; outcome: "created"; @@ -7141,7 +7375,7 @@ export type WorkRetryReceiptV1 = { export type WorkRetrySourceV1 = "runtime"; -export type WorkRouteDecisionV1 = +export type WorkRouteDecisionV1 = | { decision: "abstained"; reason: string; @@ -7170,7 +7404,7 @@ export type WorkRoutePlanV1 = { export type WorkRunControlAuthorityV1 = number; -export type WorkRunControlReadingV1 = +export type WorkRunControlReadingV1 = | { control: WorkRunControlV1; live_attempts: Array; @@ -7215,7 +7449,7 @@ export type WorkRuntimeAttemptProjectionV1 = { state: WorkAttemptStateV1; }; -export type WorkRuntimeProjectionCoverageV1 = +export type WorkRuntimeProjectionCoverageV1 = | { coverage: "complete"; } @@ -7262,7 +7496,7 @@ export type WorkSubtaskSketchV1 = { summary: string; }; -export type WorkSynthesisAttemptV1 = +export type WorkSynthesisAttemptV1 = | { attempt: WorkAttemptV1; draft: WorkflowSynthesisDraft; @@ -7289,7 +7523,7 @@ export type WorkSynthesisSourceEnvelopeV1 = { source: WorkAttemptIdentityV1; }; -export type WorkSynthesisSourceOutcomeV1 = +export type WorkSynthesisSourceOutcomeV1 = | { evidence: ManifestDigest; outcome: "cancelled"; @@ -7384,7 +7618,7 @@ export type WorkTaskShapeV1 = { kind: WorkTaskShapeKindV1; }; -export type WorkTerminalEvidenceV1 = +export type WorkTerminalEvidenceV1 = | { evidence_digest: ManifestDigest; observed_at: UtcMicros; @@ -7467,7 +7701,7 @@ export type WorktreeCleanlinessRequirementV1 = "allow_untracked_only_for_preflig export type WorktreeId = string; -export type WorktreePlacementModeV1 = +export type WorktreePlacementModeV1 = | { kind: "configured_root"; root_id: string; diff --git a/dashboard/src/test/fixtures/settings-patch.json b/dashboard/src/test/fixtures/settings-patch.json new file mode 100644 index 0000000000..6c5d8473a9 --- /dev/null +++ b/dashboard/src/test/fixtures/settings-patch.json @@ -0,0 +1,639 @@ +{ + "cancelled": { + "body": { + "contract": { + "schema_id": "schema.configuration.batch.result", + "schema_revision": 1 + }, + "problem": { + "cancellation_stage": "before_admission", + "code": "cancelled", + "committed_receipt": null, + "coverage": null, + "detail": null, + "details": [], + "diagnostic": null, + "execution_failure_classification": null, + "kind": "cancelled", + "legal_actions": [], + "message": "The request was cancelled before admission", + "owning_layer": "application", + "request_id": "request.configuration.settings-fixture", + "retry": "never", + "retry_after_millis": null, + "retry_scope": null, + "retryable": false, + "revision": 1, + "terminality": "pre_admission", + "trace_id": "request.configuration.settings-fixture", + "unavailable_classification": null + }, + "request_id": "request.configuration.settings-fixture" + }, + "status": 409 + }, + "changed": { + "body": { + "application_outcome": { + "outcome": "effect", + "value": { + "authority": { + "authorized_scope_digest": "sha256:cb520e3731e8dbab143f4fc32b885fc977d467f4648a6429bf0ed9bf0294873e", + "disclosure": "metadata", + "grant_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "grant_id": "grant.settings", + "grant_revision": 1, + "policy": { + "decision_id": "policy.settings", + "digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "evaluator_revision": "policy.settings.v1", + "revision": 1 + }, + "revalidated_at": 10 + }, + "effect_class": "administrative", + "effect_id": "effect.configuration.settings", + "execution": { + "budget": { + "bytes_consumed": 0, + "elapsed_micros": 0, + "units_consumed": 0 + }, + "cancellation": null, + "effective_deadline": { + "expires_at": 1000 + }, + "ended_at": 20, + "started_at": 10, + "termination": "completed" + }, + "expected_state": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "idempotency_key": "settings-idempotency-fixture", + "payload": null, + "receipt": { + "actor": "actor.settings", + "catalog_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "committed_state": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "configuration_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "effect_class": "administrative", + "expected_state": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "external_proof": null, + "idempotency_key": "settings-idempotency-fixture", + "input_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "operation": "use-case.configuration.batch", + "outcome": "completed", + "policy_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "privacy_digest": "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "request_id": "request.configuration.settings-fixture", + "scope": { + "project_id": "project.settings.scout", + "reference": null, + "repository_id": "repository.settings", + "scope_digest": "sha256:cb520e3731e8dbab143f4fc32b885fc977d467f4648a6429bf0ed9bf0294873e", + "worktree_id": "worktree.settings" + } + }, + "reconciliation": "reconciled" + } + }, + "current": { + "authorization": { + "outcome": "authorized" + }, + "coverage": { + "completeness": "complete", + "denominator": 1, + "eligible": 1, + "examined": 1, + "excluded": 0, + "matched": 1, + "omission_reasons": [], + "omitted": 0, + "unit": "configuration_control_plane", + "unknown": 0 + }, + "domain_state": "ready", + "freshness": { + "observed_at_micros": 100, + "state": "fresh", + "watermark": null + }, + "legal_actions": [ + { + "kind": "request_apply", + "operation": "configuration_batch" + }, + { + "kind": "refresh", + "operation": "configuration_list" + } + ], + "payload": { + "automation": { + "availability": { + "available": true + }, + "backend": "codex_app_server", + "config_endpoint": "/api/plugins/holographic/curation/config", + "configuration_revision_id": "rev-43", + "enabled": true, + "host_mode": "standalone" + }, + "environment": { + "global_accounting_enabled": false, + "global_accounting_mode": "disabled", + "pricing_offline": true, + "variables": [] + }, + "project": { + "config": { + "context_scout": false, + "exclude": [ + ".git/**", + ".tracedecay/**", + "**/*.min.*", + ".cache/**", + "**/.cache/**", + ".gradle/**", + "**/.gradle/**", + ".next/**", + "**/.next/**", + ".turbo/**", + "**/.turbo/**", + ".venv/**", + "**/.venv/**", + ".worktrees/**", + "**/.worktrees/**", + "__pycache__/**", + "**/__pycache__/**", + "build/**", + "**/build/**", + "coverage/**", + "**/coverage/**", + "dist/**", + "**/dist/**", + "node_modules/**", + "**/node_modules/**", + "out/**", + "**/out/**", + "target/**", + "**/target/**", + "vendor/**", + "**/vendor/**", + "venv/**", + "**/venv/**" + ], + "extract_docstrings": true, + "include": [], + "max_file_size": 2097152, + "sync": { + "auto_track_pr_branches": false, + "auto_track_pr_poll_secs": 300 + }, + "telemetry": { + "timings": true + }, + "track_call_sites": true + }, + "configuration_revision_id": "rev-43", + "configuration_snapshot_id": "tracedecay.configuration.snapshot.v1.fd56d9635ef3558e8743096f4bcc8dc5f6e38a457b3b225086af5bf4b01b75ab", + "pr_autotrack": { + "tracked": [] + } + }, + "resync_recommended": true, + "storage": { + "dashboard_root": "/project", + "graph_db": "/project/.tracedecay/graph.db", + "lcm_db": "/project/.tracedecay/lcm.db", + "lcm_scope": "project", + "memory_db": "/project/.tracedecay/memory.db", + "project_id": "project.settings.scout", + "project_root": "/project", + "savings_db": "/project/.tracedecay/savings.db", + "storage_mode": "project_local", + "store_root": "/project/.tracedecay" + }, + "user": { + "code_index_worker_configuration_revision_id": "workers-rev-3", + "code_index_worker_configuration_snapshot_id": "workers-snapshot-3", + "code_index_worker_status": null, + "code_index_workers": { + "mode": "automatic" + }, + "configuration_revision_id": "user-rev-7", + "configuration_snapshot_id": "user-snapshot-7", + "extraction_timeout_secs": 30, + "installed_agents": [], + "upload_enabled": false, + "watcher_debounce": "15s" + }, + "version": { + "cached_latest_version": null, + "channel": "stable", + "version": "0.1.0" + } + }, + "schema_revision": 1, + "scope": { + "project_id": "project.settings.scout", + "storage_mode": "project_local", + "store_root": "/project/.tracedecay" + }, + "source_watermark": null, + "time": { + "observation_time_micros": 100, + "valid_time_micros": null + }, + "version": { + "entity_version": null, + "graph_version": null + } + } + }, + "status": 200 + }, + "idempotency_conflict": { + "body": { + "contract": { + "schema_id": "schema.configuration.batch.result", + "schema_revision": 1 + }, + "problem": { + "cancellation_stage": null, + "code": "configuration.conflict", + "committed_receipt": null, + "coverage": null, + "detail": null, + "details": [], + "diagnostic": { + "code": "configuration.conflict", + "message": "The idempotency key was already used for a different settings change." + }, + "execution_failure_classification": null, + "kind": "conflict", + "legal_actions": [ + "refresh" + ], + "message": "The idempotency key was already used for a different settings change.", + "owning_layer": "application", + "request_id": "request.configuration.settings-fixture", + "retry": "after_revalidate", + "retry_after_millis": null, + "retry_scope": "fresh_request", + "retryable": true, + "revision": 1, + "terminality": "pre_admission", + "trace_id": "request.configuration.settings-fixture", + "unavailable_classification": null + }, + "request_id": "request.configuration.settings-fixture" + }, + "status": 409 + }, + "read": { + "body": { + "authorization": { + "outcome": "authorized" + }, + "coverage": { + "completeness": "complete", + "denominator": 1, + "eligible": 1, + "examined": 1, + "excluded": 0, + "matched": 1, + "omission_reasons": [], + "omitted": 0, + "unit": "configuration_control_plane", + "unknown": 0 + }, + "domain_state": "ready", + "freshness": { + "observed_at_micros": 100, + "state": "fresh", + "watermark": null + }, + "legal_actions": [ + { + "kind": "request_apply", + "operation": "configuration_batch" + }, + { + "kind": "refresh", + "operation": "configuration_list" + } + ], + "payload": { + "automation": { + "availability": { + "available": true + }, + "backend": "codex_app_server", + "config_endpoint": "/api/plugins/holographic/curation/config", + "configuration_revision_id": "rev-42", + "enabled": true, + "host_mode": "standalone" + }, + "environment": { + "global_accounting_enabled": false, + "global_accounting_mode": "disabled", + "pricing_offline": true, + "variables": [] + }, + "project": { + "config": { + "context_scout": false, + "exclude": [ + ".git/**", + ".tracedecay/**", + "**/*.min.*", + ".cache/**", + "**/.cache/**", + ".gradle/**", + "**/.gradle/**", + ".next/**", + "**/.next/**", + ".turbo/**", + "**/.turbo/**", + ".venv/**", + "**/.venv/**", + ".worktrees/**", + "**/.worktrees/**", + "__pycache__/**", + "**/__pycache__/**", + "build/**", + "**/build/**", + "coverage/**", + "**/coverage/**", + "dist/**", + "**/dist/**", + "node_modules/**", + "**/node_modules/**", + "out/**", + "**/out/**", + "target/**", + "**/target/**", + "vendor/**", + "**/vendor/**", + "venv/**", + "**/venv/**" + ], + "extract_docstrings": true, + "include": [], + "max_file_size": 1048576, + "sync": { + "auto_track_pr_branches": false, + "auto_track_pr_poll_secs": 300 + }, + "telemetry": { + "timings": true + }, + "track_call_sites": true + }, + "configuration_revision_id": "rev-42", + "configuration_snapshot_id": "tracedecay.configuration.snapshot.v1.fd56d9635ef3558e8743096f4bcc8dc5f6e38a457b3b225086af5bf4b01b75ab", + "pr_autotrack": { + "tracked": [] + } + }, + "resync_recommended": false, + "storage": { + "dashboard_root": "/project", + "graph_db": "/project/.tracedecay/graph.db", + "lcm_db": "/project/.tracedecay/lcm.db", + "lcm_scope": "project", + "memory_db": "/project/.tracedecay/memory.db", + "project_id": "project.settings.scout", + "project_root": "/project", + "savings_db": "/project/.tracedecay/savings.db", + "storage_mode": "project_local", + "store_root": "/project/.tracedecay" + }, + "user": { + "code_index_worker_configuration_revision_id": "workers-rev-3", + "code_index_worker_configuration_snapshot_id": "workers-snapshot-3", + "code_index_worker_status": null, + "code_index_workers": { + "mode": "automatic" + }, + "configuration_revision_id": "user-rev-7", + "configuration_snapshot_id": "user-snapshot-7", + "extraction_timeout_secs": 30, + "installed_agents": [], + "upload_enabled": false, + "watcher_debounce": "15s" + }, + "version": { + "cached_latest_version": null, + "channel": "stable", + "version": "0.1.0" + } + }, + "schema_revision": 1, + "scope": { + "project_id": "project.settings.scout", + "storage_mode": "project_local", + "store_root": "/project/.tracedecay" + }, + "source_watermark": null, + "time": { + "observation_time_micros": 100, + "valid_time_micros": null + }, + "version": { + "entity_version": null, + "graph_version": null + } + }, + "status": 200 + }, + "revision_conflict": { + "body": { + "actual_revision_id": "rev-43", + "code": "configuration_revision_conflict", + "detail": "settings changed after this edit began; refresh and retry", + "expected_revision_id": "rev-42" + }, + "status": 409 + }, + "unavailable": { + "body": { + "code": "configuration_authority_unavailable", + "detail": "configuration authority is unavailable" + }, + "status": 503 + }, + "unchanged": { + "body": { + "application_outcome": null, + "current": { + "authorization": { + "outcome": "authorized" + }, + "coverage": { + "completeness": "complete", + "denominator": 1, + "eligible": 1, + "examined": 1, + "excluded": 0, + "matched": 1, + "omission_reasons": [], + "omitted": 0, + "unit": "configuration_control_plane", + "unknown": 0 + }, + "domain_state": "ready", + "freshness": { + "observed_at_micros": 100, + "state": "fresh", + "watermark": null + }, + "legal_actions": [ + { + "kind": "request_apply", + "operation": "configuration_batch" + }, + { + "kind": "refresh", + "operation": "configuration_list" + } + ], + "payload": { + "automation": { + "availability": { + "available": true + }, + "backend": "codex_app_server", + "config_endpoint": "/api/plugins/holographic/curation/config", + "configuration_revision_id": "rev-42", + "enabled": true, + "host_mode": "standalone" + }, + "environment": { + "global_accounting_enabled": false, + "global_accounting_mode": "disabled", + "pricing_offline": true, + "variables": [] + }, + "project": { + "config": { + "context_scout": false, + "exclude": [ + ".git/**", + ".tracedecay/**", + "**/*.min.*", + ".cache/**", + "**/.cache/**", + ".gradle/**", + "**/.gradle/**", + ".next/**", + "**/.next/**", + ".turbo/**", + "**/.turbo/**", + ".venv/**", + "**/.venv/**", + ".worktrees/**", + "**/.worktrees/**", + "__pycache__/**", + "**/__pycache__/**", + "build/**", + "**/build/**", + "coverage/**", + "**/coverage/**", + "dist/**", + "**/dist/**", + "node_modules/**", + "**/node_modules/**", + "out/**", + "**/out/**", + "target/**", + "**/target/**", + "vendor/**", + "**/vendor/**", + "venv/**", + "**/venv/**" + ], + "extract_docstrings": true, + "include": [], + "max_file_size": 1048576, + "sync": { + "auto_track_pr_branches": false, + "auto_track_pr_poll_secs": 300 + }, + "telemetry": { + "timings": true + }, + "track_call_sites": true + }, + "configuration_revision_id": "rev-42", + "configuration_snapshot_id": "tracedecay.configuration.snapshot.v1.fd56d9635ef3558e8743096f4bcc8dc5f6e38a457b3b225086af5bf4b01b75ab", + "pr_autotrack": { + "tracked": [] + } + }, + "resync_recommended": false, + "storage": { + "dashboard_root": "/project", + "graph_db": "/project/.tracedecay/graph.db", + "lcm_db": "/project/.tracedecay/lcm.db", + "lcm_scope": "project", + "memory_db": "/project/.tracedecay/memory.db", + "project_id": "project.settings.scout", + "project_root": "/project", + "savings_db": "/project/.tracedecay/savings.db", + "storage_mode": "project_local", + "store_root": "/project/.tracedecay" + }, + "user": { + "code_index_worker_configuration_revision_id": "workers-rev-3", + "code_index_worker_configuration_snapshot_id": "workers-snapshot-3", + "code_index_worker_status": null, + "code_index_workers": { + "mode": "automatic" + }, + "configuration_revision_id": "user-rev-7", + "configuration_snapshot_id": "user-snapshot-7", + "extraction_timeout_secs": 30, + "installed_agents": [], + "upload_enabled": false, + "watcher_debounce": "15s" + }, + "version": { + "cached_latest_version": null, + "channel": "stable", + "version": "0.1.0" + } + }, + "schema_revision": 1, + "scope": { + "project_id": "project.settings.scout", + "storage_mode": "project_local", + "store_root": "/project/.tracedecay" + }, + "source_watermark": null, + "time": { + "observation_time_micros": 100, + "valid_time_micros": null + }, + "version": { + "entity_version": null, + "graph_version": null + } + } + }, + "status": 200 + }, + "validation": { + "body": { + "detail": "invalid project settings patch: unknown field `unknown_setting`, expected one of `expected_revision_id`, `idempotency_key`, `include`, `exclude`, `max_file_size`, `extract_docstrings`, `track_call_sites`, `telemetry`, `sync`, `context_scout`, `context_scout_intent`", + "validation_errors": [ + { + "field": "unknown_setting", + "message": "invalid project settings patch: unknown field `unknown_setting`, expected one of `expected_revision_id`, `idempotency_key`, `include`, `exclude`, `max_file_size`, `extract_docstrings`, `track_call_sites`, `telemetry`, `sync`, `context_scout`, `context_scout_intent`" + } + ] + }, + "status": 400 + } +} diff --git a/dashboard/src/workspaces/settings/SettingsPage.dom.test.tsx b/dashboard/src/workspaces/settings/SettingsPage.dom.test.tsx index 370012bae5..27715acba8 100644 --- a/dashboard/src/workspaces/settings/SettingsPage.dom.test.tsx +++ b/dashboard/src/workspaces/settings/SettingsPage.dom.test.tsx @@ -2,6 +2,7 @@ import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; import { act, fireEvent, render, screen, waitFor, within } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import settingsWire from '../../test/fixtures/settings-patch.json'; import { FIXTURES } from '../../../stories/fixtures/data.ts'; import { useScope, type ScopeWritability } from '../../data/scope/store.ts'; import { SettingsPage, findConfigSection } from './SettingsPage.tsx'; @@ -97,15 +98,7 @@ const FIXTURE_SETTINGS: ReadonlyArray = [ function projectPatchResponse(current: unknown) { return { - application_outcome: { - Effect: { - receipt: { - request_id: 'request.configuration.settings-fixture', - idempotency_key: 'settings-fixture', - outcome: 'completed', - }, - }, - }, + application_outcome: structuredClone(settingsWire.changed.body.application_outcome), current, }; } @@ -402,6 +395,35 @@ describe('SettingsPage effective configuration review', () => { ).toBeTruthy(); }); + it('requires a refresh after an uncertain effect without offering to repeat the write', async () => { + const response = structuredClone(settingsWire.changed.body); + response.application_outcome.value.reconciliation = 'pending'; + let dispatched = false; + const fetch = vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => { + if (init?.method === 'PATCH') { + dispatched = true; + return jsonResponse(response); + } + return jsonResponse(dispatched ? updatedSettings('rev-43') : settings()); + }); + vi.stubGlobal('fetch', fetch); + const user = userEvent.setup(); + renderSettings(); + await openReview(user, MAX_FILE_SIZE); + await user.clear(proposal(MAX_FILE_SIZE)); + await user.type(proposal(MAX_FILE_SIZE), '2097152'); + await user.click(screen.getByRole('button', { name: 'Review project change' })); + await user.click(screen.getByRole('checkbox', { name: /I confirm this change against configuration revision rev-42/ })); + await user.click(screen.getByRole('button', { name: 'Apply project settings' })); + expect(await screen.findByText(/Refresh current values before retrying/)).toBeTruthy(); + expect(screen.queryByText('Project settings saved')).toBeNull(); + expect(screen.queryByRole('button', { name: 'Retry project settings' })).toBeNull(); + await user.click(screen.getByRole('button', { name: 'Load current values' })); + await waitFor(() => expect(within(row(MAX_FILE_SIZE)).getByText('2,097,152')).toBeTruthy()); + expect(fetch.mock.calls.filter(([, init]) => init?.method === 'PATCH')).toHaveLength(1); + expect(screen.queryByText('Project settings saved')).toBeNull(); + }); + it('blocks a stale project change before sending the patch', async () => { let getCount = 0; const methods: string[] = []; diff --git a/dashboard/src/workspaces/settings/SettingsReviewPanel.tsx b/dashboard/src/workspaces/settings/SettingsReviewPanel.tsx index 12a5b195c2..e2ae53be1f 100644 --- a/dashboard/src/workspaces/settings/SettingsReviewPanel.tsx +++ b/dashboard/src/workspaces/settings/SettingsReviewPanel.tsx @@ -397,7 +397,10 @@ function FrozenReview({ }) { const confirmed = settingsConfirmationHeld(state); const applying = state.status === 'submitting'; - const resolvable = state.status === 'conflicted' || state.status === 'review_superseded'; + const resolvable = state.status === 'conflicted' + || state.status === 'review_superseded' + || (state.status === 'submit_failed' + && (state.failure.kind === 'protocol_error' || state.failure.kind === 'unsettled')); const retryable = state.status === 'authority_withdrawn' || state.status === 'submit_failed'; return (

diff --git a/dashboard/src/workspaces/settings/settingsEditorMachine.ts b/dashboard/src/workspaces/settings/settingsEditorMachine.ts index c893afde88..fbccddee59 100644 --- a/dashboard/src/workspaces/settings/settingsEditorMachine.ts +++ b/dashboard/src/workspaces/settings/settingsEditorMachine.ts @@ -108,7 +108,7 @@ type SettingsResting = /** A write that did not reach a verdict. Distinct from a refusal, which did. */ type SettingsSubmitFailure = | { readonly kind: 'offline'; readonly detail: string } - | { readonly kind: 'error'; readonly detail: string } + | { readonly kind: 'error' | 'unsettled'; readonly detail: string } | { readonly kind: 'protocol_error'; readonly authority: string; readonly detail: string }; interface SettingsEditable { @@ -453,6 +453,8 @@ function settleSubmit( return failed(state, { kind: 'offline', detail: result.detail }); case 'error': return failed(state, { kind: 'error', detail: result.detail }); + case 'unsettled': + return failed(state, { kind: 'unsettled', detail: result.detail }); case 'protocol_error': return failed(state, { kind: 'protocol_error', diff --git a/dashboard/src/workspaces/settings/settingsMutation.test.ts b/dashboard/src/workspaces/settings/settingsMutation.test.ts new file mode 100644 index 0000000000..dae27d29a4 --- /dev/null +++ b/dashboard/src/workspaces/settings/settingsMutation.test.ts @@ -0,0 +1,89 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import wire from '../../test/fixtures/settings-patch.json'; +import { applySettingsMutation } from './settingsMutation.ts'; + +// settings_api/tests/wire.rs serializes these responses through Axum. Keep +// the complete receipts so decoding exercises the producer contract. +const request = { + writability: { state: 'writable', target: 'tracedecay' }, + scope: 'project', + expectedRevisionId: 'rev-42', + idempotencyKey: 'settings-idempotency-fixture', + readUrl: '/api/settings', + patchUrl: '/api/settings/project', + patch: { max_file_size: 2_097_152 }, +} as const; + +async function apply(response: { status: number; body: unknown }) { + const fetch = vi.fn() + .mockResolvedValueOnce(Response.json(wire.read.body, { status: wire.read.status })) + .mockResolvedValueOnce(Response.json(response.body, { status: response.status })); + vi.stubGlobal('fetch', fetch); + const result = await applySettingsMutation(request); + expect(fetch).toHaveBeenCalledTimes(2); + return result; +} + +afterEach(() => vi.unstubAllGlobals()); + +describe('settings producer/consumer settlement', () => { + it('accepts the committed effect and preserves its complete durable receipt', async () => { + expect(await apply(wire.changed)).toMatchObject({ + outcome: 'success', + revisionId: 'rev-43', + effectReceipt: wire.changed.body.application_outcome.value.receipt, + }); + }); + + it('accepts the explicit null outcome for an already-current patch', async () => { + expect(await apply(wire.unchanged)).toMatchObject({ outcome: 'success', revisionId: 'rev-42', effectReceipt: null }); + }); + + it('requires the explicit nullable settlement and the canonical effect shape', async () => { + for (const body of [ + { current: wire.changed.body.current }, + { ...wire.changed.body, application_outcome: { Effect: wire.changed.body.application_outcome.value } }, + ]) { + expect(await apply({ status: 200, body })).toMatchObject({ outcome: 'protocol_error' }); + } + }); + + it('does not present uncertain or unproven effects as applied', async () => { + for (const mutate of [ + (effect: typeof wire.changed.body.application_outcome.value) => { effect.execution.termination = 'effect_unknown'; }, + (effect: typeof wire.changed.body.application_outcome.value) => { effect.reconciliation = 'pending'; }, + (effect: typeof wire.changed.body.application_outcome.value) => { effect.receipt.outcome = 'partial'; }, + (effect: typeof wire.changed.body.application_outcome.value) => { Object.assign(effect.receipt, { committed_state: null }); }, + ]) { + const response = structuredClone(wire.changed); + mutate(response.body.application_outcome.value); + expect(await apply(response)).toMatchObject({ outcome: 'unsettled', detail: expect.stringContaining('Refresh current values before retrying') }); + } + }); + + it('retains validation, revision conflicts and unavailable authority semantics', async () => { + expect(await apply(wire.validation)).toMatchObject({ outcome: 'validation', errors: wire.validation.body.validation_errors }); + expect(await apply(wire.revision_conflict)).toEqual({ outcome: 'conflict', expectedRevisionId: 'rev-42', actualRevisionId: 'rev-43' }); + expect(await apply(wire.unavailable)).toMatchObject({ outcome: 'unavailable' }); + }); + + it('keeps idempotency conflict and cancellation messages distinct from revision conflicts', async () => { + for (const response of [wire.idempotency_conflict, wire.cancelled]) { + expect(await apply(response)).toEqual({ outcome: 'error', detail: response.body.problem.message }); + } + }); + + it('requires current values before retrying a problem carrying a committed effect', async () => { + const response = structuredClone(wire.idempotency_conflict); + Object.assign(response.body.problem, { + kind: 'partial_effect', + committed_receipt: { ...wire.changed.body.application_outcome.value.receipt, outcome: 'partial' }, + }); + expect(await apply(response)).toMatchObject({ outcome: 'unsettled', detail: expect.stringContaining('Refresh current values before retrying') }); + }); + + it('rejects problem/status disagreement and malformed refusals', async () => { + expect(await apply({ ...wire.idempotency_conflict, status: 503 })).toMatchObject({ outcome: 'protocol_error' }); + expect(await apply({ status: 409, body: {} })).toMatchObject({ outcome: 'protocol_error' }); + }); +}); diff --git a/dashboard/src/workspaces/settings/settingsMutation.ts b/dashboard/src/workspaces/settings/settingsMutation.ts index e6b90c6a38..7ad06da6dd 100644 --- a/dashboard/src/workspaces/settings/settingsMutation.ts +++ b/dashboard/src/workspaces/settings/settingsMutation.ts @@ -1,6 +1,11 @@ import { dashboardFetch } from '../../data/auth/credential.ts'; -import { z } from 'zod'; -import type { SettingsPayloadV1 } from '../../contracts/generated.ts'; +import { + DashboardConfigurationRefusalV1Schema, + ProjectSettingsPatchResponseV1Schema, + type EffectReceipt, + type SettingsPayloadV1, +} from '../../contracts/generated.ts'; +import { applicationProblemStatusMatches } from '../../contracts/application.ts'; import { readOnlyScopeRefusal, type ScopeWritability } from '../../data/scope/store.ts'; import { buildSettingsEditor, @@ -25,7 +30,7 @@ export type SettingsMutationResult = readonly resyncRecommended: boolean; readonly restartRecommended: boolean; /** Canonical durable application receipt for project writes. */ - readonly effectReceipt: Readonly> | null; + readonly effectReceipt: EffectReceipt | null; } | { readonly outcome: 'conflict'; @@ -41,6 +46,10 @@ export type SettingsMutationResult = readonly outcome: 'offline' | 'error'; readonly detail: string; } + | { + readonly outcome: 'unsettled'; + readonly detail: string; + } /** The authority that performs this write is not currently mounted. Kept * apart from `error`: nothing was attempted, and nothing changed. */ | { @@ -127,12 +136,6 @@ export async function applySettingsMutation( }), }); if (patched.outcome !== 'response') return patched; - if (patched.response.status === 409) { - return readConflict(patched.body, request.expectedRevisionId); - } - if (patched.response.status === 503) { - return { outcome: 'unavailable', detail: unavailableDetail(patched.body) }; - } // The gateway's read-only refusal, recognized by its own body rather than by // the status alone: a 405 this dashboard cannot account for stays a plain // error below rather than borrowing the scope explanation. @@ -144,13 +147,10 @@ export async function applySettingsMutation( detail: `Nothing was applied: ${scopeRefused.detail.replace(/\.$/, '')}.`, }; } + return { outcome: 'error', detail: 'Settings update failed (HTTP 405).' }; } if (!patched.response.ok) { - const validation = readValidation(patched.body); - return validation ?? { - outcome: 'error', - detail: `Settings update failed (HTTP ${patched.response.status}).`, - }; + return settingsRefusal(patched.body, patched.response.status, patchAuthority); } const projectPatch = request.scope === 'project' ? projectSettingsPatchResponse(patched.body, patchAuthority) @@ -186,23 +186,10 @@ function projectSettingsPatchResponse( | { readonly parsed: true; readonly current: unknown; - readonly effectReceipt: Readonly> | null; + readonly effectReceipt: EffectReceipt | null; } - | { readonly parsed: false; readonly violation: Extract } { - const response = z - .object({ - application_outcome: z.union([ - z.null(), - z.object({ - Effect: z.object({ - receipt: z.record(z.string(), z.unknown()), - }).passthrough(), - }).strict(), - ]), - current: z.unknown(), - }) - .strict() - .safeParse(body); + | { readonly parsed: false; readonly violation: Extract } { + const response = ProjectSettingsPatchResponseV1Schema.safeParse(body); if (!response.success) { return { parsed: false, @@ -212,10 +199,23 @@ function projectSettingsPatchResponse( ), }; } + const outcome = response.data.application_outcome; + if (outcome !== null && ( + outcome.outcome !== 'effect' + || outcome.value.execution.termination !== 'completed' + || outcome.value.reconciliation !== 'reconciled' + || outcome.value.receipt.outcome !== 'completed' + || outcome.value.receipt.committed_state === null + )) { + return { + parsed: false, + violation: { outcome: 'unsettled', detail: 'The settings effect is not confirmed. Refresh current values before retrying.' }, + }; + } return { parsed: true, current: response.data.current, - effectReceipt: response.data.application_outcome?.Effect.receipt ?? null, + effectReceipt: outcome?.outcome === 'effect' ? outcome.value.receipt : null, }; } @@ -282,42 +282,38 @@ function contractedPayload(body: unknown, authority: string): ContractedPayload } } -/** - * CONTRACT GAP, the settings refusal bodies are not generated. - * - * `/api/settings/{project,user}` build their 400/409/503 bodies with - * `serde_json::json!` in `crates/tracedecay-api/src/configuration.rs` - * (`settings_validation_error`, `configuration_revision_conflict_error`, - * `configuration_authority_unavailable_error`), so schemars never sees them - * and `contracts/generated.ts` carries no type for them. The success payload - * is contracted and is parsed as such above; only the refusals are read here. - * - * Every member below is therefore optional and every absence resolves to a - * stated unavailable value, a missing `actual_revision_id` reads as unknown, - * a missing `detail` says the authority is unavailable without naming which. - * None of it is inferred, and none of it turns a refusal into a success. - */ -const RefusalBodySchema = z - .object({ - detail: z.string().optional(), - expected_revision_id: z.string().optional(), - actual_revision_id: z.string().optional(), - validation_errors: z - .array(z.object({ field: z.string(), message: z.string() }).passthrough()) - .optional(), - }) - .passthrough(); - -function readRefusal(body: unknown): z.infer { - const parsed = RefusalBodySchema.safeParse(body); - return parsed.success ? parsed.data : {}; -} - -function unavailableDetail(body: unknown): string { - const detail = readRefusal(body).detail; - return detail != null && detail.length > 0 - ? `Nothing was applied: ${detail.replace(/\.$/, '')}.` - : 'Nothing was applied: the authority for this write is unavailable.'; +function settingsRefusal(body: unknown, status: number, authority: string): SettingsMutationResult { + const parsed = DashboardConfigurationRefusalV1Schema.safeParse(body); + if (!parsed.success) return protocolError(authority, 'expected a configuration refusal.'); + const refusal = parsed.data; + if ('problem' in refusal) { + // The existing settings adapter preserves HTTP 409 for cancellation. + const matches = refusal.problem.kind === 'cancelled' + ? status === 409 + : applicationProblemStatusMatches(status, refusal.problem.kind); + if (!matches) return protocolError(authority, 'the application problem disagrees with its HTTP status.'); + if (refusal.problem.committed_receipt !== null) { + return { outcome: 'unsettled', detail: `${refusal.problem.message} Refresh current values before retrying.` }; + } + return { outcome: 'error', detail: refusal.problem.message }; + } + if ('validation_errors' in refusal) { + return status === 400 + ? { outcome: 'validation', detail: refusal.detail, errors: refusal.validation_errors } + : protocolError(authority, 'the validation refusal disagrees with its HTTP status.'); + } + switch (refusal.code) { + case 'configuration_revision_conflict': + return status === 409 + ? { outcome: 'conflict', expectedRevisionId: refusal.expected_revision_id, actualRevisionId: refusal.actual_revision_id } + : protocolError(authority, 'the revision conflict disagrees with its HTTP status.'); + case 'configuration_authority_unavailable': + return status === 503 + ? { outcome: 'unavailable', detail: `Nothing was applied: ${refusal.detail.replace(/\.$/, '')}.` } + : protocolError(authority, 'the authority refusal disagrees with its HTTP status.'); + case 'application_contract_violation': + return protocolError(authority, refusal.detail); + } } type JsonFetchResult = @@ -357,31 +353,6 @@ function protocolError( }; } -function readValidation(body: unknown): SettingsMutationResult | null { - const refusal = readRefusal(body); - const errors: readonly SettingsValidationError[] = (refusal.validation_errors ?? []).map( - ({ field, message }) => ({ field, message }), - ); - if (errors.length === 0) return null; - return { - outcome: 'validation', - detail: refusal.detail ?? 'Settings validation failed.', - errors, - }; -} - -function readConflict( - body: unknown, - fallbackExpectedRevisionId: string, -): SettingsMutationResult { - const refusal = readRefusal(body); - return { - outcome: 'conflict', - expectedRevisionId: refusal.expected_revision_id ?? fallbackExpectedRevisionId, - actualRevisionId: refusal.actual_revision_id ?? null, - }; -} - const JSON_HEADERS = { accept: 'application/json', 'content-type': 'application/json', From a70bf5883bb5794d4754b412f570fb61ed7ec206 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 20:31:41 +0000 Subject: [PATCH 28/89] build(evals): generate memory benchmark runtime export --- benchmark_data/BUILD.bazel | 6 ++++-- scripts/bazel/gen_builds.py | 5 +++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/benchmark_data/BUILD.bazel b/benchmark_data/BUILD.bazel index f2c7affbe9..34029c7d21 100644 --- a/benchmark_data/BUILD.bazel +++ b/benchmark_data/BUILD.bazel @@ -81,6 +81,8 @@ filegroup( exports_files(["session-temporal/workload-v1.json"], visibility = ["//visibility:public"], ) -# END generated by scripts/bazel/gen_builds.py -exports_files(["run_benchmarks.py"], visibility = ["//evals/memory:__pkg__"]) +exports_files(["run_benchmarks.py"], + visibility = ["//evals/memory:__pkg__"], +) +# END generated by scripts/bazel/gen_builds.py diff --git a/scripts/bazel/gen_builds.py b/scripts/bazel/gen_builds.py index 683f12dae2..5b35e20daa 100755 --- a/scripts/bazel/gen_builds.py +++ b/scripts/bazel/gen_builds.py @@ -1427,6 +1427,11 @@ def package_rules(pkg_dir, exports): ' srcs = glob(["fixtures/**"]),\n' ' visibility = ["//visibility:public"],\n)' ) + if pkg_dir == "benchmark_data": + rules.append( + 'exports_files(["run_benchmarks.py"],\n' + ' visibility = ["//evals/memory:__pkg__"],\n)' + ) return rules def generated_block(pkg_dir, exports): From fb85df8f111522e3c1e17581c3d74fbfd841ec3c Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 20:47:39 +0000 Subject: [PATCH 29/89] fix(mcp): preserve live graph request cancellation --- .../src/invocation/dispatch.rs | 16 ++- .../src/invocation/graph_tool.rs | 13 +- .../src/request_cancellation.rs | 124 +++++++++++++++++- .../src/daemon/invocation_executor.rs | 33 ++++- 4 files changed, 164 insertions(+), 22 deletions(-) diff --git a/crates/tracedecay-daemon-service/src/invocation/dispatch.rs b/crates/tracedecay-daemon-service/src/invocation/dispatch.rs index 02d486f67a..b6bc595321 100644 --- a/crates/tracedecay-daemon-service/src/invocation/dispatch.rs +++ b/crates/tracedecay-daemon-service/src/invocation/dispatch.rs @@ -1,6 +1,7 @@ //! The daemon invocation dispatcher: `DaemonInvocationService::invoke`. use super::*; +use tracedecay_contracts::CancellationSignal; use tracedecay_runtime_core::cancellation::CancellationToken; use tracedecay_runtime_core::path_safety::canonical_existing_identity; use tracedecay_tool_catalog::ApplicationSurfaceOperation; @@ -785,13 +786,26 @@ impl DaemonInvocationService { let Some(owner) = graph_tool_owner else { return missing_registered_owner_problem(publication, request_id); }; + let signal = match self.request_cancellations.signal(&request_id, &request_cancellation) { + Some(signal) => signal, + None => match CancellationSignal::active(cancellation.token_id.as_str()) { + Ok(signal) => signal, + Err(_) => return DaemonInvocationResponse::problem( + request_id, + DaemonInvocationProblem::InvalidRequest, + ), + }, + }; + if let CancellationState::Cancelled { requested_at } = cancellation.state { + signal.cancel(requested_at); + } Box::pin(execute_graph_tool( request_id, owner, surface_operation, arguments, deadline, - cancellation, + signal, request_cancellation, )) .await diff --git a/crates/tracedecay-daemon-service/src/invocation/graph_tool.rs b/crates/tracedecay-daemon-service/src/invocation/graph_tool.rs index c215591889..ab7880061a 100644 --- a/crates/tracedecay-daemon-service/src/invocation/graph_tool.rs +++ b/crates/tracedecay-daemon-service/src/invocation/graph_tool.rs @@ -9,8 +9,7 @@ use std::sync::Arc; use serde_json::{Map, Value}; use tracedecay_contracts::graph_tool::GraphToolCompletionV1; use tracedecay_contracts::{ - ApplicationProblem, CancellationContext, CancellationSignal, CancellationState, Deadline, - RequestId, ResolvedScope, now_micros, + ApplicationProblem, CancellationSignal, Deadline, RequestId, ResolvedScope, now_micros, }; use tracedecay_daemon_protocol::{ DaemonInvocationOutcome, DaemonInvocationProblem, DaemonInvocationResponse, @@ -92,21 +91,15 @@ pub(super) async fn execute_graph_tool( operation: ApplicationSurfaceOperation, arguments: Map, deadline: Deadline, - cancellation: CancellationContext, + signal: CancellationSignal, request_cancellation: tracedecay_runtime_core::cancellation::CancellationToken, ) -> DaemonInvocationResponse { - let (Ok(typed_request_id), Ok(signal)) = ( - RequestId::new(request_id.clone()), - CancellationSignal::active(cancellation.token_id.as_str()), - ) else { + let Ok(typed_request_id) = RequestId::new(request_id.clone()) else { return DaemonInvocationResponse::problem( request_id, DaemonInvocationProblem::InvalidRequest, ); }; - if let CancellationState::Cancelled { requested_at } = &cancellation.state { - signal.cancel(*requested_at); - } let execution = owner.port.execute(GraphToolInvocationV1 { operation, arguments, diff --git a/crates/tracedecay-daemon-service/src/request_cancellation.rs b/crates/tracedecay-daemon-service/src/request_cancellation.rs index eee4ff1dc2..90e27ad813 100644 --- a/crates/tracedecay-daemon-service/src/request_cancellation.rs +++ b/crates/tracedecay-daemon-service/src/request_cancellation.rs @@ -12,6 +12,7 @@ use std::collections::BTreeMap; use std::sync::{Arc, Mutex, MutexGuard}; use std::time::{Duration, Instant}; +use tracedecay_contracts::{CancellationSignal, clock::now_micros}; use tracedecay_runtime_core::cancellation::CancellationToken; const PENDING_CAPACITY: usize = 1_024; @@ -21,11 +22,17 @@ const COMPLETED_TTL: Duration = Duration::from_mins(1); #[derive(Default)] struct State { - active: BTreeMap, + active: BTreeMap, pending: BTreeMap, completed: BTreeMap, } +#[derive(Clone)] +struct ActiveRequest { + token: CancellationToken, + signal: Option, +} + /// The cancellation table of one invocation service generation. /// /// Clones share the same table. A [`Lease`] retains the table it was @@ -55,6 +62,37 @@ impl RequestCancellationRegistryV1 { } pub fn register(&self, request_id: &str) -> Option { + self.register_control(request_id, None) + } + + /// Retains an in-process caller's live signal alongside its request lease. + /// Reconstructing it from its wire context would require an asynchronous + /// relay before a blocking worker could observe caller cancellation. + pub fn register_with_signal( + &self, + request_id: &str, + signal: CancellationSignal, + ) -> Option { + self.register_control(request_id, Some(signal)) + } + + pub(crate) fn signal( + &self, + request_id: &str, + token: &CancellationToken, + ) -> Option { + self.lock() + .active + .get(request_id) + .filter(|request| request.token.is_same_token(token)) + .and_then(|request| request.signal.clone()) + } + + fn register_control( + &self, + request_id: &str, + signal: Option, + ) -> Option { let token = CancellationToken::for_application_request(request_id); let mut state = self.lock(); expire_ephemeral(&mut state, Instant::now()); @@ -67,8 +105,17 @@ impl RequestCancellationRegistryV1 { state.completed.remove(request_id); if state.pending.remove(request_id).is_some() { token.cancel(); + if let Some(signal) = &signal { + signal.cancel(now_micros()); + } } - state.active.insert(request_id.to_owned(), token.clone()); + state.active.insert( + request_id.to_owned(), + ActiveRequest { + token: token.clone(), + signal, + }, + ); Some(Lease { state: Arc::clone(&self.state), request_id: request_id.to_owned(), @@ -81,9 +128,12 @@ impl RequestCancellationRegistryV1 { let mut state = self.lock(); let now = Instant::now(); expire_ephemeral(&mut state, now); - if let Some(token) = state.active.get(request_id).cloned() { + if let Some(request) = state.active.get(request_id).cloned() { drop(state); - token.cancel(); + request.token.cancel(); + if let Some(signal) = request.signal { + signal.cancel(now_micros()); + } true } else if state.completed.contains_key(request_id) { true @@ -128,7 +178,7 @@ impl Drop for Lease { if state .active .get(&self.request_id) - .is_some_and(|token| token.is_same_token(&self.token)) + .is_some_and(|request| request.token.is_same_token(&self.token)) { state.active.remove(&self.request_id); record_completed(&mut state, self.request_id.clone(), Instant::now()); @@ -141,6 +191,70 @@ mod tests { use std::time::{Duration, Instant}; use super::{COMPLETED_CAPACITY, RequestCancellationRegistryV1, State, record_completed}; + use tracedecay_contracts::{CancellationSignal, clock::now_micros}; + + #[test] + fn controlled_request_shares_live_cancellation_without_polling_a_relay() { + let registry = RequestCancellationRegistryV1::default(); + let signal = CancellationSignal::active("cancel.controlled").unwrap(); + let lease = registry + .register_with_signal("request.controlled", signal.clone()) + .unwrap(); + let worker = registry + .signal("request.controlled", &lease.token()) + .unwrap(); + + assert!(signal.cancel(now_micros())); + assert!(worker.is_cancelled()); + + let foreign = RequestCancellationRegistryV1::default() + .register("request.controlled") + .unwrap(); + assert!( + registry + .signal("request.controlled", &foreign.token()) + .is_none() + ); + drop(lease); + let retry = registry.register("request.controlled").unwrap(); + assert!( + registry + .signal("request.controlled", &retry.token()) + .is_none() + ); + assert!(!retry.token().is_cancelled()); + } + + #[test] + fn daemon_cancel_reaches_the_controlled_signal_and_preserves_request_isolation() { + let registry = RequestCancellationRegistryV1::default(); + let signal = CancellationSignal::active("cancel.controlled").unwrap(); + let lease = registry + .register_with_signal("request.controlled", signal.clone()) + .unwrap(); + let other_signal = CancellationSignal::active("cancel.other").unwrap(); + let other = registry + .register_with_signal("request.other", other_signal.clone()) + .unwrap(); + + assert!(registry.cancel("request.controlled")); + assert!(signal.is_cancelled()); + assert!(lease.token().is_cancelled()); + assert!(!other_signal.is_cancelled()); + assert!(!other.token().is_cancelled()); + } + + #[test] + fn pending_cancel_reaches_a_later_controlled_registration() { + let registry = RequestCancellationRegistryV1::default(); + assert!(!registry.cancel("request.controlled")); + let signal = CancellationSignal::active("cancel.controlled").unwrap(); + let lease = registry + .register_with_signal("request.controlled", signal.clone()) + .unwrap(); + assert!(signal.is_cancelled()); + assert!(lease.token().is_cancelled()); + } #[test] fn pre_registration_cancellation_is_retained_and_cleanup_is_exact() { diff --git a/crates/tracedecay/src/daemon/invocation_executor.rs b/crates/tracedecay/src/daemon/invocation_executor.rs index 87b343f20a..d886c35740 100644 --- a/crates/tracedecay/src/daemon/invocation_executor.rs +++ b/crates/tracedecay/src/daemon/invocation_executor.rs @@ -13,8 +13,8 @@ use tracedecay_tool_catalog::ApplicationSurfaceOperation; use tracedecay_application::operation_stream::OperationRequestControls; use tracedecay_daemon_protocol::DAEMON_TOOL_RESPONSE_GRACE; use tracedecay_daemon_service::{ - DaemonInvocationOperation, DaemonInvocationProblem, ProjectRuntimeRequestLeaseV1, - WorkApplicationOutcomeV1, + DaemonInvocationOperation, DaemonInvocationPayload, DaemonInvocationProblem, + ProjectRuntimeRequestLeaseV1, WorkApplicationOutcomeV1, }; use tracedecay_domain::errors::{Result, TraceDecayError}; use tracedecay_runtime_core::native_runtime; @@ -285,7 +285,7 @@ impl InProcessDaemonInvocationExecutor { &self.store_administration, Some(&self.project_path), request, - None, + self.admitted_cancellation.clone(), ) .await } @@ -659,10 +659,29 @@ impl tracedecay_daemon_protocol::DaemonInvocationExecutor for InProcessDaemonInv stage: CancellationStage::BeforeAdmission, }, )?; - let executor = self.clone(); + let mut executor = self.clone(); let admitted_cancellation = self.admitted_cancellation.clone(); let request_cancellations = self.invocation.service.request_cancellations().clone(); let unavailable = |_| tracedecay_daemon_protocol::DaemonInvocationError::Unavailable; + // The graph owner must observe the caller's live signal directly: + // relaying through two native tasks lets a blocking file walk + // continue after the transport has acknowledged cancellation. + let cancellation_lease = if executor.admitted_cancellation.is_none() + && matches!(&request.payload, DaemonInvocationPayload::GraphTool { .. }) + { + let Some(lease) = request_cancellations + .register_with_signal(&request.request_id, cancellation.clone()) + else { + return Ok(DaemonInvocationResponse::problem( + request.request_id, + DaemonInvocationProblem::InvalidRequest, + )); + }; + executor.admitted_cancellation = Some(lease.token()); + Some(lease) + } else { + None + }; match native_runtime::NativeTask::spawn(async move { let request_id = request.request_id.clone(); let invocation = @@ -671,7 +690,7 @@ impl tracedecay_daemon_protocol::DaemonInvocationExecutor for InProcessDaemonInv tracing::trace_span!("daemon.invocation.invoke_once"), )) .map_err(unavailable)?; - settle_in_process_invocation( + let result = settle_in_process_invocation( &request_cancellations, &request_id, invocation, @@ -681,7 +700,9 @@ impl tracedecay_daemon_protocol::DaemonInvocationExecutor for InProcessDaemonInv policy, DAEMON_TOOL_RESPONSE_GRACE, ) - .await + .await; + drop(cancellation_lease); + result }) .map_err(unavailable)? .await From 19dd56dcd3d9ba773da7c218552d77752dd6575e Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 20:59:25 +0000 Subject: [PATCH 30/89] test(observability): reserve space for every live carrier --- .../tests/application_suite/observability_runtime_contract.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-application/tests/application_suite/observability_runtime_contract.rs b/crates/tracedecay-application/tests/application_suite/observability_runtime_contract.rs index f76f1bec62..98425ae3e8 100644 --- a/crates/tracedecay-application/tests/application_suite/observability_runtime_contract.rs +++ b/crates/tracedecay-application/tests/application_suite/observability_runtime_contract.rs @@ -341,7 +341,8 @@ async fn live_queued_owner_claim_is_not_recovered_as_delayed_work() { configuration_revision: "configuration.owner.v1".into(), policy_revision: "policy.owner.v1".into(), }, - 2, + // Both ordinary carriers and the owner must fit before the native worker polls. + 3, ) .expect("producer"), ); @@ -351,7 +352,6 @@ async fn live_queued_owner_claim_is_not_recovered_as_delayed_work() { .expect("first queue carrier"), ObservabilityEmissionOutcomeV1::Enqueued ); - tokio::task::yield_now().await; assert_eq!( producer .try_emit(envelope(&scope, "caller", 73, 7_300_000)) From 1022db40abc16b7431e2198dd120ef96002da352 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 21:05:42 +0000 Subject: [PATCH 31/89] test(dashboard): bound Bazel workers and isolate auth streams --- dashboard/BUILD.bazel | 3 +- .../auth/authenticatedEventSource.dom.test.ts | 48 ++++++++++--------- 2 files changed, 28 insertions(+), 23 deletions(-) diff --git a/dashboard/BUILD.bazel b/dashboard/BUILD.bazel index 436abbc228..083bb8dd86 100644 --- a/dashboard/BUILD.bazel +++ b/dashboard/BUILD.bazel @@ -28,7 +28,8 @@ vitest_bin.vitest_test( "e2e/**", "stories/**", ]), - args = ["run"], + # Remote actions reserve one CPU; Vitest otherwise sizes its pool to the host. + args = ["run", "--maxWorkers=1"], chdir = package_name(), ) diff --git a/dashboard/src/data/auth/authenticatedEventSource.dom.test.ts b/dashboard/src/data/auth/authenticatedEventSource.dom.test.ts index 9ce6659a5c..ee01189d02 100644 --- a/dashboard/src/data/auth/authenticatedEventSource.dom.test.ts +++ b/dashboard/src/data/auth/authenticatedEventSource.dom.test.ts @@ -1,4 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { AuthenticatedEventSource } from './authenticatedEventSource.ts'; +import { installDashboardAuthentication } from './install.ts'; +import { connectEvents } from '../sse/connect.ts'; const TOKEN = 'cd'.repeat(32); const BASIC = `Basic ${btoa(`tracedecay:${TOKEN}`)}`; @@ -33,28 +36,28 @@ async function eventually(check: () => void): Promise { describe('AuthenticatedEventSource', () => { const fetchMock = vi.fn<(input: RequestInfo | URL, init?: RequestInit) => Promise>(); + const sources: AuthenticatedEventSource[] = []; beforeEach(() => { sessionStorage.setItem('tracedecay.dashboard.access', TOKEN); - vi.resetModules(); fetchMock.mockReset(); vi.stubGlobal('fetch', fetchMock); }); afterEach(() => { + for (const source of sources.splice(0)) source.close(); vi.unstubAllGlobals(); + vi.useRealTimers(); sessionStorage.clear(); }); - /** Loads the class, then constructs synchronously so handlers attach before - * the first awaited step, as `connectEvents` does. */ - async function opener(url = '/api/events') { - const { AuthenticatedEventSource } = await import('./authenticatedEventSource.ts'); - return () => new AuthenticatedEventSource(url); + function open(url = '/api/events') { + const source = new AuthenticatedEventSource(url); + sources.push(source); + return source; } it('authenticates the stream with a request header and no cookie', async () => { - fetchMock.mockResolvedValue(openResponse()); - const open = await opener(); + fetchMock.mockImplementation(async () => openResponse()); const source = open(); const opened = vi.fn(); source.onopen = opened; @@ -78,8 +81,7 @@ describe('AuthenticatedEventSource', () => { '\ndata: two\n\n', ]), ); - fetchMock.mockResolvedValue(openResponse()); - const open = await opener(); + fetchMock.mockImplementation(async () => openResponse()); const source = open(); const named = vi.fn(); const plain = vi.fn(); @@ -93,14 +95,18 @@ describe('AuthenticatedEventSource', () => { }); it('reconnects with Last-Event-ID and the server retry hint after the stream drops', async () => { + vi.useFakeTimers(); fetchMock.mockResolvedValueOnce(streamResponse(['retry: 1\nid: 7\ndata: x\n\n'])); - fetchMock.mockResolvedValue(openResponse()); - const open = await opener(); + fetchMock.mockImplementation(async () => openResponse()); const source = open(); const errors = vi.fn(); source.onerror = errors; - await eventually(() => expect(fetchMock).toHaveBeenCalledTimes(2)); + await vi.advanceTimersByTimeAsync(0); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(source.readyState).toBe(AuthenticatedEventSource.CONNECTING); + await vi.advanceTimersByTimeAsync(1); + expect(fetchMock).toHaveBeenCalledTimes(2); expect(new Headers(fetchMock.mock.calls[1]![1]!.headers).get('last-event-id')).toBe('7'); expect(errors).toHaveBeenCalled(); source.close(); @@ -108,7 +114,6 @@ describe('AuthenticatedEventSource', () => { it('closes for good on a refusal instead of retrying with the same credential', async () => { fetchMock.mockResolvedValue(new Response('{}', { status: 401 })); - const open = await opener(); const source = open(); const errors = vi.fn(); source.onerror = errors; @@ -120,8 +125,7 @@ describe('AuthenticatedEventSource', () => { }); it('close() aborts the in-flight request', async () => { - fetchMock.mockResolvedValue(openResponse()); - const open = await opener(); + fetchMock.mockImplementation(async () => openResponse()); const source = open(); await eventually(() => expect(fetchMock).toHaveBeenCalled()); @@ -133,24 +137,24 @@ describe('AuthenticatedEventSource', () => { }); describe('installDashboardAuthentication', () => { + let connection: ReturnType | undefined; afterEach(() => { + connection?.close(); vi.unstubAllGlobals(); sessionStorage.clear(); window.history.replaceState(null, '', '/'); }); it('routes the app event stream through an authenticated fetch', async () => { - vi.resetModules(); const fetchMock = vi.fn(async () => openResponse()); vi.stubGlobal('fetch', fetchMock); window.history.replaceState(null, '', `/#token=${TOKEN}`); - const { installDashboardAuthentication } = await import('./install.ts'); - const { connectEvents } = await import('../sse/connect.ts'); - + vi.stubGlobal('EventSource', globalThis.EventSource); installDashboardAuthentication(); - const connection = connectEvents('/api/events'); + const active = connectEvents('/api/events'); + connection = active; - await eventually(() => expect(connection.state()).toBe('live')); + await eventually(() => expect(active.state()).toBe('live')); const calls = fetchMock.mock.calls as unknown as Array<[string, RequestInit]>; expect(new Headers(calls[0]![1].headers).get('authorization')).toBe(BASIC); expect(window.location.hash).toBe(''); From f8c42c65183863527634af93a133b5f0baf2a548 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 21:32:33 +0000 Subject: [PATCH 32/89] fix(hooks): preserve lifecycle evidence during replay --- .../src/daemon/hook_v2_replay_consumer.rs | 178 +++++++++++++++++- .../intent_pass_journey_test.rs | 16 +- 2 files changed, 188 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs b/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs index 7d2e2e7ebc..607d4c45f0 100644 --- a/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs +++ b/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs @@ -209,10 +209,6 @@ async fn drain_all_hosts( Ok(more) => sweep.more_pending |= more, Err(cause) => sweep.fail(*host, StatusHookReplaySpoolV1::DeliveryReceipts, cause), } - Box::pin(redrive_pending_work( - graph, data_root, *host, project_id, now, &mut sweep, - )) - .await; let report = match Box::pin(drain_admitted_host_spool( *host, project_id, @@ -229,6 +225,11 @@ async fn drain_all_hosts( None } }; + // The host spool retains native lifecycle evidence the ledger cannot carry. + Box::pin(redrive_pending_work( + graph, data_root, *host, project_id, now, &mut sweep, + )) + .await; if let Some(report) = report && (report.committed > 0 || report.duplicates > 0 || report.tombstoned > 0) { @@ -295,6 +296,18 @@ async fn redrive_pending_work( .await }, |envelope, native_session_id| async move { + // A timed-out admission may have committed its ledger before its + // lifecycle. Only the richer host spool can finish that mount. + if native_session_id.is_none() + && tracedecay_hooks::NativeContextScoutLifecycleV1::host_mounts_lifecycle(host) + && matches!( + envelope.event, + tracedecay_hooks::HookEventV2::ToolLifecycle { .. } + | tracedecay_hooks::HookEventV2::SavedEdit { .. } + ) + { + return HookV2AdmissionOutcomeV1::Backpressured; + } admit_hook_v2_envelope(graph, &envelope, native_session_id, hook_replay_now()) .await }, @@ -655,6 +668,163 @@ mod tests { } } + #[tokio::test] + async fn pending_edit_waits_for_the_lifecycle_spool_before_settling() { + let (fixture, graph, _newer) = replay_owner_fixture().await; + let data_root = &graph.hook_store_layout().data_root; + let host = NativeHostIdentityV1::KimiCode; + let project_id = + tracedecay_agent_hosts::hooks::hook_project_id_for_layout(graph.hook_store_layout()) + .unwrap(); + let now = hook_replay_now(); + let binding = HookScopeBindingV1 { + host, + project_id, + repository_id: [2; 16], + worktree_id: [3; 16], + worktree_epoch: 1, + binding_token: [4; 32], + capabilities: vec![tracedecay_hooks::HookCapabilityV1 { + family: tracedecay_hooks::HookEventFamily::SavedEdit, + support: tracedecay_hooks::stock_event_support( + host, + tracedecay_hooks::HookEventFamily::SavedEdit, + ), + }], + }; + tracedecay_hooks::HookConfigurationFileWriterV1::new( + tracedecay_hooks::hook_configuration_path(data_root, binding.worktree_id, host), + ) + .publish(tracedecay_hooks::HookConfigurationSnapshotV1 { + schema_version: tracedecay_hooks::HOOK_CONFIGURATION_SCHEMA_VERSION, + revision: 1, + published_at: now, + expires_at: UtcMicros(now.0 + 60_000_000), + binding: binding.clone(), + }) + .unwrap(); + let lifecycle = tracedecay_hooks::NativeContextScoutLifecycleV1::new( + "session.replay.edit", + "call.replay.edit", + [8; 16], + ) + .unwrap(); + let envelope = tracedecay_hooks::HookEventEnvelopeV2 { + schema_version: tracedecay_hooks::HOOK_EVENT_SCHEMA_VERSION, + event_id: lifecycle.event_id, + producer: host, + protected_session_id: tracedecay_agent_hosts::hooks::protected_native_session_id( + lifecycle.session_id.as_str(), + ), + project_id, + repository_id: binding.repository_id, + worktree_id: binding.worktree_id, + worktree_epoch: binding.worktree_epoch, + binding_token: binding.binding_token, + ordering: tracedecay_hooks::HookOrderingV1::Unknown, + observed_at: now, + event: tracedecay_hooks::HookEventV2::SavedEdit { + file_id: [9; 16], + changed_range_count: 1, + }, + }; + // Commit admission while no orchestrator is mounted, reproducing the + // durable state left by a caller cancelled before lifecycle persistence. + assert!(matches!( + admit_hook_v2_envelope(&graph, &envelope, None, now).await, + HookV2AdmissionOutcomeV1::Admitted { .. } + )); + let pending = + || hook_v2_pending_work_envelopes(data_root, host, hook_replay_now()).unwrap(); + assert_eq!(pending(), vec![envelope.clone()]); + assert_eq!( + tracedecay_daemon_service::context_scout_lifecycle::register_context_scout_lifecycle_authority( + project_id, + binding.worktree_id, + tracedecay_domain::ProjectId::new("project.replay.owner").unwrap(), + tracedecay_domain::WorktreeId::new("worktree.replay.edit").unwrap(), + &fixture.project_sessions, + ), + tracedecay_daemon_service::context_scout_lifecycle::AuthorityRegistrationV1::Registered + ); + let (ran, mut runs) = tokio::sync::mpsc::unbounded_channel(); + let runtime = tracedecay_daemon_service::BoundedHookOrchestratorV1::new( + 1, + move |request: tracedecay_daemon_service::HookOrchestrationRequestV1, _| { + let ran = ran.clone(); + async move { ran.send(request.lifecycle).unwrap() } + }, + ) + .unwrap(); + assert!( + tracedecay_daemon_service::register_hook_orchestration_runtime( + project_id, + binding.worktree_id, + &runtime, + ) + ); + let sweep = drain_all_hosts( + &graph, + data_root, + &fixture.settlements, + &fixture.project_sessions, + &fixture.background_cpu, + ) + .await; + assert!( + sweep.failures.iter().any(|failure| failure.host == host + && failure.spool == StatusHookReplaySpoolV1::PendingWork) + ); + assert_eq!(pending(), vec![envelope.clone()]); + assert!( + runs.try_recv().is_err(), + "ledger replay must not execute without lifecycle" + ); + + let root = hook_v2_spool_root(data_root, host); + let (mut spool, _) = HookSpoolV1::open(&root, HookSpoolConfigV1::stock(host), now).unwrap(); + spool + .append_with_native_lifecycle(envelope.clone(), Some(lifecycle.clone()), &binding, now) + .unwrap(); + drop(spool); + drain_all_hosts( + &graph, + data_root, + &fixture.settlements, + &fixture.project_sessions, + &fixture.background_cpu, + ) + .await; + let mounted = tokio::time::timeout(Duration::from_secs(10), runs.recv()) + .await + .unwrap() + .unwrap() + .expect("spool replay must mount lifecycle"); + assert_eq!(mounted.session_id, lifecycle.session_id); + tokio::time::timeout(Duration::from_secs(10), async { + while !pending().is_empty() { + tokio::task::yield_now().await; + } + }) + .await + .expect("mounted work must settle its ledger obligation"); + assert!(runtime.shutdown().await); + assert!( + pending().is_empty(), + "completed lifecycle work settles its durable obligation" + ); + assert!(!HookSpoolV1::has_records(&root).unwrap()); + assert!(runs.try_recv().is_err(), "the edit executes exactly once"); + tracedecay_daemon_service::unregister_hook_orchestration_runtime( + project_id, + binding.worktree_id, + &runtime, + ); + assert!(tracedecay_daemon_service::context_scout_lifecycle::unregister_context_scout_lifecycle_authority( + project_id, binding.worktree_id, &fixture.project_sessions, + )); + } + #[test] fn only_a_missing_spool_root_is_absent() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/tracedecay/tests/hooks_lsp_suite/intent_pass_journey_test.rs b/crates/tracedecay/tests/hooks_lsp_suite/intent_pass_journey_test.rs index 5ebc0200c3..99761dfb34 100644 --- a/crates/tracedecay/tests/hooks_lsp_suite/intent_pass_journey_test.rs +++ b/crates/tracedecay/tests/hooks_lsp_suite/intent_pass_journey_test.rs @@ -10,6 +10,7 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use serde_json::{Value, json}; +use tracedecay_contracts::retrieval::StatusHookReplayV1; use tracedecay_domain::configuration::{ CONTEXT_SCOUT_SETTINGS_SETTING_KEY, ContextScoutConfigurationStateV1, ContextScoutIntentKindV1, ContextScoutIntentSettingsV1, ContextScoutSettingsV1, DecisionEngineSettingsV1, @@ -501,8 +502,19 @@ fn claude_stop_delivers_the_intent_suggestion_the_daemon_judged_at_the_tool_boun let stop_event = stop(&project, ENABLED_SESSION, &enabled_transcript); let delivers = |context: &String| context.contains(&suggestion_text(YES_INTENT)); - let mut stop_contexts = Vec::new(); - while stop_contexts.len() < 5 && !stop_contexts.last().is_some_and(delivers) { + let mut stop_contexts = vec![hook_context(&run_hook( + home, + &project, + "hook-stop", + &stop_event, + ))]; + if !stop_contexts.last().is_some_and(delivers) { + wait_for("the timed-out stop callback's replay to drain", || { + let status = run_tool(home, &project, "tracedecay_status", &json!({})); + let replay: StatusHookReplayV1 = serde_json::from_value(status["hook_replay"].clone()) + .unwrap_or_else(|error| panic!("hook replay status ({error}): {status}")); + replay == StatusHookReplayV1::Drained && intent_outcomes(&log).len() > before + 1 + }); stop_contexts.push(hook_context(&run_hook( home, &project, From 4c0e1fc6a7594992a1f24641784bf10b0e5009aa Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 21:49:08 +0000 Subject: [PATCH 33/89] fix(sessions): preserve canonical message directory scopes --- .../src/observation_projection/apply.rs | 21 +- .../src/observation_projection/rebuild.rs | 107 ++++++- .../src/observation_projection/state.rs | 24 +- .../src/schema_contract/invariants/audit.rs | 25 +- .../invariants/released_rendering.rs | 52 +++- .../src/session_project_rebind_tests.rs | 263 +++++++++++++++++- crates/tracedecay-lcm/src/raw.rs | 28 +- .../raw/ingest_protection_defaults_tests.rs | 125 +++++++-- crates/tracedecay-lcm/src/schema.rs | 69 +++++ .../src/runtime/store_access/sessions.rs | 32 ++- 10 files changed, 674 insertions(+), 72 deletions(-) diff --git a/crates/tracedecay-global-db/src/observation_projection/apply.rs b/crates/tracedecay-global-db/src/observation_projection/apply.rs index 55a0a41ba6..6151ce1d92 100644 --- a/crates/tracedecay-global-db/src/observation_projection/apply.rs +++ b/crates/tracedecay-global-db/src/observation_projection/apply.rs @@ -23,9 +23,10 @@ use tracedecay_runtime_core::db::{ use tracedecay_sessions::runtime::store_access::find_preceding_codex_goal_response; use super::state::{ - canonicalize_session_project_paths, message_projection, output_ownership_sql, read_message, - read_observation, read_output_state, read_session, reconcile_session_rows_detailed, storage, - storage_message, stored_output_digest, verify_output_state, verify_projection_rows, + canonical_message_location, canonicalize_session_project_paths, message_projection, + output_ownership_sql, read_message, read_observation, read_output_state, read_session, + reconcile_session_rows_detailed, storage, storage_message, stored_output_digest, + verify_output_state, verify_projection_rows, }; use super::transition::{ MessageTransition, MessageTransitionState, WorkflowFactTarget, WorkflowFactTransition, @@ -1065,7 +1066,8 @@ async fn upsert_projected_raw_message( conn: &impl Executor, message: &SessionMessageRecord, ) -> ProjectionStoreResult<()> { - tracedecay_lcm::raw::upsert_projection_raw_message(conn, message) + let location = canonical_message_location(message)?; + tracedecay_lcm::raw::upsert_projection_raw_message(conn, message, location.as_deref()) .await .map_err(|error| match error { LcmError::SanitizationRefused { @@ -1154,9 +1156,9 @@ pub(super) async fn retire_replaced_codex_output( /// Replaces the stored output row with the one this binary derives. /// -/// The projected message row is derived state, so every field but its identity -/// is rewritten from the record. A projector that owns the output also owns the -/// row's session, so the row is first moved onto the projection's session. +/// The projected rendering is derived state. The canonical origin is captured +/// once and survives rendering repairs. A projector that owns the output also +/// owns the row's session, so the row is first moved onto that session. /// A Hermes body belongs to the Hermes LCM turn authority: an existing Hermes /// row keeps its body and takes only the projection's session columns. Shared /// with released-rendering convergence, which reaches the same row through a @@ -1167,11 +1169,13 @@ pub(super) async fn supersede_projected_message( ) -> ProjectionStoreResult<()> { adopt_owned_projection_raw_session(conn, message).await?; if message.provider == "hermes" { + let location = canonical_message_location(message)?; let updated = conn .execute( "UPDATE lcm_raw_messages SET role = ?3, timestamp = ?4, ordinal = ?5, kind = ?6, model = ?7, - tool_names = ?8, source_path = ?9, source_offset = ?10 + tool_names = ?8, source_path = ?9, source_offset = ?10, + location_path = COALESCE(location_path, ?11) WHERE provider = ?1 AND message_id = ?2", params![ message.provider.as_str(), @@ -1184,6 +1188,7 @@ pub(super) async fn supersede_projected_message( message.tool_names.as_deref(), message.source_path.as_deref(), message.source_offset, + location.as_deref(), ], ) .await diff --git a/crates/tracedecay-global-db/src/observation_projection/rebuild.rs b/crates/tracedecay-global-db/src/observation_projection/rebuild.rs index 1234a622ea..773df53880 100644 --- a/crates/tracedecay-global-db/src/observation_projection/rebuild.rs +++ b/crates/tracedecay-global-db/src/observation_projection/rebuild.rs @@ -23,10 +23,11 @@ use super::apply::{ derive_projection_with_alias, stage_provider_usage_effects, verify_effect, }; use super::state::{ - canonicalize_session_project_paths, consume_projection_queue_item, decode_observation_row, - decode_sequence, projected_stored_message, projection_retry_state, queued_sequence, - read_checkpoint, read_message, read_observation, read_session, reconcile_session_rows_detailed, - schedule_projection_retry, storage, storage_message, stored_output_digest, write_checkpoint, + canonical_message_location, canonicalize_session_project_paths, consume_projection_queue_item, + decode_observation_row, decode_sequence, projected_stored_message, projection_retry_state, + queued_sequence, read_checkpoint, read_message, read_observation, read_session, + reconcile_session_rows_detailed, schedule_projection_retry, storage, storage_message, + stored_output_digest, write_checkpoint, }; use super::transition::{ MessageTransition, MessageTransitionState, WorkflowFactTarget, WorkflowFactTransition, @@ -947,9 +948,13 @@ async fn activate_projection_rebuild_transaction( "projection rebuild generation is incomplete", )); } + retire_projection_predecessor_state(transaction, &job.generation).await?; + // Classify active outputs before clearing their current ownership. A + // compatible protected rendering can differ from the rebuilt row while + // still belonging to this projector; foreign outputs remain collisions. + prepare_rebuild_output_activation(transaction, &job.generation).await?; clear_active_projection(transaction, &job.generation).await?; activate_rebuild_sessions(transaction, &job.generation).await?; - prepare_rebuild_output_activation(transaction, &job.generation).await?; activate_rebuild_messages(transaction, &job.generation).await?; activate_rebuild_provenance(transaction, &job.generation).await?; activate_rebuild_workflow_facts(transaction, &job.generation).await?; @@ -1490,7 +1495,17 @@ async fn write_staged_message( generation: &str, message: &SessionMessageRecord, ) -> ProjectionStoreResult<()> { - let json = encode_json(message, "encode staged projection message")?; + #[derive(serde::Serialize)] + struct StagedMessage<'a> { + #[serde(flatten)] + message: &'a SessionMessageRecord, + location_path: Option, + } + let staged = StagedMessage { + message, + location_path: canonical_message_location(message)?, + }; + let json = encode_json(&staged, "encode staged projection message")?; let content_hash = projected_content_hash(&message.text); conn.execute( "INSERT INTO observation_projection_rebuild_messages ( @@ -1956,7 +1971,6 @@ async fn clear_active_projection( conn: &impl Executor, generation: &str, ) -> ProjectionStoreResult<()> { - retire_projection_predecessor_state(conn, generation).await?; conn.execute_batch( "CREATE TEMP TABLE IF NOT EXISTS observation_projection_rebuild_retained_outputs ( output_provider TEXT NOT NULL, @@ -2519,23 +2533,26 @@ async fn activate_rebuild_messages( .join(",\n "); let text = json_extract_expr(MESSAGE_JSON_COLUMN, "text"); let metadata = json_extract_expr(MESSAGE_JSON_COLUMN, "metadata_json"); + let location = json_extract_expr(MESSAGE_JSON_COLUMN, "location_path"); // A Hermes body belongs to the Hermes LCM turn authority, so an existing // Hermes row takes only the rebuilt session columns. conn.execute( &format!( "INSERT INTO lcm_raw_messages ( provider, message_id, {session_columns}, content, content_hash, storage_kind, - payload_ref, placeholder_text, metadata_json + payload_ref, placeholder_text, metadata_json, location_path ) SELECT output_provider, output_message_id, {session_extracts}, - {text}, content_hash, 'inline', NULL, NULL, {metadata} + {text}, content_hash, 'inline', NULL, NULL, {metadata}, {location} FROM observation_projection_rebuild_messages WHERE projector_version = ?1 AND generation = ?2 ON CONFLICT(provider, message_id) DO UPDATE SET {session_updates}, - {body_updates} + {body_updates}, + location_path = COALESCE(lcm_raw_messages.location_path, excluded.location_path) WHERE {session_changed} + OR (lcm_raw_messages.location_path IS NULL AND excluded.location_path IS NOT NULL) OR (lcm_raw_messages.provider <> 'hermes' AND ( COALESCE(lcm_raw_messages.content, lcm_raw_messages.placeholder_text, '') IS NOT excluded.content @@ -2650,12 +2667,13 @@ async fn activate_rebuild_dispositions( #[allow(clippy::unwrap_used)] mod activation_tests { use super::{ - REBUILD_PAGE_SIZE, SESSION_MESSAGE_PROJECTOR_VERSION, activate_rebuild_sessions, - reconcile_overlapping_rebuild_sessions, + REBUILD_PAGE_SIZE, SESSION_MESSAGE_PROJECTOR_VERSION, + activate_projection_rebuild_transaction, activate_rebuild_sessions, + reconcile_overlapping_rebuild_sessions, write_staged_message, }; use crate::tests::harness::RegisteredGlobalDbHarness; use tracedecay_runtime_core::db::engine::{Executor, params}; - use tracedecay_store::{ProjectionStoreError, SessionRecord}; + use tracedecay_store::{ProjectionStoreError, SessionMessageRecord, SessionRecord}; const SESSION_ID: &str = "002bd803-dc62-46e2-b66a-a61cc282f0dc"; /// One past the exact-SQL transport's `MAX_QUERY_ROWS` @@ -2775,6 +2793,69 @@ mod activation_tests { ); } + #[asupersync::test] + async fn activation_refuses_an_unowned_message_and_rolls_back() { + const GENERATION: &str = "generation.foreign-output"; + let harness = RegisteredGlobalDbHarness::open("foreign-rebuild-output").await; + let active = session(None, None); + assert!(harness.registered.upsert_session(&active).await); + let foreign = SessionMessageRecord { + provider: active.provider.clone(), + session_id: active.session_id.clone(), + message_id: "message.foreign".to_owned(), + role: "assistant".to_owned(), + timestamp: Some(1), + ordinal: 1, + text: "retained foreign body".to_owned(), + kind: None, + model: None, + tool_names: None, + source_path: None, + source_offset: None, + metadata_json: None, + }; + let transaction = harness.registered.begin_write_transaction().await.unwrap(); + tracedecay_lcm::raw::upsert_projection_raw_message( + &transaction, + &foreign, + Some("/foreign"), + ) + .await + .unwrap(); + transaction.commit().await.unwrap(); + let transaction = harness.registered.begin_write_transaction().await.unwrap(); + stage(&transaction, GENERATION, &active).await; + let rebuilt = SessionMessageRecord { + text: "unrelated rebuilt body".to_owned(), + ..foreign.clone() + }; + write_staged_message(&transaction, GENERATION, &rebuilt) + .await + .unwrap(); + let error = activate_projection_rebuild_transaction(&transaction, 0) + .await + .unwrap_err(); + assert!( + matches!(error, ProjectionStoreError::OutputCollision { provider, message_id } if provider == foreign.provider && message_id == foreign.message_id) + ); + transaction.rollback().await.unwrap(); + let snapshot = harness.registered.read_snapshot().await.unwrap(); + let mut rows = snapshot.query("SELECT content, location_path FROM lcm_raw_messages WHERE provider = ?1 AND message_id = ?2", params![foreign.provider.as_str(), foreign.message_id.as_str()]).await.unwrap(); + let row = rows.next().await.unwrap().unwrap(); + assert_eq!(row.get::(0).unwrap(), foreign.text); + assert_eq!(row.get::(1).unwrap(), "/foreign"); + drop(rows); + let mut rows = snapshot + .query("SELECT COUNT(*) FROM observation_projection_rebuilds", ()) + .await + .unwrap(); + assert_eq!( + rows.next().await.unwrap().unwrap().get::(0).unwrap(), + 0, + "failed activation staging rolls back with the transaction" + ); + } + #[asupersync::test] async fn activation_names_the_session_field_instead_of_a_message_collision() { let harness = RegisteredGlobalDbHarness::open("session-collision-field").await; diff --git a/crates/tracedecay-global-db/src/observation_projection/state.rs b/crates/tracedecay-global-db/src/observation_projection/state.rs index c5884d47d5..720fe3b53a 100644 --- a/crates/tracedecay-global-db/src/observation_projection/state.rs +++ b/crates/tracedecay-global-db/src/observation_projection/state.rs @@ -769,6 +769,7 @@ pub(in super::super) struct ProjectionStorageColumns { pub(in super::super) content_hash: String, pub(in super::super) snippet_text: String, pub(in super::super) index_text: String, + pub(in super::super) location_path: Option, } pub(in super::super) struct ProjectionRowsBatch { @@ -825,7 +826,8 @@ pub(in super::super) async fn read_projection_rows_batch( .map_err(|error| storage("encode projected message request", error))?; let sql = format!( "SELECT {}, message.storage_kind, COALESCE(message.content, ''), - message.content_hash, message.snippet_text, message.index_text + message.content_hash, message.snippet_text, message.index_text, + message.location_path FROM json_each(?1) AS requested CROSS JOIN lcm_raw_messages AS message WHERE message.provider = json_extract(requested.value, '$.provider') @@ -860,6 +862,9 @@ pub(in super::super) async fn read_projection_rows_batch( content_hash: decode(21)?, snippet_text: decode(22)?, index_text: decode(23)?, + location_path: row + .get(24) + .map_err(|error| storage("decode projected message location", error))?, }; let key = (message.provider.clone(), message.message_id.clone()); storage_columns.insert(key.clone(), columns); @@ -1082,6 +1087,23 @@ pub(in super::super) async fn resolve_output_projection( Ok(owner_projection) } +/// Canonical location captured at the storage write boundary, outside the +/// immutable provider rendering and its provenance digest. +pub(in super::super) fn canonical_message_location( + message: &SessionMessageRecord, +) -> ProjectionStoreResult> { + let Some(json) = message.metadata_json.as_deref() else { + return Ok(None); + }; + let metadata: serde_json::Value = serde_json::from_str(json) + .map_err(|error| storage("decode projected message location", error))?; + Ok(metadata + .get(format!("{}_session_cwd", message.provider)) + .and_then(serde_json::Value::as_str) + .filter(|path| !path.is_empty()) + .map(durable_project_path_key)) +} + /// Normalize projection rows through the same authority as runtime session writes /// and project-scoped reads. Host/display spellings are not durable identity. /// Reconciliation remains pure over the normalized stored strings. diff --git a/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs b/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs index 51c336d4d3..68ff08dce1 100644 --- a/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs +++ b/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs @@ -9,6 +9,7 @@ use tracedecay_store::{ stored_message_is_shipped_release_rendering, }; +use crate::observation_projection::state::canonical_message_location; use crate::observation_projection::{ProjectionOutputAuthority, ProjectionRowsBatch}; use tracedecay_runtime_core::db::engine::{Executor, QueryExecutor, params}; @@ -20,7 +21,8 @@ use super::rows::{audit_read_error, authority_violation, decode_authority_observ use super::{AUDIT_PAGE_ROWS, INCOMPLETE_EXHAUSTIVE_PASS, projection_checkpoint}; const AUDIT_NAME: &str = "observation-authority"; -const AUDIT_VERSION: i64 = 2; +// Storage locations were added without changing immutable output digests. +const AUDIT_VERSION: i64 = 3; pub(super) const MAX_BOUNDED_AUDIT_PASSES: i64 = 64; const DETAILED_AUDIT_CONCURRENCY: usize = 32; // Amortize the page query across several bounded validation chunks. @@ -729,8 +731,8 @@ async fn validate_message_projection_row( // Message equality is not the whole output. The row's hash and // storage kind are not compared by the message record, so a // matching message can still sit on stale storage columns. - // Protected rows are not this arm: they differ from the - // projected row, and that compatibility already checked them. + // A compatible protected body retains its own rendering; + // backfill only its missing storage location below. let stored_matches = match resolved .projection_rows .message(&owner_message.provider, &owner_message.message_id) @@ -746,6 +748,23 @@ async fn validate_message_projection_row( } None => false, }; + // Once normalized, a location remains historical identity even + // when the provider's original symlink is moved or removed. + if resolved + .projection_rows + .storage_columns(&owner_message.provider, &owner_message.message_id) + .is_some_and(|raw| raw.location_path.is_none()) + && let Some(location) = + canonical_message_location(owner_message).map_err(|error| { + authority_violation(format!( + "invalid projected message location: {error}" + )) + })? + { + resolved + .released + .record_location(&owner_projection, location); + } if stored_matches && owned_storage_columns_need_rewrite( &owner_projection, diff --git a/crates/tracedecay-global-db/src/schema_contract/invariants/released_rendering.rs b/crates/tracedecay-global-db/src/schema_contract/invariants/released_rendering.rs index eabd20eacb..30ca077638 100644 --- a/crates/tracedecay-global-db/src/schema_contract/invariants/released_rendering.rs +++ b/crates/tracedecay-global-db/src/schema_contract/invariants/released_rendering.rs @@ -62,7 +62,7 @@ use std::sync::Mutex; -use tracedecay_runtime_core::db::engine::Executor; +use tracedecay_runtime_core::db::engine::{Executor, params}; use tracedecay_store::{ ProjectionStoreError, SESSION_MESSAGE_PROJECTOR_VERSION, SessionMessageProjection, message_output_digest, @@ -94,9 +94,23 @@ pub(super) enum StoredProvenanceRendering { #[derive(Default)] pub(super) struct ReleasedRenderingLedger { outputs: Mutex>, + locations: Mutex>, } impl ReleasedRenderingLedger { + /// Storage-only backfill for a verified row whose protected rendering is current. + pub(super) fn record_location(&self, projection: &SessionMessageProjection, location: String) { + let message = projection.message(); + self.locations + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .push(( + message.provider.clone(), + message.message_id.clone(), + location, + )); + } + pub(super) fn record(&self, projection: &SessionMessageProjection) { let mut outputs = self .outputs @@ -127,8 +141,36 @@ impl ReleasedRenderingLedger { .lock() .unwrap_or_else(std::sync::PoisonError::into_inner), ); + let locations = std::mem::take( + &mut *self + .locations + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner), + ); + if !locations.is_empty() { + let encoded = serde_json::to_string(&locations).map_err(|error| { + authority_violation(format!( + "failed to encode projected message locations: {error}" + )) + })?; + conn.execute( + "UPDATE lcm_raw_messages AS message + SET location_path = json_extract(location.value, '$[2]') + FROM json_each(?1) AS location + WHERE message.provider = json_extract(location.value, '$[0]') + AND message.message_id = json_extract(location.value, '$[1]') + AND message.location_path IS NULL", + params![encoded.as_str()], + ) + .await + .map_err(|error| { + authority_violation(format!( + "failed to backfill projected message locations: {error}" + )) + })?; + } if outputs.is_empty() { - return Ok(0); + return Ok(locations.len()); } // Retiring a quarantined observation removes every output it created, // so its remaining recorded outputs are already converged; re-running @@ -176,7 +218,7 @@ impl ReleasedRenderingLedger { retired = retired.len(), "converged released projection output renderings to this binary's rendering" ); - Ok(outputs.len()) + Ok(outputs.len() + locations.len()) } } @@ -533,7 +575,7 @@ mod tests { message.session_id, session.session_id, "the fixture's released output must belong to its released session" ); - tracedecay_lcm::raw::upsert_projection_raw_message(conn, &message) + tracedecay_lcm::raw::upsert_projection_raw_message(conn, &message, None) .await .expect("restore the released message row"); conn.execute( @@ -967,7 +1009,7 @@ mod tests { ) .await .expect("install the released session row"); - tracedecay_lcm::raw::upsert_projection_raw_message(conn, &message) + tracedecay_lcm::raw::upsert_projection_raw_message(conn, &message, None) .await .expect("the released rendering must still be servable by this binary's sanitizer"); let digest = stored_release_digest(&session, &message); diff --git a/crates/tracedecay-global-db/src/session_project_rebind_tests.rs b/crates/tracedecay-global-db/src/session_project_rebind_tests.rs index 6ee48632b2..b87f65b268 100644 --- a/crates/tracedecay-global-db/src/session_project_rebind_tests.rs +++ b/crates/tracedecay-global-db/src/session_project_rebind_tests.rs @@ -745,6 +745,16 @@ async fn session_seen_from_another_cwd_projects_every_message() { let second_cwd = tmp.path().join("repo-worktree"); std::fs::create_dir_all(&first_cwd).unwrap(); std::fs::create_dir_all(&second_cwd).unwrap(); + #[cfg(unix)] + let (first_observed, second_observed) = { + let first_alias = tmp.path().join("first-alias"); + let second_alias = tmp.path().join("second-alias"); + std::os::unix::fs::symlink(&first_cwd, &first_alias).unwrap(); + std::os::unix::fs::symlink(&second_cwd, &second_alias).unwrap(); + (first_alias, second_alias) + }; + #[cfg(not(unix))] + let (first_observed, second_observed) = (first_cwd.clone(), second_cwd.clone()); let runtime = HostAdmissionTestRuntimeV1::profile(&profile).await.unwrap(); let database = runtime .registered_database(HostAdmissionScope::Profile) @@ -758,17 +768,17 @@ async fn session_seen_from_another_cwd_projects_every_message() { ( "record.codex.cwd-a", "message from the first cwd", - &first_cwd, + &first_observed, ), ( "record.codex.cwd-b-1", "message from the second cwd", - &second_cwd, + &second_observed, ), ( "record.codex.cwd-b-2", "another message from the second cwd", - &second_cwd, + &second_observed, ), ] { let write = observation_from_source( @@ -784,12 +794,25 @@ async fn session_seen_from_another_cwd_projects_every_message() { ids.push(write.observation_id().clone()); persist(&store, write).await; } + let legacy = observation_from_source( + "codex", + CODEX_SESSION, + Some("rollout.legacy"), + ObservationScopeV1::Profile, + "record.codex.legacy", + "legacy message without directory evidence", + "receipt.codex.legacy", + None, + ); + ids.push(legacy.observation_id().clone()); + persist(&store, legacy).await; drain_projection_queue(&store).await; let texts = [ "message from the first cwd", "message from the second cwd", "another message from the second cwd", + "legacy message without directory evidence", ]; let expected_path = durable_project_path_key(&first_cwd.to_string_lossy()); for id in &ids { @@ -807,6 +830,96 @@ async fn session_seen_from_another_cwd_projects_every_message() { } let (_, project_path) = session_project_binding(database, "codex", CODEX_SESSION).await; assert_eq!(project_path, expected_path); + assert_cwd_scoped_messages(database, &first_cwd, &second_cwd).await; + assert_cwd_scoped_messages(database, &first_observed, &second_observed).await; + + // Protection re-ingest owns the body, while projection retains location. + let mut message = database + .search_session_messages("codex", Some(second_cwd.to_str().unwrap()), "message", 1) + .await + .unwrap() + .remove(0) + .message; + let mut metadata: Value = + serde_json::from_str(message.metadata_json.as_deref().unwrap()).unwrap(); + metadata["protection_rescan"] = json!("retained provider metadata"); + message.metadata_json = Some(metadata.to_string()); + let storage_root = database.db_path().parent().unwrap(); + let mut rollback = + tracedecay_lcm::payload::PayloadFileRollback::begin_cancellation_safe(storage_root); + let transaction = database + .runtime_database() + .begin_write_transaction("protect a projected body") + .await + .unwrap(); + tracedecay_lcm::raw::upsert_raw_message_with_payload_tracked( + &transaction, + storage_root, + &message, + &mut rollback, + ) + .await + .unwrap(); + transaction.commit().await.unwrap(); + assert_cwd_scoped_messages(database, &first_cwd, &second_cwd).await; + + // The shipped v15 schema lacked the storage-only location. Opening that + // shape preserves rows and lets the bounded authority audit fill locations. + crate::schema_contract::ensure_authority_invariants(database.runtime_database(), false, false) + .await + .unwrap(); + let immutable_before = message_renderings(database).await; + let transaction = database + .runtime_database() + .begin_write_transaction("restore released raw message shape") + .await + .unwrap(); + transaction + .execute_batch( + "ALTER TABLE lcm_raw_messages DROP COLUMN location_path; + UPDATE authority_audit_checkpoints SET audit_version = 2;", + ) + .await + .unwrap(); + transaction.commit().await.unwrap(); + drop(store); + drop(runtime); + let runtime = HostAdmissionTestRuntimeV1::profile(&profile).await.unwrap(); + let database = runtime + .registered_database(HostAdmissionScope::Profile) + .unwrap(); + crate::schema_contract::ensure_authority_invariants(database.runtime_database(), false, false) + .await + .unwrap(); + assert_eq!(message_renderings(database).await, immutable_before); + assert_cwd_scoped_messages(database, &first_cwd, &second_cwd).await; + assert_cwd_scoped_messages(database, &first_observed, &second_observed).await; + + let store = runtime + .observation_store(HostAdmissionScope::Profile) + .unwrap(); + + // A prior capture can know an origin a later rendering no longer carries. + // Rebuild must preserve that materialized identity for the same message. + let legacy_message = database + .search_session_messages("codex", Some("user"), "legacy", 1) + .await + .unwrap() + .remove(0) + .message; + let transaction = database + .runtime_database() + .begin_write_transaction("retain earlier message origin") + .await + .unwrap(); + tracedecay_lcm::raw::upsert_projection_raw_message( + &transaction, + &legacy_message, + Some(&expected_path), + ) + .await + .unwrap(); + transaction.commit().await.unwrap(); // A rebuild replays the same records through the staging path. let frontier = store.projection_checkpoint().await.unwrap().last_sequence(); @@ -829,4 +942,148 @@ async fn session_seen_from_another_cwd_projects_every_message() { } let (_, project_path) = session_project_binding(database, "codex", CODEX_SESSION).await; assert_eq!(project_path, expected_path); + let snapshot = database.read_snapshot().await.unwrap(); + let mut rows = snapshot.query("SELECT location_path FROM lcm_raw_messages WHERE provider = 'codex' AND message_id = ?1", params![legacy_message.message_id.as_str()]).await.unwrap(); + assert_eq!( + rows.next() + .await + .unwrap() + .unwrap() + .get::(0) + .unwrap(), + expected_path, + "a rebuild without cwd evidence retains the captured origin" + ); + drop(rows); + drop(snapshot); + assert_cwd_scoped_messages(database, &first_cwd, &second_cwd).await; + #[cfg(unix)] + { + let unrelated = tmp.path().join("unrelated"); + std::fs::create_dir_all(&unrelated).unwrap(); + std::fs::remove_file(&second_observed).unwrap(); + std::os::unix::fs::symlink(&unrelated, &second_observed).unwrap(); + crate::schema_contract::ensure_authority_invariants( + database.runtime_database(), + true, + false, + ) + .await + .unwrap(); + while !store + .rebuild_projection(frontier) + .await + .unwrap() + .is_complete() + {} + assert_cwd_scoped_messages(database, &first_cwd, &second_cwd).await; + assert_eq!( + database + .session_message_count_for_project(unrelated.to_str().unwrap()) + .await + .unwrap(), + 0 + ); + std::fs::remove_file(&second_observed).unwrap(); + crate::schema_contract::ensure_authority_invariants( + database.runtime_database(), + true, + false, + ) + .await + .unwrap(); + while !store + .rebuild_projection(frontier) + .await + .unwrap() + .is_complete() + {} + assert_cwd_scoped_messages(database, &first_cwd, &second_cwd).await; + } +} + +async fn assert_cwd_scoped_messages( + database: &RegisteredGlobalDb, + first_cwd: &std::path::Path, + second_cwd: &std::path::Path, +) { + for (cwd, expected) in [ + ( + first_cwd, + vec![ + "legacy message without directory evidence", + "message from the first cwd", + ], + ), + ( + second_cwd, + vec![ + "another message from the second cwd", + "message from the second cwd", + ], + ), + ] { + let selector = cwd.to_str().unwrap(); + assert_eq!( + database + .session_message_count_for_project(selector) + .await + .unwrap(), + 2, + "directory-scoped counts use each message's cwd, with the session fallback" + ); + let mut found = database + .search_session_messages("codex", Some(selector), "message", 10) + .await + .unwrap() + .into_iter() + .map(|result| result.message.text) + .collect::>(); + found.sort(); + assert_eq!( + found, expected, + "directory scopes must not leak other messages" + ); + } + assert_eq!( + database + .session_message_count_for_project("user") + .await + .unwrap(), + 4 + ); + assert_eq!( + database + .session_message_count_for_project("/unrelated") + .await + .unwrap(), + 0 + ); +} + +async fn message_renderings(database: &RegisteredGlobalDb) -> Vec<(String, String, String)> { + let snapshot = database.read_snapshot().await.unwrap(); + let mut rows = snapshot + .query( + "SELECT m.message_id, + json_object('content', m.content, 'content_hash', m.content_hash, + 'storage_kind', m.storage_kind, 'payload_ref', m.payload_ref, + 'placeholder_text', m.placeholder_text, 'metadata_json', m.metadata_json), + p.output_digest + FROM lcm_raw_messages m JOIN observation_projection_provenance p + ON p.output_provider = m.provider AND p.output_message_id = m.message_id + ORDER BY m.message_id", + (), + ) + .await + .unwrap(); + let mut values = Vec::new(); + while let Some(row) = rows.next().await.unwrap() { + values.push(( + row.get(0).unwrap(), + row.get(1).unwrap(), + row.get(2).unwrap(), + )); + } + values } diff --git a/crates/tracedecay-lcm/src/raw.rs b/crates/tracedecay-lcm/src/raw.rs index dad81e9f94..2b2dacb370 100644 --- a/crates/tracedecay-lcm/src/raw.rs +++ b/crates/tracedecay-lcm/src/raw.rs @@ -504,6 +504,7 @@ async fn upsert_inline_raw_message( message: &SessionMessageRecord, text: &str, metadata_json: Option<&str>, + location_path: Option<&str>, ) -> Result<(), LcmError> { let content_hash = projected_content_hash(text); if text.len() >= INLINE_BODY_BYTES { @@ -519,6 +520,7 @@ async fn upsert_inline_raw_message( payload_ref: None, placeholder: Some(placeholder.as_str()), metadata_json, + location_path, }, ) .await; @@ -533,6 +535,7 @@ async fn upsert_inline_raw_message( payload_ref: None, placeholder: None, metadata_json, + location_path, }, ) .await @@ -547,6 +550,8 @@ struct OwnedRawMessageWrite<'a> { /// index columns derive from it in place of `content`. placeholder: Option<&'a str>, metadata_json: Option<&'a str>, + /// The first captured canonical origin survives rendering and privacy rewrites. + location_path: Option<&'a str>, } async fn upsert_owned_raw_message( @@ -559,9 +564,9 @@ async fn upsert_owned_raw_message( "INSERT INTO lcm_raw_messages ( provider, message_id, session_id, role, ordinal, timestamp, content, content_hash, storage_kind, payload_ref, placeholder_text, - metadata_json, kind, model, tool_names, source_path, source_offset + metadata_json, kind, model, tool_names, source_path, source_offset, location_path ) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18) ON CONFLICT(provider, message_id) DO UPDATE SET session_id = excluded.session_id, role = excluded.role, @@ -577,7 +582,8 @@ async fn upsert_owned_raw_message( model = excluded.model, tool_names = excluded.tool_names, source_path = excluded.source_path, - source_offset = excluded.source_offset + source_offset = excluded.source_offset, + location_path = COALESCE(lcm_raw_messages.location_path, excluded.location_path) WHERE lcm_raw_messages.session_id = excluded.session_id", params![ message.provider.as_str(), @@ -597,6 +603,7 @@ async fn upsert_owned_raw_message( message.tool_names.as_deref(), message.source_path.as_deref(), message.source_offset, + write.location_path, ], ) .await?; @@ -854,13 +861,22 @@ fn externalized_payload_metadata( /// before it became durable, so the projected row stays inline; this binds a /// fresh content receipt to the stored text so the canonical raw-read /// authority (`load_raw_message`) can hydrate observation-projected -/// messages instead of refusing them as receipt-less rows. +/// messages instead of refusing them as receipt-less rows. The canonical +/// location is captured once; replay and missing cwd evidence retain it. pub async fn upsert_projection_raw_message( conn: &(impl Executor + ?Sized), message: &SessionMessageRecord, + location_path: Option<&str>, ) -> Result<(), LcmError> { let stored = projection_stored_message(message)?; - upsert_inline_raw_message(conn, message, &stored.text, stored.metadata_json.as_deref()).await + upsert_inline_raw_message( + conn, + message, + &stored.text, + stored.metadata_json.as_deref(), + location_path, + ) + .await } /// The message row exactly as [`upsert_projection_raw_message`] stores it and @@ -956,6 +972,7 @@ pub async fn commit_staged_raw_message( message, &staged.prepared.text, staged.prepared.metadata_json.as_deref(), + None, ) .await?; persist_raw_predecessor_range(conn, message).await?; @@ -975,6 +992,7 @@ pub async fn commit_staged_raw_message( payload_ref: Some(whole_message.payload_ref.payload_ref.as_str()), placeholder: Some(whole_message.placeholder.as_str()), metadata_json: Some(whole_message.metadata_json.as_str()), + location_path: None, }, ) .await?; diff --git a/crates/tracedecay-lcm/src/raw/ingest_protection_defaults_tests.rs b/crates/tracedecay-lcm/src/raw/ingest_protection_defaults_tests.rs index 04549a6bd4..0d0957a14b 100644 --- a/crates/tracedecay-lcm/src/raw/ingest_protection_defaults_tests.rs +++ b/crates/tracedecay-lcm/src/raw/ingest_protection_defaults_tests.rs @@ -129,38 +129,20 @@ async fn predecessor_range_skips_policy_anchor_roles() { let temp = tempfile::tempdir().expect("temporary directory"); let conn = TestConnection::open(&temp.path().join("sessions.db")); conn.execute_batch( - "CREATE TABLE lcm_raw_messages ( - store_id INTEGER PRIMARY KEY, + "CREATE TABLE sessions ( provider TEXT NOT NULL, - message_id TEXT NOT NULL, session_id TEXT NOT NULL, - role TEXT NOT NULL, - ordinal INTEGER NOT NULL, - timestamp INTEGER, - content TEXT, - content_hash TEXT NOT NULL, - storage_kind TEXT NOT NULL, - payload_ref TEXT, - placeholder_text TEXT, - metadata_json TEXT, - kind TEXT, - model TEXT, - tool_names TEXT, - source_path TEXT, - source_offset INTEGER, - UNIQUE(provider, message_id) - ); - CREATE TABLE lcm_raw_predecessor_ranges ( - provider TEXT NOT NULL, - message_id TEXT NOT NULL, - session_id TEXT NOT NULL, - from_store_id INTEGER NOT NULL, - to_store_id INTEGER NOT NULL, - PRIMARY KEY(provider, message_id) - );", + project_key TEXT NOT NULL, + project_path TEXT NOT NULL, + PRIMARY KEY(provider, session_id) + ); + INSERT INTO sessions VALUES ('claude', 'session-1', 'project', '/project');", ) .await - .expect("predecessor schema"); + .expect("session authority fixture"); + crate::schema::ensure_lcm_schema(&conn) + .await + .expect("canonical raw-message schema"); let storage_root = temp.path().join("storage"); let mut rollback = super::payload::PayloadFileRollback::begin_cancellation_safe(&storage_root); for (message_id, role, ordinal, text) in [ @@ -271,3 +253,90 @@ async fn message_record_readers_restore_compacted_bodies_and_reject_missing_byte Err(super::LcmError::PayloadIntegrityMismatch) ); } + +#[asupersync::test] +async fn privacy_rewrites_preserve_projection_location_authority() { + let temp = tempfile::tempdir().expect("temporary directory"); + let conn = TestConnection::open(&temp.path().join("sessions.db")); + conn.execute_batch( + "CREATE TABLE sessions ( + provider TEXT NOT NULL, session_id TEXT NOT NULL, + project_key TEXT NOT NULL, project_path TEXT NOT NULL, + PRIMARY KEY(provider, session_id) + ); + INSERT INTO sessions VALUES ('claude', 'session-1', 'project', '/first');", + ) + .await + .unwrap(); + crate::schema::ensure_lcm_schema(&conn).await.unwrap(); + let storage_root = temp.path().join("storage"); + std::fs::create_dir_all(&storage_root).unwrap(); + let mut rollback = super::payload::PayloadFileRollback::begin_cancellation_safe(&storage_root); + for (id, text, expected_storage) in [ + ("inline", "ordinary body".to_owned(), "inline"), + ("external", "repeated word ".repeat(10_000), "external"), + ] { + let message = SessionMessageRecord { + provider: "claude".to_owned(), + message_id: id.to_owned(), + session_id: "session-1".to_owned(), + role: "assistant".to_owned(), + timestamp: Some(1), + ordinal: 1, + text, + kind: None, + model: None, + tool_names: None, + source_path: None, + source_offset: None, + metadata_json: None, + }; + super::upsert_projection_raw_message(&*conn, &message, Some("/second")) + .await + .unwrap(); + // Replaying the same native message through a moved alias cannot + // replace its already materialized canonical origin. + super::upsert_projection_raw_message(&*conn, &message, Some("/retargeted")) + .await + .unwrap(); + super::upsert_raw_message_with_payload_tracked( + &*conn, + &storage_root, + &message, + &mut rollback, + ) + .await + .unwrap(); + let mut rows = conn + .query( + "SELECT location_path, storage_kind FROM lcm_raw_messages WHERE message_id = ?1", + tracedecay_runtime_core::db::engine::params![id], + ) + .await + .unwrap(); + let row = rows.next().await.unwrap().unwrap(); + assert_eq!(row.get::(0).unwrap(), "/second"); + assert_eq!(row.get::(1).unwrap(), expected_storage); + drop(rows); + super::upsert_projection_raw_message(&*conn, &message, None) + .await + .unwrap(); + let mut rows = conn + .query( + "SELECT location_path FROM lcm_raw_messages WHERE message_id = ?1", + tracedecay_runtime_core::db::engine::params![id], + ) + .await + .unwrap(); + assert_eq!( + rows.next() + .await + .unwrap() + .unwrap() + .get::>(0) + .unwrap(), + Some("/second".to_owned()), + "missing cwd evidence cannot erase a captured historical origin" + ); + } +} diff --git a/crates/tracedecay-lcm/src/schema.rs b/crates/tracedecay-lcm/src/schema.rs index 8b239e9ce9..8af0fe9573 100644 --- a/crates/tracedecay-lcm/src/schema.rs +++ b/crates/tracedecay-lcm/src/schema.rs @@ -433,6 +433,7 @@ fn schema_stamp_error(error: store_schema::SchemaVersionError) -> LcmError { } async fn ensure_raw_identity_schema(conn: &(impl Executor + ?Sized)) -> Result<(), LcmError> { + ensure_raw_message_location_schema(conn).await?; conn.execute_batch( "CREATE TABLE IF NOT EXISTS lcm_raw_predecessor_ranges ( provider TEXT NOT NULL, @@ -454,6 +455,74 @@ async fn ensure_raw_identity_schema(conn: &(impl Executor + ?Sized)) -> Result<( Ok(()) } +/// Add the storage-only location to the exact shipped v15 raw-row shape. +/// Bodies, metadata and their immutable provenance receipts are unchanged. +async fn ensure_raw_message_location_schema( + conn: &(impl Executor + ?Sized), +) -> Result<(), LcmError> { + // name, declared type, not-null, primary-key position, generated-column kind + const RELEASED_COLUMNS: &[(&str, &str, i64, i64, i64)] = &[ + ("provider", "TEXT", 1, 0, 0), + ("message_id", "TEXT", 1, 0, 0), + ("session_id", "TEXT", 1, 0, 0), + ("store_id", "INTEGER", 0, 1, 0), + ("role", "TEXT", 1, 0, 0), + ("ordinal", "INTEGER", 1, 0, 0), + ("timestamp", "INTEGER", 0, 0, 0), + ("content", "TEXT", 0, 0, 0), + ("content_hash", "TEXT", 1, 0, 0), + ("storage_kind", "TEXT", 1, 0, 0), + ("payload_ref", "TEXT", 0, 0, 0), + ("placeholder_text", "TEXT", 0, 0, 0), + ("snippet_text", "TEXT", 1, 0, 2), + ("index_text", "TEXT", 1, 0, 2), + ("metadata_json", "TEXT", 0, 0, 0), + ("kind", "TEXT", 0, 0, 0), + ("model", "TEXT", 0, 0, 0), + ("tool_names", "TEXT", 0, 0, 0), + ("source_path", "TEXT", 0, 0, 0), + ("source_offset", "INTEGER", 0, 0, 0), + ]; + let mut rows = conn + .query( + "SELECT name, type, [notnull], pk, hidden, dflt_value + FROM pragma_table_xinfo('lcm_raw_messages') ORDER BY cid", + (), + ) + .await?; + let mut count = 0; + while let Some(row) = rows.next().await? { + let name = row.get::(0)?; + let declared_type = row.get::(1)?; + let actual = ( + name.as_str(), + declared_type.as_str(), + row.get(2)?, + row.get(3)?, + row.get(4)?, + ); + let expected = RELEASED_COLUMNS.get(count).copied().or_else(|| { + (count == RELEASED_COLUMNS.len()).then_some(("location_path", "TEXT", 0, 0, 0)) + }); + if expected != Some(actual) || row.get::>(5)?.is_some() { + return Err(LcmError::Db( + "unsupported LCM raw message column shape".to_owned(), + )); + } + count += 1; + } + drop(rows); + if count == RELEASED_COLUMNS.len() { + conn.execute_batch("ALTER TABLE lcm_raw_messages ADD COLUMN location_path TEXT;") + .await?; + } else if count != RELEASED_COLUMNS.len() + 1 { + return Err(LcmError::Db( + "incomplete LCM raw message column shape".to_owned(), + )); + } + Ok(()) +} + /// The LCM schema version the store actually persists. /// /// An unreadable marker is the read error and a missing one is the same diff --git a/crates/tracedecay-sessions/src/runtime/store_access/sessions.rs b/crates/tracedecay-sessions/src/runtime/store_access/sessions.rs index 582d995c68..0d6480ab59 100644 --- a/crates/tracedecay-sessions/src/runtime/store_access/sessions.rs +++ b/crates/tracedecay-sessions/src/runtime/store_access/sessions.rs @@ -44,6 +44,10 @@ pub(crate) const EXISTING_SESSION_MESSAGE_IDS_SQL: &str = "SELECT messages.messa AND messages.provider = ?1 AND messages.message_id = requested.value"; +// A resumed session can move directories. Only messages without a canonical +// storage location inherit the session path. +const MESSAGE_PROJECT_PATH_SQL: &str = "COALESCE(m.location_path, s.project_path)"; + /// Appends the project-scope predicate. /// /// `project_key` is an opaque authority and stays byte-exact. `project_path` @@ -53,6 +57,7 @@ fn push_project_identity_predicate( sql: &mut String, query_params: &mut Vec, project_selector: &str, + project_path_sql: &str, ) { query_params.push(Value::Text(project_selector.to_owned())); let key_parameter = query_params.len(); @@ -62,7 +67,7 @@ fn push_project_identity_predicate( let opened_parameter = query_params.len(); let _ = write!( sql, - " AND (s.project_key = ?{key_parameter} OR s.project_path IN (?{path_parameter}, ?{opened_parameter}))" + " AND (s.project_key = ?{key_parameter} OR {project_path_sql} IN (?{path_parameter}, ?{opened_parameter}))" ); } @@ -481,7 +486,12 @@ impl SessionStoreAccess<'_, D> { WHERE 1 = 1" .to_owned(); let mut query_params = Vec::with_capacity(2); - push_project_identity_predicate(&mut sql, &mut query_params, project_key); + push_project_identity_predicate( + &mut sql, + &mut query_params, + project_key, + MESSAGE_PROJECT_PATH_SQL, + ); let mut rows = self .read_connection() .query(sql.as_str(), query_params) @@ -705,7 +715,12 @@ impl SessionStoreAccess<'_, D> { ]; let _ = write!(sql, " AND m.provider = ?{}", query_params.len()); if let Some(project_key) = project_key { - push_project_identity_predicate(&mut sql, &mut query_params, project_key); + push_project_identity_predicate( + &mut sql, + &mut query_params, + project_key, + MESSAGE_PROJECT_PATH_SQL, + ); } for term in &literal_terms { query_params.push(Value::Text(term.clone())); @@ -849,7 +864,12 @@ impl SessionStoreAccess<'_, D> { .to_owned(); let mut query_params = vec![Value::Text(SESSION_MESSAGE_PROJECTOR_VERSION.to_owned())]; if let Some(project_key) = project_key { - push_project_identity_predicate(&mut sql, &mut query_params, project_key); + push_project_identity_predicate( + &mut sql, + &mut query_params, + project_key, + "s.project_path", + ); } query_params.push(Value::Integer(i64::try_from(limit).unwrap_or(i64::MAX))); let _ = write!( @@ -1009,7 +1029,7 @@ async fn search_workflow_facts( ]; let _ = write!(sql, " AND w.provider = ?{}", query_params.len()); if let Some(project_key) = project_key { - push_project_identity_predicate(&mut sql, &mut query_params, project_key); + push_project_identity_predicate(&mut sql, &mut query_params, project_key, "s.project_path"); } let mut term_predicates = Vec::with_capacity(terms.len()); for term in terms { @@ -1345,7 +1365,7 @@ mod identity_tests { } let mut sql = "SELECT count(*) FROM sessions s WHERE 1 = 1".to_owned(); let mut params = Vec::new(); - push_project_identity_predicate(&mut sql, &mut params, opened); + push_project_identity_predicate(&mut sql, &mut params, opened, "s.project_path"); let params = params.iter().map(|value| match value { Value::Text(text) => text.as_str(), _ => panic!("project identity parameters must be text"), From de2fa31ffc68080cc1f9f2d857f9c0381ef79613 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 21:51:08 +0000 Subject: [PATCH 34/89] test(dashboard): reserve CPU capacity beyond Vitest workers CI races local and remote TestRunner execution. The one-worker hosted run timed out after 172 of 221 files, with only 103 seconds in reported test bodies. Keep two workers to avoid cold-import assertion failures seen with four. Reserve four CPUs for the workers, transforms, coordinator, and runtime helpers, following the existing Rust test allocation. A two-CPU pinned full run timed out after 111 files. With four pinned CPUs, all 221 files and 2,137 tests pass in 215.0 seconds, averaging 241% CPU. The focused six files and 43 tests also pass. Timeout budgets are unchanged. --- dashboard/BUILD.bazel | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dashboard/BUILD.bazel b/dashboard/BUILD.bazel index 083bb8dd86..e282d69310 100644 --- a/dashboard/BUILD.bazel +++ b/dashboard/BUILD.bazel @@ -28,9 +28,11 @@ vitest_bin.vitest_test( "e2e/**", "stories/**", ]), - # Remote actions reserve one CPU; Vitest otherwise sizes its pool to the host. - args = ["run", "--maxWorkers=1"], + # Leave CPU capacity for transforms and the coordinator beside the workers. + args = ["run", "--maxWorkers=2"], chdir = package_name(), + exec_properties = {"cpu_count": "4"}, + tags = ["cpu:4"], ) rsbuild_bin.rsbuild( From 1ca3f8e6dcfdff7373f0da9733514012aae58c90 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 22:01:37 +0000 Subject: [PATCH 35/89] test(cli): preserve fixture modes and typed capacity retries --- .../core_cli_suite/source_provenance_test.rs | 19 ++----------------- .../exact_restore.rs | 13 +++++++++++-- .../tracedecay-cli/tests/work_loop_journey.rs | 6 +++--- 3 files changed, 16 insertions(+), 22 deletions(-) diff --git a/crates/tracedecay-cli/tests/core_cli_suite/source_provenance_test.rs b/crates/tracedecay-cli/tests/core_cli_suite/source_provenance_test.rs index f59b31095b..b8b5db1d79 100644 --- a/crates/tracedecay-cli/tests/core_cli_suite/source_provenance_test.rs +++ b/crates/tracedecay-cli/tests/core_cli_suite/source_provenance_test.rs @@ -410,23 +410,8 @@ impl CargoFixture { let watched_input = directory.path().join("release-sha.txt"); std::fs::create_dir_all(&cargo_home).expect("fixture cargo home"); std::fs::write(&watched_input, ENV_SHA).expect("write watched input"); - std::fs::create_dir_all(root.join("src")).expect("fixture source directory"); - std::fs::create_dir_all(root.join("provenance/src")) - .expect("fixture build dependency source directory"); - - for relative in [ - "Cargo.toml", - "Cargo.lock", - "build.rs", - "README.md", - "provenance/Cargo.toml", - "provenance/src/lib.rs", - ] { - std::fs::copy(fixture.join(relative), root.join(relative)) - .unwrap_or_else(|error| panic!("copy checked-in fixture {relative}: {error}")); - } - std::fs::copy(fixture.join("src/main.rs"), root.join("src/main.rs")) - .expect("copy checked-in fixture src/main.rs"); + crate::common::fixture::copy_fixture_tree(&fixture, &root) + .expect("copy writable provenance fixture"); Self { _directory: directory, diff --git a/crates/tracedecay-cli/tests/host_journeys_suite/host_lifecycle_cli_acceptance/exact_restore.rs b/crates/tracedecay-cli/tests/host_journeys_suite/host_lifecycle_cli_acceptance/exact_restore.rs index 4e3e1a64fb..35297ad53b 100644 --- a/crates/tracedecay-cli/tests/host_journeys_suite/host_lifecycle_cli_acceptance/exact_restore.rs +++ b/crates/tracedecay-cli/tests/host_journeys_suite/host_lifecycle_cli_acceptance/exact_restore.rs @@ -470,6 +470,15 @@ fn uninstall_after_an_upgrade_removes_tracedecay_residue_and_keeps_unprovable_st run_phase(&cli, id, &["install", "--agent", id], false); } strip_creation_records(&cli); + // Unproven directories must retain operator modes across update/uninstall, + // regardless of the umask that created them during the old installation. + for (relative, mode) in [(".cursor/plugins", 0o750), (".cursor/plugins/local", 0o710)] { + fs::set_permissions( + cli.home.path().join(relative), + fs::Permissions::from_mode(mode), + ) + .unwrap(); + } deposit_legacy_cursor_extension(&cli); let upgraded = home_snapshot(&cli); for residue in [ @@ -503,8 +512,8 @@ fn uninstall_after_an_upgrade_removes_tracedecay_residue_and_keeps_unprovable_st " + .cline/mcp.json: file 600 {}", hex::encode(Sha256::digest(skeleton)) ), - " + .cursor/plugins: dir 755".to_string(), - " + .cursor/plugins/local: dir 755".to_string(), + " + .cursor/plugins: dir 750".to_string(), + " + .cursor/plugins/local: dir 710".to_string(), ] ); } diff --git a/crates/tracedecay-cli/tests/work_loop_journey.rs b/crates/tracedecay-cli/tests/work_loop_journey.rs index 8a138c445e..906fb5b155 100644 --- a/crates/tracedecay-cli/tests/work_loop_journey.rs +++ b/crates/tracedecay-cli/tests/work_loop_journey.rs @@ -376,13 +376,13 @@ impl ProductionDaemon { (status, parsed) } - /// Follows only the daemon's explicit same-request warming contract. A - /// terminal problem or malformed retry response is returned immediately, + /// Follows the daemon's explicit same-request warming and capacity retry + /// contract. A terminal problem or malformed retry response returns immediately, /// so callers still prove their exact success or refusal below. fn post_settled(&self, label: &str, route_path: &str, body: &Value) -> (u16, Value) { poll_until(label, || { let (status, answer) = self.post(route_path, body); - if status == 503 + if matches!(status, 429 | 503) && answer["kind"] == "problem" && answer["value"]["problem"]["retryable"] == true && answer["value"]["problem"]["retry_scope"] == "same_request" From 2971cacb5cd364b43e5f91606d0f450de7214091 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 22:02:16 +0000 Subject: [PATCH 36/89] test(sessions): distinguish exited zombies during shutdown --- .../src/runtime/hosts/codex_app_server.rs | 32 ++++++++++++++++--- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex_app_server.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex_app_server.rs index d74d64d4d0..7a5537aaf8 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex_app_server.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex_app_server.rs @@ -1578,6 +1578,7 @@ mod tests { let _process_guard = APP_SERVER_PROCESS_TEST_LOCK .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); + #[cfg(not(target_os = "linux"))] unsafe extern "C" { fn kill(pid: i32, signal: i32) -> i32; } @@ -1608,6 +1609,30 @@ mod tests { .trim() .parse() .expect("descendant pid"); + let descendant_is_live = || { + // A killed orphan can remain a zombie until the host reaps it; + // kill(pid, 0) still succeeds although it has already exited. + #[cfg(target_os = "linux")] + { + let stat = match std::fs::read_to_string(format!("/proc/{descendant_pid}/stat")) { + Ok(stat) => stat, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return false, + Err(error) => panic!("read descendant process state: {error}"), + }; + stat.rsplit_once(')') + .expect("descendant process stat must contain a command terminator") + .1 + .split_whitespace() + .next() + .expect("descendant process stat must contain a state") + != "Z" + } + #[cfg(not(target_os = "linux"))] + unsafe { + kill(descendant_pid, 0) == 0 + } + }; + assert!(descendant_is_live(), "descendant must run before shutdown"); let shutdown = begin_codex_app_server_shutdown(); let deadline = Instant::now() + Duration::from_secs(1); @@ -1619,12 +1644,11 @@ mod tests { "active child should exit during shutdown" ); let descendant_deadline = Instant::now() + Duration::from_secs(1); - while unsafe { kill(descendant_pid, 0) } == 0 && Instant::now() < descendant_deadline { + while descendant_is_live() && Instant::now() < descendant_deadline { std::thread::sleep(Duration::from_millis(10)); } - assert_ne!( - unsafe { kill(descendant_pid, 0) }, - 0, + assert!( + !descendant_is_live(), "app-server descendant should exit during shutdown" ); From 6914cc8d77cc202bfeeb01216a74fbcafb71da7a Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 22:15:08 +0000 Subject: [PATCH 37/89] fix(memory): separate process and cgroup admission budgets --- .../tracedecay-application/src/code_index.rs | 2 +- .../src/code_index_scheduler/memory_tests.rs | 16 +- .../tests/publication_store.rs | 2 +- .../code_index_scheduler/tests/residency.rs | 2 +- .../tests/code_index_scheduler/serving.rs | 2 +- .../candidate_producers.rs | 2 +- .../src/resident_memory.rs | 196 +++++++++--------- .../src/resident_memory/tests.rs | 116 +++++++++-- .../code_graph/sealed_publication_tests.rs | 10 +- ...de_index_runtime_graph_activation_tests.rs | 2 +- 10 files changed, 221 insertions(+), 129 deletions(-) diff --git a/crates/tracedecay-application/src/code_index.rs b/crates/tracedecay-application/src/code_index.rs index d660d5dd30..d540914272 100644 --- a/crates/tracedecay-application/src/code_index.rs +++ b/crates/tracedecay-application/src/code_index.rs @@ -268,7 +268,7 @@ mod tests { resident_bytes: bytes, unreclaimable_bytes: bytes, swapped_bytes: 0, - cgroup_committed_bytes: None, + cgroup_headroom_bytes: None, }) }), )); diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/memory_tests.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/memory_tests.rs index bca70e7f82..0ebe34ada8 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/memory_tests.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/memory_tests.rs @@ -490,7 +490,7 @@ fn measured_rss_pressure_refuses_worker_admission_and_readmits_as_it_falls() { resident_bytes: unreclaimable_bytes, unreclaimable_bytes, swapped_bytes: 0, - cgroup_committed_bytes: None, + cgroup_headroom_bytes: None, }) }), )); @@ -577,7 +577,7 @@ fn a_refresh_after_cold_index_sizes_its_worker_slab_to_measured_headroom() { resident_bytes: unreclaimable_bytes, unreclaimable_bytes, swapped_bytes: 0, - cgroup_committed_bytes: None, + cgroup_headroom_bytes: None, }) }), )); @@ -655,7 +655,7 @@ fn a_refresh_decodes_its_parent_before_planning_its_worker_slab() { resident_bytes: 0, unreclaimable_bytes: 0, swapped_bytes: 0, - cgroup_committed_bytes: None, + cgroup_headroom_bytes: None, }) }), )); @@ -745,7 +745,7 @@ fn a_released_generation_decodes_again_only_once_its_bytes_fit_the_budget() { resident_bytes: 0, unreclaimable_bytes: 0, swapped_bytes: 0, - cgroup_committed_bytes: None, + cgroup_headroom_bytes: None, }) }), )); @@ -829,7 +829,7 @@ fn a_decode_is_admitted_against_unreclaimable_bytes_not_clean_file_pages() { resident_bytes: GIB, unreclaimable_bytes: GIB, swapped_bytes: 0, - cgroup_committed_bytes: None, + cgroup_headroom_bytes: None, })); let sampled = Arc::clone(&view); let pressure = Arc::new(ResidentMemoryPressureV1::with_sampler( @@ -853,7 +853,7 @@ fn a_decode_is_admitted_against_unreclaimable_bytes_not_clean_file_pages() { resident_bytes: high_watermark - 1, unreclaimable_bytes: 2 * GIB, swapped_bytes: 0, - cgroup_committed_bytes: None, + cgroup_headroom_bytes: None, }; let decoded = scheduler .latest_complete() @@ -879,7 +879,7 @@ fn a_decode_is_admitted_against_unreclaimable_bytes_not_clean_file_pages() { resident_bytes: high_watermark - 1, unreclaimable_bytes: high_watermark - 1, swapped_bytes: 0, - cgroup_committed_bytes: None, + cgroup_headroom_bytes: None, }; assert!(matches!( scheduler.publication.load_active_shared(), @@ -920,7 +920,7 @@ fn reconcile_stops_when_resident_memory_crosses_the_watermark() { resident_bytes: unreclaimable_bytes, unreclaimable_bytes, swapped_bytes: 0, - cgroup_committed_bytes: None, + cgroup_headroom_bytes: None, }) }), )); diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/publication_store.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/publication_store.rs index 7dd83962b2..cb62507fcc 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/publication_store.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/publication_store.rs @@ -98,7 +98,7 @@ fn selected_graph_build_bound_reserves_transient_memory_and_retries_after_releas resident_bytes: 0, unreclaimable_bytes: 0, swapped_bytes: 0, - cgroup_committed_bytes: None, + cgroup_headroom_bytes: None, }) }), )); diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/residency.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/residency.rs index 114b5106f1..4cd725b7cb 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/residency.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/residency.rs @@ -1492,7 +1492,7 @@ fn fixed_reading_authority() -> Arc { resident_bytes: 256 * 1024 * 1024, unreclaimable_bytes: 256 * 1024 * 1024, swapped_bytes: 0, - cgroup_committed_bytes: None, + cgroup_headroom_bytes: None, }; Arc::new(ProcessResidentMemoryV1::with_pressure( limit, diff --git a/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/serving.rs b/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/serving.rs index 9d057c1692..05bf0e812e 100644 --- a/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/serving.rs +++ b/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/serving.rs @@ -2207,7 +2207,7 @@ fn text_artifact_ceilings_reserve_through_process_resident_memory() { resident_bytes: live_bytes, unreclaimable_bytes: live_bytes, swapped_bytes: 0, - cgroup_committed_bytes: None, + cgroup_headroom_bytes: None, }) }), )); diff --git a/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs b/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs index 1535b0fc07..395f642ab0 100644 --- a/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs +++ b/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs @@ -1494,7 +1494,7 @@ fn concurrent_artifact_reads_are_bounded_and_keep_the_verified_file() { resident_bytes: bytes, unreclaimable_bytes: bytes, swapped_bytes: 0, - cgroup_committed_bytes: None, + cgroup_headroom_bytes: None, }) }), ); diff --git a/crates/tracedecay-runtime-core/src/resident_memory.rs b/crates/tracedecay-runtime-core/src/resident_memory.rs index 7c8a632c17..34a9c445be 100644 --- a/crates/tracedecay-runtime-core/src/resident_memory.rs +++ b/crates/tracedecay-runtime-core/src/resident_memory.rs @@ -301,31 +301,24 @@ pub struct ProcessResidentSampleV1 { pub unreclaimable_bytes: u64, /// Anonymous pages the kernel moved to swap (`VmSwap`). pub swapped_bytes: u64, - /// Bytes the kernel charges toward a finite `memory.max`: `memory.current` - /// minus `inactive_file`, plus `memory.swap.current`, on the cgroup with - /// the tightest finite ceiling. - /// - /// `None` when no cgroup has a finite `memory.max`. An unlimited cgroup's - /// file cache is not a kill line, and inactive file pages are what the - /// kernel reclaims before it kills. Active file pages of a mapped store - /// stay in this figure because they are not dropped before the kill. - pub cgroup_committed_bytes: Option, + /// Remaining committed capacity across every finite ancestor `memory.max`. + /// Committed bytes include swap and exclude reclaimable inactive file pages. + /// `None` means no finite cgroup ceiling was observed. + pub cgroup_headroom_bytes: Option, } impl ProcessResidentSampleV1 { - /// Bytes admission compares with the watermark. - /// - /// The daemon's anonymous state is the floor, swapped pages included: - /// they are live heap the next touch faults back in, so a sample taken - /// while the kernel swaps under a cgroup ceiling must not read as room. - /// A finite cgroup ceiling also counts its committed working set, so a - /// build cannot be admitted while `memory.current` is already at the kill - /// line and only the anonymous subset sits under the watermark. + /// Occupancy against a process budget, constrained by shared cgroup room. + /// Peers consume shared capacity without spending this process's budget. + /// Swapped anonymous pages remain live process allocations. #[must_use] - pub fn admission_bytes(self) -> u64 { + pub fn admission_bytes(self, limit_bytes: u64) -> u64 { self.unreclaimable_bytes .saturating_add(self.swapped_bytes) - .max(self.cgroup_committed_bytes.unwrap_or(0)) + .max( + self.cgroup_headroom_bytes + .map_or(0, |room| limit_bytes.saturating_sub(room)), + ) } } @@ -349,7 +342,7 @@ fn process_resident_sample_from_status_v1(status: &str) -> Option Option { }) } -/// Working set a finite `memory.max` will kill for: `memory.current` minus -/// `inactive_file`, plus the cgroup's swapped anonymous pages -/// (`memory.swap.current`), on the cgroup directory with the tightest finite -/// ceiling. -/// -/// `None` when every `memory.max` is absent or `max`. Counting `memory.current` -/// on an unlimited cgroup treats the machine's page cache as a kill line and -/// refuses work the kernel can reclaim. +/// The smallest remaining committed capacity across all finite ancestors. +/// A larger ancestor can have less room than a smaller child because it also +/// contains peers. Inactive file pages are reclaimable; swapped heap is not. #[cfg(target_os = "linux")] -fn cgroup_committed_bytes_v1(proc_self_cgroup: &Path, cgroup_root: &Path) -> Option { +fn cgroup_headroom_bytes_v1(proc_self_cgroup: &Path, cgroup_root: &Path) -> Option { let mut directory = cgroup_v2_process_directory_v1(proc_self_cgroup, cgroup_root)?; - let mut chosen: Option<(std::path::PathBuf, u64)> = None; + let mut headroom = None; loop { if let Some(limit) = finite_cgroup_memory_value_v1(&directory.join("memory.max")) { - let tighter = chosen.as_ref().is_none_or(|(_, current)| limit < *current); - if tighter { - chosen = Some((directory.clone(), limit)); - } + let current = std::fs::read_to_string(directory.join("memory.current")) + .ok()? + .trim() + .parse::() + .ok()?; + let inactive_file = std::fs::read_to_string(directory.join("memory.stat")) + .ok() + .and_then(|stat| memory_stat_field_bytes(&stat, "inactive_file")) + .unwrap_or(0); + let swapped = std::fs::read_to_string(directory.join("memory.swap.current")) + .ok() + .and_then(|value| value.trim().parse::().ok()) + .unwrap_or(0); + let committed = current + .saturating_sub(inactive_file) + .saturating_add(swapped); + headroom = Some(tighten(headroom, limit.saturating_sub(committed))); } if directory == cgroup_root { break; @@ -392,25 +393,7 @@ fn cgroup_committed_bytes_v1(proc_self_cgroup: &Path, cgroup_root: &Path) -> Opt } directory = parent.to_path_buf(); } - let (directory, _) = chosen?; - let current = std::fs::read_to_string(directory.join("memory.current")) - .ok()? - .trim() - .parse::() - .ok()?; - let inactive_file = std::fs::read_to_string(directory.join("memory.stat")) - .ok() - .and_then(|stat| memory_stat_field_bytes(&stat, "inactive_file")) - .unwrap_or(0); - let swapped = std::fs::read_to_string(directory.join("memory.swap.current")) - .ok() - .and_then(|value| value.trim().parse::().ok()) - .unwrap_or(0); - Some( - current - .saturating_sub(inactive_file) - .saturating_add(swapped), - ) + headroom } /// Sample this process's resident set directly from the kernel. @@ -439,8 +422,8 @@ pub fn sampled_process_resident_v1() -> Option { let mut sample = process_resident_sample_from_status_v1( &std::fs::read_to_string("/proc/self/status").ok()?, )?; - sample.cgroup_committed_bytes = - cgroup_committed_bytes_v1(Path::new(PROC_SELF_CGROUP_V1), Path::new(CGROUP_V2_ROOT_V1)); + sample.cgroup_headroom_bytes = + cgroup_headroom_bytes_v1(Path::new(PROC_SELF_CGROUP_V1), Path::new(CGROUP_V2_ROOT_V1)); Some(sample) } #[cfg(target_os = "windows")] @@ -482,7 +465,7 @@ pub(crate) fn process_resident_sample_from_macos_vm_info_v1( resident_bytes: resident_size.max(unreclaimable_bytes), unreclaimable_bytes, swapped_bytes: 0, - cgroup_committed_bytes: None, + cgroup_headroom_bytes: None, }) } @@ -548,7 +531,7 @@ fn process_resident_sample_from_counters_v1() -> Option resident_bytes: counters.WorkingSetSize as u64, unreclaimable_bytes: counters.PrivateUsage as u64, swapped_bytes: 0, - cgroup_committed_bytes: None, + cgroup_headroom_bytes: None, }) } @@ -669,7 +652,7 @@ pub struct ResidentMemoryPressureRegistrationFailureV1; /// The measured side of the memory accounting loop. /// /// One dedicated reader samples the process (`/proc/self/status` on Linux) -/// and feeds this cell the unreclaimable bytes. Admission re-measures through the same sampler; +/// and its shared cgroup room. Admission re-measures through the same sampler; /// there is no second parser or publisher. /// /// Every request refused for memory, by any authority on this cell, waits on @@ -680,8 +663,8 @@ pub struct ResidentMemoryPressureV1 { limit_bytes: NonZeroU64, high_watermark_bytes: u64, low_watermark_bytes: u64, - observed_bytes: AtomicU64, - observed: AtomicBool, + /// Retain kernel measurements so narrower ceilings can derive their own view. + observation: Mutex>, over_budget: AtomicBool, headroom: watch::Sender, /// Authorities holding a refused request, settled on every observation. @@ -754,8 +737,7 @@ impl ResidentMemoryPressureV1 { limit_bytes, high_watermark_bytes, low_watermark_bytes, - observed_bytes: AtomicU64::new(0), - observed: AtomicBool::new(false), + observation: Mutex::new(None), over_budget: AtomicBool::new(false), headroom: watch::channel(0).0, waiting: Mutex::new(Vec::new()), @@ -773,10 +755,7 @@ impl ResidentMemoryPressureV1 { &self, ) -> Option<(ProcessResidentSampleV1, ResidentMemoryPressureStateV1)> { let sample = (self.sampler)()?; - Some(( - sample, - self.publish_observed_resident_bytes(sample.admission_bytes()), - )) + Some((sample, self.publish_sample(sample))) } /// Publish a fresh admission sample without running pressure reclaimers. @@ -810,7 +789,7 @@ impl ResidentMemoryPressureV1 { Ordering::Release, ); if let Some(sample) = sample { - self.publish_observation(sample.admission_bytes()); + self.publish_observation(sample); } self.state() } @@ -818,7 +797,7 @@ impl ResidentMemoryPressureV1 { /// Refresh pressure after reclaim without running reclaimers recursively. fn resample_after_reclaim(&self) -> Option { let sample = (self.sampler)()?; - self.publish_observation(sample.admission_bytes()); + self.publish_observation(sample); Some(self.state()) } @@ -832,7 +811,7 @@ impl ResidentMemoryPressureV1 { self.sample_and_publish().map_or(0, |(sample, state)| { state .observed_bytes() - .unwrap_or_else(|| sample.admission_bytes()) + .unwrap_or_else(|| sample.admission_bytes(self.limit_bytes())) }) } @@ -863,16 +842,36 @@ impl ResidentMemoryPressureV1 { &self, observed_bytes: u64, ) -> ResidentMemoryPressureStateV1 { - self.publish_observation(observed_bytes); + self.publish_sample(ProcessResidentSampleV1 { + resident_bytes: observed_bytes, + unreclaimable_bytes: observed_bytes, + swapped_bytes: 0, + cgroup_headroom_bytes: None, + }) + } + + fn publish_sample(&self, sample: ProcessResidentSampleV1) -> ResidentMemoryPressureStateV1 { + let observed_bytes = sample.admission_bytes(self.limit_bytes()); + self.publish_observation(sample); if observed_bytes >= self.high_watermark_bytes { self.run_pressure_reclaimers(observed_bytes); } self.state() } - fn publish_observation(&self, observed_bytes: u64) { - self.observed_bytes.store(observed_bytes, Ordering::Release); - self.observed.store(true, Ordering::Release); + fn observed_bytes_for_limit(&self, limit_bytes: u64) -> Option { + self.observation + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .map(|sample| sample.admission_bytes(limit_bytes)) + } + + fn publish_observation(&self, sample: ProcessResidentSampleV1) { + let observed_bytes = sample.admission_bytes(self.limit_bytes()); + *self + .observation + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(sample); if observed_bytes >= self.high_watermark_bytes { self.over_budget.store(true, Ordering::Release); } else if observed_bytes <= self.low_watermark_bytes @@ -918,7 +917,7 @@ impl ResidentMemoryPressureV1 { authority.upgrade().is_some_and(|authority| { let mut state = authority.lock_state(); settled |= authority.settle_waiter(&mut state); - state.waiting_level.is_some() + !state.waiting_requests.is_empty() }) }); self.any_waiting @@ -931,10 +930,9 @@ impl ResidentMemoryPressureV1 { #[must_use] pub fn state(&self) -> ResidentMemoryPressureStateV1 { - if !self.observed.load(Ordering::Acquire) { + let Some(observed_bytes) = self.observed_bytes_for_limit(self.limit_bytes()) else { return ResidentMemoryPressureStateV1::Unobserved; - } - let observed_bytes = self.observed_bytes.load(Ordering::Acquire); + }; if self.over_budget.load(Ordering::Acquire) { return ResidentMemoryPressureStateV1::OverBudget { observed_bytes, @@ -1412,9 +1410,9 @@ struct ResidentMemoryStateV1 { used_bytes: u64, charges: BTreeMap, process_shared_charges: BTreeMap, - /// The highest occupancy (ledger or measured, whichever is larger) at - /// which some refused request fits; `None` while nothing waits. - waiting_level: Option, + /// The smallest refused request per ceiling is the first that can fit. + /// Shared headroom must be evaluated against each distinct ceiling. + waiting_requests: BTreeMap, } /// The single process ceiling. Callers share one pointer-identical `Arc`. @@ -1468,12 +1466,16 @@ impl ProcessResidentMemoryV1 { /// measured process. The pressure cell's headroom epoch advances once it /// would, and every refusal this authority makes registers the same way. pub fn wait_for_headroom(self: &Arc, bytes: u64, ceiling: u64) { - let Some(level) = ceiling.checked_sub(bytes) else { + if bytes > ceiling { return; - }; + } { let mut state = self.lock_state(); - state.waiting_level = Some(state.waiting_level.map_or(level, |held| held.max(level))); + state + .waiting_requests + .entry(ceiling) + .and_modify(|held| *held = (*held).min(bytes)) + .or_insert(bytes); } self.pressure.register_waiting(self); } @@ -1481,15 +1483,18 @@ impl ProcessResidentMemoryV1 { /// Clear the waiters once the most satisfiable one fits; each still /// refused registers again on its retry. fn settle_waiter(&self, state: &mut ResidentMemoryStateV1) -> bool { - let Some(level) = state.waiting_level else { - return false; - }; - let pressure = self.pressure.state(); - let occupied = state.used_bytes.max(pressure.observed_bytes().unwrap_or(0)); - if pressure.is_over_budget() || occupied > level { + if self.pressure.state().is_over_budget() + || !state.waiting_requests.iter().any(|(ceiling, bytes)| { + let observed = self + .pressure + .observed_bytes_for_limit(*ceiling) + .unwrap_or(0); + state.used_bytes.max(observed) <= ceiling.saturating_sub(*bytes) + }) + { return false; } - state.waiting_level = None; + state.waiting_requests.clear(); true } @@ -1590,7 +1595,12 @@ impl ProcessResidentMemoryV1 { /// this authority's limit and the measurement to the pressure cell's, as /// [`Self::reserve`] holds them. pub fn headroom_below(&self, ceiling: u64) -> ResidentMemoryHeadroomV1 { - let observed_bytes = self.pressure.measure_admission_bytes(); + self.pressure.measure_admission_bytes(); + let measured_ceiling = ceiling.min(self.pressure.limit_bytes()); + let observed_bytes = self + .pressure + .observed_bytes_for_limit(measured_ceiling) + .unwrap_or(0); let used_bytes = self.lock_state().used_bytes; let available_bytes = if self.pressure.state().is_over_budget() { 0 @@ -1598,9 +1608,7 @@ impl ProcessResidentMemoryV1 { let ledger_room = ceiling .min(self.limit_bytes.get()) .saturating_sub(used_bytes); - let measured_room = ceiling - .min(self.pressure.limit_bytes()) - .saturating_sub(observed_bytes); + let measured_room = measured_ceiling.saturating_sub(observed_bytes); ledger_room.min(measured_room) }; ResidentMemoryHeadroomV1 { diff --git a/crates/tracedecay-runtime-core/src/resident_memory/tests.rs b/crates/tracedecay-runtime-core/src/resident_memory/tests.rs index d9abeffb82..557293b4ec 100644 --- a/crates/tracedecay-runtime-core/src/resident_memory/tests.rs +++ b/crates/tracedecay-runtime-core/src/resident_memory/tests.rs @@ -481,7 +481,7 @@ fn additional_reservations_share_identity_but_charge_and_release_independently() resident_bytes: 0, unreclaimable_bytes: 0, swapped_bytes: 0, - cgroup_committed_bytes: None, + cgroup_headroom_bytes: None, }) }), )), @@ -958,7 +958,7 @@ fn post_reclaim_observation_replaces_pressure_state_without_reentering_reclaimer resident_bytes: after_reclaim, unreclaimable_bytes: after_reclaim, swapped_bytes: 0, - cgroup_committed_bytes: None, + cgroup_headroom_bytes: None, }) }), )); @@ -1071,7 +1071,7 @@ fn allocator_trim_readmits_growth_when_memory_recovers_inside_checkpoint_throttl resident_bytes: observed, unreclaimable_bytes: observed, swapped_bytes: 0, - cgroup_committed_bytes: None, + cgroup_headroom_bytes: None, }) }), )); @@ -1197,11 +1197,11 @@ fn process_status_splits_clean_file_pages_from_unreclaimable_bytes() { resident_bytes: 3_355_444 * 1024, unreclaimable_bytes: 2_548_172 * 1024, swapped_bytes: 491_520 * 1024, - cgroup_committed_bytes: None, + cgroup_headroom_bytes: None, }) ); assert_eq!( - sample.map(super::ProcessResidentSampleV1::admission_bytes), + sample.map(|sample| sample.admission_bytes(u64::MAX)), Some((2_548_172 + 491_520) * 1024), "swapped anonymous pages are the daemon's heap and count toward admission" ); @@ -1232,7 +1232,7 @@ fn macos_phys_footprint_is_the_admission_charge() { assert_eq!(sample.resident_bytes, 6_400 * 1024 * 1024); assert_eq!(sample.unreclaimable_bytes, 5_200 * 1024 * 1024); assert_eq!( - sample.admission_bytes(), + sample.admission_bytes(u64::MAX), 5_200 * 1024 * 1024, "jetsam charges phys_footprint, not the larger resident_size" ); @@ -1244,12 +1244,12 @@ fn macos_phys_footprint_is_the_admission_charge() { let internal_only = super::process_resident_sample_from_macos_vm_info_v1(8, 0, 5) .expect("internal is the fallback when footprint is missing"); assert_eq!(internal_only.unreclaimable_bytes, 5); - assert_eq!(internal_only.admission_bytes(), 5); + assert_eq!(internal_only.admission_bytes(u64::MAX), 5); } #[cfg(target_os = "linux")] #[test] -fn cgroup_committed_bytes_refuse_growth_that_unreclaimable_bytes_would_admit() { +fn cgroup_headroom_bytes_refuse_growth_that_unreclaimable_bytes_would_admit() { let limit_bytes = 100 * 1024 * 1024; let limit = bytes(limit_bytes); let decide = |memory_max: &str, inactive_file: u64| { @@ -1265,12 +1265,12 @@ fn cgroup_committed_bytes_refuse_growth_that_unreclaimable_bytes_would_admit() { format!("inactive_file {inactive_file}\n"), ) .expect("stat"); - let committed = super::cgroup_committed_bytes_v1(&proc_self_cgroup, &cgroup_root); + let committed = super::cgroup_headroom_bytes_v1(&proc_self_cgroup, &cgroup_root); let sample = Arc::new(Mutex::new(super::ProcessResidentSampleV1 { resident_bytes: 10, unreclaimable_bytes: 10, swapped_bytes: 0, - cgroup_committed_bytes: committed, + cgroup_headroom_bytes: committed, })); let sampled = Arc::clone(&sample); let pressure = Arc::new(ResidentMemoryPressureV1::with_sampler( @@ -1304,6 +1304,90 @@ fn cgroup_committed_bytes_refuse_growth_that_unreclaimable_bytes_would_admit() { decide("max\n", 0).is_ok(), "an unlimited cgroup does not treat memory.current as a kill line" ); + assert!( + decide(&format!("{}\n", 10 * limit_bytes), 0).is_ok(), + "a larger shared cgroup with room does not spend the daemon's smaller process budget" + ); +} + +#[cfg(target_os = "linux")] +#[test] +fn cgroup_headroom_respects_a_larger_but_fuller_ancestor() { + const MIB: u64 = 1024 * 1024; + let (_directory, membership, root) = cgroup_fixture( + Some("0::/trace.slice/daemon.scope\n"), + Some(&format!("{}\n", 400 * MIB)), + None, + ); + let leaf = root.join("trace.slice/daemon.scope"); + let parent = root.join("trace.slice"); + fs::write(leaf.join("memory.current"), (20 * MIB).to_string()).unwrap(); + fs::write(parent.join("memory.max"), (1000 * MIB).to_string()).unwrap(); + fs::write(parent.join("memory.current"), (990 * MIB).to_string()).unwrap(); + assert_eq!( + super::cgroup_headroom_bytes_v1(&membership, &root), + Some(10 * MIB) + ); + fs::write(parent.join("memory.current"), (900 * MIB).to_string()).unwrap(); + assert_eq!( + super::cgroup_headroom_bytes_v1(&membership, &root), + Some(100 * MIB) + ); +} + +#[test] +fn narrowed_budgets_and_waiters_share_cgroup_capacity_without_charging_peers() { + const MIB: u64 = 1024 * 1024; + let room = Arc::new(AtomicU64::new(560 * MIB)); + let sampled = Arc::clone(&room); + let pressure = Arc::new(ResidentMemoryPressureV1::with_sampler( + bytes(800 * MIB), + Arc::new(move || { + Some(ProcessResidentSampleV1 { + resident_bytes: 10 * MIB, + unreclaimable_bytes: 10 * MIB, + swapped_bytes: 0, + cgroup_headroom_bytes: Some(sampled.load(Ordering::SeqCst)), + }) + }), + )); + let authority = Arc::new(ProcessResidentMemoryV1::with_pressure( + bytes(80 * MIB), + Arc::clone(&pressure), + )); + assert_eq!(authority.headroom_below(80 * MIB).available_bytes, 70 * MIB); + + room.store(10 * MIB, Ordering::SeqCst); + assert_eq!(authority.headroom_below(80 * MIB).available_bytes, 0); + let epoch = pressure.subscribe_headroom(); + let before = *epoch.borrow(); + authority.wait_for_headroom(16 * MIB, 80 * MIB); + authority.wait_for_headroom(40 * MIB, 100 * MIB); + room.store(20 * MIB, Ordering::SeqCst); + pressure.sample_and_publish(); + // The shared process latch remains raised until its low watermark clears. + assert_eq!(*epoch.borrow(), before); + room.store(200 * MIB, Ordering::SeqCst); + pressure.sample_and_publish(); + assert!(*epoch.borrow() > before); + assert_eq!(authority.headroom_below(80 * MIB).available_bytes, 70 * MIB); + + // With a nominal process latch, a request still waits for its own room. + room.store(100 * MIB, Ordering::SeqCst); + pressure.sample_and_publish(); + let before = *epoch.borrow(); + authority.wait_for_headroom(200 * MIB, 400 * MIB); + authority.wait_for_headroom(160 * MIB, 400 * MIB); + authority.wait_for_headroom(240 * MIB, 400 * MIB); + authority.wait_for_headroom(200 * MIB, 300 * MIB); + pressure.sample_and_publish(); + assert_eq!(*epoch.borrow(), before); + room.store(180 * MIB, Ordering::SeqCst); + pressure.sample_and_publish(); + assert!( + *epoch.borrow() > before, + "the satisfiable request wakes the common epoch" + ); } /// A daemon the kernel swaps under its cgroup ceiling has small resident @@ -1367,8 +1451,8 @@ fn admission_counts_swapped_anonymous_pages() { fs::write(cgroup.join("memory.stat"), "inactive_file 0\n").expect("stat"); fs::write(cgroup.join("memory.swap.current"), format!("{resident}\n")).expect("swap"); assert_eq!( - super::cgroup_committed_bytes_v1(&proc_self_cgroup, &cgroup_root), - Some(2 * resident), + super::cgroup_headroom_bytes_v1(&proc_self_cgroup, &cgroup_root), + Some(limit_bytes - 2 * resident), "the cgroup's swapped pages are committed to it" ); } @@ -1392,7 +1476,7 @@ fn checkpoints_read_the_process_once_per_interval_and_then_see_growth() { resident_bytes: resident, unreclaimable_bytes: resident, swapped_bytes: 0, - cgroup_committed_bytes: None, + cgroup_headroom_bytes: None, }) }), ); @@ -1437,7 +1521,7 @@ fn headroom_is_the_ceiling_less_the_larger_of_ledger_and_measurement() { resident_bytes: observed, unreclaimable_bytes: observed, swapped_bytes: 0, - cgroup_committed_bytes: None, + cgroup_headroom_bytes: None, }) }), )), @@ -1493,7 +1577,7 @@ fn allocator_reclaim_preserves_cgroup_pressure_until_sample_recovers() { resident_bytes: 1, unreclaimable_bytes: 1, swapped_bytes: 0, - cgroup_committed_bytes: Some(limit.get()), + cgroup_headroom_bytes: Some(0), })); let sampled = Arc::clone(&sample); let pressure = Arc::new(ResidentMemoryPressureV1::with_sampler( @@ -1520,7 +1604,7 @@ fn allocator_reclaim_preserves_cgroup_pressure_until_sample_recovers() { Err(ResidentMemoryAdmissionFailureV1::ObservedOverBudget { .. }) )); - sample.lock().expect("sample").cgroup_committed_bytes = Some(1); + sample.lock().expect("sample").cgroup_headroom_bytes = Some(limit.get() - 1); assert_eq!(pressure.measure_admission_bytes(), 1); assert!(matches!( pressure.state(), diff --git a/crates/tracedecay-store-runtime/src/session_registry/code_graph/sealed_publication_tests.rs b/crates/tracedecay-store-runtime/src/session_registry/code_graph/sealed_publication_tests.rs index 9632a0e8b4..c7c377edff 100644 --- a/crates/tracedecay-store-runtime/src/session_registry/code_graph/sealed_publication_tests.rs +++ b/crates/tracedecay-store-runtime/src/session_registry/code_graph/sealed_publication_tests.rs @@ -266,7 +266,7 @@ fn pressure_from_sample( resident_bytes: 0, unreclaimable_bytes: 0, swapped_bytes: 0, - cgroup_committed_bytes: None, + cgroup_headroom_bytes: None, }, )); let sampled = Arc::clone(&sample); @@ -288,7 +288,7 @@ fn set_sample( resident_bytes: unreclaimable_bytes, unreclaimable_bytes, swapped_bytes: 0, - cgroup_committed_bytes: None, + cgroup_headroom_bytes: None, }; } @@ -2206,7 +2206,7 @@ async fn sealed_publication_refuses_over_the_resident_memory_watermark() { resident_bytes: 0, unreclaimable_bytes: 0, swapped_bytes: 0, - cgroup_committed_bytes: None, + cgroup_headroom_bytes: None, }, )); let refuse_once_projecting = Arc::new(AtomicBool::new(true)); @@ -2227,7 +2227,7 @@ async fn sealed_publication_refuses_over_the_resident_memory_watermark() { resident_bytes: limit_bytes, unreclaimable_bytes: limit_bytes, swapped_bytes: 0, - cgroup_committed_bytes: None, + cgroup_headroom_bytes: None, }, ); } @@ -2383,7 +2383,7 @@ async fn sealed_publication_samples_resident_memory_per_interval_not_per_row() { resident_bytes: 0, unreclaimable_bytes: 0, swapped_bytes: 0, - cgroup_committed_bytes: None, + cgroup_headroom_bytes: None, }) }), )); diff --git a/crates/tracedecay/src/daemon/code_index_runtime_graph_activation_tests.rs b/crates/tracedecay/src/daemon/code_index_runtime_graph_activation_tests.rs index be1502e22d..eab245fe66 100644 --- a/crates/tracedecay/src/daemon/code_index_runtime_graph_activation_tests.rs +++ b/crates/tracedecay/src/daemon/code_index_runtime_graph_activation_tests.rs @@ -2616,7 +2616,7 @@ async fn a_refresh_refused_beside_the_serving_graph_publishes_under_status_polli resident_bytes: bytes, unreclaimable_bytes: bytes, swapped_bytes: 0, - cgroup_committed_bytes: None, + cgroup_headroom_bytes: None, }) }), )); From 18513ffd2fd86bd18794294d4bc57561e89387c3 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 22:15:12 +0000 Subject: [PATCH 38/89] fix(privacy): preserve UUID directory paths in entropy scans --- .../tracedecay-privacy/src/detector_kernel.rs | 93 ++++++++++++++++++- 1 file changed, 89 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay-privacy/src/detector_kernel.rs b/crates/tracedecay-privacy/src/detector_kernel.rs index c855935678..d0c34365a8 100644 --- a/crates/tracedecay-privacy/src/detector_kernel.rs +++ b/crates/tracedecay-privacy/src/detector_kernel.rs @@ -192,7 +192,24 @@ pub(crate) fn high_entropy_ranges(text: &str) -> Vec> { let candidate = &text[start..end]; let macos_temp_path = candidate.starts_with("/private/var/folders/") || candidate.starts_with("/var/folders/"); - if macos_temp_path { + let uuid_path = !macos_temp_path + && candidate.starts_with('/') + && candidate.split('/').any(is_uuid_path_component); + let uuid_path_residual = uuid_path.then(|| { + candidate + .split('/') + .filter(|component| !is_uuid_path_component(component)) + .collect::>() + .join("/") + }); + // UUID directory names are path identity, but the remaining path must + // still be scored together so slash-bearing credentials stay visible. + if uuid_path_residual + .as_deref() + .is_some_and(looks_high_entropy_token) + { + ranges.push(start..end); + } else if uuid_path || macos_temp_path { let mut component_start = 0usize; for component in candidate.split('/') { let component_end = component_start + component.len(); @@ -209,6 +226,17 @@ pub(crate) fn high_entropy_ranges(text: &str) -> Vec> { ranges } +fn is_uuid_path_component(component: &str) -> bool { + component.len() == 36 + && component.bytes().enumerate().all(|(index, byte)| { + if matches!(index, 8 | 13 | 18 | 23) { + byte == b'-' + } else { + byte.is_ascii_hexdigit() + } + }) +} + fn is_lcm_payload_ref(candidate: &str) -> bool { candidate .strip_prefix("ref=") @@ -405,9 +433,13 @@ mod tests { #[test] fn entropy_kernel_preserves_ordinary_macos_temp_paths() { - let path = "/private/var/folders/ab/cd0123456789abcdefghijklmnopqrst/T/"; - assert!(looks_high_entropy_token(path)); - assert!(high_entropy_ranges(path).is_empty()); + for path in [ + "/private/var/folders/ab/cd0123456789abcdefghijklmnopqrst/T/", + "/private/var/folders/ab/cd0123456789abcdefghijklmnopqrst/T/fe600511-e406-4142-b74a-cbf177c5f2cb/", + ] { + assert!(looks_high_entropy_token(path)); + assert!(high_entropy_ranges(path).is_empty()); + } } #[test] @@ -428,4 +460,57 @@ mod tests { assert!(looks_high_entropy_token(secret)); assert_eq!(high_entropy_ranges(secret), vec![0..secret.len()]); } + + #[test] + fn entropy_kernel_preserves_absolute_paths_with_uuid_directories() { + let observed = "/run/nl/w/fe600511-e406-4142-b74a-cbf177c5f2cb/tmp/"; + assert!(looks_high_entropy_token(observed)); + for path in [ + observed, + "/workspace/artifacts/fe600511-e406-4142-b74a-cbf177c5f2cb/project/src/", + "/workspace/FE600511-E406-4142-B74A-CBF177C5F2CB/source/", + ] { + assert!(high_entropy_ranges(path).is_empty(), "{path}"); + } + let malformed = "/run/nl/w/fe600511-e406-4142-b74a-cbf177c5f2cg/tmp/"; + assert_eq!(high_entropy_ranges(malformed), vec![0..malformed.len()]); + } + + #[test] + fn entropy_kernel_redacts_secrets_inside_uuid_directory_paths() { + let prefix = "/run/nl/w/fe600511-e406-4142-b74a-cbf177c5f2cb/tmp/"; + for secret in [ + "Qm9vZ2llV29vZ2llMTIzNDU2Nzg5MGFiY2RlZmdoaWprbG1ub3A4OTc2NTQzMjE", + "AbCdEfGhIjKlMnOpQrStUvWxYz01234/56789aBcDeFgHiJkLmNoPqRsTuVwXy", + ] { + let path = format!("{prefix}{secret}"); + assert_eq!(high_entropy_ranges(&path), vec![0..path.len()]); + } + } + + #[test] + fn uuid_directory_paths_do_not_dilute_secret_components() { + let prefix = format!( + "/workspace/fe600511-e406-4142-b74a-cbf177c5f2cb/{}/", + "a".repeat(512) + ); + let secret = "Qm9vZ2llV29vZ2llMTIzNDU2Nzg5MGFiY2RlZmdoaWprbG1ub3A4OTc2NTQzMjE"; + let path = format!("{prefix}{secret}"); + assert!(!looks_high_entropy_token(&path)); + assert_eq!(high_entropy_ranges(&path), vec![prefix.len()..path.len()]); + } + + #[test] + fn structured_metadata_keeps_uuid_paths_but_redacts_path_credentials() { + let prefix = "/run/nl/w/fe600511-e406-4142-b74a-cbf177c5f2cb/tmp/"; + let credential = "ghp_KsY7QwT2mZ4bV9nR6cX1jH8pL3dG5fA0eUwQ"; + let metadata = json!({ + "path": format!("{prefix}.tmp4cknUf/project/src/lib.rs"), + "source": format!("{prefix}{credential}/source"), + }); + let sanitized = crate::sanitize_provider_metadata_json(&metadata.to_string(), 4096) + .expect("structured path metadata"); + assert_eq!(sanitized["path"], metadata["path"]); + assert!(!sanitized["source"].as_str().unwrap().contains(credential)); + } } From be6a6e2c0d93391891eca6da6ab027bf5d381f98 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 22:23:51 +0000 Subject: [PATCH 39/89] fix(hooks): honor lifecycle gating during ledger replay --- .../src/handlers/hook_runtime/admission.rs | 17 ++- .../src/handlers/hook_runtime/mod.rs | 3 +- .../src/daemon/hook_v2_replay_consumer.rs | 133 +++++++++++++----- 3 files changed, 112 insertions(+), 41 deletions(-) diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/admission.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/admission.rs index df2173ffc7..b5bd4023fb 100644 --- a/crates/tracedecay-mcp/src/handlers/hook_runtime/admission.rs +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/admission.rs @@ -478,9 +478,15 @@ const fn intent_gated_lifecycle(producer: tracedecay_domain::NativeHostIdentityV ) } -/// Mirrors the intent pass's own disabled check, so a project the pass would -/// skip pays no lifecycle write inside the hook's synchronous budget. -async fn context_scout_intent_enabled(cg: &TraceDecay) -> bool { +/// Whether this host's lifecycle is enabled under the same policy used by +/// admission and ledger replay. Claude omits lifecycle writes without intent. +pub async fn hook_v2_native_lifecycle_enabled( + cg: &TraceDecay, + producer: tracedecay_domain::NativeHostIdentityV1, +) -> bool { + if !intent_gated_lifecycle(producer) { + return true; + } let Ok(pinned) = cg.configuration_runtime().client().current().await else { return false; }; @@ -702,8 +708,7 @@ async fn admit_hook_v2_envelope_with_lifecycle_inner( if let Some(mount) = mount.as_ref() && let Some(native_lifecycle) = mount.lifecycle.as_ref() { - let intent_enabled = - intent_gated_lifecycle(envelope.producer) && context_scout_intent_enabled(cg).await; + let lifecycle_enabled = hook_v2_native_lifecycle_enabled(cg, envelope.producer).await; let Some(range) = hook_v2_lifecycle_range(envelope, receipt) else { return HookV2AdmissionOutcomeV1::Backpressured; }; @@ -716,7 +721,7 @@ async fn admit_hook_v2_envelope_with_lifecycle_inner( native_lifecycle, range, envelope.producer, - intent_enabled, + lifecycle_enabled, ) .await { diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/mod.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/mod.rs index 318e201f76..3ba5220125 100644 --- a/crates/tracedecay-mcp/src/handlers/hook_runtime/mod.rs +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/mod.rs @@ -24,7 +24,8 @@ mod test_support; pub use admission::{ HookV2AdmissionLedgerUnavailable, HookV2AdmissionOutcomeV1, admit_hook_v2_envelope, - admit_hook_v2_replayed_envelope_with_lifecycle, hook_v2_pending_work_envelopes, + admit_hook_v2_replayed_envelope_with_lifecycle, hook_v2_native_lifecycle_enabled, + hook_v2_pending_work_envelopes, }; pub use envelope::daemon_mint_hook_v2_file_id; pub use hermes::replay_projectless_hermes_host_admission; diff --git a/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs b/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs index 607d4c45f0..02869cff7c 100644 --- a/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs +++ b/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs @@ -26,7 +26,8 @@ use tracedecay_hooks::{ use tracedecay_mcp::handlers::hook_runtime::{ HookV2AdmissionOutcomeV1, admit_hook_v2_envelope, - admit_hook_v2_replayed_envelope_with_lifecycle, hook_v2_pending_work_envelopes, + admit_hook_v2_replayed_envelope_with_lifecycle, hook_v2_native_lifecycle_enabled, + hook_v2_pending_work_envelopes, }; mod spool_opener; @@ -305,6 +306,7 @@ async fn redrive_pending_work( tracedecay_hooks::HookEventV2::ToolLifecycle { .. } | tracedecay_hooks::HookEventV2::SavedEdit { .. } ) + && hook_v2_native_lifecycle_enabled(graph, host).await { return HookV2AdmissionOutcomeV1::Backpressured; } @@ -677,32 +679,30 @@ mod tests { tracedecay_agent_hosts::hooks::hook_project_id_for_layout(graph.hook_store_layout()) .unwrap(); let now = hook_replay_now(); - let binding = HookScopeBindingV1 { - host, - project_id, - repository_id: [2; 16], - worktree_id: [3; 16], - worktree_epoch: 1, - binding_token: [4; 32], - capabilities: vec![tracedecay_hooks::HookCapabilityV1 { - family: tracedecay_hooks::HookEventFamily::SavedEdit, - support: tracedecay_hooks::stock_event_support( - host, - tracedecay_hooks::HookEventFamily::SavedEdit, - ), - }], - }; - tracedecay_hooks::HookConfigurationFileWriterV1::new( - tracedecay_hooks::hook_configuration_path(data_root, binding.worktree_id, host), + let hook_runtime = crate::hook_runtime(tracedecay_runtime_core::config::ProfileRoot::new( + fixture._temp.path().join("profile"), + )); + let project = tracedecay_domain::ProjectId::new( + graph + .hook_store_layout() + .identity + .project_id + .clone() + .unwrap(), ) - .publish(tracedecay_hooks::HookConfigurationSnapshotV1 { - schema_version: tracedecay_hooks::HOOK_CONFIGURATION_SCHEMA_VERSION, - revision: 1, - published_at: now, - expires_at: UtcMicros(now.0 + 60_000_000), - binding: binding.clone(), - }) .unwrap(); + let scope = (hook_runtime.scope_resolver)(graph.project_root(), &project).unwrap(); + let worktree_id = tracedecay_agent_hosts::hooks::hook_worktree_id_for_layout( + &hook_runtime, + graph.hook_store_layout(), + ) + .unwrap(); + let binding = published_hook_scope_binding(data_root, worktree_id, host, now).unwrap(); + let admission_runtime = graph.test_runtime_for_test().unwrap(); + let project_sessions = admission_runtime + .registered_database_arc(tracedecay_sessions::admission::HostAdmissionScope::Project) + .unwrap(); + let background_cpu = admission_runtime.background_cpu(); let lifecycle = tracedecay_hooks::NativeContextScoutLifecycleV1::new( "session.replay.edit", "call.replay.edit", @@ -741,9 +741,9 @@ mod tests { tracedecay_daemon_service::context_scout_lifecycle::register_context_scout_lifecycle_authority( project_id, binding.worktree_id, - tracedecay_domain::ProjectId::new("project.replay.owner").unwrap(), - tracedecay_domain::WorktreeId::new("worktree.replay.edit").unwrap(), - &fixture.project_sessions, + scope.project_id.clone(), + scope.worktree_id.clone(), + &project_sessions, ), tracedecay_daemon_service::context_scout_lifecycle::AuthorityRegistrationV1::Registered ); @@ -767,8 +767,8 @@ mod tests { &graph, data_root, &fixture.settlements, - &fixture.project_sessions, - &fixture.background_cpu, + &project_sessions, + &background_cpu, ) .await; assert!( @@ -791,8 +791,8 @@ mod tests { &graph, data_root, &fixture.settlements, - &fixture.project_sessions, - &fixture.background_cpu, + &project_sessions, + &background_cpu, ) .await; let mounted = tokio::time::timeout(Duration::from_secs(10), runs.recv()) @@ -808,20 +808,85 @@ mod tests { }) .await .expect("mounted work must settle its ledger obligation"); + assert!( + runs.try_recv().is_err(), + "the mounted edit executes exactly once" + ); + // Claude deliberately omits lifecycle writes while intent is disabled; + // an acknowledged edit can therefore need ledger-only recovery. + tracedecay_daemon_service::unregister_hook_orchestration_runtime( + project_id, + binding.worktree_id, + &runtime, + ); + let claude = NativeHostIdentityV1::ClaudeCode; + let claude_binding = + published_hook_scope_binding(data_root, binding.worktree_id, claude, hook_replay_now()) + .unwrap(); + let disabled_edit = tracedecay_hooks::HookEventEnvelopeV2 { + event_id: [10; 16], + producer: claude, + protected_session_id: tracedecay_agent_hosts::hooks::protected_native_session_id( + "session.replay.disabled-edit", + ), + project_id: claude_binding.project_id, + repository_id: claude_binding.repository_id, + worktree_id: claude_binding.worktree_id, + worktree_epoch: claude_binding.worktree_epoch, + binding_token: claude_binding.binding_token, + ..envelope + }; + assert!(matches!( + admit_hook_v2_envelope(&graph, &disabled_edit, None, hook_replay_now()).await, + HookV2AdmissionOutcomeV1::Admitted { .. } + )); + let disabled_pending = + || hook_v2_pending_work_envelopes(data_root, claude, hook_replay_now()).unwrap(); + assert_eq!(disabled_pending(), vec![disabled_edit]); + assert!( + tracedecay_daemon_service::register_hook_orchestration_runtime( + project_id, + binding.worktree_id, + &runtime, + ) + ); + drain_all_hosts( + &graph, + data_root, + &fixture.settlements, + &project_sessions, + &background_cpu, + ) + .await; + assert!( + tokio::time::timeout(Duration::from_secs(10), runs.recv()) + .await + .unwrap() + .unwrap() + .is_none(), + "disabled Claude edits settle without mounting a lifecycle" + ); + tokio::time::timeout(Duration::from_secs(10), async { + while !disabled_pending().is_empty() { + tokio::task::yield_now().await; + } + }) + .await + .expect("disabled Claude work must settle without a spool"); assert!(runtime.shutdown().await); assert!( pending().is_empty(), "completed lifecycle work settles its durable obligation" ); assert!(!HookSpoolV1::has_records(&root).unwrap()); - assert!(runs.try_recv().is_err(), "the edit executes exactly once"); + assert!(runs.try_recv().is_err(), "each edit executes exactly once"); tracedecay_daemon_service::unregister_hook_orchestration_runtime( project_id, binding.worktree_id, &runtime, ); assert!(tracedecay_daemon_service::context_scout_lifecycle::unregister_context_scout_lifecycle_authority( - project_id, binding.worktree_id, &fixture.project_sessions, + project_id, binding.worktree_id, &project_sessions, )); } From 79a6779f8dbcfa1705f233926508e3d7be44e7f6 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 22:51:32 +0000 Subject: [PATCH 40/89] fix(settings): separate saved commits from reconciliation --- .../src/settings_api.rs | 2 - .../src/settings_api/tests/wire.rs | 272 ----------- .../runtime_surface_acceptance.rs | 117 ++++- .../src/test/fixtures/settings-patch.json | 450 ++++++++++-------- .../settings/SettingsPage.dom.test.tsx | 3 +- .../settings/SettingsReviewPanel.tsx | 2 + .../settings/settingsEditorMachine.test.ts | 3 + .../settings/settingsEditorMachine.ts | 3 + .../settings/settingsMutation.test.ts | 49 +- .../workspaces/settings/settingsMutation.ts | 12 +- 10 files changed, 415 insertions(+), 498 deletions(-) delete mode 100644 crates/tracedecay-dashboard-api/src/settings_api/tests/wire.rs diff --git a/crates/tracedecay-dashboard-api/src/settings_api.rs b/crates/tracedecay-dashboard-api/src/settings_api.rs index 09ee62a729..af0eed64e2 100644 --- a/crates/tracedecay-dashboard-api/src/settings_api.rs +++ b/crates/tracedecay-dashboard-api/src/settings_api.rs @@ -871,8 +871,6 @@ fn project_apply_error( #[cfg(test)] mod tests { - mod wire; - use super::*; use serde_json::json; use tracedecay_configuration::config::registry::ConfigurationRegistry; diff --git a/crates/tracedecay-dashboard-api/src/settings_api/tests/wire.rs b/crates/tracedecay-dashboard-api/src/settings_api/tests/wire.rs deleted file mode 100644 index 47682ebfde..0000000000 --- a/crates/tracedecay-dashboard-api/src/settings_api/tests/wire.rs +++ /dev/null @@ -1,272 +0,0 @@ -//! Serialized producer fixtures shared with the dashboard behavior tests. - -use axum::Json; -use axum::body::to_bytes; -use axum::response::IntoResponse; -use serde_json::{Value, json}; -use tracedecay_api::configuration::{ - configuration_application_problem_error, configuration_authority_unavailable_error, - configuration_revision_conflict_error, parse_project_settings_patch, -}; -use tracedecay_api::read_model::{ - DashboardCoverageV1, DashboardEnvelopeV1, DashboardLegalActionKindV1, - DashboardLegalActionRefV1, DashboardScopeV1, -}; -use tracedecay_contracts::{ - ApplicationOutcome, ApplicationProblem, ApplicationProblemEnvelope, AuthorityReceipt, - CapabilityGrantId, Deadline, DisclosureClass, EffectId, EffectReceipt, EffectResult, - EffectTermination, IdempotencyKey, OperationBudgetUsage, OperationReceipt, PolicyDecisionRef, - ReconciliationState, RequestId, ResolvedScope, ResultContractRef, -}; -use tracedecay_domain::configuration::{CodeIndexWorkerSelectionV1, ConfigurationRevisionId}; -use tracedecay_domain::{ - ActorId, ComponentVersion, ManifestDigest, ProjectId, RepositoryId, UtcMicros, WorktreeId, -}; -use tracedecay_tool_catalog::{EffectClass, SchemaId, UseCaseId}; - -use super::pinned_defaults; -use crate::settings_api::{ - EnvironmentSettingsPayloadV1, PrAutoTrackPayloadV1, ProjectSettingsPatchResponseV1, - ProjectSettingsPayloadV1, SettingsPayloadV1, StorageSettingsPayloadV1, UserSettingsPayloadV1, - VersionSettingsPayloadV1, automation_settings_payload, project_editable_settings, -}; - -fn settings(revision: &str, max_file_size: u64) -> DashboardEnvelopeV1 { - let pinned = pinned_defaults(&ConfigurationRevisionId::new(revision).unwrap()); - let mut config = project_editable_settings(&pinned); - config.max_file_size = max_file_size; - let payload = SettingsPayloadV1 { - project: ProjectSettingsPayloadV1 { - configuration_snapshot_id: pinned.snapshot().snapshot_id.as_str().to_owned(), - configuration_revision_id: revision.to_owned(), - config, - pr_autotrack: PrAutoTrackPayloadV1 { tracked: vec![] }, - }, - user: UserSettingsPayloadV1 { - configuration_snapshot_id: "user-snapshot-7".to_owned(), - configuration_revision_id: "user-rev-7".to_owned(), - code_index_worker_configuration_snapshot_id: "workers-snapshot-3".to_owned(), - code_index_worker_configuration_revision_id: "workers-rev-3".to_owned(), - upload_enabled: false, - code_index_workers: CodeIndexWorkerSelectionV1::Automatic {}, - code_index_worker_status: None, - watcher_debounce: "15s".to_owned(), - extraction_timeout_secs: 30, - installed_agents: vec![], - }, - automation: automation_settings_payload(&pinned), - environment: EnvironmentSettingsPayloadV1 { - global_accounting_mode: "disabled".to_owned(), - global_accounting_enabled: false, - pricing_offline: true, - variables: vec![], - }, - storage: StorageSettingsPayloadV1 { - project_id: Some("project.settings.scout".to_owned()), - project_root: "/project".to_owned(), - storage_mode: "project_local".to_owned(), - store_root: "/project/.tracedecay".to_owned(), - dashboard_root: "/project".to_owned(), - graph_db: "/project/.tracedecay/graph.db".to_owned(), - memory_db: "/project/.tracedecay/memory.db".to_owned(), - lcm_db: "/project/.tracedecay/lcm.db".to_owned(), - lcm_scope: "project".to_owned(), - savings_db: "/project/.tracedecay/savings.db".to_owned(), - }, - version: VersionSettingsPayloadV1 { - version: "0.1.0".to_owned(), - channel: "stable".to_owned(), - cached_latest_version: None, - }, - resync_recommended: Some(max_file_size != 1_048_576), - restart_recommended: None, - }; - let mut envelope = DashboardEnvelopeV1::ready( - DashboardScopeV1 { - project_id: Some("project.settings.scout".to_owned()), - storage_mode: "project_local".to_owned(), - store_root: "/project/.tracedecay".to_owned(), - }, - DashboardCoverageV1::complete(1, "configuration_control_plane"), - payload, - ) - .with_legal_actions(vec![ - DashboardLegalActionRefV1::new( - DashboardLegalActionKindV1::RequestApply, - "configuration_batch", - ), - DashboardLegalActionRefV1::new(DashboardLegalActionKindV1::Refresh, "configuration_list"), - ]); - envelope.time.observation_time_micros = 100; - envelope.freshness.observed_at_micros = Some(100); - envelope -} - -fn digest(character: char) -> ManifestDigest { - ManifestDigest::new(format!("sha256:{}", character.to_string().repeat(64))).unwrap() -} - -fn effect() -> ApplicationOutcome { - let scope = ResolvedScope::new( - ProjectId::new("project.settings.scout").unwrap(), - RepositoryId::new("repository.settings").unwrap(), - WorktreeId::new("worktree.settings").unwrap(), - None, - ) - .unwrap(); - let idempotency_key = IdempotencyKey::new("settings-idempotency-fixture").unwrap(); - let authority = AuthorityReceipt { - grant_id: CapabilityGrantId::new("grant.settings").unwrap(), - grant_revision: 1, - grant_digest: digest('a'), - authorized_scope_digest: scope.scope_digest.clone(), - disclosure: DisclosureClass::Metadata, - policy: PolicyDecisionRef::new( - "policy.settings", - 1, - digest('b'), - ComponentVersion::new("policy.settings.v1").unwrap(), - ) - .unwrap(), - revalidated_at: UtcMicros(10), - }; - authority.validate_for(&scope).unwrap(); - ApplicationOutcome::Effect( - EffectResult::new( - EffectId::new("effect.configuration.settings").unwrap(), - EffectClass::Administrative, - idempotency_key.clone(), - authority, - digest('c'), - OperationReceipt::completed( - UtcMicros(10), - UtcMicros(20), - Deadline::new(UtcMicros(1_000)).unwrap(), - OperationBudgetUsage::default(), - ) - .unwrap(), - ReconciliationState::Reconciled, - EffectReceipt { - operation: UseCaseId::new("use-case.configuration.batch").unwrap(), - request_id: RequestId::new("request.configuration.settings-fixture").unwrap(), - actor: ActorId::new("actor.settings").unwrap(), - scope, - effect_class: EffectClass::Administrative, - idempotency_key, - input_digest: digest('a'), - expected_state: digest('c'), - policy_digest: digest('b'), - configuration_digest: digest('d'), - catalog_digest: digest('e'), - privacy_digest: digest('f'), - outcome: EffectTermination::Completed, - committed_state: Some(digest('1')), - external_proof: None, - }, - None, - ) - .unwrap(), - ) -} - -async fn http_fixture(response: impl IntoResponse) -> Value { - let response = response.into_response(); - let status = response.status().as_u16(); - let bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap(); - let body: Value = serde_json::from_slice(&bytes).unwrap(); - json!({ "status": status, "body": body }) -} - -#[tokio::test] -async fn project_patch_serialization_preserves_settlement_and_refusals() { - let outcome = effect(); - let expected_receipt = match &outcome { - ApplicationOutcome::Effect(effect) => serde_json::to_value(&effect.receipt).unwrap(), - _ => unreachable!("fixture constructs an effect"), - }; - let changed = http_fixture(Json(ProjectSettingsPatchResponseV1 { - application_outcome: Some(outcome), - current: settings("rev-43", 2_097_152), - })) - .await; - assert_eq!(changed["status"], 200); - assert_eq!(changed["body"]["application_outcome"]["outcome"], "effect"); - assert_eq!( - changed["body"]["application_outcome"]["value"]["receipt"], - expected_receipt - ); - assert_eq!( - changed["body"]["current"]["payload"]["project"]["configuration_revision_id"], - "rev-43" - ); - let unchanged = http_fixture(Json(ProjectSettingsPatchResponseV1 { - application_outcome: None, - current: settings("rev-42", 1_048_576), - })) - .await; - assert!( - unchanged["body"] - .as_object() - .unwrap() - .contains_key("application_outcome") - ); - assert!(unchanged["body"]["application_outcome"].is_null()); - - let idempotency_conflict = ApplicationProblemEnvelope::new( - ResultContractRef::new( - SchemaId::new("schema.configuration.batch.result").unwrap(), - 1, - ) - .unwrap(), - RequestId::new("request.configuration.settings-fixture").unwrap(), - ApplicationProblem::conflict( - "configuration.conflict", - "The idempotency key was already used for a different settings change.", - ), - ) - .unwrap(); - let expected_problem = serde_json::to_value(&idempotency_conflict).unwrap(); - let idempotency = http_fixture(configuration_application_problem_error( - idempotency_conflict, - )) - .await; - assert_eq!(idempotency["status"], 409); - assert_eq!(idempotency["body"], expected_problem); - - let validation = parse_project_settings_patch(json!({ - "expected_revision_id": "rev-42", - "idempotency_key": "settings-idempotency-fixture", - "unknown_setting": true, - })) - .expect_err("unknown fields must be refused"); - let cancelled = ApplicationProblemEnvelope::new( - ResultContractRef::new( - SchemaId::new("schema.configuration.batch.result").unwrap(), - 1, - ) - .unwrap(), - RequestId::new("request.configuration.settings-fixture").unwrap(), - ApplicationProblem::cancelled_before_admission(), - ) - .unwrap(); - let fixtures = json!({ - "read": http_fixture(Json(settings("rev-42", 1_048_576))).await, - "changed": changed, - "unchanged": unchanged, - "revision_conflict": http_fixture(configuration_revision_conflict_error( - "settings changed after this edit began; refresh and retry", "rev-42", "rev-43" - )).await, - "validation": http_fixture(validation).await, - "unavailable": http_fixture(configuration_authority_unavailable_error()).await, - "idempotency_conflict": idempotency, - "cancelled": http_fixture(configuration_application_problem_error(cancelled)).await, - }); - // Bazel preserves this producer output for the frontend's shared fixture. - if let Some(output) = std::env::var_os("TEST_UNDECLARED_OUTPUTS_DIR") { - std::fs::write( - std::path::PathBuf::from(output).join("settings-patch.json"), - format!("{}\n", serde_json::to_string_pretty(&fixtures).unwrap()), - ) - .unwrap(); - } -} diff --git a/crates/tracedecay/tests/runtime_acceptance_suite/runtime_surface_acceptance.rs b/crates/tracedecay/tests/runtime_acceptance_suite/runtime_surface_acceptance.rs index 1e4dbbfce7..b84be1c2fe 100644 --- a/crates/tracedecay/tests/runtime_acceptance_suite/runtime_surface_acceptance.rs +++ b/crates/tracedecay/tests/runtime_acceptance_suite/runtime_surface_acceptance.rs @@ -7,6 +7,7 @@ use std::io::Write; use std::path::{Path, PathBuf}; use std::process::{Output, Stdio}; +use axum::Json; use axum::body::{Body, to_bytes}; use axum::http::{Request, StatusCode}; use axum::response::IntoResponse; @@ -14,6 +15,10 @@ use serde_json::Value; use tempfile::TempDir; use tower::ServiceExt; use tracedecay::daemon::call_default_tool; +use tracedecay_api::configuration::{ + DashboardConfigurationRouteErrorV1, configuration_application_problem_error, + configuration_authority_unavailable_error, +}; use tracedecay_api::sse_response; use tracedecay_application::ProjectSourceAccessSnapshot; use tracedecay_application::feedback::concrete::open_feedback_runtime; @@ -33,10 +38,11 @@ use tracedecay_contracts::retrieval::PrimitiveRequest; #[cfg(all(unix, feature = "test-transport"))] use tracedecay_contracts::{ApplicationEnvelope, IdempotencyKey}; use tracedecay_contracts::{ - ApplicationOutcome, ApplicationProblemKind, CancellationContext, CancellationObservation, - CancellationSignal, CancellationStage, CapabilityGrantId, CapabilityGrantSnapshot, - CoverageCompleteness, Deadline, DisclosureClass, OperationBudgetUsage, OperationReceipt, - OperationTermination, PageRequest, RequestContext, RequestId, ResolvedScope, + ApplicationOutcome, ApplicationProblem, ApplicationProblemEnvelope, ApplicationProblemKind, + CancellationContext, CancellationObservation, CancellationSignal, CancellationStage, + CapabilityGrantId, CapabilityGrantSnapshot, CoverageCompleteness, Deadline, DisclosureClass, + OperationBudgetUsage, OperationReceipt, OperationTermination, PageRequest, RequestContext, + RequestId, ResolvedScope, ResultContractRef, }; use tracedecay_daemon_protocol::{ ApplicationSurfaceInvocationResult, ApplicationSurfaceRequest, FeedbackSurfaceRequest, @@ -73,7 +79,7 @@ use tracedecay_mcp::response_handles::{ResponseHandleLookup, retrieve_response_h use tracedecay_mcp::tools::dispatch::resolve_mcp_application_surface; use tracedecay_runtime_core::config::ProfileRoot; use tracedecay_tool_catalog::ApplicationSurfaceOperation; -use tracedecay_tool_catalog::{BindingSurface, CapabilityId, UseCaseId}; +use tracedecay_tool_catalog::{BindingSurface, CapabilityId, SchemaId, UseCaseId}; static DASHBOARD_CONFIGURATION_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); @@ -1012,6 +1018,7 @@ async fn dashboard_project_settings_commit_through_the_daemon_control_plane() { let (status, envelope) = get_dashboard_json(&agent, &settings_url); assert_eq!(status, 200, "GET settings failed: {envelope}"); + let read_fixture = serde_json::json!({ "status": status, "body": envelope }); let settings = &envelope["payload"]; let revision = settings["project"]["configuration_revision_id"] .as_str() @@ -1041,7 +1048,7 @@ async fn dashboard_project_settings_commit_through_the_daemon_control_plane() { "idempotency_key": "configuration.idempotency.dashboard-project-commit", "exclude": ["target/**", "dist/**"], "include": [".github/**"], - "max_file_size": 2048 + "max_file_size": 2_097_152 }), ); assert_eq!( @@ -1054,7 +1061,7 @@ async fn dashboard_project_settings_commit_through_the_daemon_control_plane() { let applied_payload = &applied["current"]["payload"]; assert_eq!(applied_payload["resync_recommended"], true); assert_eq!( - applied_payload["project"]["config"]["max_file_size"], 2048, + applied_payload["project"]["config"]["max_file_size"], 2_097_152, "the response must publish the daemon-returned snapshot: {applied_payload}" ); let applied_revision = applied_payload["project"]["configuration_revision_id"] @@ -1065,6 +1072,14 @@ async fn dashboard_project_settings_commit_through_the_daemon_control_plane() { applied_revision, revision, "a committed mutation must publish a new configuration revision" ); + let effect = &applied["application_outcome"]["value"]; + assert_eq!(applied["application_outcome"]["outcome"], "effect"); + assert_eq!(effect["execution"]["termination"], "completed"); + assert_eq!(effect["receipt"]["outcome"], "completed"); + assert!(effect["receipt"]["committed_state"].as_str().is_some()); + assert_eq!(effect["reconciliation"], "pending"); + assert_eq!(effect["payload"]["result_revision_id"], applied_revision); + let changed_fixture = serde_json::json!({ "status": status, "body": applied }); assert_ne!( applied_payload["project"]["config"]["max_file_size"], original_max_file_size, "the committed value must differ from the pre-change reading" @@ -1076,7 +1091,7 @@ async fn dashboard_project_settings_commit_through_the_daemon_control_plane() { assert_eq!(status, 200); assert_eq!( reloaded["payload"]["project"]["config"]["max_file_size"], - 2048 + 2_097_152 ); assert_eq!( reloaded["payload"]["project"]["config"]["include"][0], @@ -1087,6 +1102,60 @@ async fn dashboard_project_settings_commit_through_the_daemon_control_plane() { applied_revision.as_str() ); + let (unchanged_status, unchanged) = patch_dashboard_json( + &agent, + &project_url, + &serde_json::json!({ + "expected_revision_id": applied_revision, + "idempotency_key": "configuration.idempotency.dashboard-project-noop", + "max_file_size": 2_097_152 + }), + ); + assert_eq!( + unchanged_status, 200, + "unchanged project PATCH: {unchanged}" + ); + assert!(unchanged["application_outcome"].is_null()); + assert_eq!( + unchanged["current"]["payload"]["project"]["configuration_revision_id"], + applied_revision + ); + + let (idempotency_status, idempotency) = patch_dashboard_json( + &agent, + &project_url, + &serde_json::json!({ + "expected_revision_id": applied_revision, + "idempotency_key": "configuration.idempotency.dashboard-project-commit", + "max_file_size": 4096 + }), + ); + assert_eq!( + idempotency_status, 409, + "idempotency conflict: {idempotency}" + ); + assert_eq!(idempotency["problem"]["kind"], "conflict"); + + let (validation_status, validation) = patch_dashboard_json( + &agent, + &project_url, + &serde_json::json!({ + "expected_revision_id": applied_revision, + "idempotency_key": "configuration.idempotency.dashboard-project-invalid", + "unknown_setting": true + }), + ); + assert_eq!( + validation_status, 400, + "invalid project patch: {validation}" + ); + assert!( + !validation["validation_errors"] + .as_array() + .unwrap() + .is_empty() + ); + // CAS still holds against the real store: the superseded revision cannot // apply a second time. let (status, stale) = patch_dashboard_json( @@ -1102,6 +1171,38 @@ async fn dashboard_project_settings_commit_through_the_daemon_control_plane() { assert_eq!(stale["code"], "configuration_revision_conflict"); assert_eq!(stale["actual_revision_id"], applied_revision.as_str()); + // The changed response above comes from the mounted daemon, including its + // durable configuration effect and refreshed revision. The two admission + // refusals below use the same typed route factories as production. + let refusal = |(status, Json(body)): DashboardConfigurationRouteErrorV1| serde_json::json!({ "status": status.as_u16(), "body": body }); + let cancelled = ApplicationProblemEnvelope::new( + ResultContractRef::new( + SchemaId::new("schema.configuration.batch.result").unwrap(), + 1, + ) + .unwrap(), + RequestId::new("request.configuration.settings-cancelled").unwrap(), + ApplicationProblem::cancelled_before_admission(), + ) + .unwrap(); + let wire_fixture = serde_json::json!({ + "read": read_fixture, + "changed": changed_fixture, + "unchanged": { "status": unchanged_status, "body": unchanged }, + "revision_conflict": { "status": status, "body": stale }, + "validation": { "status": validation_status, "body": validation }, + "idempotency_conflict": { "status": idempotency_status, "body": idempotency }, + "unavailable": refusal(configuration_authority_unavailable_error()), + "cancelled": refusal(configuration_application_problem_error(cancelled)), + }); + if let Some(output) = std::env::var_os("TEST_UNDECLARED_OUTPUTS_DIR") { + std::fs::write( + PathBuf::from(output).join("settings-patch.json"), + format!("{}\n", serde_json::to_string_pretty(&wire_fixture).unwrap()), + ) + .unwrap(); + } + let _ = call_default_tool( fixture.profile(), &fixture.handshake, diff --git a/dashboard/src/test/fixtures/settings-patch.json b/dashboard/src/test/fixtures/settings-patch.json index 6c5d8473a9..ecfb089963 100644 --- a/dashboard/src/test/fixtures/settings-patch.json +++ b/dashboard/src/test/fixtures/settings-patch.json @@ -18,17 +18,17 @@ "legal_actions": [], "message": "The request was cancelled before admission", "owning_layer": "application", - "request_id": "request.configuration.settings-fixture", + "request_id": "request.configuration.settings-cancelled", "retry": "never", "retry_after_millis": null, "retry_scope": null, "retryable": false, "revision": 1, "terminality": "pre_admission", - "trace_id": "request.configuration.settings-fixture", + "trace_id": "request.configuration.settings-cancelled", "unavailable_classification": null }, - "request_id": "request.configuration.settings-fixture" + "request_id": "request.configuration.settings-cancelled" }, "status": 409 }, @@ -38,21 +38,21 @@ "outcome": "effect", "value": { "authority": { - "authorized_scope_digest": "sha256:cb520e3731e8dbab143f4fc32b885fc977d467f4648a6429bf0ed9bf0294873e", - "disclosure": "metadata", - "grant_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "grant_id": "grant.settings", + "authorized_scope_digest": "sha256:d6fb6d0938d4aef42e4eff3c0f113d607ce1baa34893752eae1361de955150ac", + "disclosure": "sensitive", + "grant_digest": "sha256:8c3dcc220cccaec81a373e117f12fd49517706ffc3a72c677b782ad0edcf423d", + "grant_id": "grant.daemon.configuration.ed4af84f307849aaff517d4b47c8e72002febe7cec0fcac38b788596cc343815", "grant_revision": 1, "policy": { - "decision_id": "policy.settings", - "digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "evaluator_revision": "policy.settings.v1", + "decision_id": "policy.daemon.configuration.v1", + "digest": "sha256:2dde51a3729d9b5fb9cb94b4eb70b6238971cf65bdf5a2ff5ba7d30cca42a36d", + "evaluator_revision": "tracedecay.daemon.configuration-policy.v1", "revision": 1 }, - "revalidated_at": 10 + "revalidated_at": 1792363717104188 }, - "effect_class": "administrative", - "effect_id": "effect.configuration.settings", + "effect_class": "configuration_write", + "effect_id": "effect.configuration.ed4af84f307849aaff517d4b47c8e72002febe7cec0fcac38b788596cc343815", "execution": { "budget": { "bytes_consumed": 0, @@ -61,39 +61,52 @@ }, "cancellation": null, "effective_deadline": { - "expires_at": 1000 + "expires_at": 1792363732101451 }, - "ended_at": 20, - "started_at": 10, + "ended_at": 1792363717104188, + "started_at": 1792363717104188, "termination": "completed" }, - "expected_state": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", - "idempotency_key": "settings-idempotency-fixture", - "payload": null, + "expected_state": "sha256:bae10ae7b2f476357fecbfc39d06eb7a43a89eb4b25d078422c77880d2199ef5", + "idempotency_key": "configuration.idempotency.dashboard-project-commit", + "payload": { + "base_revision_id": "configuration.initial.canonical.v1", + "created_at": 1792363717104188, + "effective_deadline_at": 1792363732101451, + "operation_digest": "sha256:95e5a654d365e7fafec21bd84bf68a159d13468d16e6cb38c26ac8493593727f", + "receipt_id": "configuration.receipt.v1.eaf7df3361e7130c1b2344d5537c8fdd09d4fb1288fdd4481bf6a0b0bfd9bbc8", + "result_revision_id": "configuration.revision.v1.f1dda57fc1603bacc602ad4cdbb08de9bfb8aa867eaabaef844d45f8c779d5b4", + "settlement_authority": { + "policy_digest": "sha256:2dde51a3729d9b5fb9cb94b4eb70b6238971cf65bdf5a2ff5ba7d30cca42a36d", + "policy_epoch": 1, + "revalidated_at": 1792363717104188 + }, + "snapshot_id": "tracedecay.configuration.snapshot.v1.061944e9d3ea29c90e551a2ce8af6d42a385839a986c5fd045ed0add7dee042f" + }, "receipt": { - "actor": "actor.settings", - "catalog_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", - "committed_state": "sha256:1111111111111111111111111111111111111111111111111111111111111111", - "configuration_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", - "effect_class": "administrative", - "expected_state": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "actor": "actor.tracedecay-daemon.project-open", + "catalog_digest": "sha256:57a70599bf2413cffb73f936d4b664b08f8b17533c2dcb0a2fd2bb078e528916", + "committed_state": "sha256:e3bd41f67533fc6a4941295fc75c48af569dfcf158b0d1cb209b9fe05fbac74e", + "configuration_digest": "sha256:e3bd41f67533fc6a4941295fc75c48af569dfcf158b0d1cb209b9fe05fbac74e", + "effect_class": "configuration_write", + "expected_state": "sha256:bae10ae7b2f476357fecbfc39d06eb7a43a89eb4b25d078422c77880d2199ef5", "external_proof": null, - "idempotency_key": "settings-idempotency-fixture", - "input_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "operation": "use-case.configuration.batch", + "idempotency_key": "configuration.idempotency.dashboard-project-commit", + "input_digest": "sha256:95e5a654d365e7fafec21bd84bf68a159d13468d16e6cb38c26ac8493593727f", + "operation": "use-case.application.configuration.batch", "outcome": "completed", - "policy_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "privacy_digest": "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", - "request_id": "request.configuration.settings-fixture", + "policy_digest": "sha256:2dde51a3729d9b5fb9cb94b4eb70b6238971cf65bdf5a2ff5ba7d30cca42a36d", + "privacy_digest": "sha256:a7f16bc386eeafbe59c14451bd32da5458ced67a14d7ee7e2c641743da818eb6", + "request_id": "request.configuration.ed4af84f307849aaff517d4b47c8e72002febe7cec0fcac38b788596cc343815", "scope": { - "project_id": "project.settings.scout", - "reference": null, - "repository_id": "repository.settings", - "scope_digest": "sha256:cb520e3731e8dbab143f4fc32b885fc977d467f4648a6429bf0ed9bf0294873e", - "worktree_id": "worktree.settings" + "project_id": "proj_69173dcc8b282d79", + "reference": "refs/heads/master", + "repository_id": "repository.daemon.e684e4fa551644cdbc59164b7a49c80cee102ea4b2d784c374df6f1cc473e629", + "scope_digest": "sha256:d6fb6d0938d4aef42e4eff3c0f113d607ce1baa34893752eae1361de955150ac", + "worktree_id": "worktree.daemon.69173dcc8b282d79a6608867f4799d9d8a97885470a7019d268a97b38a01d0a3" } }, - "reconciliation": "reconciled" + "reconciliation": "pending" } }, "current": { @@ -114,7 +127,7 @@ }, "domain_state": "ready", "freshness": { - "observed_at_micros": 100, + "observed_at_micros": 1792363717173083, "state": "fresh", "watermark": null }, @@ -123,6 +136,10 @@ "kind": "request_apply", "operation": "configuration_batch" }, + { + "kind": "request_apply", + "operation": "profile_code_index_worker_selection" + }, { "kind": "refresh", "operation": "configuration_list" @@ -135,56 +152,46 @@ }, "backend": "codex_app_server", "config_endpoint": "/api/plugins/holographic/curation/config", - "configuration_revision_id": "rev-43", + "configuration_revision_id": "configuration.revision.v1.f1dda57fc1603bacc602ad4cdbb08de9bfb8aa867eaabaef844d45f8c779d5b4", "enabled": true, "host_mode": "standalone" }, "environment": { "global_accounting_enabled": false, - "global_accounting_mode": "disabled", + "global_accounting_mode": "disabled_by_env", "pricing_offline": true, - "variables": [] + "variables": [ + { + "active": true, + "description": "A truthy value disables global savings/accounting recording.", + "name": "TRACEDECAY_DISABLE_GLOBAL_DB", + "value": "1" + }, + { + "active": true, + "description": "Pins the global accounting database to an explicit path.", + "name": "TRACEDECAY_GLOBAL_DB", + "value": "/run/nl/w/4ba25a53-d0f6-4590-8e56-7dffb5d83e12/tmp/.tmpxOcJR3/home/.tracedecay/global.db" + }, + { + "active": true, + "description": "Pins the user-level TraceDecay data directory (default ~/.tracedecay).", + "name": "TRACEDECAY_DATA_DIR", + "value": "/run/nl/w/4ba25a53-d0f6-4590-8e56-7dffb5d83e12/tmp/.tmpxOcJR3/home/.tracedecay" + } + ] }, "project": { "config": { "context_scout": false, "exclude": [ - ".git/**", - ".tracedecay/**", - "**/*.min.*", - ".cache/**", - "**/.cache/**", - ".gradle/**", - "**/.gradle/**", - ".next/**", - "**/.next/**", - ".turbo/**", - "**/.turbo/**", - ".venv/**", - "**/.venv/**", - ".worktrees/**", - "**/.worktrees/**", - "__pycache__/**", - "**/__pycache__/**", - "build/**", - "**/build/**", - "coverage/**", - "**/coverage/**", - "dist/**", - "**/dist/**", - "node_modules/**", - "**/node_modules/**", - "out/**", - "**/out/**", "target/**", - "**/target/**", - "vendor/**", - "**/vendor/**", - "venv/**", - "**/venv/**" + "dist/**" ], "extract_docstrings": true, - "include": [], + "include": [ + ".github/**" + ], "max_file_size": 2097152, "sync": { "auto_track_pr_branches": false, @@ -195,54 +202,63 @@ }, "track_call_sites": true }, - "configuration_revision_id": "rev-43", - "configuration_snapshot_id": "tracedecay.configuration.snapshot.v1.fd56d9635ef3558e8743096f4bcc8dc5f6e38a457b3b225086af5bf4b01b75ab", + "configuration_revision_id": "configuration.revision.v1.f1dda57fc1603bacc602ad4cdbb08de9bfb8aa867eaabaef844d45f8c779d5b4", + "configuration_snapshot_id": "tracedecay.configuration.snapshot.v1.061944e9d3ea29c90e551a2ce8af6d42a385839a986c5fd045ed0add7dee042f", "pr_autotrack": { "tracked": [] } }, "resync_recommended": true, "storage": { - "dashboard_root": "/project", - "graph_db": "/project/.tracedecay/graph.db", - "lcm_db": "/project/.tracedecay/lcm.db", - "lcm_scope": "project", - "memory_db": "/project/.tracedecay/memory.db", - "project_id": "project.settings.scout", - "project_root": "/project", - "savings_db": "/project/.tracedecay/savings.db", - "storage_mode": "project_local", - "store_root": "/project/.tracedecay" + "dashboard_root": "/run/nl/w/4ba25a53-d0f6-4590-8e56-7dffb5d83e12/tmp/.tmpxOcJR3/home/.tracedecay/projects/proj_69173dcc8b282d79/dashboard", + "graph_db": "/run/nl/w/4ba25a53-d0f6-4590-8e56-7dffb5d83e12/tmp/.tmpxOcJR3/home/.tracedecay/projects/proj_69173dcc8b282d79/tracedecay.db", + "lcm_db": "/run/nl/w/4ba25a53-d0f6-4590-8e56-7dffb5d83e12/tmp/.tmpxOcJR3/home/.tracedecay/projects/proj_69173dcc8b282d79/sessions.db", + "lcm_scope": "profile_sharded", + "memory_db": "/run/nl/w/4ba25a53-d0f6-4590-8e56-7dffb5d83e12/tmp/.tmpxOcJR3/home/.tracedecay/projects/proj_69173dcc8b282d79/tracedecay.db", + "project_id": "proj_69173dcc8b282d79", + "project_root": "/run/nl/w/4ba25a53-d0f6-4590-8e56-7dffb5d83e12/tmp/.tmpxOcJR3/project", + "savings_db": "/run/nl/w/4ba25a53-d0f6-4590-8e56-7dffb5d83e12/tmp/.tmpxOcJR3/home/.tracedecay/global.db", + "storage_mode": "profile_sharded", + "store_root": "/run/nl/w/4ba25a53-d0f6-4590-8e56-7dffb5d83e12/tmp/.tmpxOcJR3/home/.tracedecay/projects/proj_69173dcc8b282d79" }, "user": { - "code_index_worker_configuration_revision_id": "workers-rev-3", - "code_index_worker_configuration_snapshot_id": "workers-snapshot-3", - "code_index_worker_status": null, + "code_index_worker_configuration_revision_id": "configuration.profile.initial.v1", + "code_index_worker_configuration_snapshot_id": "tracedecay.configuration.snapshot.v1.3862ddb1c67746133efde2466f16da128b79ae5879a195e7075d1d0c9fe7525c", + "code_index_worker_status": { + "available_logical_cpus": 96, + "configured": { + "mode": "automatic" + }, + "effective_workers": 48, + "environment_override_workers": null, + "limiting_reason": "automatic_half_cores", + "memory_safe_workers": 384 + }, "code_index_workers": { "mode": "automatic" }, - "configuration_revision_id": "user-rev-7", - "configuration_snapshot_id": "user-snapshot-7", - "extraction_timeout_secs": 30, + "configuration_revision_id": "configuration.profile.initial.v1", + "configuration_snapshot_id": "tracedecay.configuration.snapshot.v1.3862ddb1c67746133efde2466f16da128b79ae5879a195e7075d1d0c9fe7525c", + "extraction_timeout_secs": 60, "installed_agents": [], "upload_enabled": false, - "watcher_debounce": "15s" + "watcher_debounce": "2s" }, "version": { "cached_latest_version": null, - "channel": "stable", - "version": "0.1.0" + "channel": "beta", + "version": "1.0.0-beta.88+451721797b8ae53bc539072fc19994df9cb66d34.dirty" } }, "schema_revision": 1, "scope": { - "project_id": "project.settings.scout", - "storage_mode": "project_local", - "store_root": "/project/.tracedecay" + "project_id": "proj_69173dcc8b282d79", + "storage_mode": "profile_sharded", + "store_root": "/run/nl/w/4ba25a53-d0f6-4590-8e56-7dffb5d83e12/tmp/.tmpxOcJR3/home/.tracedecay/projects/proj_69173dcc8b282d79" }, "source_watermark": null, "time": { - "observation_time_micros": 100, + "observation_time_micros": 1792363717173087, "valid_time_micros": null }, "version": { @@ -256,7 +272,7 @@ "idempotency_conflict": { "body": { "contract": { - "schema_id": "schema.configuration.batch.result", + "schema_id": "schema.application.configuration.configuration_batch.result", "schema_revision": 1 }, "problem": { @@ -268,26 +284,26 @@ "details": [], "diagnostic": { "code": "configuration.conflict", - "message": "The idempotency key was already used for a different settings change." + "message": "The configuration request conflicts with current state" }, "execution_failure_classification": null, "kind": "conflict", "legal_actions": [ "refresh" ], - "message": "The idempotency key was already used for a different settings change.", + "message": "The configuration request conflicts with current state", "owning_layer": "application", - "request_id": "request.configuration.settings-fixture", + "request_id": "request.dashboard.settings.7558e99727f379cfb4e9b326d15cd304.4", "retry": "after_revalidate", "retry_after_millis": null, "retry_scope": "fresh_request", "retryable": true, "revision": 1, "terminality": "pre_admission", - "trace_id": "request.configuration.settings-fixture", + "trace_id": "request.dashboard.settings.7558e99727f379cfb4e9b326d15cd304.4", "unavailable_classification": null }, - "request_id": "request.configuration.settings-fixture" + "request_id": "request.dashboard.settings.7558e99727f379cfb4e9b326d15cd304.4" }, "status": 409 }, @@ -310,7 +326,7 @@ }, "domain_state": "ready", "freshness": { - "observed_at_micros": 100, + "observed_at_micros": 1792363717093059, "state": "fresh", "watermark": null }, @@ -319,6 +335,10 @@ "kind": "request_apply", "operation": "configuration_batch" }, + { + "kind": "request_apply", + "operation": "profile_code_index_worker_selection" + }, { "kind": "refresh", "operation": "configuration_list" @@ -331,15 +351,34 @@ }, "backend": "codex_app_server", "config_endpoint": "/api/plugins/holographic/curation/config", - "configuration_revision_id": "rev-42", + "configuration_revision_id": "configuration.initial.canonical.v1", "enabled": true, "host_mode": "standalone" }, "environment": { "global_accounting_enabled": false, - "global_accounting_mode": "disabled", + "global_accounting_mode": "disabled_by_env", "pricing_offline": true, - "variables": [] + "variables": [ + { + "active": true, + "description": "A truthy value disables global savings/accounting recording.", + "name": "TRACEDECAY_DISABLE_GLOBAL_DB", + "value": "1" + }, + { + "active": true, + "description": "Pins the global accounting database to an explicit path.", + "name": "TRACEDECAY_GLOBAL_DB", + "value": "/run/nl/w/4ba25a53-d0f6-4590-8e56-7dffb5d83e12/tmp/.tmpxOcJR3/home/.tracedecay/global.db" + }, + { + "active": true, + "description": "Pins the user-level TraceDecay data directory (default ~/.tracedecay).", + "name": "TRACEDECAY_DATA_DIR", + "value": "/run/nl/w/4ba25a53-d0f6-4590-8e56-7dffb5d83e12/tmp/.tmpxOcJR3/home/.tracedecay" + } + ] }, "project": { "config": { @@ -391,54 +430,62 @@ }, "track_call_sites": true }, - "configuration_revision_id": "rev-42", - "configuration_snapshot_id": "tracedecay.configuration.snapshot.v1.fd56d9635ef3558e8743096f4bcc8dc5f6e38a457b3b225086af5bf4b01b75ab", + "configuration_revision_id": "configuration.initial.canonical.v1", + "configuration_snapshot_id": "tracedecay.configuration.snapshot.v1.e027d2b7709ea9e063d5bbce4809ee1d4e72187298a22bd16df6720b2e329741", "pr_autotrack": { "tracked": [] } }, - "resync_recommended": false, "storage": { - "dashboard_root": "/project", - "graph_db": "/project/.tracedecay/graph.db", - "lcm_db": "/project/.tracedecay/lcm.db", - "lcm_scope": "project", - "memory_db": "/project/.tracedecay/memory.db", - "project_id": "project.settings.scout", - "project_root": "/project", - "savings_db": "/project/.tracedecay/savings.db", - "storage_mode": "project_local", - "store_root": "/project/.tracedecay" + "dashboard_root": "/run/nl/w/4ba25a53-d0f6-4590-8e56-7dffb5d83e12/tmp/.tmpxOcJR3/home/.tracedecay/projects/proj_69173dcc8b282d79/dashboard", + "graph_db": "/run/nl/w/4ba25a53-d0f6-4590-8e56-7dffb5d83e12/tmp/.tmpxOcJR3/home/.tracedecay/projects/proj_69173dcc8b282d79/tracedecay.db", + "lcm_db": "/run/nl/w/4ba25a53-d0f6-4590-8e56-7dffb5d83e12/tmp/.tmpxOcJR3/home/.tracedecay/projects/proj_69173dcc8b282d79/sessions.db", + "lcm_scope": "profile_sharded", + "memory_db": "/run/nl/w/4ba25a53-d0f6-4590-8e56-7dffb5d83e12/tmp/.tmpxOcJR3/home/.tracedecay/projects/proj_69173dcc8b282d79/tracedecay.db", + "project_id": "proj_69173dcc8b282d79", + "project_root": "/run/nl/w/4ba25a53-d0f6-4590-8e56-7dffb5d83e12/tmp/.tmpxOcJR3/project", + "savings_db": "/run/nl/w/4ba25a53-d0f6-4590-8e56-7dffb5d83e12/tmp/.tmpxOcJR3/home/.tracedecay/global.db", + "storage_mode": "profile_sharded", + "store_root": "/run/nl/w/4ba25a53-d0f6-4590-8e56-7dffb5d83e12/tmp/.tmpxOcJR3/home/.tracedecay/projects/proj_69173dcc8b282d79" }, "user": { - "code_index_worker_configuration_revision_id": "workers-rev-3", - "code_index_worker_configuration_snapshot_id": "workers-snapshot-3", - "code_index_worker_status": null, + "code_index_worker_configuration_revision_id": "configuration.profile.initial.v1", + "code_index_worker_configuration_snapshot_id": "tracedecay.configuration.snapshot.v1.3862ddb1c67746133efde2466f16da128b79ae5879a195e7075d1d0c9fe7525c", + "code_index_worker_status": { + "available_logical_cpus": 96, + "configured": { + "mode": "automatic" + }, + "effective_workers": 48, + "environment_override_workers": null, + "limiting_reason": "automatic_half_cores", + "memory_safe_workers": 384 + }, "code_index_workers": { "mode": "automatic" }, - "configuration_revision_id": "user-rev-7", - "configuration_snapshot_id": "user-snapshot-7", - "extraction_timeout_secs": 30, + "configuration_revision_id": "configuration.profile.initial.v1", + "configuration_snapshot_id": "tracedecay.configuration.snapshot.v1.3862ddb1c67746133efde2466f16da128b79ae5879a195e7075d1d0c9fe7525c", + "extraction_timeout_secs": 60, "installed_agents": [], "upload_enabled": false, - "watcher_debounce": "15s" + "watcher_debounce": "2s" }, "version": { "cached_latest_version": null, - "channel": "stable", - "version": "0.1.0" + "channel": "beta", + "version": "1.0.0-beta.88+451721797b8ae53bc539072fc19994df9cb66d34.dirty" } }, "schema_revision": 1, "scope": { - "project_id": "project.settings.scout", - "storage_mode": "project_local", - "store_root": "/project/.tracedecay" + "project_id": "proj_69173dcc8b282d79", + "storage_mode": "profile_sharded", + "store_root": "/run/nl/w/4ba25a53-d0f6-4590-8e56-7dffb5d83e12/tmp/.tmpxOcJR3/home/.tracedecay/projects/proj_69173dcc8b282d79" }, "source_watermark": null, "time": { - "observation_time_micros": 100, + "observation_time_micros": 1792363717093062, "valid_time_micros": null }, "version": { @@ -450,10 +497,10 @@ }, "revision_conflict": { "body": { - "actual_revision_id": "rev-43", + "actual_revision_id": "configuration.revision.v1.f1dda57fc1603bacc602ad4cdbb08de9bfb8aa867eaabaef844d45f8c779d5b4", "code": "configuration_revision_conflict", "detail": "settings changed after this edit began; refresh and retry", - "expected_revision_id": "rev-42" + "expected_revision_id": "configuration.initial.canonical.v1" }, "status": 409 }, @@ -485,7 +532,7 @@ }, "domain_state": "ready", "freshness": { - "observed_at_micros": 100, + "observed_at_micros": 1792363717205602, "state": "fresh", "watermark": null }, @@ -494,6 +541,10 @@ "kind": "request_apply", "operation": "configuration_batch" }, + { + "kind": "request_apply", + "operation": "profile_code_index_worker_selection" + }, { "kind": "refresh", "operation": "configuration_list" @@ -506,57 +557,47 @@ }, "backend": "codex_app_server", "config_endpoint": "/api/plugins/holographic/curation/config", - "configuration_revision_id": "rev-42", + "configuration_revision_id": "configuration.revision.v1.f1dda57fc1603bacc602ad4cdbb08de9bfb8aa867eaabaef844d45f8c779d5b4", "enabled": true, "host_mode": "standalone" }, "environment": { "global_accounting_enabled": false, - "global_accounting_mode": "disabled", + "global_accounting_mode": "disabled_by_env", "pricing_offline": true, - "variables": [] + "variables": [ + { + "active": true, + "description": "A truthy value disables global savings/accounting recording.", + "name": "TRACEDECAY_DISABLE_GLOBAL_DB", + "value": "1" + }, + { + "active": true, + "description": "Pins the global accounting database to an explicit path.", + "name": "TRACEDECAY_GLOBAL_DB", + "value": "/run/nl/w/4ba25a53-d0f6-4590-8e56-7dffb5d83e12/tmp/.tmpxOcJR3/home/.tracedecay/global.db" + }, + { + "active": true, + "description": "Pins the user-level TraceDecay data directory (default ~/.tracedecay).", + "name": "TRACEDECAY_DATA_DIR", + "value": "/run/nl/w/4ba25a53-d0f6-4590-8e56-7dffb5d83e12/tmp/.tmpxOcJR3/home/.tracedecay" + } + ] }, "project": { "config": { "context_scout": false, "exclude": [ - ".git/**", - ".tracedecay/**", - "**/*.min.*", - ".cache/**", - "**/.cache/**", - ".gradle/**", - "**/.gradle/**", - ".next/**", - "**/.next/**", - ".turbo/**", - "**/.turbo/**", - ".venv/**", - "**/.venv/**", - ".worktrees/**", - "**/.worktrees/**", - "__pycache__/**", - "**/__pycache__/**", - "build/**", - "**/build/**", - "coverage/**", - "**/coverage/**", - "dist/**", - "**/dist/**", - "node_modules/**", - "**/node_modules/**", - "out/**", - "**/out/**", "target/**", - "**/target/**", - "vendor/**", - "**/vendor/**", - "venv/**", - "**/venv/**" + "dist/**" ], "extract_docstrings": true, - "include": [], - "max_file_size": 1048576, + "include": [ + ".github/**" + ], + "max_file_size": 2097152, "sync": { "auto_track_pr_branches": false, "auto_track_pr_poll_secs": 300 @@ -566,54 +607,63 @@ }, "track_call_sites": true }, - "configuration_revision_id": "rev-42", - "configuration_snapshot_id": "tracedecay.configuration.snapshot.v1.fd56d9635ef3558e8743096f4bcc8dc5f6e38a457b3b225086af5bf4b01b75ab", + "configuration_revision_id": "configuration.revision.v1.f1dda57fc1603bacc602ad4cdbb08de9bfb8aa867eaabaef844d45f8c779d5b4", + "configuration_snapshot_id": "tracedecay.configuration.snapshot.v1.061944e9d3ea29c90e551a2ce8af6d42a385839a986c5fd045ed0add7dee042f", "pr_autotrack": { "tracked": [] } }, "resync_recommended": false, "storage": { - "dashboard_root": "/project", - "graph_db": "/project/.tracedecay/graph.db", - "lcm_db": "/project/.tracedecay/lcm.db", - "lcm_scope": "project", - "memory_db": "/project/.tracedecay/memory.db", - "project_id": "project.settings.scout", - "project_root": "/project", - "savings_db": "/project/.tracedecay/savings.db", - "storage_mode": "project_local", - "store_root": "/project/.tracedecay" + "dashboard_root": "/run/nl/w/4ba25a53-d0f6-4590-8e56-7dffb5d83e12/tmp/.tmpxOcJR3/home/.tracedecay/projects/proj_69173dcc8b282d79/dashboard", + "graph_db": "/run/nl/w/4ba25a53-d0f6-4590-8e56-7dffb5d83e12/tmp/.tmpxOcJR3/home/.tracedecay/projects/proj_69173dcc8b282d79/tracedecay.db", + "lcm_db": "/run/nl/w/4ba25a53-d0f6-4590-8e56-7dffb5d83e12/tmp/.tmpxOcJR3/home/.tracedecay/projects/proj_69173dcc8b282d79/sessions.db", + "lcm_scope": "profile_sharded", + "memory_db": "/run/nl/w/4ba25a53-d0f6-4590-8e56-7dffb5d83e12/tmp/.tmpxOcJR3/home/.tracedecay/projects/proj_69173dcc8b282d79/tracedecay.db", + "project_id": "proj_69173dcc8b282d79", + "project_root": "/run/nl/w/4ba25a53-d0f6-4590-8e56-7dffb5d83e12/tmp/.tmpxOcJR3/project", + "savings_db": "/run/nl/w/4ba25a53-d0f6-4590-8e56-7dffb5d83e12/tmp/.tmpxOcJR3/home/.tracedecay/global.db", + "storage_mode": "profile_sharded", + "store_root": "/run/nl/w/4ba25a53-d0f6-4590-8e56-7dffb5d83e12/tmp/.tmpxOcJR3/home/.tracedecay/projects/proj_69173dcc8b282d79" }, "user": { - "code_index_worker_configuration_revision_id": "workers-rev-3", - "code_index_worker_configuration_snapshot_id": "workers-snapshot-3", - "code_index_worker_status": null, + "code_index_worker_configuration_revision_id": "configuration.profile.initial.v1", + "code_index_worker_configuration_snapshot_id": "tracedecay.configuration.snapshot.v1.3862ddb1c67746133efde2466f16da128b79ae5879a195e7075d1d0c9fe7525c", + "code_index_worker_status": { + "available_logical_cpus": 96, + "configured": { + "mode": "automatic" + }, + "effective_workers": 48, + "environment_override_workers": null, + "limiting_reason": "automatic_half_cores", + "memory_safe_workers": 384 + }, "code_index_workers": { "mode": "automatic" }, - "configuration_revision_id": "user-rev-7", - "configuration_snapshot_id": "user-snapshot-7", - "extraction_timeout_secs": 30, + "configuration_revision_id": "configuration.profile.initial.v1", + "configuration_snapshot_id": "tracedecay.configuration.snapshot.v1.3862ddb1c67746133efde2466f16da128b79ae5879a195e7075d1d0c9fe7525c", + "extraction_timeout_secs": 60, "installed_agents": [], "upload_enabled": false, - "watcher_debounce": "15s" + "watcher_debounce": "2s" }, "version": { "cached_latest_version": null, - "channel": "stable", - "version": "0.1.0" + "channel": "beta", + "version": "1.0.0-beta.88+451721797b8ae53bc539072fc19994df9cb66d34.dirty" } }, "schema_revision": 1, "scope": { - "project_id": "project.settings.scout", - "storage_mode": "project_local", - "store_root": "/project/.tracedecay" + "project_id": "proj_69173dcc8b282d79", + "storage_mode": "profile_sharded", + "store_root": "/run/nl/w/4ba25a53-d0f6-4590-8e56-7dffb5d83e12/tmp/.tmpxOcJR3/home/.tracedecay/projects/proj_69173dcc8b282d79" }, "source_watermark": null, "time": { - "observation_time_micros": 100, + "observation_time_micros": 1792363717205605, "valid_time_micros": null }, "version": { diff --git a/dashboard/src/workspaces/settings/SettingsPage.dom.test.tsx b/dashboard/src/workspaces/settings/SettingsPage.dom.test.tsx index 27715acba8..5859f7ac60 100644 --- a/dashboard/src/workspaces/settings/SettingsPage.dom.test.tsx +++ b/dashboard/src/workspaces/settings/SettingsPage.dom.test.tsx @@ -260,6 +260,7 @@ describe('SettingsPage effective configuration review', () => { await user.click(within(panel).getByRole('button', { name: 'Apply project settings' })); expect(await screen.findByText('Project settings saved')).toBeTruthy(); + expect(screen.getByText('Runtime reconciliation pending')).toBeTruthy(); expect(screen.getByText('Resync recommended')).toBeTruthy(); expect(reviewReadout()).toBe('applied'); // The effective value is the read-back, and the proposal is gone. @@ -397,7 +398,7 @@ describe('SettingsPage effective configuration review', () => { it('requires a refresh after an uncertain effect without offering to repeat the write', async () => { const response = structuredClone(settingsWire.changed.body); - response.application_outcome.value.reconciliation = 'pending'; + response.application_outcome.value.execution.termination = 'effect_unknown'; let dispatched = false; const fetch = vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => { if (init?.method === 'PATCH') { diff --git a/dashboard/src/workspaces/settings/SettingsReviewPanel.tsx b/dashboard/src/workspaces/settings/SettingsReviewPanel.tsx index e2ae53be1f..c982d01c51 100644 --- a/dashboard/src/workspaces/settings/SettingsReviewPanel.tsx +++ b/dashboard/src/workspaces/settings/SettingsReviewPanel.tsx @@ -336,6 +336,8 @@ function WritableBody({ {applied.resyncRecommended ? Resync recommended : null} {applied.restartRecommended ? Restart recommended : null} + {applied.effectReconciliation === 'pending' ? Runtime reconciliation pending : null} + {applied.effectReconciliation === 'failed' ? Runtime reconciliation failed : null}

) : null} diff --git a/dashboard/src/workspaces/settings/settingsEditorMachine.test.ts b/dashboard/src/workspaces/settings/settingsEditorMachine.test.ts index 81a7d40796..351b3f0f5d 100644 --- a/dashboard/src/workspaces/settings/settingsEditorMachine.test.ts +++ b/dashboard/src/workspaces/settings/settingsEditorMachine.test.ts @@ -391,6 +391,7 @@ describe('settings editor: each verdict is its own state', () => { resyncRecommended: false, restartRecommended: false, effectReceipt: null, + effectReconciliation: null, }); expect(settled).toBe(editing); @@ -408,6 +409,7 @@ describe('settings editor: a save and a pending change cannot be shown at once', resyncRecommended: true, restartRecommended: false, effectReceipt: null, + effectReconciliation: null, }); it('records the save the authority reported, and nothing it did not', () => { @@ -420,6 +422,7 @@ describe('settings editor: a save and a pending change cannot be shown at once', revisionId: 'rev-43', resyncRecommended: true, restartRecommended: false, + effectReconciliation: null, }); expect(settingsRejection(state)).toBeNull(); expect(settingsReviewOf(state)).toBeNull(); diff --git a/dashboard/src/workspaces/settings/settingsEditorMachine.ts b/dashboard/src/workspaces/settings/settingsEditorMachine.ts index fbccddee59..270172e881 100644 --- a/dashboard/src/workspaces/settings/settingsEditorMachine.ts +++ b/dashboard/src/workspaces/settings/settingsEditorMachine.ts @@ -41,6 +41,7 @@ import { type UserSettingsChangeSet, type UserSettingsValues, } from './settingsModel.ts'; +import type { ReconciliationState } from '../../contracts/generated.ts'; import type { SettingsMutationRequest, SettingsMutationResult } from './settingsMutation.ts'; /** The two drafts the editor holds, always for the same authority snapshot. */ @@ -78,6 +79,7 @@ interface SettingsAppliedRecord { readonly revisionId: string; readonly resyncRecommended: boolean; readonly restartRecommended: boolean; + readonly effectReconciliation: ReconciliationState | null; } /** Field-level refusals, kept with the authority that issued them. */ @@ -418,6 +420,7 @@ function settleSubmit( revisionId: result.revisionId, resyncRecommended: result.resyncRecommended, restartRecommended: result.restartRecommended, + effectReconciliation: result.effectReconciliation, }, }); case 'validation': diff --git a/dashboard/src/workspaces/settings/settingsMutation.test.ts b/dashboard/src/workspaces/settings/settingsMutation.test.ts index dae27d29a4..fbae39842c 100644 --- a/dashboard/src/workspaces/settings/settingsMutation.test.ts +++ b/dashboard/src/workspaces/settings/settingsMutation.test.ts @@ -2,24 +2,31 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; import wire from '../../test/fixtures/settings-patch.json'; import { applySettingsMutation } from './settingsMutation.ts'; -// settings_api/tests/wire.rs serializes these responses through Axum. Keep -// the complete receipts so decoding exercises the producer contract. +// runtime_surface_acceptance.rs captures the mounted daemon HTTP journey, +// including its durable commit, pending reconciliation, and refreshed revision. const request = { writability: { state: 'writable', target: 'tracedecay' }, scope: 'project', - expectedRevisionId: 'rev-42', - idempotencyKey: 'settings-idempotency-fixture', + expectedRevisionId: wire.read.body.payload.project.configuration_revision_id, + idempotencyKey: wire.changed.body.application_outcome.value.idempotency_key, readUrl: '/api/settings', patchUrl: '/api/settings/project', - patch: { max_file_size: 2_097_152 }, + patch: { + max_file_size: wire.changed.body.current.payload.project.config.max_file_size, + include: wire.changed.body.current.payload.project.config.include, + exclude: wire.changed.body.current.payload.project.config.exclude, + }, } as const; -async function apply(response: { status: number; body: unknown }) { +async function apply( + response: { status: number; body: unknown }, + current: typeof wire.read.body | typeof wire.changed.body.current = wire.read.body, +) { const fetch = vi.fn() - .mockResolvedValueOnce(Response.json(wire.read.body, { status: wire.read.status })) + .mockResolvedValueOnce(Response.json(current, { status: wire.read.status })) .mockResolvedValueOnce(Response.json(response.body, { status: response.status })); vi.stubGlobal('fetch', fetch); - const result = await applySettingsMutation(request); + const result = await applySettingsMutation({ ...request, expectedRevisionId: current.payload.project.configuration_revision_id }); expect(fetch).toHaveBeenCalledTimes(2); return result; } @@ -27,16 +34,23 @@ async function apply(response: { status: number; body: unknown }) { afterEach(() => vi.unstubAllGlobals()); describe('settings producer/consumer settlement', () => { - it('accepts the committed effect and preserves its complete durable receipt', async () => { + it('shows the real committed write as saved while reconciliation remains pending', async () => { + expect(wire.changed.body.application_outcome.value.reconciliation).toBe('pending'); expect(await apply(wire.changed)).toMatchObject({ outcome: 'success', - revisionId: 'rev-43', + revisionId: wire.changed.body.current.payload.project.configuration_revision_id, + effectReconciliation: 'pending', effectReceipt: wire.changed.body.application_outcome.value.receipt, }); }); it('accepts the explicit null outcome for an already-current patch', async () => { - expect(await apply(wire.unchanged)).toMatchObject({ outcome: 'success', revisionId: 'rev-42', effectReceipt: null }); + expect(await apply(wire.unchanged, wire.changed.body.current)).toMatchObject({ + outcome: 'success', + revisionId: wire.unchanged.body.current.payload.project.configuration_revision_id, + effectReceipt: null, + effectReconciliation: null, + }); }); it('requires the explicit nullable settlement and the canonical effect shape', async () => { @@ -51,7 +65,6 @@ describe('settings producer/consumer settlement', () => { it('does not present uncertain or unproven effects as applied', async () => { for (const mutate of [ (effect: typeof wire.changed.body.application_outcome.value) => { effect.execution.termination = 'effect_unknown'; }, - (effect: typeof wire.changed.body.application_outcome.value) => { effect.reconciliation = 'pending'; }, (effect: typeof wire.changed.body.application_outcome.value) => { effect.receipt.outcome = 'partial'; }, (effect: typeof wire.changed.body.application_outcome.value) => { Object.assign(effect.receipt, { committed_state: null }); }, ]) { @@ -61,9 +74,19 @@ describe('settings producer/consumer settlement', () => { } }); + it('requires the returned current revision to reflect the committed change', async () => { + const response = structuredClone(wire.changed); + response.body.current.payload.project.configuration_revision_id = request.expectedRevisionId; + expect(await apply(response)).toMatchObject({ outcome: 'unsettled' }); + }); + it('retains validation, revision conflicts and unavailable authority semantics', async () => { expect(await apply(wire.validation)).toMatchObject({ outcome: 'validation', errors: wire.validation.body.validation_errors }); - expect(await apply(wire.revision_conflict)).toEqual({ outcome: 'conflict', expectedRevisionId: 'rev-42', actualRevisionId: 'rev-43' }); + expect(await apply(wire.revision_conflict)).toEqual({ + outcome: 'conflict', + expectedRevisionId: wire.revision_conflict.body.expected_revision_id, + actualRevisionId: wire.revision_conflict.body.actual_revision_id, + }); expect(await apply(wire.unavailable)).toMatchObject({ outcome: 'unavailable' }); }); diff --git a/dashboard/src/workspaces/settings/settingsMutation.ts b/dashboard/src/workspaces/settings/settingsMutation.ts index 7ad06da6dd..5c1a384274 100644 --- a/dashboard/src/workspaces/settings/settingsMutation.ts +++ b/dashboard/src/workspaces/settings/settingsMutation.ts @@ -3,6 +3,7 @@ import { DashboardConfigurationRefusalV1Schema, ProjectSettingsPatchResponseV1Schema, type EffectReceipt, + type ReconciliationState, type SettingsPayloadV1, } from '../../contracts/generated.ts'; import { applicationProblemStatusMatches } from '../../contracts/application.ts'; @@ -31,6 +32,7 @@ export type SettingsMutationResult = readonly restartRecommended: boolean; /** Canonical durable application receipt for project writes. */ readonly effectReceipt: EffectReceipt | null; + readonly effectReconciliation: ReconciliationState | null; } | { readonly outcome: 'conflict'; @@ -153,7 +155,7 @@ export async function applySettingsMutation( return settingsRefusal(patched.body, patched.response.status, patchAuthority); } const projectPatch = request.scope === 'project' - ? projectSettingsPatchResponse(patched.body, patchAuthority) + ? projectSettingsPatchResponse(patched.body, patchAuthority, request.expectedRevisionId) : null; if (projectPatch && !projectPatch.parsed) return projectPatch.violation; const patchedPayload = contractedPayload( @@ -176,17 +178,20 @@ export async function applySettingsMutation( resyncRecommended: patchedPayload.payload.resync_recommended === true, restartRecommended: patchedPayload.payload.restart_recommended === true, effectReceipt: projectPatch?.parsed ? projectPatch.effectReceipt : null, + effectReconciliation: projectPatch?.parsed ? projectPatch.effectReconciliation : null, }; } function projectSettingsPatchResponse( body: unknown, authority: string, + expectedRevisionId: string, ): | { readonly parsed: true; readonly current: unknown; readonly effectReceipt: EffectReceipt | null; + readonly effectReconciliation: ReconciliationState | null; } | { readonly parsed: false; readonly violation: Extract } { const response = ProjectSettingsPatchResponseV1Schema.safeParse(body); @@ -200,12 +205,14 @@ function projectSettingsPatchResponse( }; } const outcome = response.data.application_outcome; + // Runtime reconciliation is separate from the durable configuration commit. + // A pending runtime update does not undo a saved, returned revision. if (outcome !== null && ( outcome.outcome !== 'effect' || outcome.value.execution.termination !== 'completed' - || outcome.value.reconciliation !== 'reconciled' || outcome.value.receipt.outcome !== 'completed' || outcome.value.receipt.committed_state === null + || response.data.current.payload.project.configuration_revision_id === expectedRevisionId )) { return { parsed: false, @@ -216,6 +223,7 @@ function projectSettingsPatchResponse( parsed: true, current: response.data.current, effectReceipt: outcome?.outcome === 'effect' ? outcome.value.receipt : null, + effectReconciliation: outcome?.outcome === 'effect' ? outcome.value.reconciliation : null, }; } From a14674cb0e111cf9c89a91f0f440a32693e6bb5e Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 22:55:51 +0000 Subject: [PATCH 41/89] fix(settings): label reconciliation as receipt state --- dashboard/src/workspaces/settings/SettingsPage.dom.test.tsx | 2 +- dashboard/src/workspaces/settings/SettingsReviewPanel.tsx | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dashboard/src/workspaces/settings/SettingsPage.dom.test.tsx b/dashboard/src/workspaces/settings/SettingsPage.dom.test.tsx index 5859f7ac60..14fd52604d 100644 --- a/dashboard/src/workspaces/settings/SettingsPage.dom.test.tsx +++ b/dashboard/src/workspaces/settings/SettingsPage.dom.test.tsx @@ -260,7 +260,7 @@ describe('SettingsPage effective configuration review', () => { await user.click(within(panel).getByRole('button', { name: 'Apply project settings' })); expect(await screen.findByText('Project settings saved')).toBeTruthy(); - expect(screen.getByText('Runtime reconciliation pending')).toBeTruthy(); + expect(screen.getByText('Receipt reconciliation pending')).toBeTruthy(); expect(screen.getByText('Resync recommended')).toBeTruthy(); expect(reviewReadout()).toBe('applied'); // The effective value is the read-back, and the proposal is gone. diff --git a/dashboard/src/workspaces/settings/SettingsReviewPanel.tsx b/dashboard/src/workspaces/settings/SettingsReviewPanel.tsx index c982d01c51..a96da1ec11 100644 --- a/dashboard/src/workspaces/settings/SettingsReviewPanel.tsx +++ b/dashboard/src/workspaces/settings/SettingsReviewPanel.tsx @@ -336,8 +336,8 @@ function WritableBody({ {applied.resyncRecommended ? Resync recommended : null} {applied.restartRecommended ? Restart recommended : null} - {applied.effectReconciliation === 'pending' ? Runtime reconciliation pending : null} - {applied.effectReconciliation === 'failed' ? Runtime reconciliation failed : null} + {applied.effectReconciliation === 'pending' ? Receipt reconciliation pending : null} + {applied.effectReconciliation === 'failed' ? Receipt reconciliation failed : null}

) : null} From 8ca9c7871832d7456ad74d40f8651e18c04c44a1 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 22:56:43 +0000 Subject: [PATCH 42/89] test(storage): use canonical writers and join fixture work --- .../orphan_stores/tests/collection.rs | 36 ++++++++-- .../src/participant_freeze.rs | 70 ++++++++++++------- .../session_runtime/retained_history.rs | 59 +++++++++++----- 3 files changed, 114 insertions(+), 51 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/orphan_stores/tests/collection.rs b/crates/tracedecay-maintenance/src/retention/orphan_stores/tests/collection.rs index 73fd48b1e7..b01d65c921 100644 --- a/crates/tracedecay-maintenance/src/retention/orphan_stores/tests/collection.rs +++ b/crates/tracedecay-maintenance/src/retention/orphan_stores/tests/collection.rs @@ -611,15 +611,39 @@ async fn sweep_unregistered_stores_never_deletes_durable_memory_rows() { ) .unwrap(); - let report = sweep_unregistered_stores(&db, &profile_root, 7 * DAY, base, true) - .await - .unwrap(); - assert_eq!(report.outcome.collected.len(), 0); + let cancellation = CancellationToken::new(); + let report = sweep_unregistered_store_page( + &db, + &profile_root, + UnregisteredStoreSweepRequestV1 { + cursor: None, + limit: DEFAULT_UNREGISTERED_STORE_PAGE_LIMIT, + retention_secs: 7 * DAY, + now: base, + apply: true, + cancellation: &cancellation, + deadline: functional_sweep_deadline(), + }, + ) + .await + .unwrap(); + assert_eq!( + report.completion, + UnregisteredSweepCompletionV1::Complete, + "functional durable-memory sweep was interrupted: {report:?}" + ); + assert!(report.applied, "complete sweep was not applied: {report:?}"); + assert!( + report.outcome.collected.is_empty(), + "durable memory store was collected: {report:?}" + ); + assert_eq!(report.outcome.errors.len(), 1, "{report:?}"); assert_eq!( report.outcome.errors[0].kind, - CollectionFailureKind::DurableDataProtected + CollectionFailureKind::DurableDataProtected, + "{report:?}" ); - assert!(dir.exists()); + assert!(dir.exists(), "durable memory store was removed: {report:?}"); } /// Seeds `branches/feature.db` (carrying a durable memory row) and its WAL the diff --git a/crates/tracedecay-session-temporal-store/src/participant_freeze.rs b/crates/tracedecay-session-temporal-store/src/participant_freeze.rs index 6d86e1d60a..13f0819874 100644 --- a/crates/tracedecay-session-temporal-store/src/participant_freeze.rs +++ b/crates/tracedecay-session-temporal-store/src/participant_freeze.rs @@ -531,7 +531,9 @@ mod tests { bind_test_session_relation_graph, open_registered_test_database_fixture, publish_test_session_relation_projection, }; - use tracedecay_global_db::{RegisteredGlobalDbLeaseV1, RegisteredGlobalDbOwnerV1}; + use tracedecay_global_db::{ + RegisteredGlobalDbLeaseV1, RegisteredGlobalDbOwnerV1, RegisteredGlobalDbWriterConnection, + }; use tracedecay_runtime_core::db::TestDatabaseRuntimeScope; use tracedecay_runtime_core::db::engine::{Executor, TestConnection}; use tracedecay_temporal_query::candidates::CandidateChannel; @@ -656,7 +658,7 @@ mod tests { /// `execute()` signs its continuation cursors, so a root fixture that stops at /// `freeze()` needs no key and one that runs a query needs exactly one. - async fn seed_root_cursor_key(connection: &TestConnection) { + async fn seed_root_cursor_key(connection: &impl Executor) { connection .execute( "INSERT INTO session_query_cursor_keys ( @@ -675,7 +677,7 @@ mod tests { /// on to publish a real projection must let the publisher mint its own. async fn publish_root_relation_projection( database: &RegisteredGlobalDbLeaseV1, - connection: &TestConnection, + connection: &impl Executor, session_id: &str, ) { for table in [ @@ -784,7 +786,7 @@ mod tests { } async fn seed_root_sessions( - connection: &TestConnection, + connection: &impl Executor, count: usize, hit_count: usize, source_frontier: u64, @@ -801,7 +803,7 @@ mod tests { /// Term overlap across sessions is what separates the lexical tiers, so a /// fixture that exercises them has to choose each session's words. async fn seed_root_sessions_with_text( - connection: &TestConnection, + connection: &impl Executor, count: usize, hit_count: usize, source_frontier: u64, @@ -978,13 +980,11 @@ mod tests { } } + /// Opens the registered root without attaching a second writer to its file. + /// Callers seed through `writer_connection`, the same lane projection uses. async fn open_root_fixture( directory: &TempDir, - ) -> ( - RegisteredGlobalDbLeaseV1, - RegisteredGlobalDbOwnerV1, - TestConnection, - ) { + ) -> (RegisteredGlobalDbLeaseV1, RegisteredGlobalDbOwnerV1) { let database_path = directory.path().join("sessions.db"); let (database, owner) = open_registered_test_database_fixture( &database_path, @@ -993,12 +993,20 @@ mod tests { .await .expect("registered schema"); bind_test_session_relation_graph(&database).expect("session relation graph"); - (database, owner, TestConnection::open(&database_path)) + (database, owner) + } + + fn root_fixture_writer( + database: &RegisteredGlobalDbLeaseV1, + ) -> RegisteredGlobalDbWriterConnection<'_> { + database + .writer_connection() + .expect("registered root fixture writer") } /// Groups the first seeded session's matching occurrence together with /// `extra_members` further matching occurrences into one span evidence row. - async fn seed_root_span_evidence(connection: &TestConnection, extra_members: usize) { + async fn seed_root_span_evidence(connection: &impl Executor, extra_members: usize) { seed_root_span(connection, extra_members, "needle cohort", "anchor.000").await; } @@ -1006,14 +1014,14 @@ mod tests { /// happens to contain the one message the query matched. Each member carries /// its own anchor and text, so no channel proposes it as a result. async fn seed_root_span_over_unmatched_members( - connection: &TestConnection, + connection: &impl Executor, extra_members: usize, ) { seed_root_span(connection, extra_members, "surrounding chatter", "").await; } async fn seed_root_span( - connection: &TestConnection, + connection: &impl Executor, extra_members: usize, text: &str, shared_anchor: &str, @@ -1149,7 +1157,7 @@ mod tests { /// query's terms is common and their conjunction is rare. Two sessions carry /// all four terms; every other session carries three of them, so an OR clause /// proposes the entire root and even a drop-one clause proposes hundreds. - async fn seed_root_lexical_cohort(connection: &TestConnection, count: usize) { + async fn seed_root_lexical_cohort(connection: &impl Executor, count: usize) { seed_root_sessions_with_text(connection, count, count, 1, &|index| { match index { 0 | 1 => "alpha beta gamma delta", @@ -1167,7 +1175,7 @@ mod tests { /// One session holds three of the query's four terms and nothing holds all /// four, so the strict tier is a verified zero and the drop-one tier answers. - async fn seed_root_relaxable_cohort(connection: &TestConnection) { + async fn seed_root_relaxable_cohort(connection: &impl Executor) { seed_root_sessions_with_text(connection, 4, 4, 1, &|index| match index { 0 => "alpha beta gamma".to_string(), _ => format!("beta without the rest {index}"), @@ -1207,7 +1215,8 @@ mod tests { #[asupersync::test] async fn root_conjunctive_lexical_answers_with_the_messages_every_term_matches() { let directory = tempdir().expect("temporary directory"); - let (database, _owner, connection) = open_root_fixture(&directory).await; + let (database, _owner) = open_root_fixture(&directory).await; + let connection = root_fixture_writer(&database); seed_root_lexical_cohort(&connection, 300).await; seed_root_cursor_key(&connection).await; for session in ["session.000", "session.001"] { @@ -1256,7 +1265,8 @@ mod tests { #[asupersync::test] async fn a_verified_zero_strict_tier_relaxes_one_named_step() { let directory = tempdir().expect("temporary directory"); - let (database, _owner, connection) = open_root_fixture(&directory).await; + let (database, _owner) = open_root_fixture(&directory).await; + let connection = root_fixture_writer(&database); seed_root_relaxable_cohort(&connection).await; seed_root_cursor_key(&connection).await; publish_root_relation_projection(&database, &connection, "session.000").await; @@ -1285,7 +1295,8 @@ mod tests { #[asupersync::test] async fn a_timed_out_strict_tier_refuses_instead_of_relaxing() { let directory = tempdir().expect("temporary directory"); - let (database, _owner, connection) = open_root_fixture(&directory).await; + let (database, _owner) = open_root_fixture(&directory).await; + let connection = root_fixture_writer(&database); seed_root_relaxable_cohort(&connection).await; seed_root_cursor_key(&connection).await; publish_root_relation_projection(&database, &connection, "session.000").await; @@ -1313,7 +1324,8 @@ mod tests { #[asupersync::test] async fn root_span_matched_by_many_members_is_one_candidate_over_300_sessions() { let directory = tempdir().expect("temporary directory"); - let (database, _owner, connection) = open_root_fixture(&directory).await; + let (database, _owner) = open_root_fixture(&directory).await; + let connection = root_fixture_writer(&database); seed_root_sessions(&connection, 300, 1, 1).await; seed_root_span_evidence(&connection, 4).await; @@ -1347,7 +1359,8 @@ mod tests { #[asupersync::test] async fn root_rare_hit_executes_under_the_unchanged_record_ceiling() { let directory = tempdir().expect("temporary directory"); - let (database, _owner, connection) = open_root_fixture(&directory).await; + let (database, _owner) = open_root_fixture(&directory).await; + let connection = root_fixture_writer(&database); seed_root_sessions(&connection, 300, 1, 1).await; seed_root_cursor_key(&connection).await; publish_root_relation_projection(&database, &connection, "session.000").await; @@ -1381,7 +1394,8 @@ mod tests { #[asupersync::test] async fn root_record_read_exhaustion_is_a_typed_budget_refusal_from_execute() { let directory = tempdir().expect("temporary directory"); - let (database, _owner, connection) = open_root_fixture(&directory).await; + let (database, _owner) = open_root_fixture(&directory).await; + let connection = root_fixture_writer(&database); seed_root_sessions(&connection, 300, 8, 1).await; seed_root_cursor_key(&connection).await; for index in 0..8 { @@ -1428,7 +1442,8 @@ mod tests { #[asupersync::test] async fn root_no_hit_over_256_sessions_is_truthful_zero_not_manifest_limit() { let directory = tempdir().expect("temporary directory"); - let (database, _owner, connection) = open_root_fixture(&directory).await; + let (database, _owner) = open_root_fixture(&directory).await; + let connection = root_fixture_writer(&database); seed_root_sessions(&connection, 300, 0, 1).await; let execution = super::super::RegisteredGlobalDbSessionTemporalExecution::new(&database); @@ -1467,7 +1482,8 @@ mod tests { #[asupersync::test] async fn root_no_hit_reports_aggregate_projection_staleness() { let directory = tempdir().expect("temporary directory"); - let (database, _owner, connection) = open_root_fixture(&directory).await; + let (database, _owner) = open_root_fixture(&directory).await; + let connection = root_fixture_writer(&database); seed_root_sessions(&connection, 3, 0, 2).await; let execution = super::super::RegisteredGlobalDbSessionTemporalExecution::new(&database); @@ -1486,7 +1502,8 @@ mod tests { #[asupersync::test] async fn root_rare_hit_over_256_sessions_freezes_only_the_admitted_participant() { let directory = tempdir().expect("temporary directory"); - let (database, _owner, connection) = open_root_fixture(&directory).await; + let (database, _owner) = open_root_fixture(&directory).await; + let connection = root_fixture_writer(&database); seed_root_sessions(&connection, 300, 1, 1).await; let execution = super::super::RegisteredGlobalDbSessionTemporalExecution::new(&database); @@ -1511,7 +1528,8 @@ mod tests { #[asupersync::test] async fn root_strict_population_over_one_window_returns_truthful_partial_coverage() { let directory = tempdir().expect("temporary directory"); - let (database, _owner, connection) = open_root_fixture(&directory).await; + let (database, _owner) = open_root_fixture(&directory).await; + let connection = root_fixture_writer(&database); seed_root_sessions_with_text(&connection, 300, 280, 1, &|index| { format!("needle strict-{index:03} cohort") }) diff --git a/crates/tracedecay/tests/session_suite/session_runtime/retained_history.rs b/crates/tracedecay/tests/session_suite/session_runtime/retained_history.rs index d5268a00cb..a3d4a8874a 100644 --- a/crates/tracedecay/tests/session_suite/session_runtime/retained_history.rs +++ b/crates/tracedecay/tests/session_suite/session_runtime/retained_history.rs @@ -791,6 +791,15 @@ async fn saturated_history_admission_defers_the_pass_and_resumes() { SessionHistoricalIngestOutcome::Complete, ])); let registry = SessionTemporalRefreshSchedulerRegistry::default(); + registry + .ensure_profile( + authority.database().db_path().to_path_buf(), + authority.database.clone(), + ) + .await; + registry + .wait_profile_idle(authority.database().db_path()) + .await; let admission = registry.historical_ingest_admission(); let permits = u32::try_from(admission.available_permits()).unwrap(); assert!(permits > 0, "the historical ingest admission is bounded"); @@ -805,29 +814,30 @@ async fn saturated_history_admission_defers_the_pass_and_resumes() { ingestor.clone(), ) .await; + registry + .wait_profile_idle(authority.database().db_path()) + .await; let saturated = SessionProjectionServingState::Stale { reason: SessionProjectionStaleReason::HistoricalRetry { reason_code: "history_admission_saturated".to_owned(), }, }; + let saturated_observed = wait_until( + || wake.serving_status().state == saturated, + Duration::from_secs(2), + ) + .await; assert!( - wait_until( - || wake.serving_status().state == saturated, - Duration::from_secs(2), - ) - .await, - "the deferred pass must surface as typed retryable staleness" + saturated_observed, + "the deferred pass must surface as typed retryable staleness: {:?}", + wake.serving_status() ); assert_eq!( ingestor.passes.get(), 0, "a saturated admission must not run the historical pass" ); - registry - .wait_profile_idle(authority.database().db_path()) - .await; - assert_eq!(ingestor.passes.get(), 0); drop(held); ingestor.passes.wait_for(|passes| *passes >= 1).await; @@ -851,6 +861,15 @@ async fn a_saturated_history_pass_runs_with_the_permit_its_queue_turn_grants() { SessionHistoricalIngestOutcome::Complete, ])); let registry = SessionTemporalRefreshSchedulerRegistry::default(); + registry + .ensure_profile( + authority.database().db_path().to_path_buf(), + authority.database.clone(), + ) + .await; + registry + .wait_profile_idle(authority.database().db_path()) + .await; let admission = registry.historical_ingest_admission(); let mut held = (0..admission.available_permits()) .map(|_| { @@ -867,22 +886,24 @@ async fn a_saturated_history_pass_runs_with_the_permit_its_queue_turn_grants() { ingestor.clone(), ) .await; + registry + .wait_profile_idle(authority.database().db_path()) + .await; let saturated = SessionProjectionServingState::Stale { reason: SessionProjectionStaleReason::HistoricalRetry { reason_code: "history_admission_saturated".to_owned(), }, }; + let saturated_observed = wait_until( + || wake.serving_status().state == saturated, + Duration::from_secs(2), + ) + .await; assert!( - wait_until( - || wake.serving_status().state == saturated, - Duration::from_secs(2), - ) - .await + saturated_observed, + "the queued pass must surface as typed retryable staleness: {:?}", + wake.serving_status() ); - registry - .wait_profile_idle(authority.database().db_path()) - .await; - let next_waiter = tokio::spawn(Arc::clone(&admission).acquire_owned()); tokio::task::yield_now().await; held.pop(); From 017d27ae7d407da87ded3965fc0a62e70699143b Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 22:57:51 +0000 Subject: [PATCH 43/89] test(global-db): batch canonical observation fixtures Use production batch admission and projection while retaining every observation and receipt-order assertion. The 384-row GC test fell from 8.072s to 3.769s on fresh local runs with unchanged 4784/4807 VM-step counts. --- .../tracedecay-global-db/src/tests/harness.rs | 43 ++++++++++++++----- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/crates/tracedecay-global-db/src/tests/harness.rs b/crates/tracedecay-global-db/src/tests/harness.rs index 6b9b56c1ad..19b49ac551 100644 --- a/crates/tracedecay-global-db/src/tests/harness.rs +++ b/crates/tracedecay-global-db/src/tests/harness.rs @@ -25,8 +25,8 @@ use tracedecay_rusqlite_runtime::repository::RepositoryWriterRuntimeSnapshot; #[cfg(test)] use tracedecay_store::{ AnchoredObservationWrite, ObservationPersistOutcome, ObservationProjectionStore, - ObservationStore, ObservationWrite, build_observation_resolution_authorization_v1, - build_observation_retrieval_anchor, + ObservationStore, ObservationWrite, ProjectionPersistOutcome, + build_observation_resolution_authorization_v1, build_observation_retrieval_anchor, }; pub(super) static TEST_RUNTIME_NONCE: AtomicU64 = AtomicU64::new(1); @@ -1147,6 +1147,7 @@ pub(crate) async fn seed_projected_messages( .unwrap() }); let mut seeded = Vec::with_capacity(indexes.len()); + let mut writes = Vec::with_capacity(indexes.len()); for index in indexes { let record_id = format!("record.audit-batch-{index}"); let record = ObservationId::new(record_id.clone()).unwrap(); @@ -1221,17 +1222,39 @@ pub(crate) async fn seed_projected_messages( ) .unwrap(); let anchored = AnchoredObservationWrite::new(write, anchor.clone(), generation).unwrap(); - assert!(matches!( - store.persist_observation(anchored).await.unwrap(), - ObservationPersistOutcome::Committed(_) - )); - store - .project_observation(observation.observation_id()) - .await - .unwrap(); + writes.push(anchored); expected_cursor = Some(next_cursor); seeded.push((observation, anchor)); } + if !writes.is_empty() { + // Fixture size is the test input; admission and projection use the + // same bounded batch authorities as host ingestion. + let outcomes = store.persist_observations(writes).await.unwrap(); + assert_eq!(outcomes.len(), seeded.len()); + assert!( + outcomes.iter().all(|outcome| matches!( + outcome.outcome(), + ObservationPersistOutcome::Committed(_) + )) + ); + let projected = store + .project_queued_observations(seeded.len()) + .await + .unwrap() + .expect("registered stores support batched projection"); + assert_eq!(projected.items.len(), seeded.len()); + for (item, admitted) in projected.items.iter().zip(&outcomes) { + assert!(matches!( + &item.outcome, + ProjectionPersistOutcome::Projected(output) if output.output_count() == 1 + )); + assert_eq!( + item.outcome.checkpoint().last_sequence(), + admitted.outcome().receipt().sequence(), + "fixture projection follows its admitted observations in order" + ); + } + } seeded } From c4ffcea1b5bf75aca8e7e6103fe926f9d1fb7af3 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 23:38:21 +0000 Subject: [PATCH 44/89] test(api): consume typed configuration refusals --- .../tests/dashboard_presentations.rs | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-api/tests/dashboard_presentations.rs b/crates/tracedecay-api/tests/dashboard_presentations.rs index 962ab7d940..28d75a49af 100644 --- a/crates/tracedecay-api/tests/dashboard_presentations.rs +++ b/crates/tracedecay-api/tests/dashboard_presentations.rs @@ -1,7 +1,8 @@ use axum::http::StatusCode; use serde_json::json; use tracedecay_api::configuration::{ - configuration_revision_conflict_error, parse_project_settings_patch, parse_user_settings_patch, + ConfigurationRefusalV1, DashboardConfigurationRefusalV1, configuration_revision_conflict_error, + parse_project_settings_patch, parse_user_settings_patch, }; use tracedecay_api::feedback::{ FeedbackStatusCoverageV1, FeedbackStatusDenominatorsV1, FeedbackStatusPresentationV1, @@ -47,7 +48,12 @@ fn configuration_patch_and_cas_errors_are_api_owned() { ) .expect_err("unknown field must remain a typed bad request"); assert_eq!(shape_error.0, StatusCode::BAD_REQUEST); - assert_eq!(shape_error.1.0["validation_errors"][0]["field"], "unknown"); + assert!(matches!( + &shape_error.1.0, + DashboardConfigurationRefusalV1::Validation { .. } + )); + let shape_body = serde_json::to_value(shape_error.1.0).expect("serialize validation refusal"); + assert_eq!(shape_body["validation_errors"][0]["field"], "unknown"); let conflict = configuration_revision_conflict_error( "settings changed after this edit began; refresh and retry", @@ -55,9 +61,16 @@ fn configuration_patch_and_cas_errors_are_api_owned() { "revision.actual", ); assert_eq!(conflict.0, StatusCode::CONFLICT); - assert_eq!(conflict.1.0["code"], "configuration_revision_conflict"); - assert_eq!(conflict.1.0["expected_revision_id"], "revision.expected"); - assert_eq!(conflict.1.0["actual_revision_id"], "revision.actual"); + assert!(matches!( + &conflict.1.0, + DashboardConfigurationRefusalV1::Configuration( + ConfigurationRefusalV1::ConfigurationRevisionConflict { .. } + ) + )); + let conflict_body = serde_json::to_value(conflict.1.0).expect("serialize revision refusal"); + assert_eq!(conflict_body["code"], "configuration_revision_conflict"); + assert_eq!(conflict_body["expected_revision_id"], "revision.expected"); + assert_eq!(conflict_body["actual_revision_id"], "revision.actual"); } #[test] From e3df8d593669583972e86b41e0cdc67079fdef5a Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Mon, 19 Oct 2026 00:08:27 +0000 Subject: [PATCH 45/89] fix(build): share dashboard source copies across consumers --- MODULE.bazel | 1 + dashboard/BUILD.bazel | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/MODULE.bazel b/MODULE.bazel index 275174e15e..bc3e333a16 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -2,6 +2,7 @@ module(name = "tracedecay", version = "0.0.0") bazel_dep(name = "platforms", version = "1.1.0") bazel_dep(name = "bazel_skylib", version = "1.9.2") +bazel_dep(name = "bazel_lib", version = "3.7.0") bazel_dep(name = "rules_python", version = "2.4.2") bazel_dep(name = "rules_rust", version = "0.74.0") # Rustc adds every dependency search directory to PATH on Windows; large diff --git a/dashboard/BUILD.bazel b/dashboard/BUILD.bazel index e282d69310..30b68f5741 100644 --- a/dashboard/BUILD.bazel +++ b/dashboard/BUILD.bazel @@ -1,3 +1,4 @@ +load("@bazel_lib//lib:copy_to_bin.bzl", "copy_to_bin") load("@npm//:defs.bzl", "npm_link_all_packages") load("@npm//dashboard:@rsbuild/core/package_json.bzl", rsbuild_bin = "bin") load("@npm//dashboard:vitest/package_json.bzl", vitest_bin = "bin") @@ -6,7 +7,8 @@ package(default_visibility = ["//visibility:public"]) npm_link_all_packages(name = "node_modules") -filegroup( +# Share source copies across the bundle and tests with different CPU allocations. +copy_to_bin( name = "dashboard_sources", srcs = glob( [ From 0d513f818df1b5d97c2e0725e155286083471dcb Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 18 Oct 2026 22:57:51 +0000 Subject: [PATCH 46/89] test(global-db): batch canonical observation fixtures Use production batch admission and projection while retaining every observation and receipt-order assertion. The 384-row GC test fell from 8.072s to 3.769s on fresh local runs with unchanged 4784/4807 VM-step counts. --- .../tracedecay-global-db/src/tests/harness.rs | 43 ++++++++++++++----- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/crates/tracedecay-global-db/src/tests/harness.rs b/crates/tracedecay-global-db/src/tests/harness.rs index 6b9b56c1ad..19b49ac551 100644 --- a/crates/tracedecay-global-db/src/tests/harness.rs +++ b/crates/tracedecay-global-db/src/tests/harness.rs @@ -25,8 +25,8 @@ use tracedecay_rusqlite_runtime::repository::RepositoryWriterRuntimeSnapshot; #[cfg(test)] use tracedecay_store::{ AnchoredObservationWrite, ObservationPersistOutcome, ObservationProjectionStore, - ObservationStore, ObservationWrite, build_observation_resolution_authorization_v1, - build_observation_retrieval_anchor, + ObservationStore, ObservationWrite, ProjectionPersistOutcome, + build_observation_resolution_authorization_v1, build_observation_retrieval_anchor, }; pub(super) static TEST_RUNTIME_NONCE: AtomicU64 = AtomicU64::new(1); @@ -1147,6 +1147,7 @@ pub(crate) async fn seed_projected_messages( .unwrap() }); let mut seeded = Vec::with_capacity(indexes.len()); + let mut writes = Vec::with_capacity(indexes.len()); for index in indexes { let record_id = format!("record.audit-batch-{index}"); let record = ObservationId::new(record_id.clone()).unwrap(); @@ -1221,17 +1222,39 @@ pub(crate) async fn seed_projected_messages( ) .unwrap(); let anchored = AnchoredObservationWrite::new(write, anchor.clone(), generation).unwrap(); - assert!(matches!( - store.persist_observation(anchored).await.unwrap(), - ObservationPersistOutcome::Committed(_) - )); - store - .project_observation(observation.observation_id()) - .await - .unwrap(); + writes.push(anchored); expected_cursor = Some(next_cursor); seeded.push((observation, anchor)); } + if !writes.is_empty() { + // Fixture size is the test input; admission and projection use the + // same bounded batch authorities as host ingestion. + let outcomes = store.persist_observations(writes).await.unwrap(); + assert_eq!(outcomes.len(), seeded.len()); + assert!( + outcomes.iter().all(|outcome| matches!( + outcome.outcome(), + ObservationPersistOutcome::Committed(_) + )) + ); + let projected = store + .project_queued_observations(seeded.len()) + .await + .unwrap() + .expect("registered stores support batched projection"); + assert_eq!(projected.items.len(), seeded.len()); + for (item, admitted) in projected.items.iter().zip(&outcomes) { + assert!(matches!( + &item.outcome, + ProjectionPersistOutcome::Projected(output) if output.output_count() == 1 + )); + assert_eq!( + item.outcome.checkpoint().last_sequence(), + admitted.outcome().receipt().sequence(), + "fixture projection follows its admitted observations in order" + ); + } + } seeded } From d7a8cd6d54fd040c4e4aa31d0bf73736fdcd9c35 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Mon, 19 Oct 2026 00:12:51 +0000 Subject: [PATCH 47/89] fix(identity): preserve nested checkout scope authority --- crates/tracedecay-mcp/src/scope.rs | 39 ++++++++++++------- .../src/context/registered_scope.rs | 25 ++++++++++-- 2 files changed, 46 insertions(+), 18 deletions(-) diff --git a/crates/tracedecay-mcp/src/scope.rs b/crates/tracedecay-mcp/src/scope.rs index 20bdbe5bb2..0763b0f312 100644 --- a/crates/tracedecay-mcp/src/scope.rs +++ b/crates/tracedecay-mcp/src/scope.rs @@ -243,7 +243,8 @@ mod tests { std::fs::create_dir_all(&subdir).unwrap(); let owner = owner_for(&root, "project.mcp-scope-test"); - let from_subdir = resolve_query_scope(&owner, &subdir).unwrap(); + let (resolved_root, from_subdir) = resolve_query_scope_with_root(&owner, &subdir).unwrap(); + assert_eq!(resolved_root, root); let from_root = resolve_query_scope(&owner, &root).unwrap(); assert_eq!( @@ -284,21 +285,24 @@ mod tests { fn sibling_root_of_another_repository_fails_closed() { let temp = TempDir::new().unwrap(); let registered = temp.path().join("registered"); - let sibling = temp.path().join("sibling"); std::fs::create_dir_all(®istered).unwrap(); - std::fs::create_dir_all(&sibling).unwrap(); let registered = registered.canonicalize().unwrap(); init_repo(®istered); - init_repo(&sibling); - let sibling = sibling.canonicalize().unwrap(); + write_identity_marker(®istered, "project.mcp-scope-test"); let owner = owner_for(®istered, "project.mcp-scope-test"); - - let error = resolve_query_scope(&owner, &sibling).unwrap_err(); - - assert!( - matches!(error, QueryScopeError::UnauthorizedSiblingRoot { .. }), - "a resolution naming a different repository must fail closed: {error}" - ); + for foreign in [ + temp.path().join("sibling"), + registered.join(".claude/foreign"), + ] { + std::fs::create_dir_all(&foreign).unwrap(); + init_repo(&foreign); + write_identity_marker(&foreign, "project.foreign"); + let error = resolve_query_scope(&owner, &foreign).unwrap_err(); + assert!( + matches!(error, QueryScopeError::UnauthorizedSiblingRoot { .. }), + "a resolution naming a different repository must fail closed: {error}" + ); + } } #[test] @@ -309,7 +313,7 @@ mod tests { init_repo(&root); let root = root.canonicalize().unwrap(); write_identity_marker(&root, "project.mcp-scope-test"); - let linked = temp.path().join("linked-feature"); + let linked = root.join(".claude/worktrees/linked-feature"); git( &root, &[ @@ -324,7 +328,14 @@ mod tests { let owner = owner_for(&root, "project.mcp-scope-test"); let main_scope = resolve_query_scope(&owner, &root).unwrap(); - let linked_scope = resolve_query_scope(&owner, &linked).unwrap(); + let (resolved_root, linked_scope) = resolve_query_scope_with_root(&owner, &linked).unwrap(); + assert_eq!(resolved_root, linked); + let subdir = linked.join("src/deep"); + std::fs::create_dir_all(&subdir).unwrap(); + assert_eq!( + resolve_query_scope_with_root(&owner, &subdir).unwrap(), + (linked.clone(), linked_scope.clone()) + ); assert_eq!( main_scope.repository_id, linked_scope.repository_id, diff --git a/crates/tracedecay-session-memory/src/context/registered_scope.rs b/crates/tracedecay-session-memory/src/context/registered_scope.rs index 36d1c48d38..2c2e7f820e 100644 --- a/crates/tracedecay-session-memory/src/context/registered_scope.rs +++ b/crates/tracedecay-session-memory/src/context/registered_scope.rs @@ -5,6 +5,9 @@ use std::path::{Path, PathBuf}; use tracedecay_contracts::ResolvedScope; use tracedecay_domain::canonical_text::sha256_hex; use tracedecay_domain::{ProjectId, RefId, RepositoryId, WorktreeId}; +use tracedecay_runtime_core::git_repository::{ + GitRepositoryError, GitTopologyProbeError, try_repository_topology, +}; use tracedecay_runtime_core::path_safety::canonical_existing_identity; use tracedecay_runtime_core::storage::RepositoryIdentityMarker; @@ -84,10 +87,24 @@ fn resolve_authorized_root( ) -> Result { let registered_root = canonical_root(registered_root, "registered root")?; let requested_root = canonical_root(requested_root, "requested root")?; - let scope_root = if requested_root.starts_with(®istered_root) { - registered_root.clone() - } else { - requested_root + // A checkout may be nested under the registered checkout. Resolve its + // actual Git root before ordinary subdirectories converge on their owner; + // lexical containment alone cannot authorize another repository. + let scope_root = match try_repository_topology(&requested_root) { + Ok(topology) => topology.worktree_root.clone().ok_or_else(|| { + ApplicationScopeError::Resolution(format!( + "requested root '{}' has no Git checkout", + requested_root.display() + )) + })?, + Err(GitTopologyProbeError::Repository(GitRepositoryError::NotARepository { .. })) => { + if requested_root.starts_with(®istered_root) { + registered_root.clone() + } else { + requested_root + } + } + Err(error) => return Err(ApplicationScopeError::Resolution(error.to_string())), }; let marker = tracedecay_runtime_core::storage::read_repository_identity_marker(&scope_root) .map_err(|error| ApplicationScopeError::Resolution(error.to_string()))?; From 25cc2907bf764ac84f2b40127912dd64ec27bf3a Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Mon, 19 Oct 2026 00:38:55 +0000 Subject: [PATCH 48/89] test(cli): wait for full project server before counting calls --- .../tests/core_cli_suite/tool_daemon_test.rs | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-cli/tests/core_cli_suite/tool_daemon_test.rs b/crates/tracedecay-cli/tests/core_cli_suite/tool_daemon_test.rs index fdfd7aa638..276401137f 100644 --- a/crates/tracedecay-cli/tests/core_cli_suite/tool_daemon_test.rs +++ b/crates/tracedecay-cli/tests/core_cli_suite/tool_daemon_test.rs @@ -434,8 +434,8 @@ fn assert_scope_set_probe_answered(output: &Output, context: &str) { /// project server, then reads /// that server's tool-call counter from its typed status. Until the full /// project server takes over from the core, the call and the status can land -/// on different servers, so the pair is re-sent until the status server has -/// counted a call. +/// on different servers with independent counters. Wait for the status +/// server's project session authority and a counted call after that handoff. fn wait_for_tool_status_server_tool_calls(home: &Path, project: &Path) -> u64 { let project_arg = project.to_string_lossy().to_string(); let deadline = Instant::now() + CLI_ROUNDTRIP_TIMEOUT; @@ -478,12 +478,18 @@ fn wait_for_tool_status_server_tool_calls(home: &Path, project: &Path) -> u64 { let tool_calls = payload["server"]["tool_calls"] .as_u64() .unwrap_or_else(|| panic!("typed status must carry its server counters: {payload}")); - if tool_calls >= 1 { + let project_sessions_mounted = + payload["server"]["retained_state_proxy"]["database_authorities"]["project_sessions"] + .as_bool() + .unwrap_or_else(|| { + panic!("typed status must carry its mounted authorities: {payload}") + }); + if project_sessions_mounted && tool_calls >= 1 { return tool_calls; } assert!( Instant::now() < deadline, - "the status server never counted a tool call before the CLI roundtrip deadline: {payload}" + "the full status server never mounted project sessions and counted a tool call before the CLI roundtrip deadline: {payload}" ); std::thread::sleep(Duration::from_millis(100)); } @@ -2148,7 +2154,7 @@ fn daemon_reuses_project_engine_across_tool_clients() { let project = TempDir::new().unwrap(); let home_path = canonical_existing_path(home.path()); let project_path = canonical_existing_path(project.path()); - init_project_with_cli(&home_path, &project_path); + init_committed_git_project_with_cli(&home_path, &project_path); let _daemon = spawn_tracedecay_daemon(&home_path); let first_tool_calls = wait_for_tool_status_server_tool_calls(&home_path, &project_path); From 8c8b3085c5ade5f69c219a72ea33a915ee12c532 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Mon, 19 Oct 2026 00:40:58 +0000 Subject: [PATCH 49/89] fix(hooks): admit cold file edits before full project open --- crates/tracedecay-mcp/src/hook_events.rs | 60 +++++ crates/tracedecay-mcp/src/server/rmcp.rs | 9 +- .../src/daemon/connection_serving.rs | 80 ++++++ crates/tracedecay/src/daemon/core_hooks.rs | 145 +++++++++- .../src/mcp/server/hook_dispatch.rs | 112 ++++++++ .../src/mcp/server/host_admission_tests.rs | 250 ++++++++++++++++++ crates/tracedecay/src/mcp/server/requests.rs | 87 ++++-- .../hooks_lsp_suite/hook_spool_drain_test.rs | 85 ++++-- 8 files changed, 780 insertions(+), 48 deletions(-) diff --git a/crates/tracedecay-mcp/src/hook_events.rs b/crates/tracedecay-mcp/src/hook_events.rs index d056113e04..9e2d727a72 100644 --- a/crates/tracedecay-mcp/src/hook_events.rs +++ b/crates/tracedecay-mcp/src/hook_events.rs @@ -137,6 +137,13 @@ fn push_admission_identity_part(buffer: &mut Vec, label: &str, value: &[u8]) #[derive(Debug, Clone, PartialEq, Eq)] pub enum HookEventPlan { SyncFiles(Vec), + SyncFilesAt { + root: PathBuf, + scope: tracedecay_contracts::ResolvedScope, + agent: HostIntegrationIdV1, + route: Option, + rel_paths: Vec, + }, AddBranch(String), AddBranchAt { root: PathBuf, @@ -165,6 +172,13 @@ enum DurableHookEventPlan { SyncFiles { rel_paths: Vec, }, + SyncFilesAt { + root: PathBuf, + scope: tracedecay_contracts::ResolvedScope, + agent: String, + route: Option, + rel_paths: Vec, + }, AddBranch { branch: String, }, @@ -356,6 +370,14 @@ fn normalize_durable_effect_root(root: &Path) -> Result { bound_absolute_add_branch_at_root(root).map_err(|_| ()) } +fn durable_registered_root(root: &Path) -> Result { + let normalized = normalize_durable_effect_root(root)?; + if normalized.as_os_str().as_encoded_bytes() != root.as_os_str().as_encoded_bytes() { + return Err(()); + } + Ok(normalized) +} + fn sanitize_durable_rel_paths(rel_paths: &[String]) -> Result, ()> { let sanitized = safe_hook_rel_paths(rel_paths); if sanitized.len() != rel_paths.len() || sanitized.len() > DURABLE_MAX_REL_PATHS { @@ -375,6 +397,22 @@ fn durable_plan_from_runtime(plan: &HookEventPlan) -> Result DurableHookEventPlan::SyncFiles { rel_paths: sanitize_durable_rel_paths(rel_paths)?, }, + HookEventPlan::SyncFilesAt { + root, + scope, + agent, + route, + rel_paths, + } => DurableHookEventPlan::SyncFilesAt { + root: durable_registered_root(root)?, + scope: { + scope.validate().map_err(|_| ())?; + scope.clone() + }, + agent: agent.as_wire().to_string(), + route: sanitize_durable_route(route.as_ref())?, + rel_paths: sanitize_durable_rel_paths(rel_paths)?, + }, HookEventPlan::AddBranch(branch) => DurableHookEventPlan::AddBranch { branch: durable_bound_required_str(branch, DURABLE_MAX_BRANCH_BYTES)?, }, @@ -426,6 +464,28 @@ fn runtime_plan_from_durable( sanitize_durable_rel_paths(&rel_paths) .map_err(|()| DurableHookEventDecodeError::Malformed)?, )), + DurableHookEventPlan::SyncFilesAt { + root, + scope, + agent, + route, + rel_paths, + } => Ok(HookEventPlan::SyncFilesAt { + root: durable_registered_root(&root) + .map_err(|()| DurableHookEventDecodeError::Malformed)?, + scope: { + scope + .validate() + .map_err(|_| DurableHookEventDecodeError::Malformed)?; + scope + }, + agent: HostIntegrationIdV1::from_wire(&agent) + .ok_or(DurableHookEventDecodeError::Malformed)?, + route: sanitize_durable_route(route.as_ref()) + .map_err(|()| DurableHookEventDecodeError::Malformed)?, + rel_paths: sanitize_durable_rel_paths(&rel_paths) + .map_err(|()| DurableHookEventDecodeError::Malformed)?, + }), DurableHookEventPlan::AddBranch { branch } => Ok(HookEventPlan::AddBranch( durable_bound_required_str(&branch, DURABLE_MAX_BRANCH_BYTES) .map_err(|()| DurableHookEventDecodeError::Malformed)?, diff --git a/crates/tracedecay-mcp/src/server/rmcp.rs b/crates/tracedecay-mcp/src/server/rmcp.rs index 7e7ad793ed..777cfbbae9 100644 --- a/crates/tracedecay-mcp/src/server/rmcp.rs +++ b/crates/tracedecay-mcp/src/server/rmcp.rs @@ -764,8 +764,8 @@ where Ok(result.into()) } - /// A hook event arrives as a stateless request on its own daemon - /// connection; it is dispatched exactly like the notification form. + /// Hook requests retain their identity through dispatch so the application + /// answers only after durable admission; notifications carry no response. /// Any other custom request is an unknown method or a known one whose /// params did not fit its typed DTO; the dispatch authority answers it /// (method not found, invalid params) and accounts for it. @@ -774,11 +774,6 @@ where request: CustomRequest, context: McpRequestContext, ) -> Result { - if request.method == tracedecay_hooks::core_events::HOOK_EVENT_METHOD { - self.dispatch_notification(request.method, request.params) - .await; - return Ok(CustomResult::new(json!({}))); - } // The session lifts `params._meta` into the request context; the // object it leaves empty is a request that carried no params of its // own (the stdio proxy creates `params` only to hold that context). diff --git a/crates/tracedecay/src/daemon/connection_serving.rs b/crates/tracedecay/src/daemon/connection_serving.rs index 58b1d2e768..661c040b54 100644 --- a/crates/tracedecay/src/daemon/connection_serving.rs +++ b/crates/tracedecay/src/daemon/connection_serving.rs @@ -1507,6 +1507,40 @@ fn serve_broker_socket_client_inner( return Ok(()); } + match Box::pin(super::core_hooks::admit_unmounted_file_edit( + &engine.store_administration, + &handshake, + first_request.parsed(), + )) + .await + { + Ok(Some(routed)) => { + let response = first_request + .parsed() + .and_then(|request| request.id.clone()) + .map(|id| JsonRpcResponse::success(id, serde_json::json!({}))); + let written = match response { + Some(response) => write_daemon_response(&mut transport, &response).await, + None => Ok(()), + }; + if let Err(error) = engine.schedule_project_server_warmup(routed, None).await { + tracing::warn!(%error, "durably admitted file edit awaits project publication"); + } + written?; + return Ok(()); + } + Ok(None) => {} + Err(error) => { + write_project_open_error( + &mut transport, + &first_request, + &handshake.client_instance_id, + &error, + ) + .await?; + return Ok(()); + } + } let projectless_request = projectless_first_request(first_request.parsed()); let project_owner = boxed_broker_connection_phase(async { if handshake.project_path.is_some() && !projectless_request { @@ -2033,6 +2067,52 @@ pub(super) async fn serve_windows_broker_client_with_class_and_invocation( return Ok(()); } } + match Box::pin(super::core_hooks::admit_unmounted_file_edit( + &store_administration, + &handshake, + first_request.parsed(), + )) + .await + { + Ok(Some(routed)) => { + let response = first_request + .parsed() + .and_then(|request| request.id.clone()) + .map(|id| JsonRpcResponse::success(id, serde_json::json!({}))); + let written = match response { + Some(response) => write_daemon_response(&mut transport, &response).await, + None => Ok(()), + }; + if let Err(error) = Box::pin(schedule_portable_project_server_warmup( + lifecycle.clone(), + store_administration.clone(), + Arc::clone(&project_open_gates), + invocation.clone(), + http_application_registry.clone(), + routed, + None, + #[cfg(test)] + project_open_attempts.clone(), + )) + .await + { + tracing::warn!(%error, "durably admitted file edit awaits project publication"); + } + written?; + return Ok(()); + } + Ok(None) => {} + Err(error) => { + write_project_open_error( + &mut transport, + &first_request, + &handshake.client_instance_id, + &error, + ) + .await?; + return Ok(()); + } + } let projectless_request = projectless_first_request(first_request.parsed()); if handshake.project_path.is_some() && !projectless_request { // Heap-allocate the owner-await composition: embedded by value it diff --git a/crates/tracedecay/src/daemon/core_hooks.rs b/crates/tracedecay/src/daemon/core_hooks.rs index 5c1bbfd1fd..2352602630 100644 --- a/crates/tracedecay/src/daemon/core_hooks.rs +++ b/crates/tracedecay/src/daemon/core_hooks.rs @@ -1,23 +1,162 @@ //! Host hook events: one stateless daemon request over the broker connection. //! //! The wire metadata and event constructors are pure data and live in -//! [`tracedecay_hooks::core_events`]. Only delivery, which needs the daemon -//! connection, handshake, and preamble, remains root-coupled. +//! [`tracedecay_hooks::core_events`]. Broker admission and delivery use the +//! daemon's canonical store, connection, handshake, and preamble authorities. -use std::path::Path; +use std::path::{Path, PathBuf}; use asupersync::io::{AsyncWriteExt, BufReader}; +use rmcp::model::{ProtocolVersion, RequestMetaObject}; use tracedecay_framing::BoundedLineReader; use tracedecay_hooks::core_events::{DaemonHookEvent, HOOK_EVENT_METHOD, HookEventNotifyOutcomeV1}; +use crate::mcp::project_route::HookProjectRouteCache; use tracedecay_daemon_identity::{ ResolvedDaemonConnection, client_connection, current_daemon_connection, }; use tracedecay_daemon_protocol::{DaemonClientStream, SOCKET_ENV}; +use tracedecay_domain::errors::{Result, TraceDecayError}; +use tracedecay_mcp::hook_events::{self, HookEventKind, HookEventPlan}; use tracedecay_mcp::server::attach_stateless_request_context; use super::{JsonRpcRequest, write_daemon_preamble}; +/// An unopened worktree needs only its registered session store to retain a +/// file edit. Graph admission and full owner publication happen after the +/// durable receipt; the existing broker replay resolves the exact route. +#[tracing::instrument( + name = "daemon.engine.hooks.admit_unmounted_edit", + level = "trace", + skip_all +)] +pub(super) async fn admit_unmounted_file_edit( + stores: &super::StoreAdministration, + handshake: &super::DaemonHandshake, + request: Option<&JsonRpcRequest>, +) -> Result> { + let Some(request) = request.filter(|request| { + request.method == HOOK_EVENT_METHOD && tracedecay_mcp::server::opens_rmcp_session(request) + }) else { + return Ok(None); + }; + // Leave foreign or malformed inline contexts to the RMCP authority. This + // fast admission path must not acknowledge a request RMCP would refuse. + let Some(meta) = request + .params + .as_ref() + .and_then(|params| params.get("_meta")) + .and_then(|meta| serde_json::from_value::(meta.clone()).ok()) + else { + return Ok(None); + }; + if !meta + .missing_required_keys(&ProtocolVersion::V_2026_07_28) + .is_empty() + || meta + .protocol_version() + .is_none_or(|version| !ProtocolVersion::KNOWN_VERSIONS.contains(&version)) + { + return Ok(None); + } + let Some(event) = hook_events::parse_hook_event(request.params.as_ref()) else { + return Ok(None); + }; + if event.kind != HookEventKind::FileEdit || event.rel_paths.is_empty() { + return Ok(None); + } + let root = HookProjectRouteCache::route_cwd(&event).ok_or_else(|| { + TraceDecayError::project_route( + "project_route_unavailable", + false, + "file edit did not name its worktree", + ) + })?; + let mut routed = handshake.clone(); + routed.project_path = Some(root.to_path_buf()); + let (root, route_key) = super::project_route_for_handshake(&routed, stores.owner_home()?)?; + if stores + .project_servers() + .lock() + .await + .get_route_and_touch_for( + &route_key, + super::ProjectServerRequirement::RegisteredHostIngest, + ) + .is_some() + { + return Ok(None); + } + admit_registered_file_edit(stores, routed, root, &event) + .await + .map(Some) +} + +async fn admit_registered_file_edit( + stores: &super::StoreAdministration, + mut routed: super::DaemonHandshake, + root: PathBuf, + event: &hook_events::HookEvent, +) -> Result { + super::ensure_registered_project_route(stores, &root, false).await?; + let registry = stores.registered_profile_database().await?; + let layout = tracedecay_project::project::TraceDecay::resolve_registered_configuration_layout( + &root, + &super::handshake_open_options(&routed), + registry.as_ref(), + ) + .await?; + let project_id = layout.identity.project_id.as_deref().ok_or_else(|| { + TraceDecayError::project_route( + "project_route_unavailable", + false, + "file edit has no registered project identity", + ) + })?; + let owner = registry + .project_registry_context_by_id(project_id) + .await? + .ok_or_else(|| { + TraceDecayError::project_route( + "project_route_unavailable", + false, + "file edit project is no longer registered", + ) + })?; + let (root, scope) = tracedecay_mcp::scope::resolve_query_scope(&owner, &root) + .map_err(|error| error.into_route_failure().into_error())?; + let database = stores + .registered_project_session_database(&root, &layout) + .await?; + let broker = stores.host_admission_broker(&database).await?; + let payload = hook_events::encode_durable_hook_event_plan(&HookEventPlan::SyncFilesAt { + root: root.clone(), + scope, + agent: event.agent, + route: event.route.clone(), + rel_paths: event.rel_paths.clone(), + }) + .map_err(|_| { + TraceDecayError::project_route( + "invalid_host_event_plan", + false, + "file edit could not be encoded for durable admission", + ) + })?; + broker + .admit(&event.admission_source(), &payload) + .await + .map_err(|outcome| { + TraceDecayError::project_route( + outcome.reason_code.unwrap_or("host_admission_unavailable"), + outcome.retryable, + "file edit was not durably admitted", + ) + })?; + routed.project_path = Some(root); + Ok(routed) +} + /// Delivers `event` and resolves once the daemon acknowledges it, however /// long admission takes. A host hook that must not block its agent bounds /// this call itself. diff --git a/crates/tracedecay/src/mcp/server/hook_dispatch.rs b/crates/tracedecay/src/mcp/server/hook_dispatch.rs index 76ef2fbf0e..7ed410caac 100644 --- a/crates/tracedecay/src/mcp/server/hook_dispatch.rs +++ b/crates/tracedecay/src/mcp/server/hook_dispatch.rs @@ -79,6 +79,103 @@ impl McpServer { }) } + async fn run_routed_file_edit( + &self, + effect_root: PathBuf, + scope: tracedecay_contracts::ResolvedScope, + agent: tracedecay_domain::HostIntegrationIdV1, + route: Option, + rel_paths: Vec, + ) -> HostAdmissionOutcome { + let Some(registry) = self.registry_db.as_deref() else { + return HostAdmissionOutcome::retained_unavailable( + "project_registry_route_unavailable", + ); + }; + let owner = match registry + .project_registry_context_by_id(scope.project_id.as_str()) + .await + { + Ok(Some(owner)) => owner, + Ok(None) => return HostAdmissionOutcome::degraded("project_route_stale"), + Err(_) => { + return HostAdmissionOutcome::retained_unavailable( + "project_registry_route_unavailable", + ); + } + }; + let (root, current_scope) = + match tracedecay_mcp::scope::resolve_query_scope(&owner, &effect_root) { + Ok(resolved) => resolved, + Err(_) => return HostAdmissionOutcome::degraded("project_route_stale"), + }; + if root != effect_root || current_scope != scope { + return HostAdmissionOutcome::degraded("project_route_stale"); + } + let resolved = match crate::mcp::project_route::resolve_registered_project_route( + owner, + &root, + registry, + self.retained_project_server_resolver.clone(), + ) + .await + { + Ok(resolved) => resolved, + Err(_) => { + return HostAdmissionOutcome::retained_unavailable("project_route_unavailable"); + } + }; + if resolved.requested_root != root || resolved.scope != scope { + return HostAdmissionOutcome::degraded("project_route_stale"); + } + let event = hook_events::HookEvent { + agent, + kind: hook_events::HookEventKind::FileEdit, + rel_paths, + had_command: false, + cwd: Some(root), + route, + receipt: None, + }; + let server = match resolved.retained_server() { + Ok(server) => server, + Err(_) => { + return HostAdmissionOutcome::retained_unavailable("project_route_unavailable"); + } + }; + if server.project_route_live() == Some(false) + || server + .project_server_lifecycle + .response_revoked() + .is_cancelled() + { + return HostAdmissionOutcome::retained_unavailable("project_route_unavailable"); + } + let mut route_cache = match self.hook_project_routes.snapshot() { + Ok(cache) => cache, + Err(_) => { + return HostAdmissionOutcome::retained_unavailable( + "project_registry_route_unavailable", + ); + } + }; + route_cache.observe_workspace_route( + &event, + crate::mcp::project_route::WorkspaceProjectRoute::Resolved(Box::new(resolved.clone())), + ); + if self.hook_project_routes.store(&route_cache).is_err() { + return HostAdmissionOutcome::retained_unavailable( + "project_registry_route_unavailable", + ); + } + match server.code_index_hook_sink.as_ref() { + Some(sink) => { + code_index_host_outcome(&sink(resolved.requested_root, event.rel_paths).await) + } + None => HostAdmissionOutcome::retained_unavailable(CODE_INDEX_SCHEDULER_UNAVAILABLE), + } + } + pub(crate) async fn run_hook_event_plan( &self, cg: Arc, @@ -86,6 +183,21 @@ impl McpServer { plan: HookEventPlan, ) -> HostAdmissionOutcome { match plan { + HookEventPlan::SyncFilesAt { + root: effect_root, + scope, + agent, + route, + rel_paths, + } => { + if Some(scope.project_id.as_str()) + != cg.store_layout().identity.project_id.as_deref() + { + return HostAdmissionOutcome::degraded("project_store_mismatch"); + } + self.run_routed_file_edit(effect_root, scope, agent, route, rel_paths) + .await + } HookEventPlan::SyncFiles(rel_paths) => { if rel_paths.is_empty() { return HostAdmissionOutcome::replay_completed(false, true); diff --git a/crates/tracedecay/src/mcp/server/host_admission_tests.rs b/crates/tracedecay/src/mcp/server/host_admission_tests.rs index 3ad6b859e3..5ab100ffba 100644 --- a/crates/tracedecay/src/mcp/server/host_admission_tests.rs +++ b/crates/tracedecay/src/mcp/server/host_admission_tests.rs @@ -116,6 +116,256 @@ fn success_reconcile_sink() -> CodeIndexReconcileSink { Arc::new(|_root, _demand| Box::pin(async { super::CodeIndexDemandAdmissionV1::Queued })) } +#[tokio::test] +async fn file_edit_commit_failure_queues_no_sync() { + let (cg, project, authority) = init_indexed_repo().await; + let spool = TempDir::new().unwrap(); + let broker = default_spool_broker(spool.path()); + let effects = Arc::new(AtomicUsize::new(0)); + let context = with_broker( + registered_context(cg, &authority), + Arc::clone(&broker), + success_reconcile_sink(), + ) + .with_code_index_hook_sink({ + let effects = Arc::clone(&effects); + Arc::new(move |_, _| { + effects.fetch_add(1, Ordering::SeqCst); + Box::pin(async { super::CodeIndexDemandAdmissionV1::Queued }) + }) + }); + let server = McpServer::new_with_registered_test_context(context, Vec::new()) + .await + .unwrap(); + let metadata = spool.path().join("meta.json"); + std::fs::remove_file(&metadata).unwrap(); + std::fs::create_dir(&metadata).unwrap(); + let event = serde_json::to_value(DaemonHookEvent::post_tool_use_edit( + HostIntegrationIdV1::Claude, + vec!["src/lib.rs".to_owned()], + project.path().to_path_buf(), + )) + .unwrap(); + let response = server + .handle_request(&super::JsonRpcRequest { + jsonrpc: "2.0".to_owned(), + id: Some(json!(17)), + method: tracedecay_hooks::core_events::HOOK_EVENT_METHOD.to_owned(), + params: Some(event.clone()), + }) + .await + .unwrap(); + assert!(response.result.is_none(), "failed commit must not ACK"); + let failure = response.error.unwrap().data.unwrap(); + assert_eq!(failure["reason_code"], "spool_io_failed"); + assert_eq!(failure["status"], "unavailable"); + let outcome = server + .handle_hook_event_notification(Some(&event), &mut HookProjectRouteCache::default()) + .await; + assert_eq!(outcome.status, HostAdmissionStatus::Unavailable); + assert_eq!(outcome.reason_code, Some("spool_recovery_required")); + assert_eq!(effects.load(Ordering::SeqCst), 0); + assert_eq!(broker.pending_replay_count().await.unwrap(), 0); + server.shutdown().await; +} + +#[tokio::test] +async fn file_edit_request_acks_durable_work_while_effect_is_held() { + let (cg, project, authority) = init_indexed_repo().await; + let spool = TempDir::new().unwrap(); + let broker = default_spool_broker(spool.path()); + let entered = Arc::new(Notify::new()); + let release = Arc::new(Notify::new()); + let effects = Arc::new(AtomicUsize::new(0)); + let context = with_broker( + registered_context(cg, &authority), + Arc::clone(&broker), + success_reconcile_sink(), + ) + .with_owned_project_host_admission_replay() + .with_code_index_hook_sink({ + let entered = Arc::clone(&entered); + let release = Arc::clone(&release); + let effects = Arc::clone(&effects); + let expected_root = project.path().canonicalize().unwrap(); + Arc::new(move |root, paths| { + assert_eq!(root, expected_root); + assert_eq!(paths, ["src/lib.rs"]); + let entered = Arc::clone(&entered); + let release = Arc::clone(&release); + let effects = Arc::clone(&effects); + Box::pin(async move { + entered.notify_one(); + release.notified().await; + effects.fetch_add(1, Ordering::SeqCst); + super::CodeIndexDemandAdmissionV1::Queued + }) + }) + }); + let server = McpServer::new_with_registered_test_context(context, Vec::new()) + .await + .unwrap(); + let event = serde_json::to_value(DaemonHookEvent::post_tool_use_edit( + HostIntegrationIdV1::Claude, + vec!["src/lib.rs".to_owned()], + project.path().to_path_buf(), + )) + .unwrap(); + let response = server + .handle_request(&super::JsonRpcRequest { + jsonrpc: "2.0".to_owned(), + id: Some(json!(19)), + method: tracedecay_hooks::core_events::HOOK_EVENT_METHOD.to_owned(), + params: Some(event), + }) + .await + .unwrap(); + assert!(response.error.is_none(), "{response:?}"); + assert_eq!(response.result, Some(json!({}))); + entered.notified().await; + assert_eq!(broker.pending_count().await, 1); + assert_eq!(effects.load(Ordering::SeqCst), 0); + release.notify_one(); + server.wait_project_host_admission_replay_idle().await; + assert_eq!(broker.pending_count().await, 0); + assert_eq!(effects.load(Ordering::SeqCst), 1); + server.replay_host_admission(None).await; + assert_eq!(effects.load(Ordering::SeqCst), 1); + server.shutdown().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn retained_file_edits_replay_once_on_each_exact_linked_worktree() { + let (cg, project, authority) = init_indexed_repo().await; + let spool = TempDir::new().unwrap(); + let broker = default_spool_broker(spool.path()); + let effects = Arc::new(Mutex::new(Vec::new())); + let sink: super::CodeIndexHookSink = { + let effects = Arc::clone(&effects); + Arc::new(move |root, paths| { + let effects = Arc::clone(&effects); + Box::pin(async move { + effects.lock().unwrap().push((root, paths)); + super::CodeIndexDemandAdmissionV1::Queued + }) + }) + }; + let runtime = registered_runtime(&authority); + let owner = runtime + .project_registry_context_by_alias(project.path()) + .await + .unwrap() + .unwrap(); + let mut servers = Vec::new(); + let mut plans = Vec::new(); + let mut roots = Vec::new(); + for (name, directory) in [ + ("linked-a", "linked-a-api_key=locator-required"), + ("linked-b", "linked-b"), + ] { + let root = project.path().join(directory); + git( + project.path(), + &["worktree", "add", "-q", "-b", name, root.to_str().unwrap()], + ); + std::fs::write(root.join("src/lib.rs"), "pub fn edited() {}\n").unwrap(); + let graph = authority.reopen_project_graph(&root).await; + let (_, scope) = tracedecay_mcp::scope::resolve_query_scope(&owner, &root).unwrap(); + let mut context = + registered_context(graph, &authority).with_code_index_hook_sink(Arc::clone(&sink)); + context.host_admission_broker = Some(Arc::clone(&broker)); + servers.push( + McpServer::new_with_registered_test_context(context, Vec::new()) + .await + .unwrap(), + ); + let plan = tracedecay_mcp::hook_events::HookEventPlan::SyncFilesAt { + root: root.clone(), + scope, + agent: HostIntegrationIdV1::Claude, + route: Some(HookRouteMetadata { + session_id: Some("AKIASYNTHETICCANARY3".to_owned()), + thread_id: Some(name.to_owned()), + cwd: Some(root.clone()), + worktree: Some(root.clone()), + branch: None, + }), + rel_paths: vec!["src/lib.rs".to_owned()], + }; + let encoded = tracedecay_mcp::hook_events::encode_durable_hook_event_plan(&plan).unwrap(); + assert!(!String::from_utf8_lossy(&encoded).contains("AKIASYNTHETICCANARY3")); + let decoded = + tracedecay_mcp::hook_events::decode_durable_hook_event_plan(&encoded).unwrap(); + let tracedecay_mcp::hook_events::HookEventPlan::SyncFilesAt { + root: decoded_root, + scope: decoded_scope, + .. + } = &decoded + else { + panic!("routed edit") + }; + assert_eq!(decoded_root, &root); + let tracedecay_mcp::hook_events::HookEventPlan::SyncFilesAt { scope, .. } = &plan else { + unreachable!() + }; + assert_eq!(decoded_scope, scope); + broker.admit(name, &encoded).await.unwrap(); + plans.push(decoded); + roots.push(root); + } + let mut context = registered_context(cg, &authority); + context.host_admission_broker = Some(Arc::clone(&broker)); + context = context.with_code_index_hook_sink(Arc::clone(&sink)); + let server = McpServer::new_with_registered_test_context(context, servers.clone()) + .await + .unwrap(); + assert_eq!(broker.pending_replay_count().await.unwrap(), 2); + let drained = server.replay_host_admission(None).await; + assert_eq!(drained, HostAdmissionOutcome::accepted_for_replay()); + assert_eq!(broker.pending_replay_count().await.unwrap(), 0); + server.replay_host_admission(None).await; + assert_eq!( + *effects.lock().unwrap(), + roots + .iter() + .map(|root| (root.clone(), vec!["src/lib.rs".to_owned()])) + .collect::>() + ); + let warm_edit = serde_json::to_value(DaemonHookEvent::post_tool_use_edit( + HostIntegrationIdV1::Claude, + vec!["src/lib.rs".to_owned()], + roots[0].clone(), + )) + .unwrap(); + let outcome = server + .handle_hook_event_notification(Some(&warm_edit), &mut HookProjectRouteCache::default()) + .await; + assert_eq!(outcome.status, HostAdmissionStatus::Committed); + assert_eq!(broker.pending_replay_count().await.unwrap(), 0); + assert_eq!( + effects.lock().unwrap().len(), + 3, + "one warm admission queues one sync" + ); + assert_eq!( + effects.lock().unwrap()[2], + (roots[0].clone(), vec!["src/lib.rs".to_owned()]) + ); + let tracedecay_mcp::hook_events::HookEventPlan::SyncFilesAt { root, .. } = &mut plans[0] else { + unreachable!() + }; + *root = roots[1].clone(); + let outcome = server + .run_hook_event_plan(server.cg().await, project.path(), plans.remove(0)) + .await; + assert_eq!(outcome.reason_code, Some("project_route_stale")); + assert_eq!(effects.lock().unwrap().len(), 3); + server.shutdown().await; + for server in servers { + server.shutdown().await; + } +} + #[tokio::test] async fn hook_watch_policy_refusal_is_not_scheduler_unavailable() { let (cg, project, authority) = init_indexed_repo().await; diff --git a/crates/tracedecay/src/mcp/server/requests.rs b/crates/tracedecay/src/mcp/server/requests.rs index d7b090349c..3445da757d 100644 --- a/crates/tracedecay/src/mcp/server/requests.rs +++ b/crates/tracedecay/src/mcp/server/requests.rs @@ -342,12 +342,29 @@ impl McpServer { // survives the envelope move into `tools/call` preparation below. let method_name = request.method(); if matches!(method, McpMethod::HookEvent) { - Box::pin(self.handle_hook_event_notification( + let Some(id) = request.cloned_id() else { + Box::pin(self.handle_hook_event_notification( + request.raw_notification_params(), + &mut connection.route_cache, + )) + .await; + return None; + }; + let admitted = Box::pin(self.admit_hook_event_notification( request.raw_notification_params(), &mut connection.route_cache, + false, )) .await; - return None; + return Some(match admitted { + Ok(_) => JsonRpcResponse::success(id, json!({})), + Err(outcome) => JsonRpcResponse::error_with_data( + id, + ErrorCode::InternalError, + "hook event was not durably admitted".to_owned(), + Some(json!(outcome)), + ), + }); } if matches!(method, McpMethod::Cancelled) { self.record_request_accounting(method_name, false); @@ -437,21 +454,34 @@ impl McpServer { result } + pub(crate) async fn handle_hook_event_notification( + &self, + params: Option<&Value>, + route_cache: &mut HookProjectRouteCache, + ) -> HostAdmissionOutcome { + self.admit_hook_event_notification(params, route_cache, true) + .await + .unwrap_or_else(|outcome| outcome) + } + + /// Admission failure and postcommit replay outcome are separate: a retained + /// effect can wait for its owner after the request has a durable receipt. #[tracing::instrument(name = "mcp.server.hook_event", level = "trace", skip_all)] #[expect( clippy::too_many_lines, - reason = "Hook-event notification is one decode-admit-ack of a host envelope." + reason = "Hook-event admission is one route-admit-replay of a host envelope." )] - pub(crate) async fn handle_hook_event_notification( + async fn admit_hook_event_notification( &self, params: Option<&Value>, route_cache: &mut HookProjectRouteCache, - ) -> HostAdmissionOutcome { + replay_inline: bool, + ) -> std::result::Result { let Some(event) = hook_events::parse_hook_event(params) else { self.record_request_accounting("tracedecay/hookEvent", false); let outcome = HostAdmissionOutcome::degraded("malformed_event"); Self::report_host_admission_outcome(&outcome); - return outcome; + return Err(outcome); }; // Resolve the hook's exact project before opening a graph, publishing // activity, touching an indexing sink, or admitting durable work. The @@ -466,7 +496,7 @@ impl McpServer { "project_registry_route_unavailable", ); Self::report_host_admission_outcome(&outcome); - return outcome; + return Err(outcome); } }; let dispatch_server = match hook_route.retained_server() { @@ -484,7 +514,7 @@ impl McpServer { let outcome = HostAdmissionOutcome::retained_unavailable("project_route_unavailable"); Self::report_host_admission_outcome(&outcome); - return outcome; + return Err(outcome); } }; dispatch_server.record_request_accounting("tracedecay/hookEvent", false); @@ -517,7 +547,10 @@ impl McpServer { // event is observed. Independent of host-admission durability so an // after-edit reaches indexing even when effect processing is deferred. // Best-effort: a policy refusal or unavailable scheduler emits no activity. - if !event.rel_paths.is_empty() + if !matches!( + event.kind, + hook_events::HookEventKind::FileEdit | hook_events::HookEventKind::IncrementalSync + ) && !event.rel_paths.is_empty() && let Some(sink) = &dispatch_server.code_index_hook_sink { // An accepted admission means the paths really entered a mounted @@ -540,11 +573,22 @@ impl McpServer { } } let current_branch = live_branch.resolve_for(&root); - let plan = hook_events::plan_hook_event(&event, &root, current_branch.as_deref()); + let plan = match hook_events::plan_hook_event(&event, &root, current_branch.as_deref()) { + hook_events::HookEventPlan::SyncFiles(rel_paths) => { + hook_events::HookEventPlan::SyncFilesAt { + root: root.clone(), + scope: hook_route.scope.clone(), + agent: event.agent, + route: event.route.clone(), + rel_paths, + } + } + plan => plan, + }; let Ok(payload) = hook_events::encode_durable_hook_event_plan(&plan) else { let outcome = HostAdmissionOutcome::degraded("invalid_host_event_plan"); Self::report_host_admission_outcome(&outcome); - return outcome; + return Err(outcome); }; let admission_source = event.admission_source(); let admitted = match dispatch_server.host_admission_broker.as_ref() { @@ -557,15 +601,20 @@ impl McpServer { Ok(admitted) => admitted, Err(outcome) => { Self::report_host_admission_outcome(&outcome); - return outcome; + return Err(outcome); } }; - let outcome = Box::pin(dispatch_server.replay_host_admission(Some(admitted.seq))).await; - // Route analytics + span observations are side writes: only after the - // durable spool record is authoritatively committed (Committed or - // ExactDuplicate). Failures are best-effort and must never change the - // admission outcome already decided above. - if outcome.status.commits_replay_record() { + let outcome = if replay_inline { + Box::pin(dispatch_server.replay_host_admission(Some(admitted.seq))).await + } else { + // The mounted replay owner is woken by broker admission. Request + // delivery waits for the durable receipt, not for the effect owner. + HostAdmissionOutcome::accepted_for_replay() + }; + // Request observations follow durable admission. Notifications preserve + // their settled-effect accounting; these side writes never change the + // admission or replay outcome. + if !replay_inline || outcome.status.commits_replay_record() { if let Some(wake) = &dispatch_server.project_session_refresh_wake { wake.wake(); } @@ -581,7 +630,7 @@ impl McpServer { dispatch_server.record_hook_span_observation(&event, &hook_route); } Self::report_host_admission_outcome(&outcome); - outcome + Ok(outcome) } /// Records the caller's negotiated `clientInfo.name` (e.g. diff --git a/crates/tracedecay/tests/hooks_lsp_suite/hook_spool_drain_test.rs b/crates/tracedecay/tests/hooks_lsp_suite/hook_spool_drain_test.rs index 54d12a9d4b..689f2ab387 100644 --- a/crates/tracedecay/tests/hooks_lsp_suite/hook_spool_drain_test.rs +++ b/crates/tracedecay/tests/hooks_lsp_suite/hook_spool_drain_test.rs @@ -17,7 +17,10 @@ use tracedecay_hooks::{ }; use tracedecay_runtime_core::storage::default_profile_sharded_layout; -use crate::common::{git_program, spawn_tracedecay_daemon, tracedecay_command_with_home}; +use crate::common::{ + git_program, spawn_tracedecay_daemon, spawn_tracedecay_daemon_logged, + tracedecay_command_with_home, +}; /// Well under the replay consumer's 30 s interval sweep, so a drain inside it /// can only come from the append wake or the startup open. @@ -510,42 +513,86 @@ fn an_edit_in_a_never_opened_linked_worktree_binds_it_and_reaches_the_daemon() { let home = tempfile::TempDir::new().unwrap(); let home = home.path().canonicalize().unwrap(); let project = init_git_project(&home); - let _daemon = spawn_tracedecay_daemon(&home); + let daemon = spawn_tracedecay_daemon(&home); tracedecay(&home, &project, &["init"]); let data_root = default_profile_sharded_layout(&project, &home.join(".tracedecay")) .unwrap() .data_root; let worktree = project.join(".claude/worktrees/agent-x"); - assert!( - Command::new(git_program()) - .args(["worktree", "add", "-q", "-b", "agent-x"]) - .arg(&worktree) - .current_dir(&project) - .status() - .unwrap() - .success() - ); - std::fs::write( - worktree.join("src/lib.rs"), - "pub fn edited_in_worktree() {}\n", - ) - .unwrap(); + let other_worktree = project.join(".claude/worktrees/agent-y"); + for (root, branch) in [(&worktree, "agent-x"), (&other_worktree, "agent-y")] { + assert!( + Command::new(git_program()) + .args(["worktree", "add", "-q", "-b", branch]) + .arg(root) + .current_dir(&project) + .status() + .unwrap() + .success() + ); + std::fs::write(root.join("src/lib.rs"), "pub fn edited_in_worktree() {}\n").unwrap(); + } assert_eq!( bound_worktrees(&data_root), 1, "only the main checkout is bound" ); + drop(daemon); + let hold = home.join("hold-after-core"); + let entered = PathBuf::from(format!("{}.entered", hold.display())); + std::fs::write(&hold, b"hold").unwrap(); + let log = home.join("daemon.log"); + let _daemon = spawn_tracedecay_daemon_logged(&home, &log, |command| { + command.env("TRACEDECAY_TEST_HOLD_AFTER_CORE_PUBLISH", &hold); + }); assert_eq!( claude_edit(&home, &worktree, &data_root, "toolu_unbound_first"), "hook_v2_accepted", "the first edit in an unbound worktree must reach the daemon, not be dropped" ); + assert!( + latency_until(PROMPT_DRAIN, || entered.is_file()).is_some(), + "the ordinary project open must reach the existing core publication hold: {}", + std::fs::read_to_string(&log).unwrap() + ); + assert!( + hold.is_file(), + "the first edit was acknowledged while full publication is held" + ); assert_eq!( - bound_worktrees(&data_root), - 2, - "delivering that edit opens the worktree and publishes its binding" + claude_edit( + &home, + &other_worktree, + &data_root, + "toolu_other_unbound_first" + ), + "hook_v2_accepted", + "another linked checkout's same relative file must commit while full publication is held" + ); + // Graph admission publishes hook bindings before the core hold. Full + // publication is the capability boundary this fixture actually holds. + let full_markers = [&worktree, &other_worktree] + .map(|root| format!("project={} phase=full_published", root.display())); + let held_log = std::fs::read_to_string(&log).unwrap(); + for marker in &full_markers { + assert!( + !held_log.contains(marker), + "full publication escaped its hold: {held_log}" + ); + } + std::fs::remove_file(&hold).unwrap(); + assert!( + latency_until(PROMPT_DRAIN, || { + let published = std::fs::read_to_string(&log).unwrap(); + bound_worktrees(&data_root) == 3 + && full_markers.iter().all(|marker| published.contains(marker)) + }) + .is_some(), + "delivering that edit opens the worktree and publishes its binding: {}", + std::fs::read_to_string(&log).unwrap() ); + assert_eq!(bound_worktrees(&data_root), 3); let next = claude_edit(&home, &worktree, &data_root, "toolu_unbound_next"); assert!( matches!(next.as_str(), "hook_v2_accepted" | "hook_v2_spooled"), From 651a8221a4abbde355447045b4437e78fce5235e Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Mon, 19 Oct 2026 00:55:12 +0000 Subject: [PATCH 50/89] fix(identity): retain enrolled monorepo package scope --- crates/tracedecay-mcp/src/scope.rs | 42 +++++++++++++++++++ .../src/context/registered_scope.rs | 28 ++++++++++--- 2 files changed, 64 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay-mcp/src/scope.rs b/crates/tracedecay-mcp/src/scope.rs index 0763b0f312..48d916006b 100644 --- a/crates/tracedecay-mcp/src/scope.rs +++ b/crates/tracedecay-mcp/src/scope.rs @@ -253,6 +253,48 @@ mod tests { ); } + #[tokio::test] + async fn enrolled_monorepo_package_retains_its_path_local_scope() { + let temp = TempDir::new().unwrap(); + let repository = temp.path().canonicalize().unwrap(); + init_repo(&repository); + write_identity_marker(&repository, "project.enclosing-repository"); + let package = repository.join("packages/app"); + let subdirectory = package.join("src/deep"); + std::fs::create_dir_all(&subdirectory).unwrap(); + let project_id = tracedecay_runtime_core::storage::default_profile_project_id(&package); + assert_ne!(project_id, "project.enclosing-repository"); + let profile = TempDir::new().unwrap(); + let (registry, _registry_owner) = + tracedecay_global_db::tests::harness::open_registered_test_database_fixture( + &profile.path().join("profile.db"), + tracedecay_runtime_core::db::TestDatabaseRuntimeScope::ProfileSessions, + ) + .await + .unwrap(); + registry + .upsert_code_project(&project_id, &package, None, None, Some("main")) + .await + .unwrap(); + let owner = registry + .project_registry_context_by_id(&project_id) + .await + .unwrap() + .unwrap(); + let (root, scope) = resolve_query_scope_with_root(&owner, &subdirectory).unwrap(); + assert_eq!(root, package); + assert_eq!(scope.project_id.as_str(), project_id); + assert_eq!(scope, resolve_query_scope(&owner, &package).unwrap()); + assert_ne!( + scope, + resolve_query_scope( + &owner_for(&repository, "project.enclosing-repository"), + &repository + ) + .unwrap() + ); + } + #[test] fn relative_requested_root_fails_closed_without_cwd_fallback() { let temp = TempDir::new().unwrap(); diff --git a/crates/tracedecay-session-memory/src/context/registered_scope.rs b/crates/tracedecay-session-memory/src/context/registered_scope.rs index 2c2e7f820e..1cbe53a49c 100644 --- a/crates/tracedecay-session-memory/src/context/registered_scope.rs +++ b/crates/tracedecay-session-memory/src/context/registered_scope.rs @@ -91,12 +91,28 @@ fn resolve_authorized_root( // actual Git root before ordinary subdirectories converge on their owner; // lexical containment alone cannot authorize another repository. let scope_root = match try_repository_topology(&requested_root) { - Ok(topology) => topology.worktree_root.clone().ok_or_else(|| { - ApplicationScopeError::Resolution(format!( - "requested root '{}' has no Git checkout", - requested_root.display() - )) - })?, + Ok(topology) => { + let checkout = topology.worktree_root.clone().ok_or_else(|| { + ApplicationScopeError::Resolution(format!( + "requested root '{}' has no Git checkout", + requested_root.display() + )) + })?; + // An independently enrolled package owns its path-local identity; + // discovery of the enclosing repository cannot widen that scope. + if requested_root.starts_with(®istered_root) + && !checkout.starts_with(®istered_root) + { + validate_non_git_project_identity(®istered_root, project_id)?; + return Ok(AuthorizedRoot { + scope_root: registered_root.clone(), + registered_root, + marker: None, + }); + } else { + checkout + } + } Err(GitTopologyProbeError::Repository(GitRepositoryError::NotARepository { .. })) => { if requested_root.starts_with(®istered_root) { registered_root.clone() From 98609ed28ca660812dfd4f69288bec95211a1fa5 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 18 Oct 2026 19:57:52 -0700 Subject: [PATCH 51/89] fix(hooks): require event-specific spool evidence before ledger replay --- crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs b/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs index 02869cff7c..85d168689b 100644 --- a/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs +++ b/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs @@ -297,10 +297,10 @@ async fn redrive_pending_work( .await }, |envelope, native_session_id| async move { - // A timed-out admission may have committed its ledger before its - // lifecycle. Only the richer host spool can finish that mount. - if native_session_id.is_none() - && tracedecay_hooks::NativeContextScoutLifecycleV1::host_mounts_lifecycle(host) + // A registered session may belong to an earlier event. It does + // not prove that this event's native call/edit lifecycle was + // mounted. Only the matching host spool record can finish it. + if tracedecay_hooks::NativeContextScoutLifecycleV1::host_mounts_lifecycle(host) && matches!( envelope.event, tracedecay_hooks::HookEventV2::ToolLifecycle { .. } From 976c9572d0d9dd7c753ba6da0cfd2ca5794746dd Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 18 Oct 2026 19:59:00 -0700 Subject: [PATCH 52/89] fix(hooks): guard known-session replay and test per-edit lifecycle --- .../src/daemon/hook_v2_replay_consumer.rs | 94 ++++++++++++++++++- 1 file changed, 90 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs b/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs index 02869cff7c..1c9668ad80 100644 --- a/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs +++ b/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs @@ -297,10 +297,10 @@ async fn redrive_pending_work( .await }, |envelope, native_session_id| async move { - // A timed-out admission may have committed its ledger before its - // lifecycle. Only the richer host spool can finish that mount. - if native_session_id.is_none() - && tracedecay_hooks::NativeContextScoutLifecycleV1::host_mounts_lifecycle(host) + // A registered session may belong to an earlier event. It does + // not prove that this event's native call/edit lifecycle was + // mounted. Only the matching host spool record can finish it. + if tracedecay_hooks::NativeContextScoutLifecycleV1::host_mounts_lifecycle(host) && matches!( envelope.event, tracedecay_hooks::HookEventV2::ToolLifecycle { .. } @@ -812,6 +812,92 @@ mod tests { runs.try_recv().is_err(), "the mounted edit executes exactly once" ); + // A known session is NOT proof that another edit's native lifecycle + // reached the spool. This is the ledger-only redrive regression. + assert_eq!( + tracedecay_daemon_service::context_scout_lifecycle::lookup_registered_context_scout_native_session( + project_id, + binding.worktree_id, + envelope.protected_session_id, + ) + .await, + Some(lifecycle.session_id.clone()) + ); + tracedecay_daemon_service::unregister_hook_orchestration_runtime( + project_id, + binding.worktree_id, + &runtime, + ); + let second_lifecycle = tracedecay_hooks::NativeContextScoutLifecycleV1::new( + "session.replay.edit", + "call.replay.second", + [11; 16], + ) + .unwrap(); + let second_edit = tracedecay_hooks::HookEventEnvelopeV2 { + event_id: second_lifecycle.event_id, + observed_at: hook_replay_now(), + event: tracedecay_hooks::HookEventV2::SavedEdit { + file_id: [11; 16], + changed_range_count: 1, + }, + ..envelope.clone() + }; + assert!(matches!( + admit_hook_v2_envelope(&graph, &second_edit, None, hook_replay_now()).await, + HookV2AdmissionOutcomeV1::Admitted { .. } + )); + assert!(tracedecay_daemon_service::register_hook_orchestration_runtime( + project_id, + binding.worktree_id, + &runtime, + )); + drain_all_hosts( + &graph, + data_root, + &fixture.settlements, + &project_sessions, + &background_cpu, + ) + .await; + assert_eq!(pending(), vec![second_edit.clone()]); + assert!( + runs.try_recv().is_err(), + "a known session must not settle an edit without its call evidence" + ); + let (mut second_spool, _) = + HookSpoolV1::open(&root, HookSpoolConfigV1::stock(host), hook_replay_now()).unwrap(); + second_spool + .append_with_native_lifecycle( + second_edit, + Some(second_lifecycle.clone()), + &binding, + hook_replay_now(), + ) + .unwrap(); + drop(second_spool); + drain_all_hosts( + &graph, + data_root, + &fixture.settlements, + &project_sessions, + &background_cpu, + ) + .await; + let second_mounted = tokio::time::timeout(Duration::from_secs(10), runs.recv()) + .await + .unwrap() + .unwrap() + .expect("second spool record must mount its own lifecycle"); + assert_eq!(second_mounted.session_id, second_lifecycle.session_id); + tokio::time::timeout(Duration::from_secs(10), async { + while !pending().is_empty() { + tokio::task::yield_now().await; + } + }) + .await + .expect("second mounted edit must settle its ledger debt"); + assert!(runs.try_recv().is_err(), "second edit executes exactly once"); // Claude deliberately omits lifecycle writes while intent is disabled; // an acknowledged edit can therefore need ledger-only recovery. tracedecay_daemon_service::unregister_hook_orchestration_runtime( From 1787f22fe9f456e44fce73d4d17f6fd9207ad874 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Mon, 19 Oct 2026 00:08:27 +0000 Subject: [PATCH 53/89] fix(build): share dashboard source copies across consumers --- MODULE.bazel | 1 + dashboard/BUILD.bazel | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/MODULE.bazel b/MODULE.bazel index 275174e15e..bc3e333a16 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -2,6 +2,7 @@ module(name = "tracedecay", version = "0.0.0") bazel_dep(name = "platforms", version = "1.1.0") bazel_dep(name = "bazel_skylib", version = "1.9.2") +bazel_dep(name = "bazel_lib", version = "3.7.0") bazel_dep(name = "rules_python", version = "2.4.2") bazel_dep(name = "rules_rust", version = "0.74.0") # Rustc adds every dependency search directory to PATH on Windows; large diff --git a/dashboard/BUILD.bazel b/dashboard/BUILD.bazel index e282d69310..30b68f5741 100644 --- a/dashboard/BUILD.bazel +++ b/dashboard/BUILD.bazel @@ -1,3 +1,4 @@ +load("@bazel_lib//lib:copy_to_bin.bzl", "copy_to_bin") load("@npm//:defs.bzl", "npm_link_all_packages") load("@npm//dashboard:@rsbuild/core/package_json.bzl", rsbuild_bin = "bin") load("@npm//dashboard:vitest/package_json.bzl", vitest_bin = "bin") @@ -6,7 +7,8 @@ package(default_visibility = ["//visibility:public"]) npm_link_all_packages(name = "node_modules") -filegroup( +# Share source copies across the bundle and tests with different CPU allocations. +copy_to_bin( name = "dashboard_sources", srcs = glob( [ From a5896ffa660306618dc55fd6e3eb70a94c27815a Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 18 Oct 2026 20:01:04 -0700 Subject: [PATCH 54/89] fix(hooks): allow replay after branch label changes in same checkout --- crates/tracedecay/src/mcp/server/hook_dispatch.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay/src/mcp/server/hook_dispatch.rs b/crates/tracedecay/src/mcp/server/hook_dispatch.rs index 7ed410caac..2badd05ff7 100644 --- a/crates/tracedecay/src/mcp/server/hook_dispatch.rs +++ b/crates/tracedecay/src/mcp/server/hook_dispatch.rs @@ -109,7 +109,7 @@ impl McpServer { Ok(resolved) => resolved, Err(_) => return HostAdmissionOutcome::degraded("project_route_stale"), }; - if root != effect_root || current_scope != scope { + if root != effect_root || !current_scope.identifies_same_checkout(&scope) { return HostAdmissionOutcome::degraded("project_route_stale"); } let resolved = match crate::mcp::project_route::resolve_registered_project_route( @@ -125,7 +125,7 @@ impl McpServer { return HostAdmissionOutcome::retained_unavailable("project_route_unavailable"); } }; - if resolved.requested_root != root || resolved.scope != scope { + if resolved.requested_root != root || !resolved.scope.identifies_same_checkout(&scope) { return HostAdmissionOutcome::degraded("project_route_stale"); } let event = hook_events::HookEvent { From 38e3b22256d59cf5ba0a74a4a48e278b7b93b68c Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 18 Oct 2026 20:05:12 -0700 Subject: [PATCH 55/89] test(hooks): preserve lifecycle for edits in an existing session --- .../src/daemon/hook_v2_replay_consumer.rs | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs b/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs index 85d168689b..1c9668ad80 100644 --- a/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs +++ b/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs @@ -812,6 +812,92 @@ mod tests { runs.try_recv().is_err(), "the mounted edit executes exactly once" ); + // A known session is NOT proof that another edit's native lifecycle + // reached the spool. This is the ledger-only redrive regression. + assert_eq!( + tracedecay_daemon_service::context_scout_lifecycle::lookup_registered_context_scout_native_session( + project_id, + binding.worktree_id, + envelope.protected_session_id, + ) + .await, + Some(lifecycle.session_id.clone()) + ); + tracedecay_daemon_service::unregister_hook_orchestration_runtime( + project_id, + binding.worktree_id, + &runtime, + ); + let second_lifecycle = tracedecay_hooks::NativeContextScoutLifecycleV1::new( + "session.replay.edit", + "call.replay.second", + [11; 16], + ) + .unwrap(); + let second_edit = tracedecay_hooks::HookEventEnvelopeV2 { + event_id: second_lifecycle.event_id, + observed_at: hook_replay_now(), + event: tracedecay_hooks::HookEventV2::SavedEdit { + file_id: [11; 16], + changed_range_count: 1, + }, + ..envelope.clone() + }; + assert!(matches!( + admit_hook_v2_envelope(&graph, &second_edit, None, hook_replay_now()).await, + HookV2AdmissionOutcomeV1::Admitted { .. } + )); + assert!(tracedecay_daemon_service::register_hook_orchestration_runtime( + project_id, + binding.worktree_id, + &runtime, + )); + drain_all_hosts( + &graph, + data_root, + &fixture.settlements, + &project_sessions, + &background_cpu, + ) + .await; + assert_eq!(pending(), vec![second_edit.clone()]); + assert!( + runs.try_recv().is_err(), + "a known session must not settle an edit without its call evidence" + ); + let (mut second_spool, _) = + HookSpoolV1::open(&root, HookSpoolConfigV1::stock(host), hook_replay_now()).unwrap(); + second_spool + .append_with_native_lifecycle( + second_edit, + Some(second_lifecycle.clone()), + &binding, + hook_replay_now(), + ) + .unwrap(); + drop(second_spool); + drain_all_hosts( + &graph, + data_root, + &fixture.settlements, + &project_sessions, + &background_cpu, + ) + .await; + let second_mounted = tokio::time::timeout(Duration::from_secs(10), runs.recv()) + .await + .unwrap() + .unwrap() + .expect("second spool record must mount its own lifecycle"); + assert_eq!(second_mounted.session_id, second_lifecycle.session_id); + tokio::time::timeout(Duration::from_secs(10), async { + while !pending().is_empty() { + tokio::task::yield_now().await; + } + }) + .await + .expect("second mounted edit must settle its ledger debt"); + assert!(runs.try_recv().is_err(), "second edit executes exactly once"); // Claude deliberately omits lifecycle writes while intent is disabled; // an acknowledged edit can therefore need ledger-only recovery. tracedecay_daemon_service::unregister_hook_orchestration_runtime( From 8b95affaacda522b2d0607f4ba82204ffd2b4927 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 18 Oct 2026 20:05:14 -0700 Subject: [PATCH 56/89] test(session-runtime): eliminate CI-sensitive refresh deadline races --- .../worker.rs | 39 ++++++++++++++----- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker.rs index 489de73c34..5427dd0fa8 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker.rs @@ -1915,13 +1915,12 @@ mod tests { use tracedecay_runtime_core::db::engine::params; use tracedecay_sessions::runtime::{SessionMessageRecord, SessionRecord}; - /// Wide enough that one item's commit lands inside it on a loaded host. - const COMMIT_DEADLINE: Duration = Duration::from_secs(2); - - /// Commits `item_count` items. A group of several items never commits - /// before the deadline, and neither does a `slow` item alone; every other - /// item alone commits at once. Returns how many commits the deadline cut - /// short. + /// Commits `item_count` items. A group of several items never commits, and + /// neither does a `slow` item alone: its transaction ends in the + /// `DeadlineExceeded` that storage reports when its own deadline passes. + /// Every other item alone commits. Real writes race no wall-clock deadline + /// (`Duration::MAX` starts no timer), so host load cannot cut an item that + /// the test means to commit. Returns how many commits were cut short. async fn timed_refresh_group_progress(item_count: usize, slow: &[usize]) -> usize { let harness = RegisteredGlobalDbHarness::open("refresh-group-progress").await; let database = &harness.registered; @@ -1937,7 +1936,7 @@ mod tests { let cut_short = std::sync::atomic::AtomicUsize::new(0); let cut_short = &cut_short; - let outcomes = commit_refresh_groups(item_count, COMMIT_DEADLINE, |range| async move { + let outcomes = commit_refresh_groups(item_count, Duration::MAX, |range| async move { let transaction = database .begin_write_transaction() .await @@ -1953,7 +1952,7 @@ mod tests { } if range.len() > 1 || slow.contains(&range.start) { cut_short.fetch_add(1, Ordering::SeqCst); - std::future::pending::<()>().await; + return Err(SessionStoreError::DeadlineExceeded); } transaction .commit() @@ -2010,6 +2009,28 @@ mod tests { ); } + /// The deadline itself ends a group that never commits. No I/O races it: + /// a commit that is ready at its first poll wins even past the deadline, + /// so only the pending group is ever cut, however loaded the host. + #[tokio::test] + async fn a_refresh_group_that_outlives_its_deadline_is_retried_item_by_item() { + let outcomes = commit_refresh_groups(3, Duration::from_millis(10), |range| async move { + if range.len() > 1 { + std::future::pending::<()>().await; + } + Ok(range.map(Ok).collect()) + }) + .await + .unwrap(); + assert_eq!( + outcomes + .into_iter() + .map(|outcome| outcome.unwrap()) + .collect::>(), + [0, 1, 2] + ); + } + #[tokio::test] async fn a_timed_out_completion_stays_claimable_by_the_next_pass() { let harness = RegisteredGlobalDbHarness::open("refresh-timed-out-completion").await; From 7d1e61af6ca12f6ed6b25c22a8e471e30682b3e2 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 18 Oct 2026 20:08:39 -0700 Subject: [PATCH 57/89] style(hooks): apply Rustfmt to lifecycle regression assertions --- .../src/daemon/hook_v2_replay_consumer.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs b/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs index 1c9668ad80..4935ec3b5f 100644 --- a/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs +++ b/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs @@ -847,11 +847,13 @@ mod tests { admit_hook_v2_envelope(&graph, &second_edit, None, hook_replay_now()).await, HookV2AdmissionOutcomeV1::Admitted { .. } )); - assert!(tracedecay_daemon_service::register_hook_orchestration_runtime( - project_id, - binding.worktree_id, - &runtime, - )); + assert!( + tracedecay_daemon_service::register_hook_orchestration_runtime( + project_id, + binding.worktree_id, + &runtime, + ) + ); drain_all_hosts( &graph, data_root, @@ -897,7 +899,10 @@ mod tests { }) .await .expect("second mounted edit must settle its ledger debt"); - assert!(runs.try_recv().is_err(), "second edit executes exactly once"); + assert!( + runs.try_recv().is_err(), + "second edit executes exactly once" + ); // Claude deliberately omits lifecycle writes while intent is disabled; // an acknowledged edit can therefore need ledger-only recovery. tracedecay_daemon_service::unregister_hook_orchestration_runtime( From 7971c420fa3f00a096ed6cc42ab3d50ddb35d547 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 18 Oct 2026 20:08:42 -0700 Subject: [PATCH 58/89] style(hooks): apply Rustfmt to lifecycle regression assertions --- .../src/daemon/hook_v2_replay_consumer.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs b/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs index 1c9668ad80..4935ec3b5f 100644 --- a/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs +++ b/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs @@ -847,11 +847,13 @@ mod tests { admit_hook_v2_envelope(&graph, &second_edit, None, hook_replay_now()).await, HookV2AdmissionOutcomeV1::Admitted { .. } )); - assert!(tracedecay_daemon_service::register_hook_orchestration_runtime( - project_id, - binding.worktree_id, - &runtime, - )); + assert!( + tracedecay_daemon_service::register_hook_orchestration_runtime( + project_id, + binding.worktree_id, + &runtime, + ) + ); drain_all_hosts( &graph, data_root, @@ -897,7 +899,10 @@ mod tests { }) .await .expect("second mounted edit must settle its ledger debt"); - assert!(runs.try_recv().is_err(), "second edit executes exactly once"); + assert!( + runs.try_recv().is_err(), + "second edit executes exactly once" + ); // Claude deliberately omits lifecycle writes while intent is disabled; // an acknowledged edit can therefore need ledger-only recovery. tracedecay_daemon_service::unregister_hook_orchestration_runtime( From 600b671545e92fe406d9df823f0774a1b2c81e25 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Mon, 19 Oct 2026 03:38:01 +0000 Subject: [PATCH 59/89] fix(identity): share indexed scope identities across surfaces --- .../src/code_index_scheduler/identity.rs | 46 ++---- crates/tracedecay-runtime-core/src/lib.rs | 1 + .../src/scope_identity.rs | 52 +++++++ .../src/context/registered_scope.rs | 84 ++--------- .../src/mcp/server/query_scope_tests.rs | 136 ++++++++++++++++++ 5 files changed, 212 insertions(+), 107 deletions(-) create mode 100644 crates/tracedecay-runtime-core/src/scope_identity.rs diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs index eeb3f1d934..1fa9419b75 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs @@ -16,21 +16,19 @@ use std::time::{SystemTime, UNIX_EPOCH}; use tracedecay_contracts::{ApplicationContractError, ResolvedScope}; use tracedecay_domain::{CommitId, ProjectId, RefId, RepositoryId, TreeId, WorktreeId}; -use tracedecay_runtime_core::path_safety::canonical_existing_identity; +use tracedecay_runtime_core::scope_identity::ScopeIdentityError; + +pub use tracedecay_runtime_core::scope_identity::{ + repository_id_for, repository_id_for_common_dir, worktree_id_for, +}; /// Failure to resolve an exact indexing identity from a checkout. #[derive(Debug, thiserror::Error)] pub enum IdentityErrorV1 { #[error("code-index identity: repository open failed: {0}")] Git(String), - #[error("code-index identity: canonical worktree path unavailable: {}: {source}", path.display())] - CanonicalWorktreePath { - path: PathBuf, - #[source] - source: std::io::Error, - }, - #[error("code-index identity: canonical id construction failed: {0}")] - Domain(String), + #[error(transparent)] + Structural(#[from] ScopeIdentityError), } /// The exact project/repository/worktree/ref/commit/tree identity of one @@ -307,36 +305,10 @@ fn git_metadata_dirs(project_root: &Path) -> (PathBuf, PathBuf) { (git_dir.clone(), git_dir) } -pub fn repository_id_for(project_root: &Path) -> Result { - let common = tracedecay_runtime_core::worktree::git_common_dir(project_root) - .unwrap_or_else(|| project_root.to_path_buf()); - repository_id_for_common_dir(&common) -} - -pub fn repository_id_for_common_dir(common_dir: &Path) -> Result { - let digest = super::sha256_hex(common_dir.to_string_lossy().as_bytes()); - RepositoryId::new(format!("repository.daemon.{digest}")) - .map_err(|error| IdentityErrorV1::Domain(error.to_string())) -} - -pub fn worktree_id_for(project_root: &Path) -> Result { - let project_root = canonical_existing_identity(project_root).map_err(|source| { - IdentityErrorV1::CanonicalWorktreePath { - path: project_root.to_path_buf(), - source, - } - })?; - WorktreeId::new(format!( - "worktree.daemon.{}", - super::sha256_hex(project_root.to_string_lossy().as_bytes()) - )) - .map_err(|error| IdentityErrorV1::Domain(error.to_string())) -} - /// Resolve the code-index `ResolvedScope` for one project checkout. /// -/// Repository and worktree identities come from this module; the current -/// branch is best-effort (`None` when HEAD is unborn or detached). +/// Repository and worktree identities come from the shared runtime authority; +/// the current branch is best-effort (`None` when HEAD is unborn or detached). pub fn resolved_scope_for_project( project_root: &Path, project_id: &ProjectId, diff --git a/crates/tracedecay-runtime-core/src/lib.rs b/crates/tracedecay-runtime-core/src/lib.rs index 3519228257..73bac69e1c 100644 --- a/crates/tracedecay-runtime-core/src/lib.rs +++ b/crates/tracedecay-runtime-core/src/lib.rs @@ -124,6 +124,7 @@ pub mod path_safety; pub mod resident_memory; pub mod runtime_identity; pub mod runtime_telemetry; +pub mod scope_identity; pub mod shard_runtime; pub mod sqlite_read_snapshot; pub mod storage; diff --git a/crates/tracedecay-runtime-core/src/scope_identity.rs b/crates/tracedecay-runtime-core/src/scope_identity.rs new file mode 100644 index 0000000000..42796168e3 --- /dev/null +++ b/crates/tracedecay-runtime-core/src/scope_identity.rs @@ -0,0 +1,52 @@ +//! Structural repository and worktree identity shared by application scopes +//! and code indexing. Authorization remains with the registered-root caller. + +use std::path::{Path, PathBuf}; + +use tracedecay_domain::canonical_text::sha256_hex; +use tracedecay_domain::{RepositoryId, WorktreeId}; + +use crate::path_safety::canonical_existing_identity; + +/// Failure to resolve the structural identity of an authorized project root. +#[derive(Debug, thiserror::Error)] +pub enum ScopeIdentityError { + #[error("scope identity: canonical worktree path unavailable: {}: {source}", path.display())] + CanonicalWorktreePath { + path: PathBuf, + #[source] + source: std::io::Error, + }, + #[error("scope identity: canonical id construction failed: {0}")] + Domain(String), +} + +/// Preserve the indexed repository identity: linked checkouts share the Git +/// common directory; a root outside Git uses the supplied project path. +pub fn repository_id_for(project_root: &Path) -> Result { + let common = + crate::worktree::git_common_dir(project_root).unwrap_or_else(|| project_root.to_path_buf()); + repository_id_for_common_dir(&common) +} + +/// Resolve identity from an already-discovered repository common directory. +pub fn repository_id_for_common_dir(common_dir: &Path) -> Result { + let digest = sha256_hex(common_dir.to_string_lossy().as_bytes()); + RepositoryId::new(format!("repository.daemon.{digest}")) + .map_err(|error| ScopeIdentityError::Domain(error.to_string())) +} + +/// Resolve one exact checkout or independently enrolled package path. +pub fn worktree_id_for(project_root: &Path) -> Result { + let project_root = canonical_existing_identity(project_root).map_err(|source| { + ScopeIdentityError::CanonicalWorktreePath { + path: project_root.to_path_buf(), + source, + } + })?; + WorktreeId::new(format!( + "worktree.daemon.{}", + sha256_hex(project_root.to_string_lossy().as_bytes()) + )) + .map_err(|error| ScopeIdentityError::Domain(error.to_string())) +} diff --git a/crates/tracedecay-session-memory/src/context/registered_scope.rs b/crates/tracedecay-session-memory/src/context/registered_scope.rs index 1cbe53a49c..d3e1b85649 100644 --- a/crates/tracedecay-session-memory/src/context/registered_scope.rs +++ b/crates/tracedecay-session-memory/src/context/registered_scope.rs @@ -3,13 +3,12 @@ use std::path::{Path, PathBuf}; use tracedecay_contracts::ResolvedScope; -use tracedecay_domain::canonical_text::sha256_hex; -use tracedecay_domain::{ProjectId, RefId, RepositoryId, WorktreeId}; +use tracedecay_domain::{ProjectId, RefId}; use tracedecay_runtime_core::git_repository::{ GitRepositoryError, GitTopologyProbeError, try_repository_topology, }; use tracedecay_runtime_core::path_safety::canonical_existing_identity; -use tracedecay_runtime_core::storage::RepositoryIdentityMarker; +use tracedecay_runtime_core::scope_identity::{repository_id_for, worktree_id_for}; use super::ApplicationScopeError; @@ -36,7 +35,6 @@ impl RegisteredScopeResolver { project_id: &ProjectId, ) -> Result { resolve_authorized_root(registered_root, requested_root, project_id) - .map(|resolved| resolved.scope_root) } #[tracing::instrument(name = "usecases.context.registered_scope", level = "trace", skip_all)] @@ -45,28 +43,15 @@ impl RegisteredScopeResolver { requested_root: &Path, project_id: &ProjectId, ) -> Result { - let resolved = resolve_authorized_root(registered_root, requested_root, project_id)?; - let scope = if let Some(marker) = resolved.marker { - let identity = tracedecay_sessions::repository_provenance:: - RepositoryProvenanceAdmissionContext::from_authoritative_project_marker( - &resolved.scope_root, - project_id, - &marker, - ) - .and_then(|authority| authority.admitted_identity()) - .ok_or_else(|| { - ApplicationScopeError::Resolution(format!( - "registered identity authority rejected '{}'", - resolved.scope_root.display() - )) - })?; - let reference = tracedecay_runtime_core::branch::current_branch(&resolved.scope_root) - .and_then(|branch| RefId::new(format!("refs/heads/{branch}")).ok()); - ResolvedScope::new(identity.0, identity.1, identity.2, reference) - .map_err(|error| ApplicationScopeError::Contract(error.to_string()))? - } else { - resolve_non_git_scope(&resolved.registered_root, &resolved.scope_root, project_id)? - }; + let scope_root = resolve_authorized_root(registered_root, requested_root, project_id)?; + let repository_id = repository_id_for(&scope_root) + .map_err(|error| ApplicationScopeError::Resolution(error.to_string()))?; + let worktree_id = worktree_id_for(&scope_root) + .map_err(|error| ApplicationScopeError::Resolution(error.to_string()))?; + let reference = tracedecay_runtime_core::branch::current_branch(&scope_root) + .and_then(|branch| RefId::new(format!("refs/heads/{branch}")).ok()); + let scope = ResolvedScope::new(project_id.clone(), repository_id, worktree_id, reference) + .map_err(|error| ApplicationScopeError::Contract(error.to_string()))?; scope .validate() .map_err(|error| ApplicationScopeError::InconsistentScope(error.to_string()))?; @@ -74,17 +59,11 @@ impl RegisteredScopeResolver { } } -struct AuthorizedRoot { - registered_root: PathBuf, - scope_root: PathBuf, - marker: Option, -} - fn resolve_authorized_root( registered_root: &Path, requested_root: &Path, project_id: &ProjectId, -) -> Result { +) -> Result { let registered_root = canonical_root(registered_root, "registered root")?; let requested_root = canonical_root(requested_root, "requested root")?; // A checkout may be nested under the registered checkout. Resolve its @@ -104,11 +83,7 @@ fn resolve_authorized_root( && !checkout.starts_with(®istered_root) { validate_non_git_project_identity(®istered_root, project_id)?; - return Ok(AuthorizedRoot { - scope_root: registered_root.clone(), - registered_root, - marker: None, - }); + return Ok(registered_root); } else { checkout } @@ -148,33 +123,7 @@ fn resolve_authorized_root( } None => validate_non_git_project_identity(®istered_root, project_id)?, } - Ok(AuthorizedRoot { - registered_root, - scope_root, - marker, - }) -} - -/// Resolve an enrolled project that has no Git repository identity. A non-Git -/// project persists nothing in its working tree: its identity is -/// deterministic from the canonical project path, with the home-profile -/// registry as the durable authority. Requiring the supplied project id to -/// equal that deterministic identity keeps an arbitrary directory from -/// becoming an application scope merely because a caller supplies a -/// project id. -fn resolve_non_git_scope( - registered_root: &Path, - scope_root: &Path, - project_id: &ProjectId, -) -> Result { - let repository_id = RepositoryId::new(path_identity("repository.daemon", registered_root)) - .map_err(|error| ApplicationScopeError::Contract(error.to_string()))?; - let worktree_id = WorktreeId::new(path_identity("worktree.daemon", scope_root)) - .map_err(|error| ApplicationScopeError::Contract(error.to_string()))?; - let reference = tracedecay_runtime_core::branch::current_branch(scope_root) - .and_then(|branch| RefId::new(format!("refs/heads/{branch}")).ok()); - ResolvedScope::new(project_id.clone(), repository_id, worktree_id, reference) - .map_err(|error| ApplicationScopeError::Contract(error.to_string())) + Ok(scope_root) } fn validate_non_git_project_identity( @@ -192,11 +141,6 @@ fn validate_non_git_project_identity( Ok(()) } -fn path_identity(prefix: &str, root: &Path) -> String { - let digest = sha256_hex(root.to_string_lossy().as_bytes()); - format!("{prefix}.{digest}") -} - fn canonical_root(root: &Path, label: &str) -> Result { if !root.is_absolute() { return Err(ApplicationScopeError::RelativeRoot { diff --git a/crates/tracedecay/src/mcp/server/query_scope_tests.rs b/crates/tracedecay/src/mcp/server/query_scope_tests.rs index 9a7cad746a..97907c59f2 100644 --- a/crates/tracedecay/src/mcp/server/query_scope_tests.rs +++ b/crates/tracedecay/src/mcp/server/query_scope_tests.rs @@ -6,11 +6,147 @@ //! on the routed project reader, failing closed exactly as the unrouted //! selection already did. +use std::path::Path; + use serde_json::json; +use tempfile::TempDir; +use tracedecay_contracts::ResolvedScope; +use tracedecay_domain::ProjectId; +use tracedecay_session_memory::context::RegisteredScopeResolver; use super::McpServer; use super::writer_test_support::{init_indexed_repo, registered_context}; use crate::mcp::tools::handlers::resolve_registered_project_route_for_tool; +use crate::test_support::git::git; + +async fn assert_scope_surface_identity( + registered_root: &Path, + requested_root: &Path, + project_id: &str, +) -> ResolvedScope { + let profile = TempDir::new().expect("isolated profile"); + let (registry, _registry_owner) = + tracedecay_global_db::tests::harness::open_registered_test_database_fixture( + &profile.path().join("profile.db"), + tracedecay_runtime_core::db::TestDatabaseRuntimeScope::ProfileSessions, + ) + .await + .expect("registered profile database"); + registry + .upsert_code_project(project_id, registered_root, None, None, None) + .await + .expect("enroll exact project root"); + let owner = registry + .project_registry_context_by_id(project_id) + .await + .expect("registered owner lookup") + .expect("enrolled owner"); + let project_id = ProjectId::new(project_id).expect("project id"); + let (scope_root, mcp) = tracedecay_mcp::scope::resolve_query_scope(&owner, requested_root) + .expect("MCP exact scope"); + let indexed = + tracedecay_code_index_runtime::resolved_scope_for_project(&scope_root, &project_id) + .expect("indexed exact scope"); + let application = + RegisteredScopeResolver::resolve(registered_root, requested_root, &project_id) + .expect("application exact scope"); + let dashboard = tracedecay_dashboard_api::scope::resolve_dashboard_scope( + &scope_root, + Some(project_id.as_str()), + ) + .expect("dashboard exact scope"); + for (surface, scope) in [ + ("MCP", mcp), + ("application", application), + ("dashboard", dashboard), + ] { + scope.validate().expect("surface digest validates"); + assert_eq!(scope.scope_digest, indexed.scope_digest, "{surface} digest"); + assert_eq!(scope, indexed, "{surface} structural scope"); + } + indexed +} + +fn scope_identity_repository() -> TempDir { + let root = TempDir::new().expect("repository"); + git(root.path(), &["init", "-q", "-b", "main"]); + std::fs::write(root.path().join("lib.rs"), "pub fn scope() {}\n").expect("source"); + git(root.path(), &["add", "."]); + git(root.path(), &["commit", "-qm", "initial"]); + assert!( + tracedecay_runtime_core::storage::write_repository_identity_marker( + root.path(), + "project.scope-parity", + ) + .expect("registered repository marker") + ); + root +} + +#[tokio::test] +async fn registered_scope_surfaces_share_git_identity() { + let repository = scope_identity_repository(); + let root = repository.path().canonicalize().expect("canonical root"); + let subdir = root.join("src/deep"); + std::fs::create_dir_all(&subdir).expect("ordinary subdirectory"); + let scope = assert_scope_surface_identity(&root, &subdir, "project.scope-parity").await; + assert_eq!( + scope, + tracedecay_code_index_runtime::resolved_scope_for_project(&root, &scope.project_id) + .expect("registered root indexing scope"), + ); +} + +#[tokio::test] +async fn registered_scope_surfaces_share_linked_identity() { + let repository = scope_identity_repository(); + let root = repository.path().canonicalize().expect("canonical root"); + let linked = root.join("linked-checkouts/feature"); + git( + &root, + &["worktree", "add", linked.to_str().unwrap(), "-b", "feature"], + ); + let linked = linked.canonicalize().expect("canonical linked checkout"); + let scope = assert_scope_surface_identity(&root, &linked, "project.scope-parity").await; + let primary = + tracedecay_code_index_runtime::resolved_scope_for_project(&root, &scope.project_id) + .expect("primary indexing scope"); + assert_eq!(scope.repository_id, primary.repository_id); + assert_ne!(scope.worktree_id, primary.worktree_id); + assert_ne!(scope.scope_digest, primary.scope_digest); +} + +#[tokio::test] +async fn registered_scope_surfaces_share_enrolled_package_identity() { + let repository = scope_identity_repository(); + let root = repository.path().canonicalize().expect("canonical root"); + let package = root.join("packages/app"); + let subdir = package.join("src/deep"); + std::fs::create_dir_all(&subdir).expect("package subdirectory"); + let project_id = tracedecay_runtime_core::storage::default_profile_project_id(&package); + assert_ne!(project_id, "project.scope-parity"); + let scope = assert_scope_surface_identity(&package, &subdir, &project_id).await; + let enclosing = tracedecay_code_index_runtime::resolved_scope_for_project( + &root, + &ProjectId::new("project.scope-parity").expect("enclosing project"), + ) + .expect("enclosing indexing scope"); + assert_ne!(scope.project_id, enclosing.project_id); + assert_eq!(scope.repository_id, enclosing.repository_id); + assert_ne!(scope.worktree_id, enclosing.worktree_id); + assert_ne!(scope.scope_digest, enclosing.scope_digest); +} + +#[tokio::test] +async fn registered_scope_surfaces_share_non_git_identity() { + let directory = TempDir::new().expect("non-Git project"); + let root = directory.path().canonicalize().expect("canonical root"); + let subdir = root.join("src/deep"); + std::fs::create_dir_all(&subdir).expect("ordinary subdirectory"); + let project_id = tracedecay_runtime_core::storage::default_profile_project_id(&root); + let scope = assert_scope_surface_identity(&root, &subdir, &project_id).await; + assert!(scope.reference.is_none()); +} #[tokio::test] async fn canonical_project_id_reader_resolves_same_project_and_scope_via_application_type() { From 87c29d0bde0b7ccabd1b89468616418dc6fa3a68 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Mon, 19 Oct 2026 03:38:22 +0000 Subject: [PATCH 60/89] fix(db): deduplicate hook-route analytics with indexed lookup --- .../src/observability_outbox_tests.rs | 48 +++-- .../src/registered_analytics.rs | 57 ++++++ .../tracedecay-global-db/src/schema_stages.rs | 22 +++ crates/tracedecay-global-db/src/tests.rs | 2 + .../src/tests/hook_route_analytics.rs | 176 ++++++++++++++++++ 5 files changed, 289 insertions(+), 16 deletions(-) create mode 100644 crates/tracedecay-global-db/src/tests/hook_route_analytics.rs diff --git a/crates/tracedecay-global-db/src/observability_outbox_tests.rs b/crates/tracedecay-global-db/src/observability_outbox_tests.rs index ddbd41a5b2..0ca9246108 100644 --- a/crates/tracedecay-global-db/src/observability_outbox_tests.rs +++ b/crates/tracedecay-global-db/src/observability_outbox_tests.rs @@ -1,4 +1,6 @@ -use crate::registered_analytics::OBSERVABILITY_EVENT_BY_IDEMPOTENCY_KEY_SQL; +use crate::registered_analytics::{ + HOOK_ROUTE_BY_IDEMPOTENCY_KEY_SQL, OBSERVABILITY_EVENT_BY_IDEMPOTENCY_KEY_SQL, +}; use crate::tests::harness::RegisteredGlobalDbHarness; use crate::{ AnalyticsEventInsert, CoverageStateV1, ObservabilityEmissionClaimV1, @@ -10,22 +12,36 @@ use crate::{ async fn idempotency_lookup_searches_its_partial_index() { let harness = RegisteredGlobalDbHarness::open("observability-idempotency-plan").await; let snapshot = harness.registered.read_snapshot().await.unwrap(); - let mut rows = snapshot - .query( - &format!("EXPLAIN QUERY PLAN {OBSERVABILITY_EVENT_BY_IDEMPOTENCY_KEY_SQL}"), - ["scope:fixture", "idempotency:fixture"], - ) - .await - .unwrap(); - let mut plan = Vec::new(); - while let Some(row) = rows.next().await.unwrap() { - plan.push(row.get::(3).unwrap()); + for (query, index) in [ + ( + OBSERVABILITY_EVENT_BY_IDEMPOTENCY_KEY_SQL, + "idx_observability_event_idempotency", + ), + ( + HOOK_ROUTE_BY_IDEMPOTENCY_KEY_SQL, + "idx_hook_route_idempotency", + ), + ] { + let mut rows = snapshot + .query( + &format!("EXPLAIN QUERY PLAN {query}"), + ["scope:fixture", "idempotency:fixture"], + ) + .await + .unwrap(); + let mut plan = Vec::new(); + while let Some(row) = rows.next().await.unwrap() { + plan.push(row.get::(3).unwrap()); + } + assert!( + plan.iter() + .any(|detail| detail.starts_with("SEARCH analytics_events") + && detail.contains(index) + && detail.contains("project_id=?") + && detail.contains("hint_id=?")), + "the idempotency lookup must seek project and hint in {index}: {plan:?}" + ); } - assert!( - plan.iter() - .any(|detail| detail.contains("idx_observability_event_idempotency")), - "the idempotency lookup must search its partial index: {plan:?}" - ); } #[asupersync::test] diff --git a/crates/tracedecay-global-db/src/registered_analytics.rs b/crates/tracedecay-global-db/src/registered_analytics.rs index 609e51ecf1..8b78ab0584 100644 --- a/crates/tracedecay-global-db/src/registered_analytics.rs +++ b/crates/tracedecay-global-db/src/registered_analytics.rs @@ -19,6 +19,17 @@ const OBSERVABILITY_ROLLUP_RETENTION_SECONDS: i64 = 395 * 86_400; const MAX_OBSERVABILITY_OUTBOX_JSON_BYTES: usize = 1_048_576; const OBSERVABILITY_RETENTION_ROWS_PER_CLASS: usize = 512; pub(crate) const ANALYTICS_INSERT_ROWS_PER_STATEMENT: usize = 500; +pub(crate) const HOOK_ROUTE_PERFORMANCE_INDEX_SQL: &str = + "CREATE INDEX IF NOT EXISTS idx_hook_route_idempotency + ON analytics_events(project_id, hint_id) + WHERE provider = 'daemon_hook' AND event_kind = 'hook_route' AND hint_id IS NOT NULL"; +// Literal predicates make the partial index usable. Requiring the index keeps +// best-effort telemetry from scanning history while an existing store upgrades. +pub(crate) const HOOK_ROUTE_BY_IDEMPOTENCY_KEY_SQL: &str = + "SELECT id FROM analytics_events INDEXED BY idx_hook_route_idempotency + WHERE provider = 'daemon_hook' AND event_kind = 'hook_route' + AND project_id = ?1 AND hint_id = ?2 + ORDER BY id ASC LIMIT 1"; // The provider is a literal so `idx_observability_event_idempotency` stays // usable; the planner does not match a bound value against an index predicate. pub(crate) const OBSERVABILITY_EVENT_BY_IDEMPOTENCY_KEY_SQL: &str = @@ -142,6 +153,12 @@ impl RegisteredGlobalDb { &self, event: &AnalyticsEventInsert, ) -> Result { + if event.provider == "daemon_hook" + && event.event_kind == "hook_route" + && let Some(hint_id) = event.hint_id.as_deref() + { + return self.append_hook_route_event(event, hint_id).await; + } let writer = self .runtime_database() .writer_connection("append analytics event") @@ -191,6 +208,46 @@ impl RegisteredGlobalDb { Ok(id) } + async fn append_hook_route_event( + &self, + event: &AnalyticsEventInsert, + hint_id: &str, + ) -> Result { + let transaction = self + .begin_write_transaction() + .await + .map_err(|error| format!("failed to begin hook route append: {error}"))?; + let existing = { + let mut rows = transaction + .query( + HOOK_ROUTE_BY_IDEMPOTENCY_KEY_SQL, + tracedecay_runtime_core::db::engine::params![ + event.project_id.as_str(), + hint_id + ], + ) + .await + .map_err(|error| format!("failed to read hook route identity: {error}"))?; + rows.next() + .await + .map_err(|error| format!("failed to read hook route receipt: {error}"))? + .map(|row| row.get::(0)) + .transpose() + .map_err(|error| format!("failed to decode hook route receipt: {error}"))? + }; + // Retry observations may have a later timestamp or current branch. + // The first durable event and its ID remain authoritative. + let id = match existing { + Some(id) => id, + None => append_analytics_event_in_existing_tx(&transaction, event).await?, + }; + transaction + .commit() + .await + .map_err(|error| format!("failed to commit hook route append: {error}"))?; + Ok(id) + } + /// Canonical observability append with replay-safe idempotency. /// /// The registered writer serializes the lookup and insert; the partial diff --git a/crates/tracedecay-global-db/src/schema_stages.rs b/crates/tracedecay-global-db/src/schema_stages.rs index dfddb4d965..9f39094da3 100644 --- a/crates/tracedecay-global-db/src/schema_stages.rs +++ b/crates/tracedecay-global-db/src/schema_stages.rs @@ -792,6 +792,16 @@ pub async fn ensure_registered_schema_for_admission( indexes.await.map_err(|error| { global_db_operation_error("initialize observation projection indexes", error) })?; + // Fresh stores have no history to index. Existing stores build this on + // the independently durable convergence writer after required admission. + if is_fresh { + installation + .execute_authority_revalidated_batch( + crate::registered_analytics::HOOK_ROUTE_PERFORMANCE_INDEX_SQL, + ) + .await + .map_err(|error| global_db_operation_error("initialize hook route index", error))?; + } // Fresh installations publish the final LCM index shape. Keep each build // independently durable outside the shared schema transaction so a // real-scale index build gets the long lease without holding every other @@ -1201,6 +1211,12 @@ pub async fn converge_registered_schema( database: &Database, convergence: RegisteredSchemaConvergence, ) -> tracedecay_domain::errors::Result<()> { + converge_migration_batches( + database, + "converge hook route performance index", + &[crate::registered_analytics::HOOK_ROUTE_PERFORMANCE_INDEX_SQL], + ) + .await?; let bodies: Pin> + Send + '_>> = Box::pin(observation::converge_canonical_bodies(database)); bodies.await?; @@ -1307,6 +1323,12 @@ async fn converge_registered_schema_on( pub async fn converge_attached_registered_schema( database: &Database, ) -> tracedecay_domain::errors::Result<()> { + converge_migration_batches( + database, + "converge hook route performance index", + &[crate::registered_analytics::HOOK_ROUTE_PERFORMANCE_INDEX_SQL], + ) + .await?; if hosts_session_features(&database.registered_binding().shard_id.scope) { converge_migration_batches( database, diff --git a/crates/tracedecay-global-db/src/tests.rs b/crates/tracedecay-global-db/src/tests.rs index 9e471a6fdd..6d7242e6f2 100644 --- a/crates/tracedecay-global-db/src/tests.rs +++ b/crates/tracedecay-global-db/src/tests.rs @@ -10,6 +10,8 @@ use super::{ mod codex_response_echo; pub mod harness; #[cfg(test)] +mod hook_route_analytics; +#[cfg(test)] pub(crate) mod lcm_privacy_rescan; #[cfg(test)] mod lcm_schema; diff --git a/crates/tracedecay-global-db/src/tests/hook_route_analytics.rs b/crates/tracedecay-global-db/src/tests/hook_route_analytics.rs new file mode 100644 index 0000000000..d1c91bf908 --- /dev/null +++ b/crates/tracedecay-global-db/src/tests/hook_route_analytics.rs @@ -0,0 +1,176 @@ +use super::harness::RegisteredGlobalDbHarness; +use crate::AnalyticsEventInsert; + +fn event(project: &str, hint: Option<&str>) -> AnalyticsEventInsert { + AnalyticsEventInsert { + provider: "daemon_hook".to_owned(), + project_id: project.to_owned(), + session_id: None, + timestamp: 1, + event_kind: "hook_route".to_owned(), + hook_name: Some("file_edit".to_owned()), + tool_name: None, + tool_category: None, + skill_name: None, + hint_category: None, + hint_id: hint.map(str::to_owned), + outcome: Some("observed".to_owned()), + metadata_json: Some(r#"{"branch":"main","admission_seq":1}"#.to_owned()), + } +} + +#[asupersync::test] +async fn replay_returns_first_hook_route_row_without_mutating_observation() { + let harness = RegisteredGlobalDbHarness::open("hook-route-first-receipt").await; + let event = event("project.route", Some("root-a:1")); + // Earlier binaries appended every replay. Their rows remain valid history. + let historical = harness + .registered + .append_analytics_events(&[event.clone(), event.clone()]) + .await + .unwrap(); + assert_ne!(historical[0], historical[1]); + let mut replay = event.clone(); + replay.timestamp = 200; + replay.metadata_json = Some(r#"{"branch":"next","admission_seq":1}"#.to_owned()); + assert_eq!( + harness + .registered + .append_analytics_event(&replay) + .await + .unwrap(), + historical[0] + ); + let snapshot = harness.registered.read_snapshot().await.unwrap(); + let mut rows = snapshot + .query( + "SELECT timestamp, metadata_json FROM analytics_events WHERE id = ?1", + [historical[0]], + ) + .await + .unwrap(); + let row = rows.next().await.unwrap().unwrap(); + assert_eq!(row.get::(0).unwrap(), event.timestamp); + assert_eq!(row.get::(1).unwrap(), event.metadata_json.unwrap()); + assert_eq!( + harness + .registered + .count_analytics_events(Some("project.route"), 0) + .await + .unwrap(), + 2 + ); +} + +#[asupersync::test] +async fn concurrent_registered_handles_share_one_hook_route_receipt() { + let harness = RegisteredGlobalDbHarness::open("hook-route-concurrent").await; + let mut handles = Vec::new(); + for _ in 0..8 { + handles.push(harness.mount().await); + } + let event = event("project.route", Some("root-a:1")); + let ids = + futures_util::future::join_all(handles.iter().map(|db| db.append_analytics_event(&event))) + .await; + let ids = ids.into_iter().collect::, _>>().unwrap(); + assert!(ids.iter().all(|id| *id == ids[0])); + assert_eq!( + harness + .registered + .count_analytics_events(Some("project.route"), 0) + .await + .unwrap(), + 1 + ); +} + +#[asupersync::test] +async fn distinct_routes_and_sequences_append_and_other_events_keep_ordinary_semantics() { + let harness = RegisteredGlobalDbHarness::open("hook-route-scope").await; + let db = &harness.registered; + let mut ids = Vec::new(); + for (project, key) in [ + ("project.route", "root-a:1"), + ("project.route", "root-a:2"), + ("project.route", "root-b:1"), + ("project.other", "root-a:1"), + ] { + ids.push( + db.append_analytics_event(&event(project, Some(key))) + .await + .unwrap(), + ); + } + ids.sort_unstable(); + ids.dedup(); + assert_eq!(ids.len(), 4); + let no_hint = event("project.route", None); + let mut other_provider = event("project.route", Some("root-a:1")); + other_provider.provider = "another-provider".to_owned(); + let mut other_kind = event("project.route", Some("root-a:1")); + other_kind.event_kind = "another-kind".to_owned(); + for event in [no_hint, other_provider, other_kind] { + assert_ne!( + db.append_analytics_event(&event).await.unwrap(), + db.append_analytics_event(&event).await.unwrap() + ); + } + assert_eq!(db.count_analytics_events(None, 0).await.unwrap(), 10); +} + +#[asupersync::test] +async fn existing_store_defers_hook_index_build_then_preserves_earliest_receipt() { + let harness = RegisteredGlobalDbHarness::open("hook-route-index-upgrade").await; + let event = event("project.route", Some("root-a:1")); + let ids = harness + .registered + .append_analytics_events(&[event.clone(), event.clone()]) + .await + .unwrap(); + let transaction = harness.registered.begin_write_transaction().await.unwrap(); + transaction + .execute_batch("DROP INDEX idx_hook_route_idempotency") + .await + .unwrap(); + transaction.commit().await.unwrap(); + let (harness, convergence) = harness.restart_for_daemon().await; + // Required admission succeeds before the history-sized index build. + // Best-effort telemetry refuses an unindexed scan until convergence lands. + let error = harness + .registered + .append_analytics_event(&event) + .await + .unwrap_err(); + assert!(error.contains("idx_hook_route_idempotency"), "{error}"); + assert_eq!( + harness + .registered + .count_analytics_events(None, 0) + .await + .unwrap(), + 2 + ); + crate::schema_stages::converge_registered_schema( + harness.registered.runtime_database(), + convergence, + ) + .await + .unwrap(); + assert_eq!( + harness + .registered + .append_analytics_event(&event) + .await + .unwrap(), + ids[0] + ); + assert_eq!( + harness + .registered + .count_analytics_events(None, 0) + .await + .unwrap(), + 2 + ); +} From 4b0e015c63a550551d09d6dfc46d51e27c86df44 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Mon, 19 Oct 2026 03:39:07 +0000 Subject: [PATCH 61/89] fix(hooks): recover intent delivery after completed replay --- .../src/handlers/hook_runtime/admission.rs | 16 +++- .../project_open_owners/advisory_runtime.rs | 39 +++++++--- .../intent_pass_journey_test.rs | 74 ++++++++++++------- 3 files changed, 91 insertions(+), 38 deletions(-) diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/admission.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/admission.rs index b5bd4023fb..5922953c5a 100644 --- a/crates/tracedecay-mcp/src/handlers/hook_runtime/admission.rs +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/admission.rs @@ -647,8 +647,22 @@ async fn admit_hook_v2_envelope_with_lifecycle_inner( hook_v2_context_scout_lifecycle_for_session(envelope, native_session_id).await; match lifecycle.as_ref() { Some(lifecycle) => { - cg.resolve_mounted_context_scout_claim_authority(lifecycle) + match cg + .resolve_mounted_context_scout_claim_authority(lifecycle) .await + { + Some(authority) => Some(authority), + // A replay can complete producer work without a host + // response. Its next callback still owns delivery of + // the session's separately mounted intent suggestion. + None => match lifecycle.intent_scoped() { + Some(intent_lifecycle) => { + cg.resolve_mounted_context_scout_claim_authority(&intent_lifecycle) + .await + } + None => None, + }, + } } None => None, } diff --git a/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime.rs b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime.rs index ef417b310a..46e9014f60 100644 --- a/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime.rs +++ b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime.rs @@ -62,6 +62,7 @@ use tracedecay_contracts::{ DisclosureClass, LegalAction, RequestContext, RetryDirective, SafeDiagnostic, now_micros, }; use tracedecay_domain::GitHeadStateV1; +use tracedecay_domain::canonical_text::encode_lowercase_hex; use tracedecay_domain::feedback::{ CiFailureParserIdentityV1, FeedbackScopeV1, FeedbackTriggerV1, GitHubPullRequestIdV1, GitHubReviewReadOperationV1, @@ -1326,12 +1327,28 @@ const fn feedback_diagnostic_trigger( } } -fn log_scout_intent_outcome(project_root: &Path, outcome: &str) { +fn log_scout_intent_outcome( + project_root: &Path, + request: &HookOrchestrationRequestV1, + outcome: &str, +) { tracedecay_runtime_core::logging::log_daemon_event( "context_scout_intent_work", &[ ("project", project_root.display().to_string()), ("outcome", outcome.to_owned()), + ( + "protected_session_id", + encode_lowercase_hex(&request.hook.envelope().protected_session_id), + ), + ( + "event_id", + encode_lowercase_hex(&request.hook.envelope().event_id), + ), + ( + "lifecycle_supplied", + request.lifecycle.is_some().to_string(), + ), ], ); } @@ -1354,7 +1371,7 @@ async fn run_production_intent_cycle( let producer = Arc::clone(&cycle.producer); let project_root = producer.project_root.as_path(); if work_cancellation.is_cancelled() { - log_scout_intent_outcome(project_root, "cancelled"); + log_scout_intent_outcome(project_root, &request, "cancelled"); return HookOrchestrationWorkOutcomeV1::Completed; } let Ok(pinned_configuration) = producer @@ -1364,7 +1381,7 @@ async fn run_production_intent_cycle( .current() .await else { - log_scout_intent_outcome(project_root, "configuration_unavailable"); + log_scout_intent_outcome(project_root, &request, "configuration_unavailable"); return HookOrchestrationWorkOutcomeV1::Completed; }; let current_configuration = ConfigurationCurrentStateV1 { @@ -1374,7 +1391,7 @@ async fn run_production_intent_cycle( let Some(scout_configuration) = ContextScoutConfigurationPinV1::from_current(¤t_configuration) else { - log_scout_intent_outcome(project_root, "configuration_pin_unavailable"); + log_scout_intent_outcome(project_root, &request, "configuration_pin_unavailable"); return HookOrchestrationWorkOutcomeV1::Completed; }; // Disabled or absent: no window read, no engine request, no authority. @@ -1385,6 +1402,7 @@ async fn run_production_intent_cycle( { log_scout_intent_outcome( project_root, + &request, ContextScoutIntentPassOutcomeV1::Disabled.as_str(), ); return HookOrchestrationWorkOutcomeV1::Completed; @@ -1394,7 +1412,7 @@ async fn run_production_intent_cycle( pinned_configuration.snapshot(), ) else { - log_scout_intent_outcome(project_root, "model_configuration_unavailable"); + log_scout_intent_outcome(project_root, &request, "model_configuration_unavailable"); return HookOrchestrationWorkOutcomeV1::Completed; }; // Installing the pinned configuration also builds the decision engine @@ -1410,7 +1428,7 @@ async fn run_production_intent_cycle( .await .is_err() { - log_scout_intent_outcome(project_root, "configuration_install_failed"); + log_scout_intent_outcome(project_root, &request, "configuration_install_failed"); return HookOrchestrationWorkOutcomeV1::Completed; } let envelope = request.hook.envelope(); @@ -1435,7 +1453,7 @@ async fn run_production_intent_cycle( }, }; let Some(lifecycle) = lifecycle else { - log_scout_intent_outcome(project_root, "lifecycle_absent"); + log_scout_intent_outcome(project_root, &request, "lifecycle_absent"); return HookOrchestrationWorkOutcomeV1::Completed; }; let observed_at = now_micros(); @@ -1449,7 +1467,7 @@ async fn run_production_intent_cycle( .and_then(|access| { intent_authorization_context(&access, &cycle.feedback_scope, envelope, observed_at) }) else { - log_scout_intent_outcome(project_root, "authority_unavailable"); + log_scout_intent_outcome(project_root, &request, "authority_unavailable"); return HookOrchestrationWorkOutcomeV1::Completed; }; let Some(pin) = ContextScoutAuthorityPinV1::new( @@ -1458,7 +1476,7 @@ async fn run_production_intent_cycle( scout_configuration, observed_at, ) else { - log_scout_intent_outcome(project_root, "authority_pin_unavailable"); + log_scout_intent_outcome(project_root, &request, "authority_pin_unavailable"); return HookOrchestrationWorkOutcomeV1::Completed; }; let code_generation_id = producer @@ -1501,7 +1519,7 @@ async fn run_production_intent_cycle( lifecycle: intent_lifecycle, } = outcome else { - log_scout_intent_outcome(project_root, outcome.as_str()); + log_scout_intent_outcome(project_root, &request, outcome.as_str()); return HookOrchestrationWorkOutcomeV1::Completed; }; let mounted = producer @@ -1520,6 +1538,7 @@ async fn run_production_intent_cycle( .await; log_scout_intent_outcome( project_root, + &request, if mounted { "mounted" } else { diff --git a/crates/tracedecay/tests/hooks_lsp_suite/intent_pass_journey_test.rs b/crates/tracedecay/tests/hooks_lsp_suite/intent_pass_journey_test.rs index 99761dfb34..5f4763fd17 100644 --- a/crates/tracedecay/tests/hooks_lsp_suite/intent_pass_journey_test.rs +++ b/crates/tracedecay/tests/hooks_lsp_suite/intent_pass_journey_test.rs @@ -272,12 +272,18 @@ fn enabled_settings(engine_url: &str) -> ContextScoutSettingsV1 { } } -/// Every `context_scout_intent_work` outcome the daemon logged, in order. -fn intent_outcomes(log: &Path) -> Vec { +/// Intent outcomes for exactly one protected native session, in order. +fn intent_outcomes(log: &Path, session: &str) -> Vec { + let protected = tracedecay_domain::canonical_text::encode_lowercase_hex( + &tracedecay_agent_hosts::hooks::protected_native_session_id(session), + ); + let session_field = format!(" protected_session_id={protected}"); std::fs::read_to_string(log) .unwrap_or_default() .lines() - .filter(|line| line.contains("event=context_scout_intent_work ")) + .filter(|line| { + line.contains("event=context_scout_intent_work ") && line.contains(&session_field) + }) .filter_map(|line| line.split(" outcome=").nth(1)) .map(|outcome| outcome.split_whitespace().next().unwrap_or("").to_owned()) .collect() @@ -301,7 +307,7 @@ fn wait_for_hints(home: &Path, project: &Path, log: &Path, ready: impl Fn((i64, assert!( Instant::now() < deadline, "timed out at (emitted, suppressed) {counts:?} after intent outcomes {:?}", - intent_outcomes(log) + intent_outcomes(log, ENABLED_SESSION) ); std::thread::sleep(Duration::from_millis(100)); } @@ -418,7 +424,7 @@ fn claude_stop_delivers_the_intent_suggestion_the_daemon_judged_at_the_tool_boun &post_tool_use(&project, ABSENT_SESSION, &absent_transcript), ); wait_for("the absent-block tool boundary's intent pass", || { - !intent_outcomes(&log).is_empty() + !intent_outcomes(&log, ABSENT_SESSION).is_empty() }); let absent_stop = hook_context(&run_hook( home, @@ -427,14 +433,14 @@ fn claude_stop_delivers_the_intent_suggestion_the_daemon_judged_at_the_tool_boun &stop(&project, ABSENT_SESSION, &absent_transcript), )); wait_for("the absent-block stop boundary's intent pass", || { - intent_outcomes(&log).len() >= 2 + intent_outcomes(&log, ABSENT_SESSION).len() >= 2 }); assert!( - intent_outcomes(&log) + intent_outcomes(&log, ABSENT_SESSION) .iter() .all(|outcome| outcome == "disabled"), "without an intent block every intent pass ends disabled: {:?}", - intent_outcomes(&log) + intent_outcomes(&log, ABSENT_SESSION) ); assert_eq!( engine.requests(), @@ -479,7 +485,7 @@ fn claude_stop_delivers_the_intent_suggestion_the_daemon_judged_at_the_tool_boun "the enabled intent block commits through the configuration surface: {receipt}" ); - let before = intent_outcomes(&log).len(); + let before = intent_outcomes(&log, ENABLED_SESSION).len(); run_hook( home, &project, @@ -487,12 +493,13 @@ fn claude_stop_delivers_the_intent_suggestion_the_daemon_judged_at_the_tool_boun &post_tool_use(&project, ENABLED_SESSION, &enabled_transcript), ); wait_for("the enabled tool boundary's intent pass", || { - intent_outcomes(&log).len() > before + intent_outcomes(&log, ENABLED_SESSION).len() > before }); assert_eq!( - intent_outcomes(&log)[before..], + intent_outcomes(&log, ENABLED_SESSION)[before..], ["mounted"], - "the tool boundary's intent pass enqueues one suggestion and mounts its claim" + "the tool boundary's intent pass enqueues one suggestion and mounts its claim: {}", + std::fs::read_to_string(&log).unwrap_or_default() ); assert_eq!( engine.requests(), @@ -502,35 +509,48 @@ fn claude_stop_delivers_the_intent_suggestion_the_daemon_judged_at_the_tool_boun let stop_event = stop(&project, ENABLED_SESSION, &enabled_transcript); let delivers = |context: &String| context.contains(&suggestion_text(YES_INTENT)); - let mut stop_contexts = vec![hook_context(&run_hook( - home, - &project, - "hook-stop", - &stop_event, - ))]; + let first_stop = run_hook(home, &project, "hook-stop", &stop_event); + let mut stop_contexts = vec![hook_context(&first_stop)]; + let mut stop_outputs = vec![first_stop]; if !stop_contexts.last().is_some_and(delivers) { wait_for("the timed-out stop callback's replay to drain", || { let status = run_tool(home, &project, "tracedecay_status", &json!({})); let replay: StatusHookReplayV1 = serde_json::from_value(status["hook_replay"].clone()) .unwrap_or_else(|error| panic!("hook replay status ({error}): {status}")); - replay == StatusHookReplayV1::Drained && intent_outcomes(&log).len() > before + 1 + replay == StatusHookReplayV1::Drained + && intent_outcomes(&log, ENABLED_SESSION).len() > before + 1 }); - stop_contexts.push(hook_context(&run_hook( - home, - &project, - "hook-stop", - &stop_event, - ))); + let next_stop = run_hook(home, &project, "hook-stop", &stop_event); + stop_contexts.push(hook_context(&next_stop)); + stop_outputs.push(next_stop); } assert!( stop_contexts.last().is_some_and(delivers), "the stop boundary, or the host's retry of it past the admission ack budget, delivers \ - the session-recall suggestion: {stop_contexts:?}" + the session-recall suggestion: {stop_contexts:?}\noutputs: {:?}\ndaemon: {}\nhook receipts: {}", + stop_outputs + .iter() + .map(|output| ( + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + )) + .collect::>(), + std::fs::read_to_string(&log).unwrap_or_default(), + std::fs::read_to_string( + tracedecay_runtime_core::storage::default_profile_sharded_layout( + &project, + &home.join(".tracedecay"), + ) + .unwrap() + .data_root + .join("hook_analytics.jsonl") + ) + .unwrap_or_default(), ); wait_for_hints(home, &project, &log, |(emitted, suppressed)| { emitted == 1 && suppressed >= 1 }); - let after_mount = intent_outcomes(&log)[before + 1..].to_vec(); + let after_mount = intent_outcomes(&log, ENABLED_SESSION)[before + 1..].to_vec(); assert!( !after_mount.is_empty() && after_mount.iter().all(|outcome| outcome == "suppressed"), "the stop boundary's own intent pass withholds the suggestion it just delivered: \ From 1b6f500920763af34ba9c1abeef667eaadb05adb Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Mon, 19 Oct 2026 03:39:32 +0000 Subject: [PATCH 62/89] fix(hooks): preserve lifecycle debt and immutable replay order --- .../src/invocation.rs | 7 +- .../src/invocation/types.rs | 49 +- .../invocation/types/context_scout_tests.rs | 172 +++- crates/tracedecay-daemon-service/src/lib.rs | 12 +- .../tracedecay-hooks/src/admission_ledger.rs | 753 ++++++++++++++++-- crates/tracedecay-hooks/src/lib.rs | 4 +- .../src/handlers/hook_runtime/admission.rs | 194 ++++- .../handlers/hook_runtime/admission/tests.rs | 80 +- .../admission/tests/legacy_continuity.rs | 554 +++++++++++++ .../handlers/hook_runtime/context_scout.rs | 315 ++++++-- .../hook_runtime/context_scout/tests.rs | 27 +- .../handlers/hook_runtime/envelope/tests.rs | 3 +- .../src/daemon/hook_v2_replay_consumer.rs | 244 +++++- .../daemon/invocation_tests/types_tests.rs | 27 +- .../advisory_cycle_language_journey_test.rs | 3 +- 15 files changed, 2161 insertions(+), 283 deletions(-) create mode 100644 crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests/legacy_continuity.rs diff --git a/crates/tracedecay-daemon-service/src/invocation.rs b/crates/tracedecay-daemon-service/src/invocation.rs index 395b0931a3..2bb639dc82 100644 --- a/crates/tracedecay-daemon-service/src/invocation.rs +++ b/crates/tracedecay-daemon-service/src/invocation.rs @@ -247,9 +247,10 @@ pub use primitive::{ }; pub use types::{ BoundedHookOrchestratorV1, DaemonLspInvocationOwner, HookOrchestrationAdmissionV1, - HookOrchestrationRequestV1, HookOrchestrationTriggerV1, HookOrchestrationWorkOutcomeV1, - MAX_COALESCED_HOOK_COMPLETIONS, admit_registered_hook_orchestration, - register_hook_orchestration_runtime, unregister_hook_orchestration_runtime, + HookOrchestrationRequestV1, HookOrchestrationSettlementV1, HookOrchestrationTriggerV1, + HookOrchestrationWorkOutcomeV1, MAX_COALESCED_HOOK_COMPLETIONS, + admit_registered_hook_orchestration, register_hook_orchestration_runtime, + unregister_hook_orchestration_runtime, }; // `pub(super)` on these shapes, in their original flat-file home, meant // "visible to `daemon::service`" (their home's actual parent); nesting them diff --git a/crates/tracedecay-daemon-service/src/invocation/types.rs b/crates/tracedecay-daemon-service/src/invocation/types.rs index 387aa2c7de..1574e8177f 100644 --- a/crates/tracedecay-daemon-service/src/invocation/types.rs +++ b/crates/tracedecay-daemon-service/src/invocation/types.rs @@ -35,11 +35,13 @@ pub struct HookOrchestrationRequestV1 { pub lifecycle: Option, pub hook_configuration_revision: u64, pub trigger: HookOrchestrationTriggerV1, + /// Immutable order from the event's host admission ledger. + pub admission_order: Option, explicit_request_id: Option, #[cfg(any(test, feature = "test-helpers"))] - pub completion: Option>, + pub completion: Option>, #[cfg(not(any(test, feature = "test-helpers")))] - completion: Option>, + completion: Option>, } impl HookOrchestrationRequestV1 { @@ -74,6 +76,7 @@ impl HookOrchestrationRequestV1 { lifecycle, hook_configuration_revision: configuration_revision, trigger, + admission_order: None, explicit_request_id: None, completion: None, }) @@ -88,12 +91,20 @@ impl HookOrchestrationRequestV1 { lifecycle: self.lifecycle.clone(), hook_configuration_revision: self.hook_configuration_revision, trigger: HookOrchestrationTriggerV1::IntentBoundary, + admission_order: self.admission_order, explicit_request_id: None, completion: None, }) } } +/// Why an admitted producer obligation reached its terminal state. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum HookOrchestrationSettlementV1 { + Completed, + Superseded, +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum HookOrchestrationWorkOutcomeV1 { Completed, @@ -128,10 +139,13 @@ type HookOrchestrationEventKeyV1 = ([u8; 16], [u8; 16], [u8; 16], Option; +type HookOrchestrationCompletionV1 = + Arc; struct HookOrchestrationInFlightEntryV1 { event: HookOrchestrationEventKeyV1, + producer: tracedecay_domain::NativeHostIdentityV1, + admission_order: Option, lifecycle: Option, cancellation: tracedecay_runtime_core::cancellation::CancellationToken, superseded: std::sync::atomic::AtomicBool, @@ -288,7 +302,16 @@ impl BoundedHookOrchestratorV1 { }; if emit_terminal { for completion in completions { - completion(); + completion( + if operation + .superseded + .load(std::sync::atomic::Ordering::Acquire) + { + HookOrchestrationSettlementV1::Superseded + } else { + HookOrchestrationSettlementV1::Completed + }, + ); } } } @@ -392,6 +415,15 @@ impl BoundedHookOrchestratorV1 { { return Err(HookOrchestrationAdmissionV1::Backpressured); } + // The lock that chooses the incumbent also checks durable order. + // A delayed replay must never cancel a newer live boundary. + if in_flight.addresses.get(&address).is_some_and(|incumbent| { + incumbent.producer == request.hook.envelope().producer + && matches!((request.admission_order, incumbent.admission_order), + (Some(incoming), Some(active)) if incoming < active) + }) { + return Err(HookOrchestrationAdmissionV1::Backpressured); + } let permit = if let Some(incumbent) = in_flight.addresses.remove(&address) { // A newer boundary at the same stable address supersedes the // incumbent. A host boundary without its own call identity @@ -414,6 +446,8 @@ impl BoundedHookOrchestratorV1 { let work_cancellation = tracedecay_runtime_core::cancellation::CancellationToken::new(); let operation = Arc::new(HookOrchestrationInFlightEntryV1 { event: event.clone(), + producer: request.hook.envelope().producer, + admission_order: request.admission_order, lifecycle: request.lifecycle.clone(), cancellation: work_cancellation, superseded: std::sync::atomic::AtomicBool::new(false), @@ -660,7 +694,8 @@ pub fn admit_registered_hook_orchestration( lifecycle: Option, configuration_revision: u64, explicit: bool, - completion: Option>, + admission_order: Option, + completion: Option>, ) -> HookOrchestrationAdmissionV1 { // Any other hook of a session that now carries a lifecycle is the event a // boundary parked for that session waits on. @@ -686,6 +721,7 @@ pub fn admit_registered_hook_orchestration( { runtime.resume_parked(request.hook.envelope(), lifecycle); } + request.admission_order = admission_order; let intent_companion = request.intent_companion(); request.completion = completion; let admission = runtime.admit(request); @@ -741,8 +777,9 @@ pub(super) async fn run_registered_context_scout_request( lifecycle: Some(lifecycle), hook_configuration_revision, trigger: HookOrchestrationTriggerV1::Explicit, + admission_order: None, explicit_request_id: Some(effect_identity), - completion: Some(Arc::new(move || { + completion: Some(Arc::new(move |_settlement| { if let Ok(mut completed) = completed.lock() && let Some(completed) = completed.take() { diff --git a/crates/tracedecay-daemon-service/src/invocation/types/context_scout_tests.rs b/crates/tracedecay-daemon-service/src/invocation/types/context_scout_tests.rs index f1b8ea1ac6..315b7cc58d 100644 --- a/crates/tracedecay-daemon-service/src/invocation/types/context_scout_tests.rs +++ b/crates/tracedecay-daemon-service/src/invocation/types/context_scout_tests.rs @@ -289,7 +289,7 @@ async fn settled_cycle_admits_the_sessions_next_boundary_from_its_terminal() { let (admitted, admission) = tokio::sync::oneshot::channel(); let pending = StdMutex::new(Some((next, admitted))); let owner = Arc::downgrade(&runtime); - first.completion = Some(Arc::new(move || { + first.completion = Some(Arc::new(move |_| { if let Some((next, admitted)) = pending.lock().unwrap().take() && let Some(runtime) = owner.upgrade() { @@ -361,7 +361,8 @@ async fn lifecycle_less_boundary_settles_and_resumes_on_its_sessions_lifecycle() None, 1, false, - Some(Arc::new(move || { + None, + Some(Arc::new(move |_| { if let Some(completed) = completed.lock().unwrap().take() { let _ = completed.send(()); } @@ -384,6 +385,7 @@ async fn lifecycle_less_boundary_settles_and_resumes_on_its_sessions_lifecycle() lifecycle, 1, false, + None, None ), HookOrchestrationAdmissionV1::UnsupportedTrigger @@ -646,7 +648,8 @@ async fn intent_boundaries_run_in_their_own_lane_and_supersede_only_intent_work( None, 1, false, - Some(Arc::new(move || { + None, + Some(Arc::new(move |_| { if let Some(parked) = parked.lock().unwrap().take() { let _ = parked.send(()); } @@ -674,6 +677,7 @@ async fn intent_boundaries_run_in_their_own_lane_and_supersede_only_intent_work( 1, false, None, + None, ), HookOrchestrationAdmissionV1::Enqueued ); @@ -702,6 +706,7 @@ async fn intent_boundaries_run_in_their_own_lane_and_supersede_only_intent_work( 1, false, None, + None, ), HookOrchestrationAdmissionV1::Enqueued ); @@ -723,6 +728,7 @@ async fn intent_boundaries_run_in_their_own_lane_and_supersede_only_intent_work( 1, false, None, + None, ), HookOrchestrationAdmissionV1::Backpressured ); @@ -737,6 +743,7 @@ async fn intent_boundaries_run_in_their_own_lane_and_supersede_only_intent_work( 1, false, None, + None, ), HookOrchestrationAdmissionV1::Enqueued ); @@ -765,7 +772,8 @@ async fn intent_boundaries_run_in_their_own_lane_and_supersede_only_intent_work( Some(lifecycle), 1, false, - Some(Arc::new(move || { + None, + Some(Arc::new(move |_| { if let Some(stopped) = stopped.lock().unwrap().take() { let _ = stopped.send(()); } @@ -797,3 +805,159 @@ async fn intent_boundaries_run_in_their_own_lane_and_supersede_only_intent_work( unregister_hook_orchestration_runtime(envelope.project_id, envelope.worktree_id, &runtime); assert!(runtime.shutdown().await); } + +struct OrderedHookFixture { + runtime: Arc, + starts: tokio::sync::mpsc::UnboundedReceiver<(u64, CancellationToken)>, + terminals: tokio::sync::mpsc::UnboundedReceiver<(u64, HookOrchestrationSettlementV1)>, + settled: tokio::sync::mpsc::UnboundedSender<(u64, HookOrchestrationSettlementV1)>, + release: Arc, +} + +impl OrderedHookFixture { + fn new() -> Self { + let (started, starts) = tokio::sync::mpsc::unbounded_channel(); + let (settled, terminals) = tokio::sync::mpsc::unbounded_channel(); + let release = Arc::new(tokio::sync::Notify::new()); + let runtime = BoundedHookOrchestratorV1::new(1, { + let release = Arc::clone(&release); + move |request: HookOrchestrationRequestV1, cancellation: CancellationToken| { + let started = started.clone(); + let release = Arc::clone(&release); + async move { + started.send((request.admission_order.unwrap(), cancellation.clone())).unwrap(); + tokio::select! { + () = cancellation.cancelled() => HookOrchestrationWorkOutcomeV1::RetryableFailure, + () = release.notified() => HookOrchestrationWorkOutcomeV1::Completed, + } + } + } + }).unwrap(); + Self { + runtime, + starts, + terminals, + settled, + release, + } + } + + fn request(&self, order: u8) -> HookOrchestrationRequestV1 { + let template = native_request(111); + let mut envelope = template.hook.envelope().clone(); + envelope.event_id = [order; 16]; + let binding = HookScopeBindingV1 { + host: envelope.producer, + project_id: envelope.project_id, + repository_id: envelope.repository_id, + worktree_id: envelope.worktree_id, + worktree_epoch: envelope.worktree_epoch, + binding_token: envelope.binding_token, + capabilities: vec![tracedecay_hooks::HookCapabilityV1 { + family: tracedecay_hooks::HookEventFamily::SavedEdit, + support: tracedecay_hooks::stock_event_support( + envelope.producer, + tracedecay_hooks::HookEventFamily::SavedEdit, + ), + }], + }; + let mut request = HookOrchestrationRequestV1::from_envelope( + envelope, + &binding, + template.lifecycle, + 1, + false, + ) + .unwrap(); + request.admission_order = Some(u64::from(order)); + let settled = self.settled.clone(); + request.completion = Some(Arc::new(move |terminal| { + settled.send((u64::from(order), terminal)).unwrap(); + })); + request + } +} + +#[tokio::test] +async fn delayed_replay_preserves_newer_work_and_its_typed_terminal() { + let mut fixture = OrderedHookFixture::new(); + assert_eq!( + fixture.runtime.admit(fixture.request(1)), + HookOrchestrationAdmissionV1::Enqueued + ); + let (_, old_cancellation) = fixture.starts.recv().await.unwrap(); + assert_eq!( + fixture.runtime.admit(fixture.request(2)), + HookOrchestrationAdmissionV1::Enqueued + ); + assert_eq!( + fixture.terminals.recv().await.unwrap(), + (1, HookOrchestrationSettlementV1::Superseded) + ); + assert!(old_cancellation.is_cancelled()); + let (order, new_cancellation) = fixture.starts.recv().await.unwrap(); + assert_eq!(order, 2); + assert_eq!( + fixture.runtime.admit(fixture.request(1)), + HookOrchestrationAdmissionV1::Backpressured + ); + assert!(!new_cancellation.is_cancelled()); + fixture.release.notify_one(); + assert_eq!( + fixture.terminals.recv().await.unwrap(), + (2, HookOrchestrationSettlementV1::Completed) + ); + assert!(fixture.starts.try_recv().is_err()); + assert!(fixture.runtime.shutdown().await); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn concurrent_live_admission_and_delayed_replay_keep_newest_boundary() { + let mut fixture = OrderedHookFixture::new(); + let barrier = Arc::new(std::sync::Barrier::new(2)); + let live = tokio::task::spawn_blocking({ + let runtime = Arc::clone(&fixture.runtime); + let barrier = Arc::clone(&barrier); + let request = fixture.request(2); + move || { + barrier.wait(); + runtime.admit(request) + } + }); + let replay = tokio::task::spawn_blocking({ + let runtime = Arc::clone(&fixture.runtime); + let request = fixture.request(1); + move || { + barrier.wait(); + runtime.admit(request) + } + }); + assert_eq!(live.await.unwrap(), HookOrchestrationAdmissionV1::Enqueued); + let replay = replay.await.unwrap(); + assert!(matches!( + replay, + HookOrchestrationAdmissionV1::Enqueued | HookOrchestrationAdmissionV1::Backpressured + )); + let cancellation = loop { + let (order, cancellation) = fixture.starts.recv().await.unwrap(); + if order == 2 { + break cancellation; + } + }; + assert!(!cancellation.is_cancelled()); + fixture.release.notify_one(); + let expected = if replay == HookOrchestrationAdmissionV1::Enqueued { + 2 + } else { + 1 + }; + let mut terminals = Vec::new(); + for _ in 0..expected { + terminals.push(fixture.terminals.recv().await.unwrap()); + } + assert!(terminals.contains(&(2, HookOrchestrationSettlementV1::Completed))); + if replay == HookOrchestrationAdmissionV1::Enqueued { + assert!(terminals.contains(&(1, HookOrchestrationSettlementV1::Superseded))); + } + assert!(fixture.runtime.shutdown().await); +} diff --git a/crates/tracedecay-daemon-service/src/lib.rs b/crates/tracedecay-daemon-service/src/lib.rs index cf67070aa5..8e31200d59 100644 --- a/crates/tracedecay-daemon-service/src/lib.rs +++ b/crates/tracedecay-daemon-service/src/lib.rs @@ -104,12 +104,12 @@ pub use invocation::{ DaemonSourceEditOwnerRegistrationError, DaemonWorkProposalRoutingAuthorityV1, DaemonWorkRuntimeRegistrar, FeedbackCycleRuntimeBuilderV1, GraphToolFuture, GraphToolInvocationV1, HookOrchestrationAdmissionV1, HookOrchestrationRequestV1, - HookOrchestrationTriggerV1, HookOrchestrationWorkOutcomeV1, LSP_WORKSPACE_CAPABILITY_ID_V1, - LSP_WORKSPACE_USE_CASE_ID_V1, LspDeliverySettlementAdmissionV1, MAX_COALESCED_HOOK_COMPLETIONS, - ProfileConfigurationAuthorityV1, ProjectGraphToolPortV1, RegisteredCallableCodeRuntime, - RegisteredConfigurationRuntime, RegisteredFeedbackRuntime, RegisteredGraphToolOwnerV1, - RegisteredRetainedRequestContextError, RegisteredRetainedRuntime, RegisteredWorkRuntime, - SwitchableFeedbackCycleRuntimeV1, UnavailableFeedbackCycleRuntimeV1, + HookOrchestrationSettlementV1, HookOrchestrationTriggerV1, HookOrchestrationWorkOutcomeV1, + LSP_WORKSPACE_CAPABILITY_ID_V1, LSP_WORKSPACE_USE_CASE_ID_V1, LspDeliverySettlementAdmissionV1, + MAX_COALESCED_HOOK_COMPLETIONS, ProfileConfigurationAuthorityV1, ProjectGraphToolPortV1, + RegisteredCallableCodeRuntime, RegisteredConfigurationRuntime, RegisteredFeedbackRuntime, + RegisteredGraphToolOwnerV1, RegisteredRetainedRequestContextError, RegisteredRetainedRuntime, + RegisteredWorkRuntime, SwitchableFeedbackCycleRuntimeV1, UnavailableFeedbackCycleRuntimeV1, admit_registered_hook_orchestration, advisory_cycle_invocation_result, callable_code_request_context, daemon_operation_event_authority, execute_profile_configuration, feedback_proximity_invocation_result, register_hook_orchestration_runtime, diff --git a/crates/tracedecay-hooks/src/admission_ledger.rs b/crates/tracedecay-hooks/src/admission_ledger.rs index ba77fff77f..03b9276234 100644 --- a/crates/tracedecay-hooks/src/admission_ledger.rs +++ b/crates/tracedecay-hooks/src/admission_ledger.rs @@ -11,9 +11,10 @@ //! * otherwise: this is a first admission. //! //! Admissions that owe producer work also carry the provider envelope until -//! that work completes, so a restart redrives it. It carries no session content -//! and no application state. The daemon is the sole writer; the ledger is -//! stored beside the transport spool in the same daemon-owned hook data root +//! that work settles, so a restart redrives it. The exact native lifecycle +//! prerequisite is retained with the admission; no session content or application +//! state is stored. The daemon is the sole writer. The ledger is stored beside +//! the transport spool in the same daemon-owned hook data root //! and never touches a migrated database. //! //! Everything lives in one append-only log of checksummed frames. An admission @@ -29,13 +30,13 @@ //! Bounds (stated, not implied): at most //! [`HookAdmissionLedgerLimitsV1::max_records`] live entries per host, nothing //! older than [`HookAdmissionLedgerLimitsV1::max_age_micros`], and at most -//! [`MAX_SPOOL_BYTES_PER_HOST`] of pending work. Beyond the entry or age bound -//! the oldest entries are dropped with any work they still owed, so +//! [`MAX_SPOOL_BYTES_PER_HOST`] of pending work and retained native lifecycle +//! metadata. Beyond the entry or age bound the oldest entries are dropped with any work they still owed, so //! idempotency converges within that window and no further, a replay older //! than the window is admitted again rather than silently believed to be new //! forever. -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeMap; use std::fs::{self, File, OpenOptions}; use std::io::{self, Seek, SeekFrom, Write}; use std::path::{Path, PathBuf}; @@ -73,6 +74,12 @@ const MAX_FRAME_BODY_BYTES: usize = ADMITTED_BODY_BYTES + MAX_HOOK_PAYLOAD_BYTES const FRAME_ADMITTED: u8 = 1; const FRAME_WORK: u8 = 2; const FRAME_COMPLETED: u8 = 3; +const FRAME_ORDERED_ADMITTED: u8 = 4; +const FRAME_NEXT_ORDER: u8 = 5; +const FRAME_NATIVE_LIFECYCLE: u8 = 6; +const FRAME_SUPERSEDED: u8 = 7; +const FRAME_RECONSTRUCTED_ADMITTED: u8 = 8; +const ORDERED_ADMITTED_BODY_BYTES: usize = ADMITTED_BODY_BYTES + 8; /// Superseded frames the log may carry before an append compacts it. const COMPACTION_SLACK_BYTES: u64 = 256 * 1024; const LOG_FILE: &str = "admissions.v2.log"; @@ -176,7 +183,18 @@ pub enum HookAdmissionDecisionV1 { pub struct HookAdmissionLedgerReceiptV1 { pub decision: HookAdmissionDecisionV1, pub order: u64, - pub work_completed: bool, + /// Historical compaction retained relative position, not a comparable + /// lifecycle sequence. Only a canonical observation can recover its range. + pub order_reconstructed: bool, + pub work_settlement: Option, +} + +/// Durable terminal reason for producer work. Supersession settles the debt +/// without claiming that the displaced producer ran successfully. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum HookAdmissionWorkSettlementV1 { + Completed, + Superseded, } #[derive(Clone, Copy, Debug, Error, PartialEq, Eq)] @@ -193,6 +211,10 @@ pub enum HookAdmissionLedgerError { RecordUndecodable, #[error("hook admission ledger identity is invalid")] InvalidIdentity, + #[error("hook admission ledger admission order is exhausted")] + OrderExhausted, + #[error("hook admission order authority is unavailable")] + OrderAuthorityUnavailable, #[error("hook admission ledger is busy in another daemon")] Busy, #[error("hook admission ledger holds its bound of pending producer work")] @@ -217,6 +239,7 @@ struct LedgerEntry { digest: [u8; DIGEST_BYTES], admitted_at: UtcMicros, order: u64, + order_reconstructed: bool, } #[derive(Debug)] @@ -293,6 +316,7 @@ impl HookAdmissionCommitV1 { pub struct HookAdmissionStagedV1 { pub receipt: HookAdmissionLedgerReceiptV1, pub commit: HookAdmissionCommitV1, + pub native_lifecycle: Option, } /// Digest over canonical native-event material. A provider retry observes the @@ -317,7 +341,9 @@ pub struct HookAdmissionLedgerV1 { host: NativeHostIdentityV1, limits: HookAdmissionLedgerLimitsV1, entries: BTreeMap<[u8; IDENTITY_BYTES], LedgerEntry>, - completed_work: BTreeSet<[u8; IDENTITY_BYTES]>, + completed_work: BTreeMap<[u8; IDENTITY_BYTES], HookAdmissionWorkSettlementV1>, + native_lifecycles: BTreeMap<[u8; IDENTITY_BYTES], (crate::NativeContextScoutLifecycleV1, u64)>, + native_lifecycle_bytes: u64, pending_work: BTreeMap<[u8; IDENTITY_BYTES], PendingWork>, pending_work_bytes: u64, next_order: u64, @@ -370,7 +396,9 @@ impl HookAdmissionLedgerV1 { host, limits, entries: BTreeMap::new(), - completed_work: BTreeSet::new(), + completed_work: BTreeMap::new(), + native_lifecycles: BTreeMap::new(), + native_lifecycle_bytes: 0, pending_work: BTreeMap::new(), pending_work_bytes: 0, next_order: 0, @@ -436,15 +464,23 @@ impl HookAdmissionLedgerV1 { /// before the returned commit is awaited, and a duplicate of an admission /// still in flight waits for that admission's frames too. #[tracing::instrument(name = "hooks.admission.stage", level = "trace", skip_all)] + /// Admit or enrich one exact event. Lifecycle conflicts are checked before + /// any append, pending-work adoption, or retention mutation. The canonical + /// floor applies only after those checks to a new allocation; unavailable + /// order authority refuses new work without moving a duplicate position. pub fn stage_admission( &mut self, envelope: &HookEventEnvelopeV2, work: Option<&HookEventEnvelopeV2>, + lifecycle: Option<&crate::NativeContextScoutLifecycleV1>, + minimum_order: Option, now: UtcMicros, ) -> Result { self.ensure_writable()?; let identity = envelope.event_id; - if identity == [0; IDENTITY_BYTES] { + if identity == [0; IDENTITY_BYTES] + || lifecycle.is_some_and(|lifecycle| !lifecycle.matches_envelope(envelope)) + { return Err(HookAdmissionLedgerError::InvalidIdentity); } let digest = hook_admission_digest(envelope)?; @@ -452,53 +488,82 @@ impl HookAdmissionLedgerV1 { .map(|work| encode_work(work).map(|encoded| (work, encoded))) .transpose()?; if let Some(existing) = self.entries.get(&identity).copied() { - if is_expired(existing.admitted_at, now, self.limits.max_age_micros) { - self.forget(&identity); - } else if existing.digest == digest { - return self.stage_duplicate(identity, existing.order, work); - } else { - let work_completed = self.completed_work.contains(&identity); - return Ok(self.staged( - HookAdmissionDecisionV1::Conflict, - existing.order, - work_completed, - )); + if !is_expired(existing.admitted_at, now, self.limits.max_age_micros) { + if existing.digest != digest + || lifecycle.is_some_and(|incoming| { + self.native_lifecycles + .get(&identity) + .is_some_and(|(retained, _)| retained != incoming) + }) + { + return Ok(self.staged( + identity, + HookAdmissionDecisionV1::Conflict, + existing.order, + existing.order_reconstructed, + )); + } + return self.stage_duplicate(identity, existing, work, lifecycle); } + self.forget(&identity); } - if let Some((_, encoded)) = work.as_ref() { - self.ensure_work_capacity(encoded)?; + let minimum_order = + minimum_order.ok_or(HookAdmissionLedgerError::OrderAuthorityUnavailable)?; + let order = self.next_order.max(minimum_order); + if matches!(envelope.ordering, crate::HookOrderingV1::Unknown) + && order.checked_add(2).is_none() + { + return Err(HookAdmissionLedgerError::OrderExhausted); } + let next_order = order + .checked_add(1) + .ok_or(HookAdmissionLedgerError::OrderExhausted)?; + self.ensure_retained_capacity( + work.as_ref() + .map_or(0, |(_, bytes)| work_frame_bytes(bytes)) + + lifecycle + .map(native_lifecycle_frame_bytes) + .transpose()? + .unwrap_or(0), + )?; if self.entries.len() as u32 >= self.limits.max_records { self.trim_to((self.limits.max_records as usize).saturating_sub(1)); self.rewrite()?; } let mut frames = Vec::new(); encode_frame( - FRAME_ADMITTED, + FRAME_ORDERED_ADMITTED, &encode_admitted_body( identity, digest, now, + order, work.as_ref().map_or(&[][..], |(_, encoded)| encoded), ), &mut frames, )?; + if let Some(lifecycle) = lifecycle { + encode_native_lifecycle_frame(identity, lifecycle, &mut frames)?; + } self.append(&frames)?; - let order = self.next_order; - self.next_order = self.next_order.saturating_add(1); + self.next_order = next_order; self.entries.insert( identity, LedgerEntry { digest, admitted_at: now, order, + order_reconstructed: false, }, ); if let Some((work, _)) = work { self.insert_pending_work(identity, work.clone())?; } + if let Some(lifecycle) = lifecycle { + self.insert_native_lifecycle(identity, lifecycle.clone())?; + } self.compact_if_sparse()?; - Ok(self.staged(HookAdmissionDecisionV1::Admitted, order, false)) + Ok(self.staged(identity, HookAdmissionDecisionV1::Admitted, order, false)) } /// Stage completion of an admission's producer work. `None` means it was @@ -507,6 +572,16 @@ impl HookAdmissionLedgerV1 { pub fn stage_work_completion( &mut self, envelope: &HookEventEnvelopeV2, + ) -> Result, HookAdmissionLedgerError> { + self.stage_work_settlement(envelope, HookAdmissionWorkSettlementV1::Completed) + } + + /// Settle owed work with its actual terminal reason. The first durable + /// terminal decision is immutable, including a later completion callback. + pub fn stage_work_settlement( + &mut self, + envelope: &HookEventEnvelopeV2, + settlement: HookAdmissionWorkSettlementV1, ) -> Result, HookAdmissionLedgerError> { self.ensure_writable()?; let identity = envelope.event_id; @@ -516,14 +591,18 @@ impl HookAdmissionLedgerV1 { if entry.digest != hook_admission_digest(envelope)? { return Err(HookAdmissionLedgerError::InvalidIdentity); } - if self.completed_work.contains(&identity) { + if self.completed_work.contains_key(&identity) { return Ok(None); } let mut frames = Vec::with_capacity(COMPLETED_FRAME_BYTES); - encode_frame(FRAME_COMPLETED, &identity, &mut frames)?; + let kind = match settlement { + HookAdmissionWorkSettlementV1::Completed => FRAME_COMPLETED, + HookAdmissionWorkSettlementV1::Superseded => FRAME_SUPERSEDED, + }; + encode_frame(kind, &identity, &mut frames)?; self.append(&frames)?; self.remove_pending_work(&identity); - self.completed_work.insert(identity); + self.completed_work.insert(identity, settlement); self.compact_if_sparse()?; Ok(Some(self.commit_ticket())) } @@ -585,39 +664,67 @@ impl HookAdmissionLedgerV1 { identity, digest, admitted_at, + order, + order_reconstructed, work, } => { + let order = order.unwrap_or(self.next_order); + self.next_order = self.next_order.max( + order + .checked_add(1) + .ok_or(HookAdmissionLedgerError::OrderExhausted)?, + ); self.forget(&identity); if is_expired(admitted_at, now, self.limits.max_age_micros) { dropped_expired_records = dropped_expired_records.saturating_add(1); continue; } admitted_frames = admitted_frames.saturating_add(1); - let order = self.next_order; - self.next_order = self.next_order.saturating_add(1); self.entries.insert( identity, LedgerEntry { digest, admitted_at, order, + order_reconstructed, }, ); if let Some(work) = work { self.insert_pending_work(identity, work)?; } } + LogFrame::NextOrder(next_order) => { + self.next_order = self.next_order.max(next_order); + } LogFrame::Work { identity, envelope } => { if self.entries.contains_key(&identity) - && !self.completed_work.contains(&identity) + && !self.completed_work.contains_key(&identity) { self.insert_pending_work(identity, envelope)?; } } - LogFrame::Completed { identity } => { + LogFrame::Settled { + identity, + settlement, + } => { if self.entries.contains_key(&identity) { self.remove_pending_work(&identity); - self.completed_work.insert(identity); + self.completed_work.entry(identity).or_insert(settlement); + } + } + LogFrame::NativeLifecycle { + identity, + lifecycle, + } => { + if self.entries.contains_key(&identity) { + if self + .native_lifecycles + .get(&identity) + .is_some_and(|(retained, _)| retained != &lifecycle) + { + return Err(HookAdmissionLedgerError::RecordUndecodable); + } + self.insert_native_lifecycle(identity, lifecycle)?; } } } @@ -653,44 +760,71 @@ impl HookAdmissionLedgerV1 { fn stage_duplicate( &mut self, identity: [u8; IDENTITY_BYTES], - order: u64, + existing: LedgerEntry, work: Option<(&HookEventEnvelopeV2, Vec)>, + lifecycle: Option<&crate::NativeContextScoutLifecycleV1>, ) -> Result { - let work_completed = self.completed_work.contains(&identity); - if let Some((work, encoded)) = work - && !work_completed - && !self.pending_work.contains_key(&identity) - { - self.ensure_work_capacity(&encoded)?; - let mut frames = Vec::new(); + let work = work.filter(|_| { + !self.completed_work.contains_key(&identity) + && !self.pending_work.contains_key(&identity) + }); + let lifecycle = lifecycle.filter(|_| !self.native_lifecycles.contains_key(&identity)); + self.ensure_retained_capacity( + work.as_ref() + .map_or(0, |(_, bytes)| work_frame_bytes(bytes)) + + lifecycle + .map(native_lifecycle_frame_bytes) + .transpose()? + .unwrap_or(0), + )?; + let mut frames = Vec::new(); + if let Some((_, encoded)) = work.as_ref() { encode_frame( FRAME_WORK, - &encode_work_body(identity, &encoded), + &encode_work_body(identity, encoded), &mut frames, )?; + } + if let Some(lifecycle) = lifecycle { + encode_native_lifecycle_frame(identity, lifecycle, &mut frames)?; + } + if !frames.is_empty() { self.append(&frames)?; - self.insert_pending_work(identity, work.clone())?; + if let Some((work, _)) = work { + self.insert_pending_work(identity, work.clone())?; + } + if let Some(lifecycle) = lifecycle { + self.insert_native_lifecycle(identity, lifecycle.clone())?; + } + self.compact_if_sparse()?; } Ok(self.staged( + identity, HookAdmissionDecisionV1::ExactDuplicate, - order, - work_completed, + existing.order, + existing.order_reconstructed, )) } fn staged( &self, + identity: [u8; IDENTITY_BYTES], decision: HookAdmissionDecisionV1, order: u64, - work_completed: bool, + order_reconstructed: bool, ) -> HookAdmissionStagedV1 { HookAdmissionStagedV1 { receipt: HookAdmissionLedgerReceiptV1 { decision, order, - work_completed, + order_reconstructed, + work_settlement: self.completed_work.get(&identity).copied(), }, commit: self.commit_ticket(), + native_lifecycle: self + .native_lifecycles + .get(&identity) + .map(|(lifecycle, _)| lifecycle.clone()), } } @@ -709,10 +843,11 @@ impl HookAdmissionLedgerV1 { Ok(()) } - fn ensure_work_capacity(&self, work: &[u8]) -> Result<(), HookAdmissionLedgerError> { + fn ensure_retained_capacity(&self, additional: u64) -> Result<(), HookAdmissionLedgerError> { if self .pending_work_bytes - .saturating_add(work_frame_bytes(work)) + .saturating_add(self.native_lifecycle_bytes) + .saturating_add(additional) > MAX_SPOOL_BYTES_PER_HOST { return Err(HookAdmissionLedgerError::WorkCapacityExceeded); @@ -720,6 +855,25 @@ impl HookAdmissionLedgerV1 { Ok(()) } + fn insert_native_lifecycle( + &mut self, + identity: [u8; IDENTITY_BYTES], + lifecycle: crate::NativeContextScoutLifecycleV1, + ) -> Result<(), HookAdmissionLedgerError> { + let bytes = native_lifecycle_frame_bytes(&lifecycle)?; + let previous = self + .native_lifecycles + .get(&identity) + .map_or(0, |(_, bytes)| *bytes); + self.ensure_retained_capacity(bytes.saturating_sub(previous))?; + self.native_lifecycles.insert(identity, (lifecycle, bytes)); + self.native_lifecycle_bytes = self + .native_lifecycle_bytes + .saturating_sub(previous) + .saturating_add(bytes); + Ok(()) + } + /// Append frames without a barrier. A failed or short write poisons the /// handle: later frames must never land behind a torn one. fn append(&mut self, frames: &[u8]) -> Result<(), HookAdmissionLedgerError> { @@ -743,6 +897,11 @@ impl HookAdmissionLedgerV1 { envelope: HookEventEnvelopeV2, ) -> Result<(), HookAdmissionLedgerError> { let frame_bytes = work_frame_bytes(&encode_work(&envelope)?); + let previous = self + .pending_work + .get(&identity) + .map_or(0, |work| work.frame_bytes); + self.ensure_retained_capacity(frame_bytes.saturating_sub(previous))?; if let Some(previous) = self.pending_work.insert( identity, PendingWork { @@ -765,6 +924,9 @@ impl HookAdmissionLedgerV1 { fn forget(&mut self, identity: &[u8; IDENTITY_BYTES]) { self.entries.remove(identity); self.completed_work.remove(identity); + if let Some((_, bytes)) = self.native_lifecycles.remove(identity) { + self.native_lifecycle_bytes = self.native_lifecycle_bytes.saturating_sub(bytes); + } if self.pending_work.contains_key(identity) { tracing::debug!( event = "hook_admission_pending_work_dropped", @@ -798,8 +960,10 @@ impl HookAdmissionLedgerV1 { fn live_bytes(&self) -> u64 { HEADER_BYTES as u64 - + self.entries.len() as u64 * ADMITTED_FRAME_BYTES as u64 + + (8 + FRAME_OVERHEAD_BYTES) as u64 + + self.entries.len() as u64 * (ADMITTED_FRAME_BYTES + 8) as u64 + self.completed_work.len() as u64 * COMPLETED_FRAME_BYTES as u64 + + self.native_lifecycle_bytes + self.pending_work_bytes } @@ -835,25 +999,33 @@ impl HookAdmissionLedgerV1 { .map(|work| encode_work(&work.envelope)) .transpose()?; encode_frame( - FRAME_ADMITTED, + if entry.order_reconstructed { + FRAME_RECONSTRUCTED_ADMITTED + } else { + FRAME_ORDERED_ADMITTED + }, &encode_admitted_body( *identity, entry.digest, entry.admitted_at, + entry.order, work.as_deref().unwrap_or_default(), ), &mut bytes, )?; - if self.completed_work.contains(identity) { - encode_frame(FRAME_COMPLETED, identity, &mut bytes)?; + if let Some(settlement) = self.completed_work.get(identity) { + let kind = match settlement { + HookAdmissionWorkSettlementV1::Completed => FRAME_COMPLETED, + HookAdmissionWorkSettlementV1::Superseded => FRAME_SUPERSEDED, + }; + encode_frame(kind, identity, &mut bytes)?; } - } - for (index, (_, identity, _)) in ordered.iter().enumerate() { - if let Some(entry) = self.entries.get_mut(identity) { - entry.order = index as u64; + if let Some((lifecycle, _)) = self.native_lifecycles.get(identity) { + encode_native_lifecycle_frame(*identity, lifecycle, &mut bytes)?; } } - self.next_order = ordered.len() as u64; + // An empty retained window must not reuse an earlier lifecycle position. + encode_frame(FRAME_NEXT_ORDER, &self.next_order.to_le_bytes(), &mut bytes)?; let path = log_path(&self.root); let mut state = self .log @@ -882,22 +1054,54 @@ impl HookAdmissionLedgerV1 { #[derive(Debug)] enum LogFrame { + NextOrder(u64), Admitted { identity: [u8; IDENTITY_BYTES], digest: [u8; DIGEST_BYTES], admitted_at: UtcMicros, + order: Option, + order_reconstructed: bool, work: Option, }, Work { identity: [u8; IDENTITY_BYTES], envelope: HookEventEnvelopeV2, }, - Completed { + Settled { + identity: [u8; IDENTITY_BYTES], + settlement: HookAdmissionWorkSettlementV1, + }, + NativeLifecycle { identity: [u8; IDENTITY_BYTES], + lifecycle: crate::NativeContextScoutLifecycleV1, }, } +fn native_lifecycle_frame_bytes( + lifecycle: &crate::NativeContextScoutLifecycleV1, +) -> Result { + let bytes = + canonical_json_bytes(lifecycle).map_err(|_| HookAdmissionLedgerError::RecordUnencodable)?; + Ok((IDENTITY_BYTES + bytes.len() + FRAME_OVERHEAD_BYTES) as u64) +} + +fn encode_native_lifecycle_frame( + identity: [u8; IDENTITY_BYTES], + lifecycle: &crate::NativeContextScoutLifecycleV1, + out: &mut Vec, +) -> Result<(), HookAdmissionLedgerError> { + let mut body = identity.to_vec(); + body.extend_from_slice( + &canonical_json_bytes(lifecycle) + .map_err(|_| HookAdmissionLedgerError::RecordUnencodable)?, + ); + encode_frame(FRAME_NATIVE_LIFECYCLE, &body, out) +} + fn encode_frame(kind: u8, body: &[u8], out: &mut Vec) -> Result<(), HookAdmissionLedgerError> { + if body.len() > MAX_FRAME_BODY_BYTES { + return Err(HookAdmissionLedgerError::RecordUnencodable); + } let length = u32::try_from(body.len()).map_err(|_| HookAdmissionLedgerError::RecordUnencodable)?; let start = out.len(); @@ -913,12 +1117,14 @@ fn encode_admitted_body( identity: [u8; IDENTITY_BYTES], digest: [u8; DIGEST_BYTES], admitted_at: UtcMicros, + order: u64, work: &[u8], ) -> Vec { - let mut body = Vec::with_capacity(ADMITTED_BODY_BYTES + work.len()); + let mut body = Vec::with_capacity(ORDERED_ADMITTED_BODY_BYTES + work.len()); body.extend_from_slice(&identity); body.extend_from_slice(&digest); body.extend_from_slice(&admitted_at.0.to_le_bytes()); + body.extend_from_slice(&order.to_le_bytes()); body.extend_from_slice(work); body } @@ -982,26 +1188,65 @@ fn scan_log(bytes: &[u8]) -> Result<(Vec, usize), HookAdmissionLedgerE } let body = &bytes[offset + FRAME_PREFIX_BYTES..end]; frames.push(match (kind, body.len()) { - (FRAME_ADMITTED, length) if length >= ADMITTED_BODY_BYTES => { + (FRAME_ADMITTED | FRAME_ORDERED_ADMITTED | FRAME_RECONSTRUCTED_ADMITTED, length) + if length + >= if kind == FRAME_ADMITTED { + ADMITTED_BODY_BYTES + } else { + ORDERED_ADMITTED_BODY_BYTES + } => + { let mut digest = [0u8; DIGEST_BYTES]; digest.copy_from_slice(&body[IDENTITY_BYTES..IDENTITY_BYTES + DIGEST_BYTES]); let mut admitted = [0u8; 8]; admitted.copy_from_slice(&body[IDENTITY_BYTES + DIGEST_BYTES..ADMITTED_BODY_BYTES]); + let (order, work_start) = if kind != FRAME_ADMITTED { + let mut order = [0u8; 8]; + order.copy_from_slice(&body[ADMITTED_BODY_BYTES..ORDERED_ADMITTED_BODY_BYTES]); + (Some(u64::from_le_bytes(order)), ORDERED_ADMITTED_BODY_BYTES) + } else { + (None, ADMITTED_BODY_BYTES) + }; LogFrame::Admitted { identity: identity_at(body), digest, admitted_at: UtcMicros(i64::from_le_bytes(admitted)), - work: decode_work(&body[ADMITTED_BODY_BYTES..])?, + order, + order_reconstructed: kind != FRAME_ORDERED_ADMITTED, + work: decode_work(&body[work_start..])?, } } + (FRAME_NEXT_ORDER, 8) => { + let mut next_order = [0u8; 8]; + next_order.copy_from_slice(body); + LogFrame::NextOrder(u64::from_le_bytes(next_order)) + } (FRAME_WORK, length) if length > IDENTITY_BYTES => LogFrame::Work { identity: identity_at(body), envelope: decode_work(&body[IDENTITY_BYTES..])? .ok_or(HookAdmissionLedgerError::RecordUndecodable)?, }, - (FRAME_COMPLETED, IDENTITY_BYTES) => LogFrame::Completed { + (FRAME_COMPLETED | FRAME_SUPERSEDED, IDENTITY_BYTES) => LogFrame::Settled { identity: identity_at(body), + settlement: if kind == FRAME_COMPLETED { + HookAdmissionWorkSettlementV1::Completed + } else { + HookAdmissionWorkSettlementV1::Superseded + }, }, + (FRAME_NATIVE_LIFECYCLE, length) if length > IDENTITY_BYTES => { + let identity = identity_at(body); + let lifecycle: crate::NativeContextScoutLifecycleV1 = + serde_json::from_slice(&body[IDENTITY_BYTES..]) + .map_err(|_| HookAdmissionLedgerError::RecordUndecodable)?; + if lifecycle.event_id != identity { + return Err(HookAdmissionLedgerError::RecordUndecodable); + } + LogFrame::NativeLifecycle { + identity, + lifecycle, + } + } _ => return Err(HookAdmissionLedgerError::RecordUndecodable), }); offset = end + CHECKSUM_PREFIX_BYTES; @@ -1095,6 +1340,7 @@ mod tests { use super::*; use crate::{HOOK_EVENT_SCHEMA_VERSION, HookBoundaryV1, HookEventV2, HookOrderingV1}; use std::cell::Cell; + use std::collections::BTreeSet; use std::io::{BufRead, BufReader}; use std::process::{Command, Stdio}; use std::sync::atomic::{AtomicU64, Ordering}; @@ -1133,7 +1379,7 @@ mod tests { envelope: &HookEventEnvelopeV2, now: UtcMicros, ) -> Result { - let staged = self.stage_admission(envelope, None, now)?; + let staged = self.stage_admission(envelope, None, None, Some(0), now)?; staged.commit.wait()?; Ok(staged.receipt) } @@ -1204,6 +1450,351 @@ mod tests { .0 } + #[test] + fn admission_order_survives_compaction_with_retained_gaps() { + let root = TestDir::new("ledger-order-compaction"); + let mut ledger = open(root.path(), UtcMicros(1)); + for marker in 1..=3 { + assert_eq!( + ledger + .admit_with_receipt(&envelope(marker, 5), UtcMicros(2)) + .unwrap() + .order, + u64::from(marker - 1), + ); + } + ledger.forget(&envelope(1, 5).event_id); + ledger.rewrite().unwrap(); + assert_eq!( + ledger + .admit_with_receipt(&envelope(3, 5), UtcMicros(3)) + .unwrap() + .order, + 2, + ); + drop(ledger); + let mut reopened = open(root.path(), UtcMicros(4)); + assert_eq!( + reopened + .admit_with_receipt(&envelope(2, 5), UtcMicros(5)) + .unwrap() + .order, + 1, + ); + assert_eq!( + reopened + .admit_with_receipt(&envelope(4, 5), UtcMicros(6)) + .unwrap() + .order, + 3, + ); + } + + #[test] + fn admission_order_survives_an_empty_retained_window_and_restart() { + let root = TestDir::new("ledger-order-empty"); + let limits = HookAdmissionLedgerLimitsV1 { + max_age_micros: 10, + ..HookAdmissionLedgerLimitsV1::stock() + }; + for (marker, now) in [(1, 1), (2, 21), (3, 41)] { + let mut ledger = HookAdmissionLedgerV1::open( + root.path(), + NativeHostIdentityV1::ClaudeCode, + limits, + UtcMicros(now), + ) + .unwrap() + .0; + assert_eq!(ledger.live_records(), 0); + let admitted = ledger + .admit_with_receipt(&envelope(marker, 5), UtcMicros(now + 1)) + .unwrap(); + assert_eq!(admitted.order, u64::from(marker - 1)); + assert_eq!(ledger.expire(UtcMicros(now + 12)).unwrap(), 1); + assert_eq!(ledger.live_records(), 0); + } + } + + #[test] + fn legacy_frames_preserve_admission_order_when_rewritten() { + let root = TestDir::new("ledger-legacy-order"); + drop(open(root.path(), UtcMicros(1))); + let first = envelope(9, 5); + let second = envelope(10, 5); + let mut bytes = log_header().to_vec(); + for record in [&first, &second] { + // The master format stores identity, digest, time, then optional work. + let mut body = record.event_id.to_vec(); + body.extend_from_slice(&hook_admission_digest(record).unwrap()); + body.extend_from_slice(&2i64.to_le_bytes()); + encode_frame(FRAME_ADMITTED, &body, &mut bytes).unwrap(); + } + encode_frame( + FRAME_WORK, + &encode_work_body(second.event_id, &encode_work(&second).unwrap()), + &mut bytes, + ) + .unwrap(); + encode_frame(FRAME_COMPLETED, &first.event_id, &mut bytes).unwrap(); + fs::write(root.path().join(LOG_FILE), bytes).unwrap(); + let mut ledger = open(root.path(), UtcMicros(3)); + let first_receipt = ledger.admit_with_receipt(&first, UtcMicros(4)).unwrap(); + assert_eq!(first_receipt.order, 0); + assert!(first_receipt.order_reconstructed); + assert!(first_receipt.work_settlement.is_some()); + assert_eq!( + ledger.pending_work_envelopes(UtcMicros(4)).unwrap(), + [second.clone()] + ); + ledger.forget(&first.event_id); + ledger.rewrite().unwrap(); + drop(ledger); + let mut reopened = open(root.path(), UtcMicros(5)); + assert!( + reopened + .admit_with_receipt(&second, UtcMicros(6)) + .unwrap() + .order_reconstructed + ); + assert_eq!( + reopened + .admit_with_receipt(&second, UtcMicros(6)) + .unwrap() + .order, + 1 + ); + assert_eq!( + reopened + .admit_with_receipt(&envelope(11, 5), UtcMicros(6)) + .unwrap() + .order, + 2 + ); + assert_eq!( + reopened.pending_work_envelopes(UtcMicros(6)).unwrap(), + [second] + ); + } + + #[test] + fn canonical_order_floor_changes_only_new_admissions_and_refuses_exhaustion() { + let root = TestDir::new("ledger-canonical-order-floor"); + let now = UtcMicros(2); + let mut ledger = open(root.path(), now); + let first = envelope(1, 5); + let admitted = ledger + .stage_admission(&first, Some(&first), None, Some(99), now) + .unwrap(); + admitted.commit.wait().unwrap(); + assert_eq!(admitted.receipt.order, 99); + assert!(!admitted.receipt.order_reconstructed); + let duplicate = ledger + .stage_admission(&first, Some(&first), None, None, now) + .unwrap(); + duplicate.commit.wait().unwrap(); + assert_eq!(duplicate.receipt.order, 99); + let mut conflict = first.clone(); + conflict.worktree_epoch += 1; + let before = ledger.written; + let conflicted = ledger + .stage_admission(&conflict, Some(&conflict), None, Some(u64::MAX), now) + .unwrap(); + conflicted.commit.wait().unwrap(); + assert_eq!( + conflicted.receipt.decision, + HookAdmissionDecisionV1::Conflict + ); + assert_eq!( + ledger.written, before, + "conflict cannot move the order floor" + ); + let second = envelope(2, 5); + assert!(matches!( + ledger.stage_admission(&second, Some(&second), None, None, now), + Err(HookAdmissionLedgerError::OrderAuthorityUnavailable) + )); + for floor in [u64::MAX, u64::MAX - 1] { + assert!(matches!( + ledger.stage_admission(&second, Some(&second), None, Some(floor), now), + Err(HookAdmissionLedgerError::OrderExhausted) + )); + assert_eq!(ledger.written, before); + } + let final_order = ledger + .stage_admission(&second, Some(&second), None, Some(u64::MAX - 2), now) + .unwrap(); + final_order.commit.wait().unwrap(); + assert_eq!(final_order.receipt.order, u64::MAX - 2); + ledger.rewrite().unwrap(); + drop(ledger); + let mut ledger = open(root.path(), now); + assert_eq!( + ledger.admit_with_receipt(&second, now).unwrap().order, + u64::MAX - 2 + ); + assert!(matches!( + ledger.stage_admission(&envelope(3, 5), None, None, Some(0), now), + Err(HookAdmissionLedgerError::OrderExhausted) + )); + } + + #[test] + fn native_lifecycle_enrichment_is_durable_and_conflicts_before_work_adoption() { + let root = TestDir::new("ledger-native-lifecycle"); + let mut event = envelope(41, 5); + let lifecycle = crate::NativeContextScoutLifecycleV1::new( + "session.native-metadata", + "call.native-metadata", + event.event_id, + ) + .unwrap(); + event.protected_session_id = + ::digest(lifecycle.session_id.as_str().as_bytes()).into(); + event.event = HookEventV2::SavedEdit { + file_id: [8; 16], + changed_range_count: 1, + }; + let mut ledger = open(root.path(), UtcMicros(1)); + let first = ledger + .stage_admission(&event, None, None, Some(0), UtcMicros(2)) + .unwrap(); + let order = first.receipt.order; + assert_eq!(first.native_lifecycle, None); + first.commit.wait().unwrap(); + let enriched = ledger + .stage_admission(&event, None, Some(&lifecycle), Some(0), UtcMicros(3)) + .unwrap(); + assert_eq!( + enriched.receipt.decision, + HookAdmissionDecisionV1::ExactDuplicate + ); + assert_eq!(enriched.receipt.order, order); + assert_eq!(enriched.native_lifecycle, Some(lifecycle.clone())); + enriched.commit.wait().unwrap(); + let before = fs::metadata(log_path(root.path())).unwrap().len(); + let conflict = crate::NativeContextScoutLifecycleV1::new( + lifecycle.session_id.as_str(), + "call.different", + event.event_id, + ) + .unwrap(); + let refused = ledger + .stage_admission(&event, Some(&event), Some(&conflict), Some(0), UtcMicros(4)) + .unwrap(); + assert_eq!(refused.receipt.decision, HookAdmissionDecisionV1::Conflict); + refused.commit.wait().unwrap(); + assert_eq!(fs::metadata(log_path(root.path())).unwrap().len(), before); + assert!( + ledger + .pending_work_envelopes(UtcMicros(4)) + .unwrap() + .is_empty() + ); + let forged = crate::NativeContextScoutLifecycleV1::new( + "session.foreign", + "call.foreign", + event.event_id, + ) + .unwrap(); + assert!(matches!( + ledger.stage_admission(&event, Some(&event), Some(&forged), Some(0), UtcMicros(4)), + Err(HookAdmissionLedgerError::InvalidIdentity) + )); + assert_eq!(fs::metadata(log_path(root.path())).unwrap().len(), before); + let owed = ledger + .stage_admission( + &event, + Some(&event), + Some(&lifecycle), + Some(0), + UtcMicros(5), + ) + .unwrap(); + owed.commit.wait().unwrap(); + assert_eq!( + ledger.pending_work_envelopes(UtcMicros(5)).unwrap(), + [event.clone()] + ); + ledger + .stage_work_completion(&event) + .unwrap() + .unwrap() + .wait() + .unwrap(); + ledger.rewrite().unwrap(); + drop(ledger); + let mut reopened = open(root.path(), UtcMicros(6)); + let retained = reopened + .stage_admission(&event, Some(&event), None, Some(0), UtcMicros(6)) + .unwrap(); + assert_eq!(retained.receipt.order, order); + assert_eq!( + retained.receipt.work_settlement, + Some(HookAdmissionWorkSettlementV1::Completed) + ); + assert_eq!(retained.native_lifecycle, Some(lifecycle)); + retained.commit.wait().unwrap(); + assert!( + reopened + .pending_work_envelopes(UtcMicros(6)) + .unwrap() + .is_empty() + ); + } + + #[test] + fn producer_work_retains_its_first_typed_terminal_reason_through_restart() { + for terminal in [ + HookAdmissionWorkSettlementV1::Completed, + HookAdmissionWorkSettlementV1::Superseded, + ] { + let root = TestDir::new("ledger-work-terminal"); + let event = envelope(43, 5); + let mut ledger = open(root.path(), UtcMicros(1)); + ledger + .stage_admission(&event, Some(&event), None, Some(0), UtcMicros(2)) + .unwrap() + .commit + .wait() + .unwrap(); + ledger + .stage_work_settlement(&event, terminal) + .unwrap() + .unwrap() + .wait() + .unwrap(); + let other = match terminal { + HookAdmissionWorkSettlementV1::Completed => { + HookAdmissionWorkSettlementV1::Superseded + } + HookAdmissionWorkSettlementV1::Superseded => { + HookAdmissionWorkSettlementV1::Completed + } + }; + assert!( + ledger + .stage_work_settlement(&event, other) + .unwrap() + .is_none() + ); + ledger.rewrite().unwrap(); + drop(ledger); + let mut reopened = open(root.path(), UtcMicros(3)); + let retry = reopened + .stage_admission(&event, Some(&event), None, Some(0), UtcMicros(3)) + .unwrap(); + assert_eq!(retry.receipt.work_settlement, Some(terminal)); + retry.commit.wait().unwrap(); + assert!( + reopened + .pending_work_envelopes(UtcMicros(3)) + .unwrap() + .is_empty() + ); + } + } + #[test] fn identical_bytes_converge_on_exact_duplicate() { let root = TestDir::new("ledger"); @@ -1357,10 +1948,10 @@ mod tests { { let mut ledger = open(root.path(), UtcMicros(1)); let first = ledger - .stage_admission(&admitted, Some(&admitted), UtcMicros(2)) + .stage_admission(&admitted, Some(&admitted), None, Some(0), UtcMicros(2)) .unwrap(); first.commit.wait().unwrap(); - assert!(!first.receipt.work_completed); + assert!(first.receipt.work_settlement.is_none()); } { @@ -1373,7 +1964,7 @@ mod tests { .admit_with_receipt(&admitted, UtcMicros(4)) .unwrap(); assert_eq!(duplicate.decision, HookAdmissionDecisionV1::ExactDuplicate); - assert!(!duplicate.work_completed); + assert!(duplicate.work_settlement.is_none()); assert!(restarted.mark_work_completed(&admitted).unwrap()); } @@ -1387,7 +1978,7 @@ mod tests { let duplicate = completed .admit_with_receipt(&admitted, UtcMicros(6)) .unwrap(); - assert!(duplicate.work_completed); + assert!(duplicate.work_settlement.is_some()); assert!(!completed.mark_work_completed(&admitted).unwrap()); } @@ -1426,7 +2017,7 @@ mod tests { { let mut ledger = open(root.path(), UtcMicros(1)); let staged = ledger - .stage_admission(&admitted, Some(&admitted), UtcMicros(2)) + .stage_admission(&admitted, Some(&admitted), None, Some(0), UtcMicros(2)) .unwrap(); staged.commit.wait().unwrap(); assert_eq!( @@ -1453,10 +2044,10 @@ mod tests { let root = TestDir::new("ledger-group-commit"); let mut ledger = open(root.path(), UtcMicros(1)); let first = ledger - .stage_admission(&envelope(9, 5), None, UtcMicros(2)) + .stage_admission(&envelope(9, 5), None, None, Some(0), UtcMicros(2)) .unwrap(); let second = ledger - .stage_admission(&envelope(10, 5), None, UtcMicros(3)) + .stage_admission(&envelope(10, 5), None, None, Some(0), UtcMicros(3)) .unwrap(); let before = COMMIT_SYNCS.with(Cell::get); @@ -1487,7 +2078,13 @@ mod tests { let staged = ledger .lock() .unwrap() - .stage_admission(&admitted, Some(&admitted), UtcMicros(2)) + .stage_admission( + &admitted, + Some(&admitted), + None, + Some(0), + UtcMicros(2), + ) .unwrap(); assert_eq!(staged.receipt.decision, HookAdmissionDecisionV1::Admitted); staged.commit.wait().unwrap(); @@ -1536,7 +2133,13 @@ mod tests { let staged = ledger .lock() .unwrap() - .stage_admission(&admitted, Some(&admitted), UtcMicros(2)) + .stage_admission( + &admitted, + Some(&admitted), + None, + Some(0), + UtcMicros(2), + ) .unwrap(); staged.commit.wait().unwrap(); println!("acked {id}"); diff --git a/crates/tracedecay-hooks/src/lib.rs b/crates/tracedecay-hooks/src/lib.rs index 0e9eda86d7..ce84d522a2 100644 --- a/crates/tracedecay-hooks/src/lib.rs +++ b/crates/tracedecay-hooks/src/lib.rs @@ -21,8 +21,8 @@ pub mod spool; pub use admission_ledger::{ HookAdmissionCommitV1, HookAdmissionDecisionV1, HookAdmissionLedgerError, HookAdmissionLedgerLimitsV1, HookAdmissionLedgerReceiptV1, HookAdmissionLedgerV1, - HookAdmissionStagedV1, PRE_LEDGER_PENDING_WORK_DIR, PROFILE_HOOK_ADMISSIONS_DIR, - PROJECT_HOOK_ADMISSIONS_DIR, hook_admission_reset_required_roots, + HookAdmissionStagedV1, HookAdmissionWorkSettlementV1, PRE_LEDGER_PENDING_WORK_DIR, + PROFILE_HOOK_ADMISSIONS_DIR, PROJECT_HOOK_ADMISSIONS_DIR, hook_admission_reset_required_roots, }; pub use capture::{ NativeHookCaptureOutcomeV1, NativeHookCaptureSourceV1, capture_native_event_for_replay, diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/admission.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/admission.rs index 5922953c5a..7b2b4ef1c0 100644 --- a/crates/tracedecay-mcp/src/handlers/hook_runtime/admission.rs +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/admission.rs @@ -11,9 +11,10 @@ use tracedecay_project::project::TraceDecay; use tracedecay_runtime_core::background_cpu::ProcessBackgroundCpuV1; use super::context_scout::{ - admit_native_context_scout_lifecycle, hook_v2_context_scout_lifecycle_for_session, - hook_v2_native_context_scout_lifecycle, lookup_hook_v2_delivery_claim, - lookup_hook_v2_delivery_claim_for_event, remove_hook_v2_delivery_claim, + NativeLifecycleAdmissionV1, admit_native_context_scout_lifecycle, + hook_v2_context_scout_lifecycle_for_session, hook_v2_native_context_scout_lifecycle, + lookup_hook_v2_delivery_claim, lookup_hook_v2_delivery_claim_for_event, native_lifecycle_order, + native_lifecycle_order_floor, recovered_native_lifecycle_order, remove_hook_v2_delivery_claim, retain_hook_v2_delivery_claim, }; use super::envelope::{ @@ -108,9 +109,10 @@ fn pre_ledger_hook_v2_pending_work_root( .join(host.hook_key()) } -fn complete_hook_v2_pending_work( +fn settle_hook_v2_pending_work( data_root: &Path, envelope: &tracedecay_hooks::HookEventEnvelopeV2, + settlement: tracedecay_hooks::HookAdmissionWorkSettlementV1, ) -> bool { let ledger_root = hook_v2_admission_ledger_root(data_root, envelope.producer); // A released ledger reopens from its log; a removed one owes nothing. @@ -122,7 +124,7 @@ fn complete_hook_v2_pending_work( ledger_root, envelope.producer, hook_now(), - |ledger| ledger.stage_work_completion(envelope), + |ledger| ledger.stage_work_settlement(envelope, settlement), ) { Ok(Ok(Some(commit))) => commit.wait().is_ok(), Ok(Ok(None)) => true, @@ -133,11 +135,19 @@ fn complete_hook_v2_pending_work( fn hook_v2_work_completion( data_root: &Path, envelope: &tracedecay_hooks::HookEventEnvelopeV2, -) -> Arc { +) -> Arc { let data_root = data_root.to_path_buf(); let envelope = envelope.clone(); - Arc::new(move || { - if !complete_hook_v2_pending_work(&data_root, &envelope) { + Arc::new(move |settlement| { + let settlement = match settlement { + tracedecay_daemon_service::HookOrchestrationSettlementV1::Completed => { + tracedecay_hooks::HookAdmissionWorkSettlementV1::Completed + } + tracedecay_daemon_service::HookOrchestrationSettlementV1::Superseded => { + tracedecay_hooks::HookAdmissionWorkSettlementV1::Superseded + } + }; + if !settle_hook_v2_pending_work(&data_root, &envelope, settlement) { tracing::warn!( event = "hook_v2_work_completion_failed", host = envelope.producer.hook_key(), @@ -195,6 +205,8 @@ fn stage_hook_v2_admission( data_root: &Path, envelope: &tracedecay_hooks::HookEventEnvelopeV2, work: Option<&tracedecay_hooks::HookEventEnvelopeV2>, + lifecycle: Option<&tracedecay_hooks::NativeContextScoutLifecycleV1>, + minimum_order: Option, now: UtcMicros, ) -> Option { with_hook_v2_admission_ledger( @@ -202,7 +214,11 @@ fn stage_hook_v2_admission( hook_v2_admission_ledger_root(data_root, envelope.producer), envelope.producer, now, - |ledger| ledger.stage_admission(envelope, work, now).ok(), + |ledger| { + ledger + .stage_admission(envelope, work, lifecycle, minimum_order, now) + .ok() + }, ) .ok() .flatten() @@ -507,9 +523,9 @@ async fn admit_gated_native_lifecycle( range: tracedecay_domain::ObservationSourceRangeV1, producer: tracedecay_domain::NativeHostIdentityV1, intent_enabled: bool, -) -> bool { +) -> NativeLifecycleAdmissionV1 { if intent_gated_lifecycle(producer) && !intent_enabled { - return true; + return NativeLifecycleAdmissionV1::Ready; } admit_native_context_scout_lifecycle( mount.project_sessions, @@ -597,8 +613,17 @@ async fn admit_hook_v2_envelope_with_lifecycle_inner( return HookV2AdmissionOutcomeV1::CatchupRequired; } }; + let incoming_lifecycle = mount.as_ref().and_then(|mount| mount.lifecycle.as_ref()); + if incoming_lifecycle.is_some_and(|lifecycle| !lifecycle.matches_envelope(provider_envelope)) { + return HookV2AdmissionOutcomeV1::Conflict; + } let canonical_envelope = daemon_mint_hook_v2_envelope(envelope); let envelope = &canonical_envelope; + let incoming_lifecycle = incoming_lifecycle.map(|lifecycle| { + let mut lifecycle = lifecycle.clone(); + lifecycle.event_id = envelope.event_id; + lifecycle + }); let data_root = &cg.hook_store_layout().data_root; let requires_producer_work = hook_v2_requires_producer_work(envelope); // One ledger commit makes the admission and the producer work it owes @@ -606,15 +631,55 @@ async fn admit_hook_v2_envelope_with_lifecycle_inner( // stable daemon order when the provider has no native sequence. // Exact-duplicate retries reuse that order and retry only the lifecycle // prerequisite before suppressing downstream effects. + let native_lifecycle_enabled = + tracedecay_hooks::NativeContextScoutLifecycleV1::host_mounts_lifecycle(envelope.producer) + && hook_v2_native_lifecycle_enabled(cg, envelope.producer).await; + // Producer replay compares the existing native cursor even when policy + // suppresses new lifecycle writes (notably a Claude Stop). Its new daemon + // allocation must follow that same authority before ordering is compared. + let compares_native_order = mount.is_some() + && requires_producer_work + && incoming_lifecycle.is_none() + && native_session_id.is_some(); + let minimum_order = if (native_lifecycle_enabled || compares_native_order) + && matches!(envelope.ordering, tracedecay_hooks::HookOrderingV1::Unknown) + { + match ( + mount.as_ref(), + incoming_lifecycle + .as_ref() + .map(|lifecycle| &lifecycle.session_id) + .or(native_session_id.as_ref()), + ) { + (Some(mount), Some(session)) => { + if tracedecay_agent_hosts::hooks::protected_native_session_id(session.as_str()) + != envelope.protected_session_id + { + return HookV2AdmissionOutcomeV1::Conflict; + } + let Ok(provider) = ProviderId::new(envelope.producer.hook_key()) else { + return HookV2AdmissionOutcomeV1::Backpressured; + }; + native_lifecycle_order_floor(mount.project_sessions, provider, session.clone()) + .await + } + _ => None, + } + } else { + Some(0) + }; let Some(staged) = stage_hook_v2_admission( data_root, envelope, requires_producer_work.then_some(provider_envelope), + incoming_lifecycle.as_ref(), + minimum_order, now, ) else { return HookV2AdmissionOutcomeV1::Backpressured; }; let receipt = staged.receipt; + let native_lifecycle = staged.native_lifecycle; let commit = staged.commit; if !matches!( tracedecay_runtime_core::native_runtime::spawn_blocking(move || commit.wait()).await, @@ -629,8 +694,87 @@ async fn admit_hook_v2_envelope_with_lifecycle_inner( return HookV2AdmissionOutcomeV1::Conflict; } } + let lifecycle_required = native_lifecycle_enabled + && matches!( + envelope.event, + tracedecay_hooks::HookEventV2::ToolLifecycle { .. } + | tracedecay_hooks::HookEventV2::SavedEdit { .. } + ); + if lifecycle_required && (native_lifecycle.is_none() || mount.is_none()) { + return HookV2AdmissionOutcomeV1::Backpressured; + } + let native_session_id = native_lifecycle + .as_ref() + .map(|lifecycle| lifecycle.session_id.clone()) + .or(native_session_id); + let lifecycle_admission = if let Some(mount) = mount.as_ref() { + let Some(range) = hook_v2_lifecycle_range(envelope, receipt) else { + return HookV2AdmissionOutcomeV1::Backpressured; + }; + let Ok(provider) = ProviderId::new(envelope.producer.hook_key()) else { + return HookV2AdmissionOutcomeV1::Backpressured; + }; + if receipt.order_reconstructed + && matches!(envelope.ordering, tracedecay_hooks::HookOrderingV1::Unknown) + { + match native_lifecycle.as_ref() { + Some(lifecycle) => { + recovered_native_lifecycle_order( + mount.project_sessions, + provider, + lifecycle, + range, + ) + .await + } + None => NativeLifecycleAdmissionV1::Unavailable, + } + } else if let Some(native_lifecycle) = native_lifecycle.as_ref() { + admit_gated_native_lifecycle( + mount, + provider, + native_lifecycle, + range, + envelope.producer, + !intent_gated_lifecycle(envelope.producer) || lifecycle_required, + ) + .await + } else if requires_producer_work && let Some(session) = native_session_id.as_ref() { + native_lifecycle_order( + mount.project_sessions, + mount.background_cpu, + provider, + session.clone(), + range, + ) + .await + } else { + NativeLifecycleAdmissionV1::Ready + } + } else { + NativeLifecycleAdmissionV1::Ready + }; + match lifecycle_admission { + NativeLifecycleAdmissionV1::Unavailable => return HookV2AdmissionOutcomeV1::Backpressured, + NativeLifecycleAdmissionV1::Superseded => { + if requires_producer_work + && !settle_hook_v2_pending_work( + data_root, + envelope, + tracedecay_hooks::HookAdmissionWorkSettlementV1::Superseded, + ) + { + return HookV2AdmissionOutcomeV1::Backpressured; + } + return HookV2AdmissionOutcomeV1::ExactDuplicate { + context_scout_address: None, + ready_guidance: Value::Null, + }; + } + NativeLifecycleAdmissionV1::Ready => {} + } if receipt.decision == tracedecay_hooks::HookAdmissionDecisionV1::ExactDuplicate - && receipt.work_completed + && receipt.work_settlement.is_some() { let retained_claim = host_response_available .then(|| { @@ -719,29 +863,6 @@ async fn admit_hook_v2_envelope_with_lifecycle_inner( ready_guidance, }; } - if let Some(mount) = mount.as_ref() - && let Some(native_lifecycle) = mount.lifecycle.as_ref() - { - let lifecycle_enabled = hook_v2_native_lifecycle_enabled(cg, envelope.producer).await; - let Some(range) = hook_v2_lifecycle_range(envelope, receipt) else { - return HookV2AdmissionOutcomeV1::Backpressured; - }; - let Ok(provider) = ProviderId::new(envelope.producer.hook_key()) else { - return HookV2AdmissionOutcomeV1::Backpressured; - }; - if !admit_gated_native_lifecycle( - mount, - provider, - native_lifecycle, - range, - envelope.producer, - lifecycle_enabled, - ) - .await - { - return HookV2AdmissionOutcomeV1::Backpressured; - } - } if receipt.decision == tracedecay_hooks::HookAdmissionDecisionV1::ExactDuplicate && !requires_producer_work { @@ -858,6 +979,7 @@ async fn admit_hook_v2_envelope_with_lifecycle_inner( lifecycle, snapshot.revision, false, + Some(receipt.order), completion, ); let feedback_notice = if first_admission { @@ -1003,7 +1125,7 @@ pub(super) fn hook_v2_profile_admit( .join(binding.host.hook_key()); let now = hook_now(); let outcome = with_hook_v2_admission_ledger(None, ledger_root, binding.host, now, |ledger| { - ledger.stage_admission(&envelope, None, now) + ledger.stage_admission(&envelope, None, None, Some(0), now) }) .map(|staged| staged.and_then(|staged| staged.commit.wait().map(|()| staged.receipt.decision))); Ok(match outcome { diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests.rs index d4a5c7ceb9..2e813c384c 100644 --- a/crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests.rs +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests.rs @@ -6,6 +6,8 @@ use super::super::test_support::*; use super::super::*; use super::*; +mod legacy_continuity; + #[test] fn daemon_admission_is_idempotent_per_identity_and_conflicts_on_different_bytes() { let data_root = tempfile::tempdir().unwrap(); @@ -53,7 +55,7 @@ fn record_hook_v2_admission( envelope: &tracedecay_hooks::HookEventEnvelopeV2, now: UtcMicros, ) -> Option { - let staged = stage_hook_v2_admission(data_root, envelope, None, now)?; + let staged = stage_hook_v2_admission(data_root, envelope, None, None, Some(0), now)?; staged.commit.wait().ok()?; Some(staged.receipt) } @@ -79,14 +81,21 @@ fn producer_work_commits_with_its_admission_and_redrives_until_completed() { let data_root = tempfile::tempdir().unwrap(); let now = UtcMicros(1_000); let envelope = admission_test_envelope(31, 7); - let staged = stage_hook_v2_admission(data_root.path(), &envelope, Some(&envelope), now) - .expect("ledger available"); + let staged = stage_hook_v2_admission( + data_root.path(), + &envelope, + Some(&envelope), + None, + Some(0), + now, + ) + .expect("ledger available"); staged.commit.wait().unwrap(); assert_eq!( staged.receipt.decision, tracedecay_hooks::HookAdmissionDecisionV1::Admitted ); - assert!(!staged.receipt.work_completed); + assert!(staged.receipt.work_settlement.is_none()); restart_ledger(data_root.path()); assert_eq!(pending_work(data_root.path()), vec![envelope.clone()]); @@ -95,16 +104,21 @@ fn producer_work_commits_with_its_admission_and_redrives_until_completed() { duplicate.decision, tracedecay_hooks::HookAdmissionDecisionV1::ExactDuplicate ); - assert!(!duplicate.work_completed); + assert!(duplicate.work_settlement.is_none()); - assert!(complete_hook_v2_pending_work(data_root.path(), &envelope)); + assert!(settle_hook_v2_pending_work( + data_root.path(), + &envelope, + tracedecay_hooks::HookAdmissionWorkSettlementV1::Completed + )); assert_eq!(pending_work(data_root.path()).len(), 0); restart_ledger(data_root.path()); assert_eq!(pending_work(data_root.path()).len(), 0); assert!( record_hook_v2_admission(data_root.path(), &envelope, now) .unwrap() - .work_completed, + .work_settlement + .is_some(), "producer-work completion must survive daemon restart" ); } @@ -170,8 +184,15 @@ async fn prompt_and_tool_admissions_enqueue_an_intent_orchestration_without_owed envelope.producer = kiro; } assert!(!hook_v2_requires_producer_work(&envelope)); - let staged = stage_hook_v2_admission(data_root.path(), &envelope, None, UtcMicros(1_000)) - .expect("ledger available"); + let staged = stage_hook_v2_admission( + data_root.path(), + &envelope, + None, + None, + Some(0), + UtcMicros(1_000), + ) + .expect("ledger available"); staged.commit.wait().unwrap(); assert_eq!( tracedecay_daemon_service::admit_registered_hook_orchestration( @@ -185,6 +206,7 @@ async fn prompt_and_tool_admissions_enqueue_an_intent_orchestration_without_owed 1, false, None, + None, ), tracedecay_contracts::HookOrchestrationAdmissionV1::Enqueued ); @@ -211,6 +233,7 @@ async fn prompt_and_tool_admissions_enqueue_an_intent_orchestration_without_owed 1, false, None, + None, ), tracedecay_contracts::HookOrchestrationAdmissionV1::UnsupportedTrigger ); @@ -231,6 +254,8 @@ fn redelivered_producer_event_with_pending_work_is_an_exact_duplicate() { data_root.path(), &envelope, Some(&envelope), + None, + Some(0), UtcMicros(1_000), ) .unwrap(); @@ -240,6 +265,8 @@ fn redelivered_producer_event_with_pending_work_is_an_exact_duplicate() { data_root.path(), &redelivered, Some(&redelivered), + None, + Some(0), UtcMicros(2_000), ) .expect("a redelivery is not backpressure"); @@ -250,9 +277,10 @@ fn redelivered_producer_event_with_pending_work_is_an_exact_duplicate() { tracedecay_hooks::HookAdmissionDecisionV1::ExactDuplicate ); assert_eq!(pending_work(data_root.path()), vec![envelope.clone()]); - assert!(complete_hook_v2_pending_work( + assert!(settle_hook_v2_pending_work( data_root.path(), - &redelivered + &redelivered, + tracedecay_hooks::HookAdmissionWorkSettlementV1::Completed, )); assert_eq!(pending_work(data_root.path()).len(), 0); } @@ -267,8 +295,9 @@ fn admission_ledgers_cycle_past_the_open_bound_without_restart() { .map(|_| tempfile::tempdir().unwrap()) .collect::>(); for (pair, root) in roots.iter().enumerate() { - let staged = stage_hook_v2_admission(root.path(), &envelope, Some(&envelope), now) - .unwrap_or_else(|| panic!("project/host pair {pair} must admit without restart")); + let staged = + stage_hook_v2_admission(root.path(), &envelope, Some(&envelope), None, Some(0), now) + .unwrap_or_else(|| panic!("project/host pair {pair} must admit without restart")); staged.commit.wait().unwrap(); assert_eq!( staged.receipt.decision, @@ -293,7 +322,11 @@ fn admission_ledgers_cycle_past_the_open_bound_without_restart() { assert_eq!(pending_work(first), vec![envelope.clone()]); restart_ledger(first); assert!( - complete_hook_v2_pending_work(first, &envelope), + settle_hook_v2_pending_work( + first, + &envelope, + tracedecay_hooks::HookAdmissionWorkSettlementV1::Completed + ), "a released ledger reopens to record completion" ); assert_eq!(pending_work(first).len(), 0); @@ -315,7 +348,7 @@ fn a_ledger_with_a_commit_in_flight_keeps_its_writer_fence() { hook_v2_admission_ledger_root(root, host), host, now, - |ledger| ledger.stage_admission(envelope, Some(envelope), now), + |ledger| ledger.stage_admission(envelope, Some(envelope), None, Some(0), now), ) .map(|staged| staged.unwrap()) }; @@ -353,7 +386,7 @@ fn a_ledger_with_a_commit_in_flight_keeps_its_writer_fence() { duplicate.receipt.decision, tracedecay_hooks::HookAdmissionDecisionV1::ExactDuplicate ); - assert!(!duplicate.receipt.work_completed); + assert!(duplicate.receipt.work_settlement.is_none()); let conflict = stage(a.path(), &admission_test_envelope(36, 8)).unwrap(); conflict.commit.wait().unwrap(); assert_eq!( @@ -685,24 +718,31 @@ async fn claude_tool_calls_mint_a_lifecycle_only_while_the_intent_pass_is_enable } }; - assert!( + assert_eq!( tool_call( NativeHostIdentityV1::ClaudeCode, "session.claude.off", false ) - .await + .await, + NativeLifecycleAdmissionV1::Ready, ); assert!( !minted("session.claude.off").await, "a Claude tool call with the intent pass disabled writes no lifecycle" ); - assert!(tool_call(NativeHostIdentityV1::ClaudeCode, "session.claude.on", true).await); + assert_eq!( + tool_call(NativeHostIdentityV1::ClaudeCode, "session.claude.on", true).await, + NativeLifecycleAdmissionV1::Ready + ); assert!( minted("session.claude.on").await, "a Claude tool call with the intent pass enabled mints its lifecycle" ); - assert!(tool_call(NativeHostIdentityV1::KimiCode, "session.kimi.off", false).await); + assert_eq!( + tool_call(NativeHostIdentityV1::KimiCode, "session.kimi.off", false).await, + NativeLifecycleAdmissionV1::Ready + ); assert!( minted("session.kimi.off").await, "Kimi keeps minting its lifecycle whatever the intent pass says" diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests/legacy_continuity.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests/legacy_continuity.rs new file mode 100644 index 0000000000..12ad1a62ff --- /dev/null +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests/legacy_continuity.rs @@ -0,0 +1,554 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Duration; + +use tracedecay_domain::{ObservationSourceRangeV1, ProviderId}; +use tracedecay_hooks::{HookAdmissionWorkSettlementV1, NativeContextScoutLifecycleV1}; +use tracedecay_project::test_support::host_admission::HostAdmissionTestRuntimeV1; +use tracedecay_sessions::admission::HostAdmissionScope; + +use super::*; +use crate::handlers::hook_runtime::context_scout::{ + NativeLifecycleAdmissionV1, admit_native_context_scout_lifecycle, +}; + +struct LegacyLifecycleFixture { + _temporary: tempfile::TempDir, + graph: TraceDecay, + runtime: Arc, + envelope: tracedecay_hooks::HookEventEnvelopeV2, + lifecycle: NativeContextScoutLifecycleV1, + sessions: tracedecay_global_db::RegisteredGlobalDbLeaseV1, +} + +impl LegacyLifecycleFixture { + async fn open() -> Self { + Self::open_for_host(tracedecay_domain::NativeHostIdentityV1::KimiCode).await + } + + async fn open_for_host(host: tracedecay_domain::NativeHostIdentityV1) -> Self { + let temporary = tempfile::tempdir().unwrap(); + let profile = temporary.path().join("profile"); + let root = temporary.path().join("project"); + std::fs::create_dir_all(root.join("src")).unwrap(); + std::fs::write(root.join("src/lib.rs"), "pub fn saved() {}\n").unwrap(); + let (graph, runtime) = TraceDecay::init_test_fixture_with_registered_runtime( + &profile, + &root, + "project.legacy-hook-continuity", + ) + .await + .unwrap(); + let hook_runtime = tracedecay_project::runtime_ports::hook_runtime( + tracedecay_runtime_core::config::ProfileRoot::new(profile), + ) + .unwrap(); + let worktree = tracedecay_agent_hosts::hooks::hook_worktree_id_for_layout( + &hook_runtime, + graph.hook_store_layout(), + ) + .unwrap(); + let now = hook_now(); + let binding = tracedecay_hooks::published_hook_scope_binding( + &graph.hook_store_layout().data_root, + worktree, + host, + now, + ) + .unwrap(); + let lifecycle = NativeContextScoutLifecycleV1::new( + "session.legacy-hook", + "call.legacy-saved", + [61; 16], + ) + .unwrap(); + let envelope = tracedecay_hooks::HookEventEnvelopeV2 { + schema_version: tracedecay_hooks::HOOK_EVENT_SCHEMA_VERSION, + event_id: lifecycle.event_id, + producer: host, + protected_session_id: tracedecay_agent_hosts::hooks::protected_native_session_id( + lifecycle.session_id.as_str(), + ), + project_id: binding.project_id, + repository_id: binding.repository_id, + worktree_id: binding.worktree_id, + worktree_epoch: binding.worktree_epoch, + binding_token: binding.binding_token, + observed_at: now, + ordering: tracedecay_hooks::HookOrderingV1::Unknown, + event: tracedecay_hooks::HookEventV2::SavedEdit { + file_id: [62; 16], + changed_range_count: 1, + }, + }; + let project = graph + .store_layout() + .identity + .project_id + .as_ref() + .unwrap() + .clone() + .try_into() + .unwrap(); + let scope = (hook_runtime.scope_resolver)(&root, &project).unwrap(); + let sessions = runtime + .registered_database_arc(HostAdmissionScope::Project) + .unwrap(); + assert_eq!(tracedecay_daemon_service::context_scout_lifecycle::register_context_scout_lifecycle_authority( + envelope.project_id, envelope.worktree_id, scope.project_id, scope.worktree_id, &sessions, + ), tracedecay_daemon_service::context_scout_lifecycle::AuthorityRegistrationV1::Registered); + Self { + _temporary: temporary, + graph, + runtime, + envelope, + lifecycle, + sessions, + } + } + + async fn commit_lifecycle(&self, call: &str, start: u64) { + let lifecycle = + NativeContextScoutLifecycleV1::new(self.lifecycle.session_id.as_str(), call, [60; 16]) + .unwrap(); + assert_eq!( + admit_native_context_scout_lifecycle( + &self.sessions, + Some(&self.runtime.background_cpu()), + ProviderId::new(self.envelope.producer.hook_key()).unwrap(), + &lifecycle, + ObservationSourceRangeV1::new(start, start + 1).unwrap(), + ) + .await, + NativeLifecycleAdmissionV1::Ready + ); + } + + async fn commit_conflicting_lifecycle(&self) { + use tracedecay_domain::{ + CanonicalBoundaryKindV1, CanonicalObservationEnvelopeV1, + CanonicalObservationEvidenceV1, CanonicalObservationFactV1, + CanonicalObservationRelationsV1, ObservationId, ObservationIdentityMaterialV1, + ObservationOrderingDomainV1, ObservationScopeV1, ObservationSourceGenerationV1, + ObservationSourceIdentityV1, RetentionClass, + }; + use tracedecay_host_admission::{HostAdmissionAuthorities, HostAdmissionFacade}; + use tracedecay_sessions::observation::{ + CaptureObservationOutcome, CaptureObservationRequest, ObservationCancellation, + }; + use tracedecay_store::StoreShardScopeV1; + + let binding = self.sessions.binding(); + let StoreShardScopeV1::ProjectSessions { project_id } = &binding.shard_id.scope else { + panic!("project fixture"); + }; + let provider = ProviderId::new("kimi").unwrap(); + let range = ObservationSourceRangeV1::new(99, 100).unwrap(); + let scope = ObservationScopeV1::Project { + project_id: project_id.clone(), + }; + let source = ObservationSourceIdentityV1::for_provider( + provider.clone(), + self.lifecycle.session_id.clone(), + ) + .unwrap(); + let agent = ObservationId::new(self.lifecycle.session_id.as_str().to_owned()).unwrap(); + let parsed = tracedecay_privacy::parse_normalized_observation_record_v1( + &serde_json::to_vec(&self.lifecycle).unwrap(), + range, + ObservationOrderingDomainV1::DaemonSequence, + |_| { + CanonicalObservationEnvelopeV1::new( + provider, + "hook_tool_after", + self.lifecycle.call_id.clone(), + CanonicalObservationRelationsV1::new(self.lifecycle.session_id.clone()) + .with_thread_id(agent.clone()) + .with_agent_id(agent) + .with_turn_id(self.lifecycle.call_id.clone()) + .with_message_id(ObservationId::new("call.conflicting-lifecycle").unwrap()), + vec![CanonicalObservationFactV1::Boundary { + boundary_kind: CanonicalBoundaryKindV1::TurnEnd, + }], + CanonicalObservationEvidenceV1::new( + ObservationOrderingDomainV1::DaemonSequence, + range, + ), + ) + .map_err(|_| tracedecay_privacy::ObservationRecordParseErrorV1::NormalizationFailed) + }, + ) + .unwrap(); + let identity = ObservationIdentityMaterialV1::for_native_record( + source, + scope, + ObservationSourceGenerationV1::new(1).unwrap(), + range, + ObservationOrderingDomainV1::DaemonSequence, + self.lifecycle.call_id.clone(), + ) + .unwrap(); + let facade = HostAdmissionFacade::new( + HostAdmissionAuthorities::for_project( + binding.shard_id.brain_id.clone(), + binding.shard_id.profile_id.clone(), + project_id.clone(), + &self.sessions, + ) + .with_background_cpu(self.runtime.background_cpu()), + ); + let outcome = facade + .capture_observation( + CaptureObservationRequest::new( + parsed, + identity, + None, + RetentionClass::new("transcript.hook-lifecycle.v1").unwrap(), + ObservationCancellation::default(), + ) + .unwrap(), + ) + .await + .unwrap(); + assert!(matches!( + outcome, + CaptureObservationOutcome::Persisted { .. } + | CaptureObservationOutcome::AcceptedForReplay { .. } + )); + } + + fn write_shipped_ledger(&self, retained: bool, owed: bool) { + let root = hook_v2_admission_ledger_root( + &self.graph.hook_store_layout().data_root, + self.envelope.producer, + ); + drop( + tracedecay_hooks::HookAdmissionLedgerV1::open( + &root, + self.envelope.producer, + tracedecay_hooks::HookAdmissionLedgerLimitsV1::stock(), + hook_now(), + ) + .unwrap(), + ); + let mut bytes = b"TDL2".to_vec(); + bytes.extend_from_slice(&2u16.to_le_bytes()); + if retained { + let canonical = daemon_mint_hook_v2_envelope(&self.envelope); + let mut body = canonical.event_id.to_vec(); + body.extend_from_slice( + &tracedecay_hooks::admission_ledger::hook_admission_digest(&canonical).unwrap(), + ); + body.extend_from_slice(&hook_now().0.to_le_bytes()); + if owed { + body.extend_from_slice( + &tracedecay_domain::canonical_json_bytes(&self.envelope).unwrap(), + ); + } + append_shipped_frame(&mut bytes, 1, &body); + // An expired admission forces the real legacy-open rewrite, which + // must preserve reconstructed provenance across the next restart. + let mut expired = body[..56].to_vec(); + expired[..16].copy_from_slice(&[59; 16]); + expired[48..56].copy_from_slice( + &(hook_now().0 - tracedecay_hooks::MAX_SPOOL_AGE_MICROS - 1).to_le_bytes(), + ); + append_shipped_frame(&mut bytes, 1, &expired); + } + std::fs::write(root.join("admissions.v2.log"), bytes).unwrap(); + } + + async fn admit(&self) -> HookV2AdmissionOutcomeV1 { + admit_hook_v2_replayed_envelope_with_lifecycle( + &self.graph, + &self.envelope, + Some(self.lifecycle.session_id.clone()), + Some(self.lifecycle.clone()), + &self.sessions, + &self.runtime.background_cpu(), + hook_now(), + ) + .await + } + + fn pending(&self) -> Vec { + hook_v2_pending_work_envelopes( + &self.graph.hook_store_layout().data_root, + self.envelope.producer, + hook_now(), + ) + .unwrap() + } + + fn restart(&self) { + forget_hook_v2_admission_ledger_for_test( + &self.graph.hook_store_layout().data_root, + self.envelope.producer, + ); + } + + fn producer( + &self, + effects: Arc, + ) -> Arc { + let producer = tracedecay_daemon_service::BoundedHookOrchestratorV1::new( + 1, + move |request: tracedecay_daemon_service::HookOrchestrationRequestV1, _| { + let effects = Arc::clone(&effects); + async move { + if request.trigger + == tracedecay_daemon_service::HookOrchestrationTriggerV1::IntentBoundary + { + assert!( + matches!( + request.hook.envelope().event, + tracedecay_hooks::HookEventV2::SessionBoundary { + boundary: tracedecay_hooks::HookBoundaryV1::End + | tracedecay_hooks::HookBoundaryV1::TurnComplete, + } + ), + "only the real Stop intent companion is outside the producer count" + ); + assert!(request.completion.is_none()); + return; + } + assert!(matches!( + request.trigger, + tracedecay_daemon_service::HookOrchestrationTriggerV1::SavedEdit + | tracedecay_daemon_service::HookOrchestrationTriggerV1::Stop + )); + assert!( + request.lifecycle.is_some(), + "owed edit requires its exact lifecycle" + ); + effects.fetch_add(1, Ordering::SeqCst); + } + }, + ) + .unwrap(); + assert!( + tracedecay_daemon_service::register_hook_orchestration_runtime( + self.envelope.project_id, + self.envelope.worktree_id, + &producer + ) + ); + producer + } + + async fn settled(&self) { + tokio::time::timeout(Duration::from_secs(10), async { + while !self.pending().is_empty() { + tokio::task::yield_now().await; + } + }) + .await + .expect("producer completion settles the durable event"); + } +} + +impl Drop for LegacyLifecycleFixture { + fn drop(&mut self) { + self.restart(); + assert!(tracedecay_daemon_service::context_scout_lifecycle::unregister_context_scout_lifecycle_authority(self.envelope.project_id, self.envelope.worktree_id, &self.sessions)); + } +} + +fn append_shipped_frame(bytes: &mut Vec, kind: u8, body: &[u8]) { + let start = bytes.len(); + bytes.extend_from_slice(&(body.len() as u32).to_le_bytes()); + bytes.push(kind); + bytes.extend_from_slice(body); + let checksum = tracedecay_domain::framed_log::checksum(&bytes[start..]); + bytes.extend_from_slice(&checksum[..8]); +} + +#[tokio::test] +async fn fresh_saved_edit_runs_after_shipped_compacted_or_header_only_ledger() { + for retained in [false, true] { + let mut fixture = LegacyLifecycleFixture::open().await; + fixture.commit_lifecycle("call.legacy-previous", 99).await; + fixture.write_shipped_ledger(retained, false); + // A compacted unrelated admission is not this fresh native event. + fixture.envelope.event_id = [63; 16]; + fixture.lifecycle.event_id = fixture.envelope.event_id; + let effects = Arc::new(AtomicUsize::new(0)); + let producer = fixture.producer(Arc::clone(&effects)); + assert!(matches!( + fixture.admit().await, + HookV2AdmissionOutcomeV1::Admitted { .. } + )); + fixture.settled().await; + assert_eq!(effects.load(Ordering::SeqCst), 1); + fixture.restart(); + assert!(matches!( + fixture.admit().await, + HookV2AdmissionOutcomeV1::ExactDuplicate { .. } + )); + assert_eq!(effects.load(Ordering::SeqCst), 1); + assert!(producer.shutdown().await); + } +} + +#[tokio::test] +async fn retained_legacy_receipt_keeps_producer_work_owed_until_execution() { + let fixture = LegacyLifecycleFixture::open().await; + fixture + .commit_lifecycle(fixture.lifecycle.call_id.as_str(), 99) + .await; + fixture.write_shipped_ledger(true, true); + assert_eq!(fixture.pending(), [fixture.envelope.clone()]); + fixture.restart(); + let effects = Arc::new(AtomicUsize::new(0)); + let producer = fixture.producer(Arc::clone(&effects)); + assert!(matches!( + fixture.admit().await, + HookV2AdmissionOutcomeV1::Admitted { .. } + )); + fixture.settled().await; + assert_eq!(effects.load(Ordering::SeqCst), 1); + let canonical = daemon_mint_hook_v2_envelope(&fixture.envelope); + let receipt = stage_hook_v2_admission( + &fixture.graph.hook_store_layout().data_root, + &canonical, + None, + None, + None, + hook_now(), + ) + .unwrap(); + assert!(receipt.receipt.order_reconstructed); + assert_eq!(receipt.receipt.order, 0); + assert_eq!( + receipt.receipt.work_settlement, + Some(HookAdmissionWorkSettlementV1::Completed) + ); + receipt.commit.wait().unwrap(); + assert!(producer.shutdown().await); +} + +#[tokio::test] +async fn missing_or_conflicting_legacy_receipt_stays_owed_through_rewrite_and_restart() { + for conflicting in [false, true] { + let fixture = LegacyLifecycleFixture::open().await; + if conflicting { + fixture.commit_conflicting_lifecycle().await; + } else { + fixture.commit_lifecycle("call.other-committed", 99).await; + } + fixture.write_shipped_ledger(true, true); + for _ in 0..2 { + assert!(matches!( + fixture.admit().await, + HookV2AdmissionOutcomeV1::Backpressured + )); + assert_eq!(fixture.pending(), [fixture.envelope.clone()]); + fixture.restart(); + } + } +} + +#[tokio::test] +async fn a_later_canonical_receipt_truthfully_supersedes_retained_legacy_work() { + let fixture = LegacyLifecycleFixture::open().await; + fixture + .commit_lifecycle(fixture.lifecycle.call_id.as_str(), 99) + .await; + fixture.commit_lifecycle("call.actual-newer", 100).await; + fixture.write_shipped_ledger(true, true); + let effects = Arc::new(AtomicUsize::new(0)); + let producer = fixture.producer(Arc::clone(&effects)); + assert!(matches!( + fixture.admit().await, + HookV2AdmissionOutcomeV1::ExactDuplicate { .. } + )); + assert!(fixture.pending().is_empty()); + assert_eq!(effects.load(Ordering::SeqCst), 0); + let canonical = daemon_mint_hook_v2_envelope(&fixture.envelope); + let receipt = stage_hook_v2_admission( + &fixture.graph.hook_store_layout().data_root, + &canonical, + None, + None, + None, + hook_now(), + ) + .unwrap(); + assert_eq!( + receipt.receipt.work_settlement, + Some(HookAdmissionWorkSettlementV1::Superseded) + ); + receipt.commit.wait().unwrap(); + assert!(producer.shutdown().await); +} + +#[tokio::test] +async fn disabled_claude_stop_follows_the_prior_session_cursor_without_minting_lifecycle() { + let mut fixture = + LegacyLifecycleFixture::open_for_host(tracedecay_domain::NativeHostIdentityV1::ClaudeCode) + .await; + assert!(!hook_v2_native_lifecycle_enabled(&fixture.graph, fixture.envelope.producer).await); + fixture + .commit_lifecycle("call.prior-real-session", 99) + .await; + fixture.write_shipped_ledger(false, false); + fixture.envelope.event = tracedecay_hooks::HookEventV2::SessionBoundary { + boundary: tracedecay_hooks::HookBoundaryV1::TurnComplete, + }; + let effects = Arc::new(AtomicUsize::new(0)); + let producer = fixture.producer(Arc::clone(&effects)); + for duplicate in [false, true] { + let outcome = admit_hook_v2_replayed_envelope_with_lifecycle( + &fixture.graph, + &fixture.envelope, + Some(fixture.lifecycle.session_id.clone()), + None, + &fixture.sessions, + &fixture.runtime.background_cpu(), + hook_now(), + ) + .await; + if duplicate { + assert!(matches!( + outcome, + HookV2AdmissionOutcomeV1::ExactDuplicate { .. } + )); + } else { + assert!(matches!(outcome, HookV2AdmissionOutcomeV1::Admitted { .. })); + } + fixture.settled().await; + assert_eq!(effects.load(Ordering::SeqCst), 1); + fixture.restart(); + } + let canonical = daemon_mint_hook_v2_envelope(&fixture.envelope); + let receipt = stage_hook_v2_admission( + &fixture.graph.hook_store_layout().data_root, + &canonical, + None, + None, + None, + hook_now(), + ) + .unwrap(); + assert_eq!(receipt.receipt.order, 99); + assert_eq!( + receipt.receipt.work_settlement, + Some(HookAdmissionWorkSettlementV1::Completed) + ); + assert!( + receipt.native_lifecycle.is_none(), + "Stop cannot attach a tool lifecycle" + ); + receipt.commit.wait().unwrap(); + assert_eq!( + super::super::super::context_scout::native_lifecycle_order_floor( + &fixture.sessions, + ProviderId::new(fixture.envelope.producer.hook_key()).unwrap(), + fixture.lifecycle.session_id.clone(), + ) + .await, + Some(99), + "disabled policy must not mint another lifecycle or advance the source cursor", + ); + assert!(producer.shutdown().await); +} diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/context_scout.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/context_scout.rs index 4c33586d44..ec6d6e6b4a 100644 --- a/crates/tracedecay-mcp/src/handlers/hook_runtime/context_scout.rs +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/context_scout.rs @@ -14,8 +14,8 @@ use tracedecay_contracts::retrieval::{ use tracedecay_domain::errors::Result; use tracedecay_domain::{ CanonicalBoundaryKindV1, CanonicalObservationEnvelopeV1, CanonicalObservationEvidenceV1, - CanonicalObservationFactV1, CanonicalObservationRelationsV1, ObservationId, - ObservationIdentityMaterialV1, ObservationOrderingDomainV1, ObservationScopeV1, + CanonicalObservationFactV1, CanonicalObservationIdV1, CanonicalObservationRelationsV1, + ObservationId, ObservationIdentityMaterialV1, ObservationOrderingDomainV1, ObservationScopeV1, ObservationSourceGenerationV1, ObservationSourceIdentityV1, ObservationSourceRangeV1, ProviderId, RetentionClass, SessionId, UtcMicros, }; @@ -27,7 +27,7 @@ use tracedecay_runtime_core::background_cpu::ProcessBackgroundCpuV1; use tracedecay_sessions::observation::{ CaptureObservationOutcome, CaptureObservationRequest, ObservationCancellation, }; -use tracedecay_store::{ObservationPersistOutcome, StoreShardScopeV1}; +use tracedecay_store::{ObservationPersistOutcome, ObservationStore, StoreShardScopeV1}; use super::admission::{HookV2BindingAdmission, hook_v2_binding_admission}; use super::envelope::{hook_now, hook_v2_envelope}; @@ -54,46 +54,231 @@ pub(super) fn hook_v2_native_context_scout_lifecycle( lifecycle.matches_envelope(envelope).then_some(lifecycle) } -#[tracing::instrument(name = "mcp.hook_runtime.scout_lifecycle", level = "trace", skip_all)] -#[expect( - clippy::too_many_lines, - reason = "Context-scout admission is one native lifecycle bind for the scout claim." -)] -pub(super) async fn admit_native_context_scout_lifecycle( - sessions: &RegisteredGlobalDb, +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum NativeLifecycleAdmissionV1 { + Ready, + Superseded, + Unavailable, +} + +fn native_lifecycle_authorities<'a>( + sessions: &'a RegisteredGlobalDb, background_cpu: Option<&std::sync::Arc>, provider: ProviderId, - lifecycle: &tracedecay_agent_hosts::hooks::NativeContextScoutLifecycleV1, - range: ObservationSourceRangeV1, -) -> bool { + session: SessionId, +) -> Option<( + HostAdmissionFacade<'a>, + ObservationScopeV1, + ObservationSourceIdentityV1, +)> { let StoreShardScopeV1::ProjectSessions { project_id } = &sessions.binding().shard_id.scope else { - return false; + return None; }; - let project_id = project_id.clone(); let scope = ObservationScopeV1::Project { project_id: project_id.clone(), }; - let source = match ObservationSourceIdentityV1::for_provider( - provider.clone(), - lifecycle.session_id.clone(), - ) { - Ok(source) => source, - Err(_) => return false, - }; + let source = ObservationSourceIdentityV1::for_provider(provider, session).ok()?; let binding = sessions.binding(); let authorities = HostAdmissionAuthorities::for_project( binding.shard_id.brain_id.clone(), binding.shard_id.profile_id.clone(), - project_id, + project_id.clone(), sessions, ); let facade = HostAdmissionFacade::new(match background_cpu { - Some(background_cpu) => { - authorities.with_background_cpu(std::sync::Arc::clone(background_cpu)) - } + Some(cpu) => authorities.with_background_cpu(std::sync::Arc::clone(cpu)), None => authorities, }); + Some((facade, scope, source)) +} + +/// The compatible durable source cursor is a floor only for a new daemon +/// allocation. A missing cursor starts this source; incompatible or unavailable +/// authority cannot be converted into a zero floor. +pub(super) async fn native_lifecycle_order_floor( + sessions: &RegisteredGlobalDb, + provider: ProviderId, + session: SessionId, +) -> Option { + let (facade, scope, source) = native_lifecycle_authorities(sessions, None, provider, session)?; + match facade.get_source_cursor(&source, &scope).await { + Ok(None) => Some(0), + Ok(Some(cursor)) + if cursor.source() == &source + && cursor.scope() == &scope + && cursor.generation().file_id() == 1 + && cursor.ordering_domain() == ObservationOrderingDomainV1::DaemonSequence + && cursor.position().checked_add(1).is_some() => + { + Some(cursor.position().saturating_sub(1)) + } + Ok(Some(_)) | Err(_) => None, + } +} + +fn native_lifecycle_observation_envelope( + provider: ProviderId, + lifecycle: &tracedecay_agent_hosts::hooks::NativeContextScoutLifecycleV1, + range: ObservationSourceRangeV1, +) -> std::result::Result { + let session = lifecycle.session_id.clone(); + let call = lifecycle.call_id.clone(); + let agent = ObservationId::new(session.as_str().to_owned()) + .map_err(|_| ObservationRecordParseErrorV1::NormalizationFailed)?; + CanonicalObservationEnvelopeV1::new( + provider, + "hook_tool_after", + call.clone(), + CanonicalObservationRelationsV1::new(session) + .with_thread_id(agent.clone()) + .with_turn_id(call.clone()) + .with_agent_id(agent) + .with_message_id(call), + vec![CanonicalObservationFactV1::Boundary { + boundary_kind: CanonicalBoundaryKindV1::TurnEnd, + }], + CanonicalObservationEvidenceV1::new(ObservationOrderingDomainV1::DaemonSequence, range), + ) + .map_err(|_| ObservationRecordParseErrorV1::NormalizationFailed) +} + +/// A reconstructed legacy position never proves that producer work ran or +/// became obsolete. Recover its immutable range from the canonical native +/// observation, and compare only that range with the compatible source cursor. +pub(super) async fn recovered_native_lifecycle_order( + sessions: &RegisteredGlobalDb, + provider: ProviderId, + lifecycle: &tracedecay_agent_hosts::hooks::NativeContextScoutLifecycleV1, + reconstructed_range: ObservationSourceRangeV1, +) -> NativeLifecycleAdmissionV1 { + let Some((facade, scope, source)) = native_lifecycle_authorities( + sessions, + None, + provider.clone(), + lifecycle.session_id.clone(), + ) else { + return NativeLifecycleAdmissionV1::Unavailable; + }; + let Ok(generation) = ObservationSourceGenerationV1::new(1) else { + return NativeLifecycleAdmissionV1::Unavailable; + }; + // Native identity derivation uses source/scope/call, not the candidate's + // range. No reconstructed range is persisted or used for cursor ordering. + let Ok(candidate) = ObservationIdentityMaterialV1::for_native_record( + source.clone(), + scope.clone(), + generation, + reconstructed_range, + ObservationOrderingDomainV1::DaemonSequence, + lifecycle.call_id.clone(), + ) else { + return NativeLifecycleAdmissionV1::Unavailable; + }; + let Ok(id) = CanonicalObservationIdV1::derive(&candidate) else { + return NativeLifecycleAdmissionV1::Unavailable; + }; + let Ok(Some(stored)) = sessions.observation_store().get_observation(&id).await else { + return NativeLifecycleAdmissionV1::Unavailable; + }; + let observation = stored.observation(); + let identity = observation.identity(); + if identity.source() != &source + || identity.scope() != &scope + || identity.generation() != generation + || identity.ordering_domain() != ObservationOrderingDomainV1::DaemonSequence + || identity.native_record_id() != Some(&lifecycle.call_id) + { + return NativeLifecycleAdmissionV1::Unavailable; + } + let range = identity.position(); + let Ok(expected) = + native_lifecycle_observation_envelope(provider, lifecycle, range).and_then(|envelope| { + serde_json::to_value(envelope) + .map_err(|_| ObservationRecordParseErrorV1::NormalizationFailed) + }) + else { + return NativeLifecycleAdmissionV1::Unavailable; + }; + let committed = stored.committed_cursor(); + if observation.payload() != &expected + || committed.source() != &source + || committed.scope() != &scope + || committed.generation() != generation + || committed.ordering_domain() != ObservationOrderingDomainV1::DaemonSequence + || committed.position() != range.end() + { + return NativeLifecycleAdmissionV1::Unavailable; + } + match facade.get_source_cursor(&source, &scope).await { + Ok(Some(cursor)) + if cursor.source() == &source + && cursor.scope() == &scope + && cursor.generation() == generation + && cursor.ordering_domain() == ObservationOrderingDomainV1::DaemonSequence => + { + match cursor.position().cmp(&range.end()) { + std::cmp::Ordering::Equal => NativeLifecycleAdmissionV1::Ready, + std::cmp::Ordering::Greater => NativeLifecycleAdmissionV1::Superseded, + std::cmp::Ordering::Less => NativeLifecycleAdmissionV1::Unavailable, + } + } + Ok(None) | Ok(Some(_)) | Err(_) => NativeLifecycleAdmissionV1::Unavailable, + } +} + +/// A later canonical lifecycle is proof that an older boundary is superseded, +/// including after its successor has left the in-flight orchestration maps. +pub(super) async fn native_lifecycle_order( + sessions: &RegisteredGlobalDb, + background_cpu: Option<&std::sync::Arc>, + provider: ProviderId, + session: SessionId, + range: ObservationSourceRangeV1, +) -> NativeLifecycleAdmissionV1 { + let Some((facade, scope, source)) = + native_lifecycle_authorities(sessions, background_cpu, provider, session) + else { + return NativeLifecycleAdmissionV1::Unavailable; + }; + match facade.get_source_cursor(&source, &scope).await { + Ok(None) => NativeLifecycleAdmissionV1::Ready, + Ok(Some(cursor)) + if cursor.source() == &source + && cursor.scope() == &scope + && cursor.generation().file_id() == 1 + && cursor.ordering_domain() == ObservationOrderingDomainV1::DaemonSequence => + { + if cursor.position() > range.end() { + NativeLifecycleAdmissionV1::Superseded + } else { + NativeLifecycleAdmissionV1::Ready + } + } + Ok(Some(_)) | Err(_) => NativeLifecycleAdmissionV1::Unavailable, + } +} + +#[tracing::instrument(name = "mcp.hook_runtime.scout_lifecycle", level = "trace", skip_all)] +#[expect( + clippy::too_many_lines, + reason = "Context-scout admission is one native lifecycle bind for the scout claim." +)] +pub(super) async fn admit_native_context_scout_lifecycle( + sessions: &RegisteredGlobalDb, + background_cpu: Option<&std::sync::Arc>, + provider: ProviderId, + lifecycle: &tracedecay_agent_hosts::hooks::NativeContextScoutLifecycleV1, + range: ObservationSourceRangeV1, +) -> NativeLifecycleAdmissionV1 { + let Some((facade, scope, source)) = native_lifecycle_authorities( + sessions, + background_cpu, + provider.clone(), + lifecycle.session_id.clone(), + ) else { + return NativeLifecycleAdmissionV1::Unavailable; + }; // Admission order is per host, so another session's events, or this // session's events without a lifecycle, leave gaps between this session's // lifecycle positions. The observation covers from the session cursor to @@ -102,73 +287,50 @@ pub(super) async fn admit_native_context_scout_lifecycle( let (range, expected_cursor) = match facade.get_source_cursor(&source, &scope).await { Ok(None) => (range, None), Ok(Some(cursor)) - if cursor.generation().file_id() == 1 + if cursor.source() == &source + && cursor.scope() == &scope + && cursor.generation().file_id() == 1 && cursor.ordering_domain() == ObservationOrderingDomainV1::DaemonSequence => { - if cursor.position() >= range.end() { - return true; + if cursor.position() > range.end() { + return NativeLifecycleAdmissionV1::Superseded; + } + if cursor.position() == range.end() { + return NativeLifecycleAdmissionV1::Ready; } let Ok(covered) = ObservationSourceRangeV1::new(cursor.position(), range.end()) else { - return false; + return NativeLifecycleAdmissionV1::Unavailable; }; (covered, Some(cursor)) } - Ok(Some(_)) | Err(_) => return false, + Ok(Some(_)) | Err(_) => return NativeLifecycleAdmissionV1::Unavailable, }; let raw = match serde_json::to_vec(lifecycle) { Ok(raw) => raw, - Err(_) => return false, + Err(_) => return NativeLifecycleAdmissionV1::Unavailable, }; - let session_id = lifecycle.session_id.clone(); - let call_id = lifecycle.call_id.clone(); - let canonical_provider = provider.clone(); let parsed = match parse_normalized_observation_record_v1( &raw, range, ObservationOrderingDomainV1::DaemonSequence, - move |_| { - CanonicalObservationEnvelopeV1::new( - canonical_provider, - "hook_tool_after", - call_id.clone(), - CanonicalObservationRelationsV1::new(session_id.clone()) - .with_thread_id( - ObservationId::new(session_id.as_str().to_owned()) - .map_err(|_| ObservationRecordParseErrorV1::NormalizationFailed)?, - ) - .with_turn_id(call_id.clone()) - .with_agent_id( - ObservationId::new(session_id.as_str().to_owned()) - .map_err(|_| ObservationRecordParseErrorV1::NormalizationFailed)?, - ) - .with_message_id(call_id.clone()), - vec![CanonicalObservationFactV1::Boundary { - boundary_kind: CanonicalBoundaryKindV1::TurnEnd, - }], - CanonicalObservationEvidenceV1::new( - ObservationOrderingDomainV1::DaemonSequence, - range, - ), - ) - .map_err(|_| ObservationRecordParseErrorV1::NormalizationFailed) - }, + move |_| native_lifecycle_observation_envelope(provider, lifecycle, range), ) { Ok(parsed) => parsed, - Err(_) => return false, + Err(_) => return NativeLifecycleAdmissionV1::Unavailable, }; let identity = match ObservationIdentityMaterialV1::for_native_record( source, scope, match ObservationSourceGenerationV1::new(1) { Ok(generation) => generation, - Err(_) => return false, + Err(_) => return NativeLifecycleAdmissionV1::Unavailable, }, range, ObservationOrderingDomainV1::DaemonSequence, lifecycle.call_id.clone(), ) { Ok(identity) => identity, - Err(_) => return false, + Err(_) => return NativeLifecycleAdmissionV1::Unavailable, }; let request = match CaptureObservationRequest::new( parsed, @@ -176,12 +338,12 @@ pub(super) async fn admit_native_context_scout_lifecycle( expected_cursor, match RetentionClass::new("transcript.hook-lifecycle.v1") { Ok(retention) => retention, - Err(_) => return false, + Err(_) => return NativeLifecycleAdmissionV1::Unavailable, }, ObservationCancellation::default(), ) { Ok(request) => request, - Err(_) => return false, + Err(_) => return NativeLifecycleAdmissionV1::Unavailable, }; // Admission is the durable commit of the lifecycle observation. Providers // routed through the external-source replay path commit the same durable @@ -189,16 +351,17 @@ pub(super) async fn admit_native_context_scout_lifecycle( // queued projection never blocks Scout lifecycle admission. Idempotent // re-admission of the exact same record remains admitted. match facade.capture_observation(request).await { - Ok(CaptureObservationOutcome::Persisted { .. }) => true, - Ok(CaptureObservationOutcome::AcceptedForReplay { outcome, .. }) => matches!( - *outcome, - ObservationPersistOutcome::Committed(_) | ObservationPersistOutcome::ExactDuplicate(_) - ), - Ok( - CaptureObservationOutcome::Rejected { .. } - | CaptureObservationOutcome::Quarantined { .. }, - ) - | Err(_) => false, + Ok(CaptureObservationOutcome::Persisted { .. }) => NativeLifecycleAdmissionV1::Ready, + Ok(CaptureObservationOutcome::AcceptedForReplay { outcome, .. }) + if matches!( + *outcome, + ObservationPersistOutcome::Committed(_) + | ObservationPersistOutcome::ExactDuplicate(_) + ) => + { + NativeLifecycleAdmissionV1::Ready + } + Ok(_) | Err(_) => NativeLifecycleAdmissionV1::Unavailable, } } diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/context_scout/tests.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/context_scout/tests.rs index 079c58cab4..f47b7ca903 100644 --- a/crates/tracedecay-mcp/src/handlers/hook_runtime/context_scout/tests.rs +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/context_scout/tests.rs @@ -192,7 +192,7 @@ async fn kimi_and_opencode_queued_lifecycle_delivery_prepares_scout_lookup() { u64::try_from(order).unwrap() + 2, ) .unwrap(); - assert!( + assert_eq!( admit_native_context_scout_lifecycle( &sessions, Some(&runtime.background_cpu()), @@ -200,9 +200,10 @@ async fn kimi_and_opencode_queued_lifecycle_delivery_prepares_scout_lookup() { &identity, range, ) - .await + .await, + NativeLifecycleAdmissionV1::Ready, ); - assert!( + assert_eq!( admit_native_context_scout_lifecycle( &sessions, Some(&runtime.background_cpu()), @@ -210,7 +211,8 @@ async fn kimi_and_opencode_queued_lifecycle_delivery_prepares_scout_lookup() { &identity, range, ) - .await + .await, + NativeLifecycleAdmissionV1::Ready, ); } let lifecycle = @@ -279,14 +281,23 @@ async fn lifecycle_positions_with_gaps_or_late_arrivals_settle() { } }; - assert!(admit("call.kimi.first", 1).await); - assert!(admit("call.kimi.after-gap", 5).await, "a gap must commit"); - assert!( + assert_eq!( + admit("call.kimi.first", 1).await, + NativeLifecycleAdmissionV1::Ready + ); + assert_eq!( admit("call.kimi.after-gap", 5).await, + NativeLifecycleAdmissionV1::Ready, + "a gap must commit" + ); + assert_eq!( + admit("call.kimi.after-gap", 5).await, + NativeLifecycleAdmissionV1::Ready, "a retry stays admitted" ); - assert!( + assert_eq!( admit("call.kimi.late", 3).await, + NativeLifecycleAdmissionV1::Superseded, "an event older than the committed cursor is already covered" ); diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/envelope/tests.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/envelope/tests.rs index de6eeed00f..bdbbadcafd 100644 --- a/crates/tracedecay-mcp/src/handlers/hook_runtime/envelope/tests.rs +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/envelope/tests.rs @@ -44,7 +44,8 @@ fn lifecycle_range_prefers_native_sequence_and_reuses_unknown_ledger_order() { let receipt = tracedecay_hooks::HookAdmissionLedgerReceiptV1 { decision: tracedecay_hooks::HookAdmissionDecisionV1::Admitted, order: 7, - work_completed: false, + order_reconstructed: false, + work_settlement: None, }; let unknown = admission_test_envelope(21, 7); assert_eq!( diff --git a/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs b/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs index 02869cff7c..b6b716a583 100644 --- a/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs +++ b/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs @@ -25,8 +25,7 @@ use tracedecay_hooks::{ }; use tracedecay_mcp::handlers::hook_runtime::{ - HookV2AdmissionOutcomeV1, admit_hook_v2_envelope, - admit_hook_v2_replayed_envelope_with_lifecycle, hook_v2_native_lifecycle_enabled, + HookV2AdmissionOutcomeV1, admit_hook_v2_replayed_envelope_with_lifecycle, hook_v2_pending_work_envelopes, }; @@ -210,6 +209,18 @@ async fn drain_all_hosts( Ok(more) => sweep.more_pending |= more, Err(cause) => sweep.fail(*host, StatusHookReplaySpoolV1::DeliveryReceipts, cause), } + // Retained admission order precedes richer, as-yet-unadmitted host records. + // An event missing its own lifecycle remains owed until its spool copy + // supplies that prerequisite. + Box::pin(redrive_pending_work( + graph, + *host, + now, + &mut sweep, + project_sessions, + background_cpu, + )) + .await; let report = match Box::pin(drain_admitted_host_spool( *host, project_id, @@ -226,11 +237,6 @@ async fn drain_all_hosts( None } }; - // The host spool retains native lifecycle evidence the ledger cannot carry. - Box::pin(redrive_pending_work( - graph, data_root, *host, project_id, now, &mut sweep, - )) - .await; if let Some(report) = report && (report.committed > 0 || report.duplicates > 0 || report.tombstoned > 0) { @@ -260,12 +266,15 @@ async fn drain_all_hosts( /// redrive cannot admit stays owed, and the sweep names it. async fn redrive_pending_work( graph: &tracedecay_project::project::TraceDecay, - data_root: &Path, host: NativeHostIdentityV1, - project_id: Option<[u8; 16]>, now: UtcMicros, sweep: &mut HookReplaySweepV1, + project_sessions: &tracedecay_global_db::RegisteredGlobalDb, + background_cpu: &Arc, ) { + let data_root = &graph.hook_store_layout().data_root; + let project_id = + tracedecay_agent_hosts::hooks::hook_project_id_for_layout(graph.hook_store_layout()); let pending_work = match hook_v2_pending_work_envelopes(data_root, host, now) { Ok(pending_work) => pending_work, Err(error) => { @@ -297,21 +306,10 @@ async fn redrive_pending_work( .await }, |envelope, native_session_id| async move { - // A timed-out admission may have committed its ledger before its - // lifecycle. Only the richer host spool can finish that mount. - if native_session_id.is_none() - && tracedecay_hooks::NativeContextScoutLifecycleV1::host_mounts_lifecycle(host) - && matches!( - envelope.event, - tracedecay_hooks::HookEventV2::ToolLifecycle { .. } - | tracedecay_hooks::HookEventV2::SavedEdit { .. } - ) - && hook_v2_native_lifecycle_enabled(graph, host).await - { - return HookV2AdmissionOutcomeV1::Backpressured; - } - admit_hook_v2_envelope(graph, &envelope, native_session_id, hook_replay_now()) - .await + admit_hook_v2_replayed_envelope_with_lifecycle( + graph, &envelope, native_session_id, None, + project_sessions, background_cpu, hook_replay_now(), + ).await }, ) .await; @@ -646,6 +644,7 @@ async fn stop_replay_consumer( #[cfg(test)] mod tests { use super::*; + use tracedecay_mcp::handlers::hook_runtime::admit_hook_v2_envelope; #[test] fn only_an_admitted_redrive_settles_pending_work() { @@ -728,15 +727,6 @@ mod tests { changed_range_count: 1, }, }; - // Commit admission while no orchestrator is mounted, reproducing the - // durable state left by a caller cancelled before lifecycle persistence. - assert!(matches!( - admit_hook_v2_envelope(&graph, &envelope, None, now).await, - HookV2AdmissionOutcomeV1::Admitted { .. } - )); - let pending = - || hook_v2_pending_work_envelopes(data_root, host, hook_replay_now()).unwrap(); - assert_eq!(pending(), vec![envelope.clone()]); assert_eq!( tracedecay_daemon_service::context_scout_lifecycle::register_context_scout_lifecycle_authority( project_id, @@ -747,6 +737,58 @@ mod tests { ), tracedecay_daemon_service::context_scout_lifecycle::AuthorityRegistrationV1::Registered ); + // An earlier call establishes the session, but cannot supply this + // edit's own native call identity when its spool copy is delayed. + let mut prior_envelope = envelope.clone(); + prior_envelope.event_id = [7; 16]; + prior_envelope.event = tracedecay_hooks::HookEventV2::ToolLifecycle { + tool_id: [7; 16], + phase: tracedecay_hooks::HookLifecyclePhaseV1::Completed, + effect_receipt_id: None, + }; + let prior_lifecycle = tracedecay_hooks::NativeContextScoutLifecycleV1::new( + lifecycle.session_id.as_str(), + "call.replay.previous", + prior_envelope.event_id, + ) + .unwrap(); + assert!(matches!( + admit_hook_v2_replayed_envelope_with_lifecycle( + &graph, + &prior_envelope, + Some(prior_lifecycle.session_id.clone()), + Some(prior_lifecycle.clone()), + &project_sessions, + &background_cpu, + now, + ) + .await, + HookV2AdmissionOutcomeV1::Admitted { .. } + )); + assert_eq!( + tracedecay_daemon_service::context_scout_lifecycle::lookup_registered_context_scout_native_session( + project_id, binding.worktree_id, envelope.protected_session_id, + ).await, + Some(prior_lifecycle.session_id), + ); + // Commit admission while no orchestrator is mounted, reproducing the + // durable state left by a caller cancelled before lifecycle persistence. + assert!(matches!( + admit_hook_v2_replayed_envelope_with_lifecycle( + &graph, + &envelope, + Some(lifecycle.session_id.clone()), + None, + &project_sessions, + &background_cpu, + now + ) + .await, + HookV2AdmissionOutcomeV1::Backpressured + )); + let pending = + || hook_v2_pending_work_envelopes(data_root, host, hook_replay_now()).unwrap(); + assert_eq!(pending(), vec![envelope.clone()]); let (ran, mut runs) = tokio::sync::mpsc::unbounded_channel(); let runtime = tracedecay_daemon_service::BoundedHookOrchestratorV1::new( 1, @@ -801,6 +843,10 @@ mod tests { .unwrap() .expect("spool replay must mount lifecycle"); assert_eq!(mounted.session_id, lifecycle.session_id); + assert_eq!( + mounted.logical_message_id.as_str(), + lifecycle.call_id.as_str() + ); tokio::time::timeout(Duration::from_secs(10), async { while !pending().is_empty() { tokio::task::yield_now().await; @@ -812,6 +858,16 @@ mod tests { runs.try_recv().is_err(), "the mounted edit executes exactly once" ); + superseded_edit_settles_after_newer_completion( + &graph, + &envelope, + &lifecycle, + &project_sessions, + &background_cpu, + &fixture.settlements, + &mut runs, + ) + .await; // Claude deliberately omits lifecycle writes while intent is disabled; // an acknowledged edit can therefore need ledger-only recovery. tracedecay_daemon_service::unregister_hook_orchestration_runtime( @@ -890,6 +946,128 @@ mod tests { )); } + /// A newer live admission can complete before the older host spool arrives. + /// Canonical source order settles the old debt without running its producer. + async fn superseded_edit_settles_after_newer_completion( + graph: &tracedecay_project::project::TraceDecay, + template: &tracedecay_hooks::HookEventEnvelopeV2, + session: &tracedecay_hooks::NativeContextScoutLifecycleV1, + project_sessions: &tracedecay_global_db::RegisteredGlobalDb, + background_cpu: &Arc, + settlements: &tracedecay_application::observability::DeliverySettlementAuthorityV1, + runs: &mut tokio::sync::mpsc::UnboundedReceiver>, + ) { + let data_root = &graph.hook_store_layout().data_root; + let mut older = template.clone(); + older.event_id = [11; 16]; + let older_lifecycle = tracedecay_hooks::NativeContextScoutLifecycleV1::new( + session.session_id.as_str(), + "call.replay.older", + older.event_id, + ) + .unwrap(); + assert!(matches!( + admit_hook_v2_replayed_envelope_with_lifecycle( + graph, + &older, + Some(session.session_id.clone()), + None, + project_sessions, + background_cpu, + hook_replay_now() + ) + .await, + HookV2AdmissionOutcomeV1::Backpressured, + )); + let mut newer = template.clone(); + newer.event_id = [12; 16]; + let newer_lifecycle = tracedecay_hooks::NativeContextScoutLifecycleV1::new( + session.session_id.as_str(), + "call.replay.newer", + newer.event_id, + ) + .unwrap(); + assert!(matches!( + admit_hook_v2_replayed_envelope_with_lifecycle( + graph, + &newer, + Some(newer_lifecycle.session_id.clone()), + Some(newer_lifecycle.clone()), + project_sessions, + background_cpu, + hook_replay_now(), + ) + .await, + HookV2AdmissionOutcomeV1::Admitted { .. }, + )); + let mounted = tokio::time::timeout(Duration::from_secs(10), runs.recv()) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!( + mounted.logical_message_id.as_str(), + newer_lifecycle.call_id.as_str() + ); + let pending = || { + hook_v2_pending_work_envelopes(data_root, older.producer, hook_replay_now()).unwrap() + }; + tokio::time::timeout(Duration::from_secs(10), async { + while pending() != vec![older.clone()] { + tokio::task::yield_now().await; + } + }) + .await + .expect("newer producer completed while older event remains owed"); + let binding = published_hook_scope_binding( + data_root, + older.worktree_id, + older.producer, + hook_replay_now(), + ) + .unwrap(); + let root = hook_v2_spool_root(data_root, older.producer); + let (mut spool, _) = HookSpoolV1::open( + &root, + HookSpoolConfigV1::stock(older.producer), + hook_replay_now(), + ) + .unwrap(); + spool + .append_with_native_lifecycle( + older.clone(), + Some(older_lifecycle), + &binding, + hook_replay_now(), + ) + .unwrap(); + drop(spool); + drain_all_hosts( + graph, + data_root, + settlements, + project_sessions, + background_cpu, + ) + .await; + assert!( + pending().is_empty(), + "superseded old work settles instead of staying pending forever" + ); + assert!( + runs.try_recv().is_err(), + "superseded old work must not invoke a producer" + ); + let current = tracedecay_daemon_service::context_scout_lifecycle::lookup_registered_context_scout_lifecycle( + newer.project_id, newer.worktree_id, &newer_lifecycle.session_id, + ).await.unwrap(); + assert_eq!( + current.logical_message_id.as_str(), + newer_lifecycle.call_id.as_str() + ); + assert!(!HookSpoolV1::has_records(&root).unwrap()); + } + #[test] fn only_a_missing_spool_root_is_absent() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/tracedecay/src/daemon/invocation_tests/types_tests.rs b/crates/tracedecay/src/daemon/invocation_tests/types_tests.rs index 7e6bfee6f3..fb83458677 100644 --- a/crates/tracedecay/src/daemon/invocation_tests/types_tests.rs +++ b/crates/tracedecay/src/daemon/invocation_tests/types_tests.rs @@ -118,7 +118,7 @@ async fn hook_orchestration_backpressures_without_waiting() { let completed = Arc::new(tokio::sync::Notify::new()); let completion_notification = Arc::clone(&completed); let mut request = saved_edit_request(None); - request.completion = Some(Arc::new(move || { + request.completion = Some(Arc::new(move |_| { observed_completions.fetch_add(1, std::sync::atomic::Ordering::Relaxed); completion_notification.notify_one(); })); @@ -201,6 +201,7 @@ async fn hook_orchestration_runs_feedback_work_without_scout_lifecycle() { 1, false, None, + None, ); ADMITTING_HOOK.set(false); assert_eq!(admission, HookOrchestrationAdmissionV1::Enqueued); @@ -259,14 +260,14 @@ async fn hook_orchestration_coalesces_duplicate_work_and_completes_every_admissi let mut request = saved_edit_request(Some(hook_lifecycle())); let first_completions = Arc::clone(&completions); let first_completed = Arc::clone(&completed); - request.completion = Some(Arc::new(move || { + request.completion = Some(Arc::new(move |_| { first_completions.fetch_add(1, std::sync::atomic::Ordering::Relaxed); first_completed.notify_one(); })); let mut duplicate = request.clone(); let duplicate_completions = Arc::clone(&completions); let duplicate_completed = Arc::clone(&completed); - duplicate.completion = Some(Arc::new(move || { + duplicate.completion = Some(Arc::new(move |_| { duplicate_completions.fetch_add(1, std::sync::atomic::Ordering::Relaxed); duplicate_completed.notify_one(); })); @@ -327,7 +328,7 @@ async fn hook_orchestration_supersedes_same_address_without_replay_delay() { let first_completions = Arc::new(std::sync::atomic::AtomicUsize::new(0)); let observed_first_completions = Arc::clone(&first_completions); let mut first = saved_edit_request(Some(hook_lifecycle())); - first.completion = Some(Arc::new(move || { + first.completion = Some(Arc::new(move |_| { observed_first_completions.fetch_add(1, std::sync::atomic::Ordering::Relaxed); })); assert_eq!(runtime.admit(first), HookOrchestrationAdmissionV1::Enqueued); @@ -354,7 +355,7 @@ async fn hook_orchestration_supersedes_same_address_without_replay_delay() { false, ) .unwrap(); - second.completion = Some(Arc::new(move || { + second.completion = Some(Arc::new(move |_| { observed_second_completions.fetch_add(1, std::sync::atomic::Ordering::Relaxed); observed_second_completed.notify_one(); })); @@ -475,7 +476,7 @@ async fn retryable_hook_work_does_not_acknowledge_the_durable_admission() { let completions = Arc::new(std::sync::atomic::AtomicUsize::new(0)); let observed_completions = Arc::clone(&completions); let mut request = saved_edit_request(Some(hook_lifecycle())); - request.completion = Some(Arc::new(move || { + request.completion = Some(Arc::new(move |_| { observed_completions.fetch_add(1, std::sync::atomic::Ordering::Relaxed); })); @@ -541,7 +542,8 @@ async fn replayed_admission_after_retryable_failure_completes_a_fresh_cycle() { Some(hook_lifecycle()), 1, false, - Some(Arc::new(move || { + None, + Some(Arc::new(move |_| { first_acknowledged.fetch_add(1, std::sync::atomic::Ordering::Relaxed); })), ), @@ -570,7 +572,8 @@ async fn replayed_admission_after_retryable_failure_completes_a_fresh_cycle() { Some(hook_lifecycle()), 1, false, - Some(Arc::new(move || { + None, + Some(Arc::new(move |_| { replay_acknowledged.fetch_add(1, std::sync::atomic::Ordering::Relaxed); replay_terminal.notify_one(); })), @@ -640,7 +643,7 @@ async fn superseded_blocking_work_is_joined_before_its_receipt_terminal() { let observed_nested_finished = Arc::clone(&nested_finished_at_terminal); let completion_finished = Arc::clone(&blocking_finished); let mut first = saved_edit_request(Some(hook_lifecycle())); - first.completion = Some(Arc::new(move || { + first.completion = Some(Arc::new(move |_| { observed_nested_finished.store( completion_finished.load(std::sync::atomic::Ordering::Acquire), std::sync::atomic::Ordering::Release, @@ -737,7 +740,7 @@ async fn superseded_queued_work_emits_its_own_receipt_terminal() { assert_eq!( runtime.admit(request( [2; 16], - Some(Arc::new(move || observed_queued_terminal.notify_one())), + Some(Arc::new(move |_| observed_queued_terminal.notify_one())), )), HookOrchestrationAdmissionV1::Enqueued ); @@ -770,7 +773,7 @@ async fn hook_orchestration_bounds_coalesced_completion_waiters() { } }) .unwrap(); - let completion = Arc::new(|| {}) as Arc; + let completion = Arc::new(|_| {}) as Arc; let request = |completion| { let mut request = saved_edit_request(Some(hook_lifecycle())); request.completion = Some(completion); @@ -809,7 +812,7 @@ async fn dropping_hook_orchestrator_cancels_daemon_owned_work_without_false_comp let completions = Arc::new(std::sync::atomic::AtomicUsize::new(0)); let observed_completions = Arc::clone(&completions); let mut request = saved_edit_request(Some(hook_lifecycle())); - request.completion = Some(Arc::new(move || { + request.completion = Some(Arc::new(move |_| { observed_completions.fetch_add(1, std::sync::atomic::Ordering::Relaxed); })); assert_eq!( diff --git a/crates/tracedecay/src/daemon/production_harness/advisory_cycle_language_journey_test.rs b/crates/tracedecay/src/daemon/production_harness/advisory_cycle_language_journey_test.rs index dd83b7a94a..c7ad72c237 100644 --- a/crates/tracedecay/src/daemon/production_harness/advisory_cycle_language_journey_test.rs +++ b/crates/tracedecay/src/daemon/production_harness/advisory_cycle_language_journey_test.rs @@ -413,7 +413,8 @@ async fn hook_work_completes( None, 1, false, - Some(std::sync::Arc::new(move || { + None, + Some(std::sync::Arc::new(move |_| { if let Some(completed) = completed.lock().expect("completion").take() { let _ = completed.send(()); } From d33a12867bd2175b61210e0d721a07b97e56a060 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Mon, 19 Oct 2026 03:40:00 +0000 Subject: [PATCH 63/89] fix(hooks): reconcile durable routes and observe queued edits --- crates/tracedecay-mcp/src/hook_events.rs | 21 +- .../tracedecay/src/mcp/server/connection.rs | 3 +- .../hook_boundary_failure_matrix_tests.rs | 8 +- .../src/mcp/server/hook_dispatch.rs | 115 ++++++--- .../src/mcp/server/host_admission_tests.rs | 218 +++++++++++++++++- crates/tracedecay/src/mcp/server/requests.rs | 58 +++-- 6 files changed, 364 insertions(+), 59 deletions(-) diff --git a/crates/tracedecay-mcp/src/hook_events.rs b/crates/tracedecay-mcp/src/hook_events.rs index 9e2d727a72..4d6229db37 100644 --- a/crates/tracedecay-mcp/src/hook_events.rs +++ b/crates/tracedecay-mcp/src/hook_events.rs @@ -12,7 +12,8 @@ use serde_json::Value; use tracedecay_domain::HostIntegrationIdV1; use tracedecay_domain::canonical_text::sha256_hex; -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] pub enum HookEventKind { FileEdit, Shell, @@ -141,6 +142,8 @@ pub enum HookEventPlan { root: PathBuf, scope: tracedecay_contracts::ResolvedScope, agent: HostIntegrationIdV1, + event_kind: HookEventKind, + had_command: bool, route: Option, rel_paths: Vec, }, @@ -176,6 +179,8 @@ enum DurableHookEventPlan { root: PathBuf, scope: tracedecay_contracts::ResolvedScope, agent: String, + event_kind: HookEventKind, + had_command: bool, route: Option, rel_paths: Vec, }, @@ -401,6 +406,8 @@ fn durable_plan_from_runtime(plan: &HookEventPlan) -> Result DurableHookEventPlan::SyncFilesAt { @@ -410,6 +417,11 @@ fn durable_plan_from_runtime(plan: &HookEventPlan) -> Result *event_kind, + _ => return Err(()), + }, + had_command: *had_command, route: sanitize_durable_route(route.as_ref())?, rel_paths: sanitize_durable_rel_paths(rel_paths)?, }, @@ -468,6 +480,8 @@ fn runtime_plan_from_durable( root, scope, agent, + event_kind, + had_command, route, rel_paths, } => Ok(HookEventPlan::SyncFilesAt { @@ -481,6 +495,11 @@ fn runtime_plan_from_durable( }, agent: HostIntegrationIdV1::from_wire(&agent) .ok_or(DurableHookEventDecodeError::Malformed)?, + event_kind: match event_kind { + HookEventKind::FileEdit | HookEventKind::IncrementalSync => event_kind, + _ => return Err(DurableHookEventDecodeError::Malformed), + }, + had_command, route: sanitize_durable_route(route.as_ref()) .map_err(|()| DurableHookEventDecodeError::Malformed)?, rel_paths: sanitize_durable_rel_paths(&rel_paths) diff --git a/crates/tracedecay/src/mcp/server/connection.rs b/crates/tracedecay/src/mcp/server/connection.rs index 69fcf2f938..f3ab5c7493 100644 --- a/crates/tracedecay/src/mcp/server/connection.rs +++ b/crates/tracedecay/src/mcp/server/connection.rs @@ -459,7 +459,8 @@ impl McpServer { }; let cg = self.reopen_if_branch_drifted().await; let root = cg.project_root().to_path_buf(); - let canonical_outcome = Box::pin(self.run_hook_event_plan(cg, &root, plan)).await; + let canonical_outcome = + Box::pin(self.run_hook_event_plan(cg, &root, plan, Some(record.seq))).await; let outcome = if canonical_outcome.reason_code == Some("stale_branch_authorization") && !canonical_outcome.retryable { diff --git a/crates/tracedecay/src/mcp/server/hook_boundary_failure_matrix_tests.rs b/crates/tracedecay/src/mcp/server/hook_boundary_failure_matrix_tests.rs index c4dce5ab21..964eafe370 100644 --- a/crates/tracedecay/src/mcp/server/hook_boundary_failure_matrix_tests.rs +++ b/crates/tracedecay/src/mcp/server/hook_boundary_failure_matrix_tests.rs @@ -296,7 +296,10 @@ async fn after_edit_hook_delivers_touched_paths_to_code_index_sink() { super::CodeIndexDemandAdmissionV1::Queued }) }); - let context = registered_context(cg, &authority).with_code_index_hook_sink(sink); + let spool = TempDir::new().unwrap(); + let broker = default_spool_broker(spool.path()); + let mut context = registered_context(cg, &authority).with_code_index_hook_sink(sink); + context.host_admission_broker = Some(broker); let server = McpServer::new_with_registered_test_context(context, Vec::new()) .await .expect("registered test server"); @@ -308,7 +311,8 @@ async fn after_edit_hook_delivers_touched_paths_to_code_index_sink() { )) .unwrap(); - Box::pin(server.handle_hook_event_notification(Some(&event), &mut routes)).await; + let outcome = Box::pin(server.handle_hook_event_notification(Some(&event), &mut routes)).await; + assert_eq!(outcome.status, HostAdmissionStatus::Committed); { let recorded = recorded.lock().unwrap(); diff --git a/crates/tracedecay/src/mcp/server/hook_dispatch.rs b/crates/tracedecay/src/mcp/server/hook_dispatch.rs index 7ed410caac..267ac05871 100644 --- a/crates/tracedecay/src/mcp/server/hook_dispatch.rs +++ b/crates/tracedecay/src/mcp/server/hook_dispatch.rs @@ -79,38 +79,42 @@ impl McpServer { }) } - async fn run_routed_file_edit( + async fn resolve_file_edit_route( &self, - effect_root: PathBuf, - scope: tracedecay_contracts::ResolvedScope, - agent: tracedecay_domain::HostIntegrationIdV1, - route: Option, - rel_paths: Vec, - ) -> HostAdmissionOutcome { + effect_root: &Path, + scope: &tracedecay_contracts::ResolvedScope, + ) -> std::result::Result + { let Some(registry) = self.registry_db.as_deref() else { - return HostAdmissionOutcome::retained_unavailable( + return Err(HostAdmissionOutcome::retained_unavailable( "project_registry_route_unavailable", - ); + )); }; let owner = match registry .project_registry_context_by_id(scope.project_id.as_str()) .await { Ok(Some(owner)) => owner, - Ok(None) => return HostAdmissionOutcome::degraded("project_route_stale"), + Ok(None) => return Err(HostAdmissionOutcome::degraded("project_route_stale")), Err(_) => { - return HostAdmissionOutcome::retained_unavailable( + return Err(HostAdmissionOutcome::retained_unavailable( "project_registry_route_unavailable", - ); + )); } }; let (root, current_scope) = - match tracedecay_mcp::scope::resolve_query_scope(&owner, &effect_root) { + match tracedecay_mcp::scope::resolve_query_scope(&owner, effect_root) { Ok(resolved) => resolved, - Err(_) => return HostAdmissionOutcome::degraded("project_route_stale"), + Err(_) => return Err(HostAdmissionOutcome::degraded("project_route_stale")), }; - if root != effect_root || current_scope != scope { - return HostAdmissionOutcome::degraded("project_route_stale"); + // Ref is mutable checkout state. Fixed route identity must still match + // exactly; indexing reconciles the canonical current branch below. + if root.as_path() != effect_root + || current_scope.project_id != scope.project_id + || current_scope.repository_id != scope.repository_id + || current_scope.worktree_id != scope.worktree_id + { + return Err(HostAdmissionOutcome::degraded("project_route_stale")); } let resolved = match crate::mcp::project_route::resolve_registered_project_route( owner, @@ -122,21 +126,29 @@ impl McpServer { { Ok(resolved) => resolved, Err(_) => { - return HostAdmissionOutcome::retained_unavailable("project_route_unavailable"); + return Err(HostAdmissionOutcome::retained_unavailable( + "project_route_unavailable", + )); } }; - if resolved.requested_root != root || resolved.scope != scope { - return HostAdmissionOutcome::degraded("project_route_stale"); + if resolved.requested_root != root || resolved.scope != current_scope { + return Err(HostAdmissionOutcome::degraded("project_route_stale")); } - let event = hook_events::HookEvent { - agent, - kind: hook_events::HookEventKind::FileEdit, - rel_paths, - had_command: false, - cwd: Some(root), - route, - receipt: None, + Ok(resolved) + } + + async fn run_routed_file_edit( + &self, + effect_root: PathBuf, + scope: tracedecay_contracts::ResolvedScope, + event: hook_events::HookEvent, + admission_seq: Option, + ) -> HostAdmissionOutcome { + let resolved = match self.resolve_file_edit_route(&effect_root, &scope).await { + Ok(resolved) => resolved, + Err(outcome) => return outcome, }; + let root = resolved.requested_root.clone(); let server = match resolved.retained_server() { Ok(server) => server, Err(_) => { @@ -168,9 +180,42 @@ impl McpServer { "project_registry_route_unavailable", ); } + let (cg, branch) = server.reopen_if_branch_drifted_memoized().await; + if cg.project_root() != root { + return HostAdmissionOutcome::degraded("project_route_stale"); + } + if let Some(seq) = admission_seq { + if let Some(wake) = &server.project_session_refresh_wake { + wake.wake(); + } + if let Some(wake) = &server.user_session_refresh_wake { + wake.wake(); + } + server.record_hook_route_analytics( + &root, + &event, + branch.resolve_for(&root).as_deref(), + seq, + ); + server.record_hook_span_observation(&event, &resolved); + } match server.code_index_hook_sink.as_ref() { Some(sink) => { - code_index_host_outcome(&sink(resolved.requested_root, event.rel_paths).await) + let admission = sink(root.clone(), event.rel_paths.clone()).await; + if admission.is_queued() + && let Some(db) = server.project_session_db.as_deref() + { + tracedecay_session_memory::event_lane::publish( + db, + tracedecay_session_memory::event_lane::ActivityFamilyV1::CodeIndex, + &root, + Some(scope.project_id.as_str()), + event.rel_paths.len() as u64, + Some(event.kind.as_key()), + ) + .await; + } + code_index_host_outcome(&admission) } None => HostAdmissionOutcome::retained_unavailable(CODE_INDEX_SCHEDULER_UNAVAILABLE), } @@ -181,12 +226,15 @@ impl McpServer { cg: Arc, root: &Path, plan: HookEventPlan, + admission_seq: Option, ) -> HostAdmissionOutcome { match plan { HookEventPlan::SyncFilesAt { root: effect_root, scope, agent, + event_kind, + had_command, route, rel_paths, } => { @@ -195,7 +243,16 @@ impl McpServer { { return HostAdmissionOutcome::degraded("project_store_mismatch"); } - self.run_routed_file_edit(effect_root, scope, agent, route, rel_paths) + let event = hook_events::HookEvent { + agent, + kind: event_kind, + rel_paths, + had_command, + cwd: Some(effect_root.clone()), + route, + receipt: None, + }; + self.run_routed_file_edit(effect_root, scope, event, admission_seq) .await } HookEventPlan::SyncFiles(rel_paths) => { diff --git a/crates/tracedecay/src/mcp/server/host_admission_tests.rs b/crates/tracedecay/src/mcp/server/host_admission_tests.rs index 5ab100ffba..afd6295aae 100644 --- a/crates/tracedecay/src/mcp/server/host_admission_tests.rs +++ b/crates/tracedecay/src/mcp/server/host_admission_tests.rs @@ -21,6 +21,7 @@ use tracedecay_host_admission::{ }; use tracedecay_private_fs::framed_log::sync_latency; use tracedecay_project::test_support::host_admission::HostAdmissionTestRuntimeV1; +use tracedecay_session_memory::event_lane::{self, ActivityFamilyV1}; use tracedecay_sessions::admission::{ HostAdmissionOutcome, HostAdmissionScope, HostAdmissionStatus, }; @@ -29,6 +30,23 @@ use tracedecay_sessions::runtime::git_correlation::{ }; use tracedecay_sessions::runtime::{SessionMessageRecord, SessionRecord}; +async fn code_index_activity(server: &McpServer) -> Vec { + let graph = server.cg_snapshot().await; + let project_id = graph.store_layout().identity.project_id.as_deref().unwrap(); + event_lane::replay_after( + server.project_session_db.as_deref().unwrap(), + project_id, + None, + ) + .await + .unwrap() + .unwrap() + .records + .into_iter() + .filter(|record| record.pulse.family == ActivityFamilyV1::CodeIndex) + .collect() +} + /// Builds the terminal-receipt wire event the Hermes plugin sends. Production /// only deserializes these, so the tests own the sender-side construction. fn hermes_terminal_receipt_event( @@ -166,12 +184,14 @@ async fn file_edit_commit_failure_queues_no_sync() { assert_eq!(outcome.reason_code, Some("spool_recovery_required")); assert_eq!(effects.load(Ordering::SeqCst), 0); assert_eq!(broker.pending_replay_count().await.unwrap(), 0); + assert!(code_index_activity(&server).await.is_empty()); server.shutdown().await; } #[tokio::test] async fn file_edit_request_acks_durable_work_while_effect_is_held() { let (cg, project, authority) = init_indexed_repo().await; + let mut live_activity = event_lane::subscribe().unwrap(); let spool = TempDir::new().unwrap(); let broker = default_spool_broker(spool.path()); let entered = Arc::new(Notify::new()); @@ -225,18 +245,194 @@ async fn file_edit_request_acks_durable_work_while_effect_is_held() { entered.notified().await; assert_eq!(broker.pending_count().await, 1); assert_eq!(effects.load(Ordering::SeqCst), 0); + assert!(code_index_activity(&server).await.is_empty()); release.notify_one(); server.wait_project_host_admission_replay_idle().await; assert_eq!(broker.pending_count().await, 0); assert_eq!(effects.load(Ordering::SeqCst), 1); + let activity = code_index_activity(&server).await; + assert_eq!(activity.len(), 1); + assert_eq!(activity[0].pulse.units, 1); + assert_eq!(activity[0].pulse.detail, None); + let mut queued_activity = Vec::new(); + while let Ok(record) = live_activity.try_recv() { + if record.pulse.family == ActivityFamilyV1::CodeIndex + && record.pulse.project_root == project.path() + { + queued_activity.push(record); + } + } + assert_eq!(queued_activity.len(), 1); + assert_eq!( + queued_activity[0].pulse.project_id, + activity[0].pulse.project_id + ); server.replay_host_admission(None).await; assert_eq!(effects.load(Ordering::SeqCst), 1); + assert_eq!(code_index_activity(&server).await, activity); + server.shutdown().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn incremental_path_hints_emit_activity_only_for_queued_paths() { + let (cg, project, authority) = init_indexed_repo().await; + let spool = TempDir::new().unwrap(); + let broker = default_spool_broker(spool.path()); + let effects = Arc::new(AtomicUsize::new(0)); + let mut context = registered_context(cg, &authority); + context.host_admission_broker = Some(broker); + context = context + .with_code_index_reconcile_sink(success_reconcile_sink()) + .with_code_index_hook_sink({ + let effects = Arc::clone(&effects); + let root = project.path().to_path_buf(); + Arc::new(move |effect_root, paths| { + assert_eq!(effect_root, root); + assert_eq!(paths, ["src/lib.rs"]); + effects.fetch_add(1, Ordering::SeqCst); + Box::pin(async { super::CodeIndexDemandAdmissionV1::Queued }) + }) + }); + let server = McpServer::new_with_registered_test_context(context, Vec::new()) + .await + .unwrap(); + for paths in [json!(["src/lib.rs"]), json!([])] { + let event = json!({"agent":"claude", "event":"postToolUse", "rel_paths":paths, "cwd":project.path()}); + let outcome = server + .handle_hook_event_notification(Some(&event), &mut HookProjectRouteCache::default()) + .await; + assert_eq!(outcome.status, HostAdmissionStatus::Committed); + } + assert_eq!(effects.load(Ordering::SeqCst), 1); + let activity = code_index_activity(&server).await; + assert_eq!(activity.len(), 1); + assert_eq!(activity[0].pulse.detail, None); + server.shutdown().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn retained_file_edits_reconcile_branch_drift_without_blocking_the_source() { + let (cg, project, authority) = init_indexed_repo().await; + let broker_root = TempDir::new().unwrap(); + let broker = default_spool_broker(broker_root.path()); + let owner = registered_runtime(&authority) + .project_registry_context_by_alias(project.path()) + .await + .unwrap() + .unwrap(); + let effects = Arc::new(AtomicUsize::new(0)); + let available = Arc::new(AtomicBool::new(false)); + let root = project.path().canonicalize().unwrap(); + let mut context = registered_context(cg, &authority); + context.host_admission_broker = Some(Arc::clone(&broker)); + context = context.with_code_index_hook_sink({ + let effects = Arc::clone(&effects); + let available = Arc::clone(&available); + let root = root.clone(); + Arc::new(move |effect_root, paths| { + assert_eq!(effect_root, root); + assert_eq!(paths, ["src/lib.rs"]); + if !available.load(Ordering::SeqCst) { + return Box::pin(async { super::CodeIndexDemandAdmissionV1::RefusedByPolicy }); + } + effects.fetch_add(1, Ordering::SeqCst); + Box::pin(async { super::CodeIndexDemandAdmissionV1::Queued }) + }) + }); + let server = McpServer::new_with_registered_test_context(context, Vec::new()) + .await + .unwrap(); + let mut seqs = Vec::new(); + for branch in ["main", "next"] { + if branch == "next" { + git(&root, &["checkout", "-q", "-b", branch]); + } + let (_, scope) = tracedecay_mcp::scope::resolve_query_scope(&owner, &root).unwrap(); + let plan = tracedecay_mcp::hook_events::HookEventPlan::SyncFilesAt { + root: root.clone(), + scope, + agent: HostIntegrationIdV1::Claude, + event_kind: tracedecay_mcp::hook_events::HookEventKind::FileEdit, + had_command: false, + route: Some(HookRouteMetadata { + session_id: Some("AKIASYNTHETICCANARY3".to_owned()), + thread_id: Some("branch-session".to_owned()), + cwd: Some(root.clone()), + worktree: Some(root.clone()), + branch: Some(branch.to_owned()), + }), + rel_paths: vec!["src/lib.rs".to_owned()], + }; + let payload = tracedecay_mcp::hook_events::encode_durable_hook_event_plan(&plan).unwrap(); + seqs.push(broker.admit("same-source", &payload).await.unwrap().seq); + } + assert_eq!(broker.pending_replay_count().await.unwrap(), 2); + assert_eq!( + server.replay_host_admission(None).await.status, + HostAdmissionStatus::Degraded + ); + assert_eq!(broker.pending_replay_count().await.unwrap(), 2); + assert_eq!(effects.load(Ordering::SeqCst), 0); + assert!(code_index_activity(&server).await.is_empty()); + available.store(true, Ordering::SeqCst); + assert_eq!( + server.replay_host_admission(None).await, + HostAdmissionOutcome::accepted_for_replay() + ); + assert_eq!(broker.pending_replay_count().await.unwrap(), 0); + assert_eq!(effects.load(Ordering::SeqCst), 2); + assert_eq!(code_index_activity(&server).await.len(), 2); + server.replay_host_admission(None).await; + assert_eq!(effects.load(Ordering::SeqCst), 2); + server.ledger_writes_settled().await; + let runtime = server.host_admission_test_runtime_for_test().unwrap(); + let rows = runtime + .query_profile_analytics_events_for_test(&tracedecay_global_db::AnalyticsEventQuery { + provider: Some("daemon_hook".to_owned()), + project_id: None, + session_id: None, + event_kind: Some("hook_route".to_owned()), + since: None, + until: None, + before_id: None, + limit: 16, + }) + .await + .unwrap(); + assert_eq!(rows.len(), 2); + let mut observed_seqs = rows + .iter() + .map(|row| { + let metadata = row.metadata_json.as_deref().unwrap(); + assert!(!metadata.contains("AKIASYNTHETICCANARY3")); + assert_ne!(row.session_id.as_deref(), Some("AKIASYNTHETICCANARY3")); + serde_json::from_str::(metadata).unwrap()["admission_seq"] + .as_u64() + .unwrap() + }) + .collect::>(); + observed_seqs.sort_unstable(); + assert_eq!(observed_seqs, seqs); + let spans = runtime + .git_sessions_for_for_test( + &SessionsForQuery { + git_ref: GitRefFilter::Branch("next".to_owned()), + since: None, + until: None, + limit: 16, + }, + CommitRelationFilter::Produced, + ) + .await + .unwrap(); + assert_eq!(spans.len(), 1); server.shutdown().await; } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn retained_file_edits_replay_once_on_each_exact_linked_worktree() { let (cg, project, authority) = init_indexed_repo().await; + let mut live_activity = event_lane::subscribe().unwrap(); let spool = TempDir::new().unwrap(); let broker = default_spool_broker(spool.path()); let effects = Arc::new(Mutex::new(Vec::new())); @@ -283,6 +479,8 @@ async fn retained_file_edits_replay_once_on_each_exact_linked_worktree() { root: root.clone(), scope, agent: HostIntegrationIdV1::Claude, + event_kind: tracedecay_mcp::hook_events::HookEventKind::FileEdit, + had_command: false, route: Some(HookRouteMetadata { session_id: Some("AKIASYNTHETICCANARY3".to_owned()), thread_id: Some(name.to_owned()), @@ -324,6 +522,22 @@ async fn retained_file_edits_replay_once_on_each_exact_linked_worktree() { assert_eq!(drained, HostAdmissionOutcome::accepted_for_replay()); assert_eq!(broker.pending_replay_count().await.unwrap(), 0); server.replay_host_admission(None).await; + let mut activity_roots = Vec::new(); + while let Ok(record) = live_activity.try_recv() { + if record.pulse.family == ActivityFamilyV1::CodeIndex + && roots.contains(&record.pulse.project_root) + { + assert_eq!( + record.pulse.project_id.as_deref(), + Some(owner.project.project_id.as_str()) + ); + activity_roots.push(record.pulse.project_root); + } + } + activity_roots.sort(); + let mut expected_roots = roots.clone(); + expected_roots.sort(); + assert_eq!(activity_roots, expected_roots); assert_eq!( *effects.lock().unwrap(), roots @@ -356,7 +570,7 @@ async fn retained_file_edits_replay_once_on_each_exact_linked_worktree() { }; *root = roots[1].clone(); let outcome = server - .run_hook_event_plan(server.cg().await, project.path(), plans.remove(0)) + .run_hook_event_plan(server.cg().await, project.path(), plans.remove(0), None) .await; assert_eq!(outcome.reason_code, Some("project_route_stale")); assert_eq!(effects.lock().unwrap().len(), 3); @@ -388,7 +602,7 @@ async fn hook_watch_policy_refusal_is_not_scheduler_unavailable() { }, ] { let outcome = server - .run_hook_event_plan(Arc::clone(&cg), project.path(), plan) + .run_hook_event_plan(Arc::clone(&cg), project.path(), plan, None) .await; assert_eq!(outcome.status, HostAdmissionStatus::Degraded); assert_eq!(outcome.reason_code, Some("linked_worktree_disabled")); diff --git a/crates/tracedecay/src/mcp/server/requests.rs b/crates/tracedecay/src/mcp/server/requests.rs index 3445da757d..2b3d74530e 100644 --- a/crates/tracedecay/src/mcp/server/requests.rs +++ b/crates/tracedecay/src/mcp/server/requests.rs @@ -342,29 +342,7 @@ impl McpServer { // survives the envelope move into `tools/call` preparation below. let method_name = request.method(); if matches!(method, McpMethod::HookEvent) { - let Some(id) = request.cloned_id() else { - Box::pin(self.handle_hook_event_notification( - request.raw_notification_params(), - &mut connection.route_cache, - )) - .await; - return None; - }; - let admitted = Box::pin(self.admit_hook_event_notification( - request.raw_notification_params(), - &mut connection.route_cache, - false, - )) - .await; - return Some(match admitted { - Ok(_) => JsonRpcResponse::success(id, json!({})), - Err(outcome) => JsonRpcResponse::error_with_data( - id, - ErrorCode::InternalError, - "hook event was not durably admitted".to_owned(), - Some(json!(outcome)), - ), - }); + return Box::pin(self.dispatch_hook_event(request, &mut connection.route_cache)).await; } if matches!(method, McpMethod::Cancelled) { self.record_request_accounting(method_name, false); @@ -454,6 +432,35 @@ impl McpServer { result } + async fn dispatch_hook_event( + &self, + request: McpDispatchRequest<'_>, + route_cache: &mut HookProjectRouteCache, + ) -> Option { + let Some(id) = request.cloned_id() else { + Box::pin( + self.handle_hook_event_notification(request.raw_notification_params(), route_cache), + ) + .await; + return None; + }; + let admitted = Box::pin(self.admit_hook_event_notification( + request.raw_notification_params(), + route_cache, + false, + )) + .await; + Some(match admitted { + Ok(_) => JsonRpcResponse::success(id, json!({})), + Err(outcome) => JsonRpcResponse::error_with_data( + id, + ErrorCode::InternalError, + "hook event was not durably admitted".to_owned(), + Some(json!(outcome)), + ), + }) + } + pub(crate) async fn handle_hook_event_notification( &self, params: Option<&Value>, @@ -579,12 +586,15 @@ impl McpServer { root: root.clone(), scope: hook_route.scope.clone(), agent: event.agent, + event_kind: event.kind, + had_command: event.had_command, route: event.route.clone(), rel_paths, } } plan => plan, }; + let routed_file_edit = matches!(plan, HookEventPlan::SyncFilesAt { .. }); let Ok(payload) = hook_events::encode_durable_hook_event_plan(&plan) else { let outcome = HostAdmissionOutcome::degraded("invalid_host_event_plan"); Self::report_host_admission_outcome(&outcome); @@ -614,7 +624,7 @@ impl McpServer { // Request observations follow durable admission. Notifications preserve // their settled-effect accounting; these side writes never change the // admission or replay outcome. - if !replay_inline || outcome.status.commits_replay_record() { + if !routed_file_edit && (!replay_inline || outcome.status.commits_replay_record()) { if let Some(wake) = &dispatch_server.project_session_refresh_wake { wake.wake(); } From 68306ef03a20e3787106e7d37f14f9eddfb8eb93 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Mon, 19 Oct 2026 03:40:33 +0000 Subject: [PATCH 64/89] fix(hooks): fence source currency until routed edits reach indexing --- .../src/code_index_scheduler.rs | 20 +++ .../src/code_index_scheduler/reconcile.rs | 27 +++ .../src/code_index_scheduler/registry.rs | 6 + .../code_index_scheduler/registry/mount.rs | 11 +- .../registry/owner_signals.rs | 69 +++++++- .../registry/serving_reads.rs | 52 ++++-- .../src/broker_group_commit_test.rs | 90 ++++++++++ crates/tracedecay-host-admission/src/lib.rs | 80 +++++++-- .../tracedecay-host-admission/src/runtime.rs | 14 ++ crates/tracedecay/src/daemon/branch_add.rs | 7 +- ...de_index_runtime_graph_activation_tests.rs | 41 ++++- crates/tracedecay/src/daemon/core_hooks.rs | 14 +- .../tracedecay/src/daemon/invocation_state.rs | 6 +- crates/tracedecay/src/daemon/pr_autotrack.rs | 89 ++++++++-- .../src/daemon/pr_autotrack/runtime.rs | 4 +- .../src/daemon/pr_autotrack/tests.rs | 154 +++++++++++++--- .../src/daemon/project_composition.rs | 18 +- .../code_index_activation.rs | 4 + .../hook_source_pending.rs | 132 ++++++++++++++ .../hook_source_pending/tests.rs | 166 ++++++++++++++++++ .../ignored_dependency_admission_tests.rs | 6 +- crates/tracedecay/src/daemon/tests.rs | 2 +- .../tracedecay/src/daemon/tests/rmcp_route.rs | 3 + .../tests/rmcp_route/host_index_handoff.rs | 152 ++++++++++++++++ .../code_index_ignored_dependencies_test.rs | 5 +- .../branch_publication_tests.rs | 4 +- .../tests/daemon_suite/code_index_journey.rs | 6 +- 27 files changed, 1074 insertions(+), 108 deletions(-) create mode 100644 crates/tracedecay/src/daemon/project_composition/hook_source_pending.rs create mode 100644 crates/tracedecay/src/daemon/project_composition/hook_source_pending/tests.rs create mode 100644 crates/tracedecay/src/daemon/tests/rmcp_route/host_index_handoff.rs diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs index 401d14a393..05db158734 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs @@ -29,6 +29,26 @@ use tracedecay_contracts::now_micros; const DEFAULT_STALENESS_THRESHOLD: Duration = Duration::from_secs(30); pub use tracedecay_code_index_retention::code_index_generations::scoped_code_index_store_root; +/// Route-bound input already durably admitted outside the index scheduler. +/// The reader borrows its owning broker; progress only wakes callers to reread +/// that authority. It neither owns work nor grants source currency. +pub trait CodeIndexSourcePendingReaderV1: Send + Sync { + fn read_pending( + &self, + ) -> Result; + fn revision(&self) -> u64; + fn changed(&self, observed: u64) -> CodeIndexSourcePendingChangedFutureV1<'_>; +} + +pub type CodeIndexSourcePendingChangedFutureV1<'a> = + std::pin::Pin + Send + 'a>>; + +/// Source inputs installed together before a worktree becomes visible. +pub struct CodeIndexSourceInputsV1 { + pub path_policy: tracedecay_domain::IndexPathPolicyV1, + pub pending_admission: Option>, +} + /// How the scheduler is hinted about changes. /// /// `TraceDecay`'s edits are agent-driven, so the daemon already learns about diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs index d603b25b81..10ca1218db 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs @@ -530,6 +530,7 @@ pub(crate) struct SourceFreshnessFenceV1 { source_epoch: Arc, /// Held only by source sweeps, never across a scheduler pass. sweep_cache: Arc>, + pending_source: Option>, } #[derive(Clone)] @@ -567,9 +568,25 @@ impl SourceFreshnessFenceV1 { last_reconciled_at_micros: Arc::new(AtomicI64::new(0)), source_epoch, sweep_cache: Arc::new(Mutex::new(SourceSweepCacheV1::new(path_policy))), + pending_source: None, } } + pub(super) fn pending_source_reader( + &self, + ) -> Option> { + self.pending_source.clone() + } + + pub(super) fn read_pending_source( + &self, + ) -> Result + { + self.pending_source + .as_ref() + .map_or(Ok(false), |reader| reader.read_pending()) + } + fn snapshot(&self) -> SourceFreshnessFenceStateV1 { self.state .lock() @@ -669,6 +686,9 @@ impl SourceFreshnessFenceV1 { project_root: &Path, shutting_down: &AtomicBool, ) -> bool { + if !self.read_pending_source().is_ok_and(|pending| !pending) { + return false; + } let state = self.snapshot(); self.proof_is_unmoved(&state, project_root, shutting_down) } @@ -1400,6 +1420,13 @@ impl CodeIndexWorktreeSchedulerV1 { Arc::clone(&self.freshness_fence.last_reconciled_at_micros) } + pub(super) fn bind_pending_source_reader( + &mut self, + reader: Option>, + ) { + self.freshness_fence.pending_source = reader; + } + pub(crate) fn freshness_fence(&self) -> SourceFreshnessFenceV1 { self.freshness_fence.clone() } diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry.rs index 1d254e46e2..cf07902d1f 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry.rs @@ -4156,6 +4156,12 @@ impl CodeIndexSchedulerRegistryV1 { Arc::clone(&worktree.shutting_down), ) }; + let admission_pending = source_freshness.read_pending_source().map_err(|error| { + CodeIndexSchedulerErrorV1::Identity(format!("source admission read failed: {error:?}")) + })?; + if admission_pending { + return Ok(None); + } let known = known.cloned(); let metadata = native_runtime::spawn_blocking(move || { owner.active_publication_text_metadata(known.as_ref()) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/mount.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/mount.rs index 14f33b6c65..9306dc6508 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/mount.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/mount.rs @@ -353,7 +353,7 @@ impl CodeIndexSchedulerRegistryV1 { graph_runtime: Arc, project_database: Arc, graph_activation_policy: CodeGraphActivationPolicyV1, - path_policy: IndexPathPolicyV1, + source_inputs: super::super::CodeIndexSourceInputsV1, ) -> Result { self.mount_worktree_inner( project_id, @@ -365,7 +365,8 @@ impl CodeIndexSchedulerRegistryV1 { policy: Arc::new(AtomicBool::new(graph_activation_policy.is_enabled())), seated: Arc::default(), }, - path_policy, + source_inputs.path_policy, + source_inputs.pending_admission, ) .await } @@ -386,6 +387,7 @@ impl CodeIndexSchedulerRegistryV1 { policy: Arc::new(AtomicBool::new(true)), }, crate::config::registry_default_index_path_policy(), + None, ) .await } @@ -406,6 +408,7 @@ impl CodeIndexSchedulerRegistryV1 { policy: Arc::new(AtomicBool::new(policy.is_enabled())), }, crate::config::registry_default_index_path_policy(), + None, ) .await } @@ -540,6 +543,7 @@ impl CodeIndexSchedulerRegistryV1 { store_root: PathBuf, graph_activation: CodeGraphActivationAuthorityV1, path_policy: IndexPathPolicyV1, + pending_source: Option>, ) -> Result { let project_root = canonical_existing_identity(project_root)?; #[cfg(test)] @@ -606,7 +610,7 @@ impl CodeIndexSchedulerRegistryV1 { let progress_producer_incarnation = self.mint_progress_producer_incarnation()?; let mounted_path_policy = path_policy.clone(); let mut cold_mount_reservation = cold_mount_reservation; - let opened = loop { + let mut opened = loop { let open_project_id = project_id.clone(); let open_project_root = project_root.clone(); let open_store_root = scoped_store_root.clone(); @@ -656,6 +660,7 @@ impl CodeIndexSchedulerRegistryV1 { opened => break opened?, } }; + opened.bind_pending_source_reader(pending_source); let repository_id = opened.identity().repository_id().clone(); let worktree_id = opened.identity().worktree_id().clone(); let reconcile_in_progress = opened.reconcile_in_progress(); diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/owner_signals.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/owner_signals.rs index 70d66658c3..6bc68025ea 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/owner_signals.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/owner_signals.rs @@ -89,11 +89,20 @@ pub struct CodeIndexOwnerSignalsV1 { publications: tokio::sync::broadcast::Receiver, serving: Option>, activity: Option, + pending_source: Option<(Arc, u64)>, } impl CodeIndexOwnerSignalsV1 { pub async fn subscribe(registry: &CodeIndexSchedulerRegistryV1, path: &Path) -> Self { + let pending_source = registry + .pending_source_reader_for_root(path) + .await + .map(|reader| { + let observed = reader.revision(); + (reader, observed) + }); Self { + pending_source, registry: registry.clone(), path: canonical_existing_identity(path).unwrap_or_else(|_| path.to_path_buf()), seats: registry.subscribe_serving_seats(), @@ -141,7 +150,31 @@ impl CodeIndexOwnerSignalsV1 { return Ok(()); } } + if self.pending_source.is_none() { + self.pending_source = self + .registry + .pending_source_reader_for_root(&self.path) + .await + .map(|reader| { + let observed = reader.revision(); + (reader, observed) + }); + if self.pending_source.is_some() { + return Ok(()); + } + } + let pending_source = self.pending_source.clone(); tokio::select! { + () = async move { + match pending_source { + Some((reader, observed)) => reader.changed(observed).await, + None => std::future::pending().await, + } + } => { + if let Some((reader, observed)) = self.pending_source.as_mut() { + *observed = reader.revision(); + } + } changed = self.seats.changed() => changed.map_err(|_| CodeIndexOwnerSignalsClosedV1)?, changed = self.root_mounted.changed() => { changed.map_err(|_| CodeIndexOwnerSignalsClosedV1)?; @@ -233,6 +266,19 @@ impl CodeIndexOwnerSignalsV1 { } impl CodeIndexSchedulerRegistryV1 { + async fn pending_source_reader_for_root( + &self, + root: &Path, + ) -> Option> { + let root = canonical_existing_identity(root).ok()?; + self.mounted + .lock() + .await + .get(&root)? + .source_freshness + .pending_source_reader() + } + /// The one wait for a worktree's seat, whatever the seat is: a decoded /// generation, a current text owner, text serving under a query /// authority, or a readiness target. @@ -453,7 +499,10 @@ impl CodeIndexSchedulerRegistryV1 { let deadline = Instant::now() + budget; if target != CodeIndexReadinessTargetV1::Fresh && let Some(reading) = self - .dashboard_freshness_read(project_root) + .dashboard_freshness_read_with_source_admission( + project_root, + target != CodeIndexReadinessTargetV1::GraphReady, + ) .await? .filter(|freshness| freshness.readiness(target) == CodeIndexReadinessV1::Reached) && (target == CodeIndexReadinessTargetV1::GraphReady @@ -489,7 +538,10 @@ impl CodeIndexSchedulerRegistryV1 { Err(_) => { return Ok(CodeIndexReadinessWaitReadV1::TimedOut { last: self - .dashboard_freshness_read(project_root) + .dashboard_freshness_read_with_source_admission( + project_root, + target != CodeIndexReadinessTargetV1::GraphReady, + ) .await? .map(Box::new), }); @@ -497,7 +549,13 @@ impl CodeIndexSchedulerRegistryV1 { } let waited = self .wait_for_seat(project_root, deadline, |owner_settled| async move { - let Some(freshness) = self.dashboard_freshness_read(project_root).await? else { + let Some(freshness) = self + .dashboard_freshness_read_with_source_admission( + project_root, + target != CodeIndexReadinessTargetV1::GraphReady, + ) + .await? + else { return Ok(None); }; Ok(match freshness.readiness(target) { @@ -530,7 +588,10 @@ impl CodeIndexSchedulerRegistryV1 { }, CodeIndexSeatWaitV1::Deadline => CodeIndexReadinessWaitReadV1::TimedOut { last: self - .dashboard_freshness_read(project_root) + .dashboard_freshness_read_with_source_admission( + project_root, + target != CodeIndexReadinessTargetV1::GraphReady, + ) .await? .map(Box::new), }, diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/serving_reads.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/serving_reads.rs index 5627618270..d2ddd3076c 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/serving_reads.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/serving_reads.rs @@ -261,6 +261,18 @@ impl CodeIndexSchedulerRegistryV1 { ) -> Result< Option, tracedecay_contracts::code_index_freshness::CodeIndexFreshnessReadFailureV1, + > { + self.dashboard_freshness_read_with_source_admission(project_root, true) + .await + } + + pub(super) async fn dashboard_freshness_read_with_source_admission( + &self, + project_root: &Path, + include_source_admission: bool, + ) -> Result< + Option, + tracedecay_contracts::code_index_freshness::CodeIndexFreshnessReadFailureV1, > { let canonical_root = match canonical_existing_identity(project_root) { Ok(root) => root, @@ -301,6 +313,9 @@ impl CodeIndexSchedulerRegistryV1 { ) }; native_runtime::spawn_blocking(move || { + // Sample the broker before any generation/fence value. A record + // disappears only after the scheduler accepted the source hint. + let admission_pending = include_source_admission && source_freshness.read_pending_source()?; let mut progress = { let _span = tracing::trace_span!("daemon.code_index.dashboard.progress").entered(); build_progress @@ -313,7 +328,7 @@ impl CodeIndexSchedulerRegistryV1 { || pending_wake.lock() .micros != 0; - let source_change_pending = source_freshness.source_change_pending(); + let source_change_pending = admission_pending || source_freshness.source_change_pending(); let parked = convergence_park .read() .unwrap_or_else(std::sync::PoisonError::into_inner) @@ -396,7 +411,7 @@ impl CodeIndexSchedulerRegistryV1 { parked: parked.as_ref(), // The scheduler lock is held by the pass this read // could not join, so verification is unknown here. - source_verified: None, + source_verified: admission_pending.then_some(false), hook_hint_count, sources_omitted: identity.omitted_sources.is_some(), }, @@ -418,7 +433,7 @@ impl CodeIndexSchedulerRegistryV1 { micros => Some(micros), }; project_graph_publication_phase(&mut progress, &code_graph_serving); - return tracedecay_contracts::code_index_freshness::CodeIndexWorktreeFreshnessV1 { + return Ok(tracedecay_contracts::code_index_freshness::CodeIndexWorktreeFreshnessV1 { worktree_root: canonical_root.display().to_string(), code_graph_serving, clone_index: Some(clone_index), @@ -432,10 +447,10 @@ impl CodeIndexSchedulerRegistryV1 { parked, generation_recovery, ..identity - }; + }); } }; - let verified = scheduler.verified_against_source(); + let verified = !admission_pending && scheduler.verified_against_source(); let latest = serving_generation .read() .unwrap_or_else(std::sync::PoisonError::into_inner) @@ -506,7 +521,7 @@ impl CodeIndexSchedulerRegistryV1 { ) }); project_graph_publication_phase(&mut progress, &code_graph_serving); - tracedecay_contracts::code_index_freshness::CodeIndexWorktreeFreshnessV1 { + Ok(tracedecay_contracts::code_index_freshness::CodeIndexWorktreeFreshnessV1 { worktree_root: canonical_root.display().to_string(), code_graph_serving, clone_index: Some(clone_index), @@ -520,13 +535,13 @@ impl CodeIndexSchedulerRegistryV1 { parked, generation_recovery, ..identity - } + }) }) .await - .map(Some) .map_err(|_| { tracedecay_contracts::code_index_freshness::CodeIndexFreshnessReadFailureV1::ReadFailed - }) + })? + .map(Some) } /// The deterministic contract violation currently parking background @@ -765,6 +780,7 @@ impl CodeIndexSchedulerRegistryV1 { }; let freshness_root = project_root.clone(); let (request_reconcile, latest) = native_runtime::spawn_blocking(move || { + let source_ready = source_freshness.ready_without_stat(&freshness_root, &shutting_down); let serving = serving_generation .read() .unwrap_or_else(std::sync::PoisonError::into_inner); @@ -777,8 +793,7 @@ impl CodeIndexSchedulerRegistryV1 { witness.generation_id == serving.generation().manifest().generation_id }) }); - let source_ready = witness_matches_seat - && source_freshness.ready_without_stat(&freshness_root, &shutting_down); + let source_ready = witness_matches_seat && source_ready; let latest = source_ready.then(|| serving.clone()).flatten(); if latest.is_none() { tracing::info!( @@ -1103,6 +1118,7 @@ impl CodeIndexSchedulerRegistryV1 { // before entering the exact freshness probe: probing an unseated slot // cannot produce a decoded owner, and on an initial lightweight mount // it would turn an unverified fence into a fabricated overflow wake. + let source_ready = source_freshness.ready_without_stat(project_root, &shutting_down); let serving = serving_generation .read() .unwrap_or_else(std::sync::PoisonError::into_inner) @@ -1123,7 +1139,7 @@ impl CodeIndexSchedulerRegistryV1 { Self::note_wake_if_idle(&pending_wake, &wake, wake_trigger); return None; } - if !source_freshness.ready_without_stat(project_root, &shutting_down) { + if !source_ready { Self::note_wake_if_idle(&pending_wake, &wake, wake_trigger); return None; } @@ -1525,6 +1541,7 @@ impl CodeIndexSchedulerRegistryV1 { Arc::clone(&worktree.shutting_down), ) }; + let admission_pending = source_freshness.read_pending_source().ok()?; let scope = scope.clone(); let latest = text_generation .read() @@ -1534,11 +1551,12 @@ impl CodeIndexSchedulerRegistryV1 { (!require_serving_ready || latest.query_owners_are_ready()) && text_matches_scope_identity(latest, &scope) })?; - let current = source_freshness.serves_verified_source( - &latest.metadata().snapshot().content_identity, - &root, - &shutting_down, - ); + let current = !admission_pending + && source_freshness.serves_verified_source( + &latest.metadata().snapshot().content_identity, + &root, + &shutting_down, + ); if !current { // Attribution, not admission. A read that cannot verify the owner // while a pass already owns the worktree is a follow-up to that diff --git a/crates/tracedecay-host-admission/src/broker_group_commit_test.rs b/crates/tracedecay-host-admission/src/broker_group_commit_test.rs index 7bb7b031fb..8a5be5a4ca 100644 --- a/crates/tracedecay-host-admission/src/broker_group_commit_test.rs +++ b/crates/tracedecay-host-admission/src/broker_group_commit_test.rs @@ -173,3 +173,93 @@ fn replay_requests_coalesce_into_one_pending_wake() { "a delivered request must leave no stored wake behind" ); } + +#[asupersync::test] +async fn pending_reads_include_leased_work_and_exclude_completed_records() { + let spool = tempfile::tempdir().unwrap(); + let broker = open_broker(spool.path()); + broker.admit("first", b"first").await.unwrap(); + broker.admit("second", b"second").await.unwrap(); + let replay = broker.begin_replay().await.unwrap(); + let first = replay.lease_next().await.unwrap().unwrap(); + let second = replay.lease_next().await.unwrap().unwrap(); + assert!( + broker + .pending_replay_matches(|bytes| Ok(bytes == b"first")) + .unwrap() + ); + assert!( + broker + .pending_replay_matches(|bytes| Ok(bytes == b"second")) + .unwrap() + ); + assert_eq!(replay.commit(second.seq).await.unwrap(), 0); + assert!( + !broker + .pending_replay_matches(|bytes| Ok(bytes == b"second")) + .unwrap() + ); + assert!( + broker + .pending_replay_matches(|bytes| Ok(bytes == b"first")) + .unwrap() + ); + assert_eq!(replay.commit(first.seq).await.unwrap(), 2); + assert!(!broker.pending_replay_matches(|_| Ok(true)).unwrap()); +} + +#[asupersync::test] +async fn progress_is_non_consuming_for_waiters_before_and_after_a_transition() { + let spool = tempfile::tempdir().unwrap(); + let broker = open_broker(spool.path()); + let observed = broker.progress_revision(); + let mut first = pin!(broker.wait_for_progress(observed)); + let mut second = pin!(broker.wait_for_progress(observed)); + let mut context = Context::from_waker(Waker::noop()); + assert!(first.as_mut().poll(&mut context).is_pending()); + assert!(second.as_mut().poll(&mut context).is_pending()); + broker.admit("source", b"event").await.unwrap(); + assert!(first.as_mut().poll(&mut context).is_ready()); + assert!(second.as_mut().poll(&mut context).is_ready()); + assert!( + pin!(broker.wait_for_progress(observed)) + .poll(&mut context) + .is_ready() + ); + assert!( + pin!(broker.wait_for_replay_request()) + .poll(&mut context) + .is_ready() + ); + let admitted = broker.progress_revision(); + let replay = broker.begin_replay().await.unwrap(); + let record = replay.lease_next().await.unwrap().unwrap(); + replay.commit(record.seq).await.unwrap(); + assert!( + pin!(broker.wait_for_progress(admitted)) + .poll(&mut context) + .is_ready() + ); + assert!(!broker.pending_replay_matches(|_| Ok(true)).unwrap()); +} + +#[test] +fn unavailable_pending_reads_are_typed_failures() { + let spool = tempfile::tempdir().unwrap(); + let broker = open_broker(spool.path()); + let guard = broker.runtime.lock().unwrap(); + assert_eq!( + broker.pending_replay_matches(|_| Ok(false)), + Err(spool_runtime_unavailable()) + ); + drop(guard); + let runtime = Arc::clone(&broker.runtime); + let _ = std::panic::catch_unwind(|| { + let _guard = runtime.lock().unwrap(); + panic!("poison isolated test runtime"); + }); + assert_eq!( + broker.pending_replay_matches(|_| Ok(false)), + Err(spool_runtime_unavailable()) + ); +} diff --git a/crates/tracedecay-host-admission/src/lib.rs b/crates/tracedecay-host-admission/src/lib.rs index 497800c23d..6bb42c1477 100644 --- a/crates/tracedecay-host-admission/src/lib.rs +++ b/crates/tracedecay-host-admission/src/lib.rs @@ -5,7 +5,7 @@ //! capture path, and depends on remaining usecases for those edges. use std::path::PathBuf; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Mutex, mpsc}; use tracedecay_domain::{ @@ -71,6 +71,8 @@ pub struct HostAdmissionBroker { /// Coalesced wake for daemon-owned profile/project replay workers: at most /// one request is pending however many admissions raise it. replay_requested: AtomicBool, + progress_revision: Arc, + progress_changed: Arc, replay_wake: asupersync::sync::Notify, } @@ -90,20 +92,22 @@ fn spool_runtime_unavailable() -> HostAdmissionOutcome { } /// Append every queued admission as one batch and answer each admitter. -fn commit_queued_appends(runtime: &mut HostAdmissionRuntime, queued: Vec) { +fn commit_queued_appends(runtime: &mut HostAdmissionRuntime, queued: Vec) -> bool { if queued.is_empty() { - return; + return false; } let items = queued .iter() .map(|append| (append.source.as_str(), append.payload.as_slice())) .collect::>(); let admitted = runtime.admit_batch(&items); + let changed = admitted.iter().any(Result::is_ok); for (append, admitted) in queued.into_iter().zip(admitted) { // A dropped receiver is a cancelled admitter; its durable record // still replays through the worker. let _ = append.admitted.send(admitted); } + changed } impl HostAdmissionBroker { @@ -113,6 +117,8 @@ impl HostAdmissionBroker { appends: Arc::new(Mutex::new(Vec::new())), replay: futures_util::lock::Mutex::new(()), replay_requested: AtomicBool::new(false), + progress_revision: Arc::new(AtomicU64::new(0)), + progress_changed: Arc::new(asupersync::sync::Notify::new()), replay_wake: asupersync::sync::Notify::new(), } } @@ -122,11 +128,33 @@ impl HostAdmissionBroker { where T: Send + 'static, F: FnOnce(&mut HostAdmissionRuntime) -> Result + Send + 'static, + { + self.with_runtime_change(move |runtime| operation(runtime).map(|value| (value, false))) + .await + } + + async fn with_runtime_change(&self, operation: F) -> Result + where + T: Send + 'static, + F: FnOnce(&mut HostAdmissionRuntime) -> Result<(T, bool), HostAdmissionOutcome> + + Send + + 'static, { let runtime = Arc::clone(&self.runtime); + let revision = Arc::clone(&self.progress_revision); + let changed = Arc::clone(&self.progress_changed); tracedecay_runtime_core::native_runtime::spawn_blocking(move || { - let mut runtime = runtime.lock().map_err(|_| spool_runtime_unavailable())?; - operation(&mut runtime) + let (value, progressed) = { + let mut runtime = runtime.lock().map_err(|_| spool_runtime_unavailable())?; + operation(&mut runtime)? + }; + // The canonical state is readable before waiters are woken. This + // runs even when the admitting/replaying future has been dropped. + if progressed { + revision.fetch_add(1, Ordering::Release); + changed.notify_waiters(); + } + Ok(value) }) .await .unwrap_or_else(|_| Err(spool_runtime_unavailable())) @@ -150,11 +178,11 @@ impl HostAdmissionBroker { admitted, }); let appends = Arc::clone(&self.appends); - self.with_runtime(move |runtime| { + self.with_runtime_change(move |runtime| { let queued = std::mem::take(&mut *appends.lock().map_err(|_| spool_runtime_unavailable())?); - commit_queued_appends(runtime, queued); - Ok(()) + let changed = commit_queued_appends(runtime, queued); + Ok(((), changed)) }) .await?; // This admission was queued before its own runtime turn, so that turn @@ -188,6 +216,30 @@ impl HostAdmissionBroker { .await; } + /// Capture before reading pending state. Each waiter owns its revision; + /// neither this subscription nor its wake consumes the replay worker's flag. + pub fn progress_revision(&self) -> u64 { + self.progress_revision.load(Ordering::Acquire) + } + + pub async fn wait_for_progress(&self, observed: u64) { + self.progress_changed + .wait_until(|| self.progress_revision() != observed) + .await; + } + + /// Inspect bounded canonical payloads without the replay lease or copies. + /// Contention and poison remain unavailable rather than empty pending work. + pub fn pending_replay_matches( + &self, + predicate: impl FnMut(&[u8]) -> Result, + ) -> Result { + self.runtime + .try_lock() + .map_err(|_| spool_runtime_unavailable())? + .pending_matches(predicate) + } + pub async fn pending_replay_count(&self) -> Result { self.with_runtime(|runtime| Ok(runtime.pending_count())) .await @@ -202,8 +254,10 @@ impl HostAdmissionBroker { #[tracing::instrument(name = "usecases.admission.begin_replay", level = "trace", skip_all)] pub async fn begin_replay(&self) -> Result, HostAdmissionOutcome> { let guard = self.replay.lock().await; - self.with_runtime(HostAdmissionRuntime::recover_leases) - .await?; + self.with_runtime_change(|runtime| { + runtime.recover_leases().map(|count| (count, count > 0)) + }) + .await?; Ok(HostAdmissionReplay { broker: self, _guard: guard, @@ -241,7 +295,7 @@ impl HostAdmissionReplay<'_> { #[tracing::instrument(name = "usecases.admission.replay.commit", level = "trace", skip_all)] pub async fn commit(&self, seq: u64) -> Result { self.broker - .with_runtime(move |runtime| runtime.commit(seq)) + .with_runtime_change(move |runtime| runtime.commit(seq).map(|count| (count, true))) .await } @@ -256,7 +310,9 @@ impl HostAdmissionReplay<'_> { reason: TerminalReason, ) -> Result { self.broker - .with_runtime(move |runtime| runtime.quarantine(seq, reason)) + .with_runtime_change(move |runtime| { + runtime.quarantine(seq, reason).map(|count| (count, true)) + }) .await } } diff --git a/crates/tracedecay-host-admission/src/runtime.rs b/crates/tracedecay-host-admission/src/runtime.rs index 24b25f4feb..77e2853594 100644 --- a/crates/tracedecay-host-admission/src/runtime.rs +++ b/crates/tracedecay-host-admission/src/runtime.rs @@ -288,6 +288,20 @@ impl HostAdmissionRuntime { } } + /// Inspect the canonical uncompleted records in place, including leased + /// records. A completed sequence can remain behind an older prefix. + pub(super) fn pending_matches( + &self, + mut predicate: impl FnMut(&[u8]) -> Result, + ) -> Result { + for record in self.spool.pending_records() { + if !self.completed.contains(&record.seq) && predicate(&record.payload)? { + return Ok(true); + } + } + Ok(false) + } + pub(super) fn pending_count(&self) -> usize { self.spool.pending_count() } diff --git a/crates/tracedecay/src/daemon/branch_add.rs b/crates/tracedecay/src/daemon/branch_add.rs index b4b75ba050..b1d5008ca3 100644 --- a/crates/tracedecay/src/daemon/branch_add.rs +++ b/crates/tracedecay/src/daemon/branch_add.rs @@ -162,6 +162,7 @@ async fn activate_and_track_manual_branch( let published_data_root = data_root.clone(); let published_schedulers = schedulers.clone(); let published_registries = administration.session_runtime_registries(); + let owned_administration = administration.clone(); administration .admit_manual_branch_publication(|cancellation, admitted| async move { @@ -195,7 +196,7 @@ async fn activate_and_track_manual_branch( graph, schedulers, branch.clone(), - data_root.clone(), + owned_administration, lifecycle, cancellation, ) @@ -352,12 +353,14 @@ pub(super) async fn activate_and_track_manual_branch_owned( graph: Arc, schedulers: CodeIndexSchedulerRegistryV1, branch: String, - data_root: std::path::PathBuf, + administration: StoreAdministration, lifecycle: ManualBranchLifecycleLeaseV1, cancellation: CancellationToken, ) -> Result { + let data_root = graph.store_layout().data_root.clone(); let publication = branch_publication_context(&graph)?; let activation = super::pr_autotrack::activate_manual_branch_head_with_lifecycle( + &administration, &project_root, &graph, Some(&schedulers), diff --git a/crates/tracedecay/src/daemon/code_index_runtime_graph_activation_tests.rs b/crates/tracedecay/src/daemon/code_index_runtime_graph_activation_tests.rs index be1502e22d..810dd44a8e 100644 --- a/crates/tracedecay/src/daemon/code_index_runtime_graph_activation_tests.rs +++ b/crates/tracedecay/src/daemon/code_index_runtime_graph_activation_tests.rs @@ -256,7 +256,10 @@ async fn failed_cold_mount_graph_replay_preserves_retained_text_generation() { graph_runtime.code_graph_seat_port(), read_only_project_database, CodeGraphActivationPolicyV1::Enabled, - tracedecay_code_index_runtime::registry_default_index_path_policy(), + tracedecay_code_index_runtime::code_index_scheduler::CodeIndexSourceInputsV1 { + path_policy: tracedecay_code_index_runtime::registry_default_index_path_policy(), + pending_admission: None, + }, ) .await .expect("mount retained generation"); @@ -564,7 +567,10 @@ async fn persistent_callers_cursor_keeps_generation_a_without_repointing_generat graph_runtime.code_graph_seat_port(), Arc::clone(&project_database), CodeGraphActivationPolicyV1::Enabled, - tracedecay_code_index_runtime::registry_default_index_path_policy(), + tracedecay_code_index_runtime::code_index_scheduler::CodeIndexSourceInputsV1 { + path_policy: tracedecay_code_index_runtime::registry_default_index_path_policy(), + pending_admission: None, + }, ) .await .expect("mount persistent generation A"); @@ -982,7 +988,10 @@ async fn restart_status_case(corrupt_graph: bool, dirty_before_restart: bool) { graph_runtime.code_graph_seat_port(), project_database, CodeGraphActivationPolicyV1::Enabled, - tracedecay_code_index_runtime::registry_default_index_path_policy(), + tracedecay_code_index_runtime::code_index_scheduler::CodeIndexSourceInputsV1 { + path_policy: tracedecay_code_index_runtime::registry_default_index_path_policy(), + pending_admission: None, + }, ) .await .expect("mount persistent graph generation"); @@ -1357,7 +1366,10 @@ async fn restart_seats_the_retained_graph_while_its_text_owner_still_projects() graph_runtime.code_graph_seat_port(), project_database, CodeGraphActivationPolicyV1::Enabled, - tracedecay_code_index_runtime::registry_default_index_path_policy(), + tracedecay_code_index_runtime::code_index_scheduler::CodeIndexSourceInputsV1 { + path_policy: tracedecay_code_index_runtime::registry_default_index_path_policy(), + pending_admission: None, + }, ) .await .expect("mount restarted retained generation"); @@ -1766,7 +1778,10 @@ async fn first_search_after_restart_serves_text_while_the_graph_head_recovers() graph_runtime.code_graph_seat_port(), project_database, CodeGraphActivationPolicyV1::Enabled, - tracedecay_code_index_runtime::registry_default_index_path_policy(), + tracedecay_code_index_runtime::code_index_scheduler::CodeIndexSourceInputsV1 { + path_policy: tracedecay_code_index_runtime::registry_default_index_path_policy(), + pending_admission: None, + }, ) .await .expect("mount restarted retained generation"); @@ -1937,7 +1952,10 @@ async fn first_index_serves_graph_reads_without_decoding_the_generation() { graph_runtime.code_graph_seat_port(), project_database, CodeGraphActivationPolicyV1::Enabled, - tracedecay_code_index_runtime::registry_default_index_path_policy(), + tracedecay_code_index_runtime::code_index_scheduler::CodeIndexSourceInputsV1 { + path_policy: tracedecay_code_index_runtime::registry_default_index_path_policy(), + pending_admission: None, + }, ) .await .expect("mount first index"); @@ -2240,7 +2258,11 @@ impl PersistentGraphMountV1 { graph_runtime.code_graph_seat_port(), project_database, CodeGraphActivationPolicyV1::Enabled, - tracedecay_code_index_runtime::registry_default_index_path_policy(), + tracedecay_code_index_runtime::code_index_scheduler::CodeIndexSourceInputsV1 { + path_policy: tracedecay_code_index_runtime::registry_default_index_path_policy( + ), + pending_admission: None, + }, ) .await .expect("mount worktree"); @@ -2870,7 +2892,10 @@ async fn ignored_dependency_admission_answers_once_the_successor_catalog_serves( graph_runtime.code_graph_seat_port(), project_database, CodeGraphActivationPolicyV1::Enabled, - tracedecay_code_index_runtime::registry_default_index_path_policy(), + tracedecay_code_index_runtime::code_index_scheduler::CodeIndexSourceInputsV1 { + path_policy: tracedecay_code_index_runtime::registry_default_index_path_policy(), + pending_admission: None, + }, ) .await .expect("mount admission fixture"); diff --git a/crates/tracedecay/src/daemon/core_hooks.rs b/crates/tracedecay/src/daemon/core_hooks.rs index 2352602630..b1994fe82a 100644 --- a/crates/tracedecay/src/daemon/core_hooks.rs +++ b/crates/tracedecay/src/daemon/core_hooks.rs @@ -133,6 +133,8 @@ async fn admit_registered_file_edit( root: root.clone(), scope, agent: event.agent, + event_kind: event.kind, + had_command: event.had_command, route: event.route.clone(), rel_paths: event.rel_paths.clone(), }) @@ -157,9 +159,9 @@ async fn admit_registered_file_edit( Ok(routed) } -/// Delivers `event` and resolves once the daemon acknowledges it, however -/// long admission takes. A host hook that must not block its agent bounds -/// this call itself. +/// Delivers `event` and resolves after the daemon commits durable admission. +/// Index effects may still be owed; source currency includes that canonical +/// pending work until scheduler acceptance. A host bounds this call itself. #[tracing::instrument(name = "daemon.engine.hooks.notify", level = "trace", skip_all)] pub async fn notify_hook_event( profile: &tracedecay_runtime_core::config::ProfileRoot, @@ -199,9 +201,9 @@ async fn notify_hook_event_to_connection( }; // A stateless request, not a notification: its own daemon connection has // no `initialize` session for a notification to ride. The daemon answers - // once it has admitted the event, the edited paths into the code-index - // queue included, so `Delivered` means a status read issued afterwards - // already sees the save. + // after the broker commits admission. `Delivered` does not wait for + // scheduler acceptance; subsequent source-currency reads account for the + // exact routed edit while the broker still owes that handoff. let mut request = JsonRpcRequest { jsonrpc: "2.0".to_string(), id: Some(serde_json::Value::from(1)), diff --git a/crates/tracedecay/src/daemon/invocation_state.rs b/crates/tracedecay/src/daemon/invocation_state.rs index dd344afe7a..0b5ac7eefc 100644 --- a/crates/tracedecay/src/daemon/invocation_state.rs +++ b/crates/tracedecay/src/daemon/invocation_state.rs @@ -367,6 +367,7 @@ impl DaemonInvocationState { index_paths: tracedecay_domain::IndexPathPolicyV1, graph_runtime: Arc, graph_publication_database: Arc, + pending_source: Arc, ) -> Result<()> { // Code-index identity is anchored on the project root's own git // repository (`IndexingIdentityV1::resolve` uses `git_open::open` on the @@ -396,7 +397,10 @@ impl DaemonInvocationState { code_index_scheduler::CodeGraphActivationPolicyV1::from_enabled( native_graph_activation, ), - index_paths, + code_index_scheduler::CodeIndexSourceInputsV1 { + path_policy: index_paths, + pending_admission: Some(pending_source), + }, ) .await .map_err(|error| TraceDecayError::Config { diff --git a/crates/tracedecay/src/daemon/pr_autotrack.rs b/crates/tracedecay/src/daemon/pr_autotrack.rs index 11d0791519..b0d2ad4d3f 100644 --- a/crates/tracedecay/src/daemon/pr_autotrack.rs +++ b/crates/tracedecay/src/daemon/pr_autotrack.rs @@ -67,7 +67,6 @@ async fn git_authority_available(repo_root: &Path) -> bool { .unwrap_or(false) } -#[cfg(test)] use super::branch_admin::StoreAdministration; mod runtime; @@ -79,10 +78,26 @@ struct PrStoreAdministration<'a> { schedulers: Option<&'a CodeIndexSchedulerRegistryV1>, graph: Option<&'a Arc>, command_control: &'a PrCommandControl, + stores: Option<&'a StoreAdministration>, } impl<'a> PrStoreAdministration<'a> { fn with_control( + stores: &'a StoreAdministration, + schedulers: &'a CodeIndexSchedulerRegistryV1, + graph: &'a Arc, + command_control: &'a PrCommandControl, + ) -> Self { + Self { + schedulers: Some(schedulers), + graph: Some(graph), + command_control, + stores: Some(stores), + } + } + + #[cfg(test)] + fn unhosted_with_control( schedulers: &'a CodeIndexSchedulerRegistryV1, graph: &'a Arc, command_control: &'a PrCommandControl, @@ -91,6 +106,7 @@ impl<'a> PrStoreAdministration<'a> { schedulers: Some(schedulers), graph: Some(graph), command_control, + stores: None, } } @@ -100,6 +116,7 @@ impl<'a> PrStoreAdministration<'a> { schedulers: None, graph: None, command_control: default_pr_command_control(), + stores: None, } } } @@ -138,26 +155,30 @@ pub(crate) async fn activate_manual_branch_head( schedulers: Option<&CodeIndexSchedulerRegistryV1>, branch: &str, ) -> std::result::Result { - if schedulers.is_none() { + let Some(schedulers) = schedulers else { return Err(ManualBranchActivationError::scheduler_unavailable( "code-index scheduler authority is unavailable for branch activation", )); - } + }; let lifecycle = acquire_manual_branch_lifecycle(&graph.store_layout().data_root, branch).await?; - activate_manual_branch_head_with_lifecycle( + activate_manual_branch_with_administration( repo_root, - graph, - schedulers, + &graph.store_layout().data_root, branch, + PrStoreAdministration::unhosted_with_control( + schedulers, + graph, + default_pr_command_control(), + ), &lifecycle, - default_pr_command_control(), ) .await } #[tracing::instrument(name = "daemon.pr_autotrack.activate", level = "trace", skip_all)] -pub(crate) async fn activate_manual_branch_head_with_lifecycle( +pub(in crate::daemon) async fn activate_manual_branch_head_with_lifecycle( + stores: &StoreAdministration, repo_root: &Path, graph: &Arc, schedulers: Option<&CodeIndexSchedulerRegistryV1>, @@ -165,17 +186,15 @@ pub(crate) async fn activate_manual_branch_head_with_lifecycle( lifecycle: &ManualBranchLifecycleLeaseV1, command_control: &PrCommandControl, ) -> std::result::Result { - if !lifecycle.matches_branch(branch) { - return Err(ManualBranchActivationError::activation_failed( - "manual branch lifecycle lease does not match requested branch", - )); - } let administration = match schedulers { - Some(schedulers) => PrStoreAdministration::with_control(schedulers, graph, command_control), + Some(schedulers) => { + PrStoreAdministration::with_control(stores, schedulers, graph, command_control) + } None => PrStoreAdministration { schedulers: None, graph: Some(graph), command_control, + stores: Some(stores), }, }; activate_manual_branch_with_administration( @@ -204,6 +223,11 @@ async fn activate_manual_branch_with_administration( administration: PrStoreAdministration<'_>, lifecycle: &ManualBranchLifecycleLeaseV1, ) -> std::result::Result { + if !lifecycle.matches_branch(branch) { + return Err(ManualBranchActivationError::activation_failed( + "manual branch lifecycle lease does not match requested branch", + )); + } let Some(schedulers) = administration.schedulers else { return Err(ManualBranchActivationError::scheduler_unavailable( "code-index scheduler authority is unavailable for branch activation", @@ -264,7 +288,7 @@ async fn activate_manual_branch_with_administration( .await? { if !schedulers.is_worktree_mounted(&worktree).await { - activate_linked_worktree(schedulers, graph, &worktree) + activate_linked_worktree(administration.stores, schedulers, graph, &worktree) .await .map_err(ManualBranchActivationError::activation_failed)?; } @@ -336,7 +360,7 @@ async fn activate_manual_branch_with_administration( } } - match activate_linked_worktree(schedulers, graph, &worktree).await { + match activate_linked_worktree(administration.stores, schedulers, graph, &worktree).await { Ok(()) => Ok(ManualBranchActivation { branch: branch.to_string(), head_sha, @@ -830,7 +854,7 @@ async fn track_pr( } } - match activate_linked_worktree(schedulers, graph, &worktree).await { + match activate_linked_worktree(administration.stores, schedulers, graph, &worktree).await { Ok(()) => Ok(ManagedPr { pr: pr.number, head_branch: pr.head_branch.clone(), @@ -859,6 +883,7 @@ async fn track_pr( skip_all )] async fn activate_linked_worktree( + stores: Option<&StoreAdministration>, schedulers: &CodeIndexSchedulerRegistryV1, graph: &tracedecay_project::project::TraceDecay, worktree: &Path, @@ -879,6 +904,31 @@ async fn activate_linked_worktree( let store_root = graph.store_layout().data_root.join("code-index-v1"); let graph_runtime = graph.retained_store_runtime_registry(); let project_database = Arc::new(graph.db().clone()); + let pending_source = match stores { + Some(stores) => { + let scope = + tracedecay_code_index_runtime::resolved_scope_for_project(worktree, &project_id) + .map_err(|error| scheduler_unavailable(&error.to_string()))?; + let (_, reader) = super::project_composition::project_hook_source_pending( + stores, graph, worktree, &scope, + ) + .await + .map_err(|error| scheduler_unavailable(&error.to_string()))?; + Some(reader) + } + None => { + #[cfg(test)] + { + None + } + #[cfg(not(test))] + { + return Err(scheduler_unavailable( + "host admission authority is unavailable for worktree activation", + )); + } + } + }; schedulers .mount_worktree_with_graph_runtime( project_id, @@ -889,7 +939,10 @@ async fn activate_linked_worktree( tracedecay_code_index_runtime::code_index_scheduler::CodeGraphActivationPolicyV1::from_enabled( graph.get_config().native_graph_activation, ), - graph.get_config().index_paths.clone(), + tracedecay_code_index_runtime::code_index_scheduler::CodeIndexSourceInputsV1 { + path_policy: graph.get_config().index_paths.clone(), + pending_admission: pending_source, + }, ) .await .map(|_| ()) diff --git a/crates/tracedecay/src/daemon/pr_autotrack/runtime.rs b/crates/tracedecay/src/daemon/pr_autotrack/runtime.rs index 41c35af77c..e52d7c212e 100644 --- a/crates/tracedecay/src/daemon/pr_autotrack/runtime.rs +++ b/crates/tracedecay/src/daemon/pr_autotrack/runtime.rs @@ -237,7 +237,7 @@ async fn poll_project( &data_root, &discovery, MAX_NEW_TRACKS_PER_CYCLE, - PrStoreAdministration::with_control(schedulers, &graph, &command_control), + PrStoreAdministration::with_control(administration, schedulers, &graph, &command_control), ) .await { @@ -294,7 +294,7 @@ async fn teardown_disabled_project_with_administration( &data_root, &PrDiscovery::default(), MAX_NEW_TRACKS_PER_CYCLE, - PrStoreAdministration::with_control(schedulers, &graph, &command_control), + PrStoreAdministration::with_control(administration, schedulers, &graph, &command_control), ) .await { diff --git a/crates/tracedecay/src/daemon/pr_autotrack/tests.rs b/crates/tracedecay/src/daemon/pr_autotrack/tests.rs index cdf283c9e7..bac3d68667 100644 --- a/crates/tracedecay/src/daemon/pr_autotrack/tests.rs +++ b/crates/tracedecay/src/daemon/pr_autotrack/tests.rs @@ -167,6 +167,7 @@ async fn cancelled_pr_teardown_preserves_artifacts_and_retries_exactly() { schedulers: Some(&schedulers), graph: None, command_control: &cancelled_control, + stores: None, }; let report = reconcile_project_with_administration( repo.path(), @@ -196,6 +197,7 @@ async fn cancelled_pr_teardown_preserves_artifacts_and_retries_exactly() { schedulers: Some(&schedulers), graph: None, command_control: &retry_control, + stores: None, }; let report = reconcile_project_with_administration( repo.path(), @@ -415,7 +417,7 @@ async fn assert_pr_11_head_reconciles(repo: &Path) { &data_root, &discovery, 10, - PrStoreAdministration::with_control(&schedulers, &graph, &command_control), + PrStoreAdministration::unhosted_with_control(&schedulers, &graph, &command_control), ) .await .expect("load managed PR state"); @@ -767,7 +769,7 @@ async fn retained_linked_worktree_honors_parent_native_graph_refusal() { ) .expect("linked-worktree scope"); let schedulers = CodeIndexSchedulerRegistryV1::new(1); - activate_linked_worktree(&schedulers, &graph, &linked) + activate_linked_worktree(None, &schedulers, &graph, &linked) .await .expect("mount retained linked-worktree generation"); let latest = tokio::time::timeout(Duration::from_secs(5), async { @@ -853,23 +855,12 @@ async fn manual_branch_identity_keeps_slashed_and_underscored_names_disjoint() { #[cfg(unix)] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn manual_branch_stages_new_head_without_replacing_published_worktree() { - use tracedecay_code_index_runtime::code_index_scheduler::CodeIndexSchedulerRegistryV1; - let repo = tempfile::tempdir().unwrap(); init_manual_branch_repo(repo.path(), "feature/advance"); - let graph = Arc::new( - tracedecay_project::project::TraceDecay::open_with_options_for_test( - repo.path(), - tracedecay_project::project::TraceDecayOpenOptions::default(), - ) - .await - .unwrap(), - ); - let schedulers = CodeIndexSchedulerRegistryV1::new(2); - let initial = - activate_manual_branch_head(repo.path(), &graph, Some(&schedulers), "feature/advance") - .await - .expect("initial activation"); + let hosted = HostedManualBranchFixture::open(repo.path()).await; + let graph = Arc::clone(&hosted.graph); + let schedulers = hosted.engine.invocation.code_index_schedulers.clone(); + let initial = hosted.activate(&schedulers, "feature/advance").await; let publication = crate::daemon::branch_add::branch_publication_context(&graph).unwrap(); publication @@ -904,6 +895,7 @@ async fn manual_branch_stages_new_head_without_replacing_published_worktree() { let staged_graph = Arc::clone(&graph); let staged_schedulers = schedulers.clone(); let staged_repo = repo.path().to_path_buf(); + let staged_stores = hosted.engine.store_administration.clone(); let (staged_sender, staged_receiver) = tokio::sync::oneshot::channel(); let owner = tokio::spawn(async move { let lifecycle = acquire_manual_branch_lifecycle( @@ -913,6 +905,7 @@ async fn manual_branch_stages_new_head_without_replacing_published_worktree() { .await .unwrap(); let staged = activate_manual_branch_head_with_lifecycle( + &staged_stores, &staged_repo, &staged_graph, Some(&staged_schedulers), @@ -953,7 +946,7 @@ async fn manual_branch_stages_new_head_without_replacing_published_worktree() { Arc::clone(&graph), schedulers.clone(), "feature/advance".to_owned(), - data_root.clone(), + hosted.engine.store_administration.clone(), acquire_manual_branch_lifecycle(&data_root, "feature/advance") .await .unwrap(), @@ -981,7 +974,7 @@ async fn manual_branch_stages_new_head_without_replacing_published_worktree() { Arc::clone(&graph), schedulers.clone(), "feature/advance".to_owned(), - data_root.clone(), + hosted.engine.store_administration.clone(), acquire_manual_branch_lifecycle(&data_root, "feature/advance") .await .unwrap(), @@ -1016,6 +1009,8 @@ async fn manual_branch_stages_new_head_without_replacing_published_worktree() { "the new candidate must carry the newly resolved branch head" ); schedulers.shutdown().await; + let shutdown = hosted.engine.shutdown_all().await; + assert!(shutdown.project_servers.is_clean(), "{shutdown:?}"); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -1089,7 +1084,7 @@ async fn failed_manual_branch_sealing_retires_the_exact_mount_worktree_and_track let lifecycle = acquire_manual_branch_lifecycle(&data_root, "feature/failure-cleanup") .await .expect("lifecycle owner"); - let activation = activate_manual_branch_head_with_lifecycle( + let activation = unhosted_manual_activation_with_lifecycle( repo.path(), &graph, Some(&schedulers), @@ -1256,7 +1251,7 @@ async fn manual_branch_head_git_failure_is_retryable_not_missing_ref() { let lifecycle = acquire_manual_branch_lifecycle(&data_root, branch) .await .expect("lifecycle owner"); - let error = activate_manual_branch_head_with_lifecycle( + let error = unhosted_manual_activation_with_lifecycle( repo.path(), &graph, Some(&schedulers), @@ -1289,7 +1284,7 @@ async fn manual_branch_head_git_failure_is_retryable_not_missing_ref() { let lifecycle = acquire_manual_branch_lifecycle(&data_root, branch) .await .expect("lifecycle owner"); - let activation = activate_manual_branch_head_with_lifecycle( + let activation = unhosted_manual_activation_with_lifecycle( repo.path(), &graph, Some(&schedulers), @@ -1391,7 +1386,7 @@ async fn cancelled_activation_keeps_its_lifecycle_owner_bounded_during_stalled_e &owner_repo, &owner_data_root, &owner_branch, - PrStoreAdministration::with_control( + PrStoreAdministration::unhosted_with_control( &owner_schedulers, &owner_graph, &owner_control, @@ -1587,3 +1582,116 @@ fn dashboard_managed_summary_reader_is_empty_without_state() { 0 ); } + +async fn unhosted_manual_activation_with_lifecycle( + repo_root: &Path, + graph: &Arc, + schedulers: Option<&CodeIndexSchedulerRegistryV1>, + branch: &str, + lifecycle: &ManualBranchLifecycleLeaseV1, + command_control: &PrCommandControl, +) -> std::result::Result { + let administration = match schedulers { + Some(schedulers) => { + PrStoreAdministration::unhosted_with_control(schedulers, graph, command_control) + } + None => PrStoreAdministration { + schedulers: None, + graph: Some(graph), + command_control, + stores: None, + }, + }; + activate_manual_branch_with_administration( + repo_root, + &graph.store_layout().data_root, + branch, + administration, + lifecycle, + ) + .await +} + +#[cfg(unix)] +struct HostedManualBranchFixture { + engine: crate::daemon::DaemonEngine, + graph: Arc, + _profile: tempfile::TempDir, + _scope: tracedecay_runtime_core::db::DaemonDatabaseScope, +} + +#[cfg(unix)] +impl HostedManualBranchFixture { + async fn open(repo: &Path) -> Self { + tracedecay_project::product_runtime::register_fixture_product_runtime(); + let temporary = tempfile::tempdir().unwrap(); + let profile = + tracedecay_runtime_core::config::ProfileRoot::new(temporary.path().join("profile")); + let identity = + tracedecay_daemon_identity::profile_identity::load_or_create(profile.data_dir()) + .unwrap(); + let handshake = crate::daemon::handshake_for_current_client( + &profile, + Some(repo.to_path_buf()), + None, + false, + true, + ) + .unwrap(); + crate::daemon::tests::initialize_test_project(repo, &handshake.client_identity).await; + let scope = tracedecay_runtime_core::db::enter_daemon_database_scope( + profile.data_dir(), + 1, + "manual-branch-fixture", + ) + .unwrap(); + let engine = crate::daemon::DaemonEngine::default() + .with_owner_profile(profile.clone()) + .with_profile_identity(identity); + engine + .invocation + .install_worker_selection( + &engine.store_administration, + tracedecay_domain::configuration::CodeIndexWorkerSelectionV1::default(), + ) + .unwrap(); + let graph = Arc::new( + Box::pin(crate::daemon::open_project_for_handshake( + repo, + &handshake, + &engine.store_administration, + &tracedecay_runtime_core::cancellation::CancellationToken::new(), + )) + .await + .unwrap(), + ); + Self { + engine, + graph, + _profile: temporary, + _scope: scope, + } + } + + async fn activate( + &self, + schedulers: &CodeIndexSchedulerRegistryV1, + branch: &str, + ) -> ManualBranchActivation { + let lifecycle = + acquire_manual_branch_lifecycle(&self.graph.store_layout().data_root, branch) + .await + .unwrap(); + activate_manual_branch_head_with_lifecycle( + &self.engine.store_administration, + self.graph.project_root(), + &self.graph, + Some(schedulers), + branch, + &lifecycle, + default_pr_command_control(), + ) + .await + .unwrap() + } +} diff --git a/crates/tracedecay/src/daemon/project_composition.rs b/crates/tracedecay/src/daemon/project_composition.rs index 5b28d35b8b..379addffae 100644 --- a/crates/tracedecay/src/daemon/project_composition.rs +++ b/crates/tracedecay/src/daemon/project_composition.rs @@ -26,6 +26,7 @@ mod code_index_activation; mod full_owner_mount; #[cfg(test)] mod future_size_tests; +mod hook_source_pending; mod runtime; mod session_database_admission; use code_index_activation::{ @@ -33,6 +34,7 @@ use code_index_activation::{ code_index_freshness_probe_sink, code_index_hook_sink, code_index_reconcile_sink, }; use full_owner_mount::join_independent_full_owner_mounts; +pub(in crate::daemon) use hook_source_pending::project_hook_source_pending; pub(in crate::daemon) use runtime::ProductionProjectCompositionRuntime; use runtime::bind_verified_project_graph_runtime; use session_database_admission::{join_independent_session_opens, log_session_database_admission}; @@ -574,6 +576,7 @@ struct OpenedProjectGraph { /// Ports both of this route's MCP servers publish. Built once before the core /// server; the full server that replaces the core publishes the same ports. struct ProjectRoutePorts { + host_admission_broker: tracedecay_host_admission::SharedHostAdmissionBroker, code_index: ProjectCodeIndexAuthorities, dashboard_code_index_freshness_reader: tracedecay_contracts::code_index_freshness::CodeIndexFreshnessReader, @@ -937,6 +940,13 @@ impl ProjectOpenInputs<'_> { &route_registered, *project_database_is_read_only, )?; + let (host_admission_broker, pending_source) = project_hook_source_pending( + self.store_administration, + cg, + self.canonical_project_path, + &code_index.scope, + ) + .await?; let code_index_mount = code_index_activation_mount(CodeIndexActivationMountInputs { invocation: self.invocation.clone(), project_id: code_index.project_id.clone(), @@ -949,6 +959,7 @@ impl ProjectOpenInputs<'_> { cancellation: route_cancellation.clone(), graph_runtime: Arc::clone(&graph_runtime), graph_publication_database: Arc::new(cg.db().clone()), + pending_source, }); let code_index_hint_sink = code_index_activation_hint_sink( self.invocation.code_index_schedulers.clone(), @@ -1023,6 +1034,7 @@ impl ProjectOpenInputs<'_> { transcript_source_profile, code_index_activation, ports: ProjectRoutePorts { + host_admission_broker, code_index, dashboard_code_index_freshness_reader, code_index_readiness_waiter, @@ -1371,11 +1383,7 @@ impl ProjectOpenInputs<'_> { &delivery_access, ) .await?; - let host_admission_broker = Some( - self.store_administration - .host_admission_broker(&session_db) - .await?, - ); + let host_admission_broker = Some(Arc::clone(&core.ports.host_admission_broker)); let refresh_schedulers = self .store_administration .session_temporal_refresh_schedulers(); diff --git a/crates/tracedecay/src/daemon/project_composition/code_index_activation.rs b/crates/tracedecay/src/daemon/project_composition/code_index_activation.rs index f64fdb75fe..5f979c1371 100644 --- a/crates/tracedecay/src/daemon/project_composition/code_index_activation.rs +++ b/crates/tracedecay/src/daemon/project_composition/code_index_activation.rs @@ -31,6 +31,7 @@ pub(super) struct CodeIndexActivationMountInputs { pub(super) cancellation: CancellationToken, pub(super) graph_runtime: Arc, pub(super) graph_publication_database: Arc, + pub(super) pending_source: Arc, } /// Build the deferred code-index mount for this route. The closure is fenced on @@ -52,6 +53,7 @@ pub(super) fn code_index_activation_mount( cancellation, graph_runtime, graph_publication_database, + pending_source, } = inputs; let mount: code_index_scheduler::CodeIndexActivationMountV1 = Arc::new(move || { let invocation = invocation.clone(); @@ -65,6 +67,7 @@ pub(super) fn code_index_activation_mount( let cancellation = cancellation.clone(); let graph_runtime = Arc::clone(&graph_runtime); let graph_publication_database = Arc::clone(&graph_publication_database); + let pending_source = Arc::clone(&pending_source); Box::pin(tracing::Instrument::instrument( async move { if cancellation.is_cancelled() || !route_registered.load(Ordering::Acquire) { @@ -80,6 +83,7 @@ pub(super) fn code_index_activation_mount( index_paths, graph_runtime, graph_publication_database, + pending_source, ); tokio::select! { biased; diff --git a/crates/tracedecay/src/daemon/project_composition/hook_source_pending.rs b/crates/tracedecay/src/daemon/project_composition/hook_source_pending.rs new file mode 100644 index 0000000000..7510b881e6 --- /dev/null +++ b/crates/tracedecay/src/daemon/project_composition/hook_source_pending.rs @@ -0,0 +1,132 @@ +//! Canonical host-admission input for an exact code-index worktree route. +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use tracedecay_code_index_runtime::code_index_scheduler::{ + CodeIndexSourcePendingChangedFutureV1, CodeIndexSourcePendingReaderV1, +}; +use tracedecay_contracts::ResolvedScope; +use tracedecay_contracts::code_index_freshness::CodeIndexFreshnessReadFailureV1; +use tracedecay_domain::errors::{Result, TraceDecayError}; +use tracedecay_host_admission::SharedHostAdmissionBroker; +use tracedecay_mcp::hook_events::{ + DurableHookEventDecodeError, HookEventPlan, decode_durable_hook_event_plan, +}; +use tracedecay_project::project::TraceDecay; +use tracedecay_sessions::admission::HostAdmissionOutcome; +use tracedecay_store::StoreShardScopeV1; + +use crate::daemon::branch_admin::StoreAdministration; + +#[cfg(test)] +mod tests; + +struct HookSourcePending { + broker: SharedHostAdmissionBroker, + root: PathBuf, + scope: ResolvedScope, +} + +impl CodeIndexSourcePendingReaderV1 for HookSourcePending { + fn read_pending(&self) -> std::result::Result { + self.broker + .pending_replay_matches(|payload| { + let plan = decode_durable_hook_event_plan(payload).map_err(|error| { + HostAdmissionOutcome::degraded(match error { + DurableHookEventDecodeError::Malformed => "malformed_host_event_plan", + DurableHookEventDecodeError::UnsupportedVersion => { + "unsupported_host_event_plan" + } + }) + })?; + match plan { + HookEventPlan::SyncFilesAt { root, scope, .. } => { + // Reference is mutable attribution. The canonical route's + // project, repository, worktree and byte-exact locator own + // the event through checkout changes. + Ok(root.as_os_str() == self.root.as_os_str() + && scope.project_id == self.scope.project_id + && scope.repository_id == self.scope.repository_id + && scope.worktree_id == self.scope.worktree_id) + } + // Shipped unscoped file edits cannot prove a different route. + // Keep their source currency unresolved until replay settles. + HookEventPlan::SyncFiles(_) => Ok(true), + HookEventPlan::AddBranch(_) + | HookEventPlan::AddBranchAt { .. } + | HookEventPlan::SyncCurrentBranch { .. } + | HookEventPlan::DebouncedIncrementalSync(_) + | HookEventPlan::RecordTerminalReceipt { .. } + | HookEventPlan::MarkTurnIngested { .. } + | HookEventPlan::Noop => Ok(false), + } + }) + .map_err(|_| CodeIndexFreshnessReadFailureV1::ReadFailed) + } + + fn revision(&self) -> u64 { + self.broker.progress_revision() + } + + fn changed(&self, observed: u64) -> CodeIndexSourcePendingChangedFutureV1<'_> { + Box::pin(self.broker.wait_for_progress(observed)) + } +} + +/// Reuse the same registered project writer/broker as cold admission and full +/// session hosting. Opening it does not settle history or publish host-ingest +/// capabilities on the core server. +pub(in crate::daemon) async fn project_hook_source_pending( + stores: &StoreAdministration, + graph: &TraceDecay, + root: &Path, + expected_scope: &ResolvedScope, +) -> Result<( + SharedHostAdmissionBroker, + Arc, +)> { + let registry = stores.registered_profile_database().await?; + let owner = registry + .project_registry_context_by_id(expected_scope.project_id.as_str()) + .await? + .ok_or_else(|| { + TraceDecayError::project_route( + "project_route_unavailable", + false, + "hook source project is not registered", + ) + })?; + let (root, scope) = tracedecay_mcp::scope::resolve_query_scope(&owner, root) + .map_err(|error| error.into_route_failure().into_error())?; + expected_scope.validate().map_err(|error| { + TraceDecayError::project_route("project_route_stale", false, error.to_string()) + })?; + if scope.project_id != expected_scope.project_id + || scope.repository_id != expected_scope.repository_id + || scope.worktree_id != expected_scope.worktree_id + { + return Err(TraceDecayError::project_route( + "project_route_stale", + false, + "hook source route identity changed", + )); + } + let database = stores + .registered_project_session_database(graph.project_root(), graph.store_layout()) + .await?; + if !matches!(&database.binding().shard_id.scope, StoreShardScopeV1::ProjectSessions { project_id } if project_id == &scope.project_id) + { + return Err(TraceDecayError::project_route( + "project_route_stale", + false, + "hook source writer belongs to a different project", + )); + } + let broker = stores.host_admission_broker(&database).await?; + let reader: Arc = Arc::new(HookSourcePending { + broker: Arc::clone(&broker), + root, + scope, + }); + Ok((broker, reader)) +} diff --git a/crates/tracedecay/src/daemon/project_composition/hook_source_pending/tests.rs b/crates/tracedecay/src/daemon/project_composition/hook_source_pending/tests.rs new file mode 100644 index 0000000000..04e5da2c4c --- /dev/null +++ b/crates/tracedecay/src/daemon/project_composition/hook_source_pending/tests.rs @@ -0,0 +1,166 @@ +use tracedecay_domain::HostIntegrationIdV1; +use tracedecay_host_admission::{HostAdmissionBroker, HostAdmissionRuntime, SpoolBounds}; +use tracedecay_mcp::hook_events::{HookEventKind, encode_durable_hook_event_plan}; + +use super::*; +use crate::mcp::server::writer_test_support::init_indexed_repo; +use crate::test_support::git::git; + +fn reader( + broker: &SharedHostAdmissionBroker, + root: &Path, + scope: &ResolvedScope, +) -> HookSourcePending { + HookSourcePending { + broker: Arc::clone(broker), + root: root.to_owned(), + scope: scope.clone(), + } +} + +fn edit(root: &Path, scope: ResolvedScope) -> Vec { + encode_durable_hook_event_plan(&HookEventPlan::SyncFilesAt { + root: root.to_owned(), + scope, + agent: HostIntegrationIdV1::Codex, + event_kind: HookEventKind::FileEdit, + had_command: false, + route: None, + rel_paths: vec!["src/a.rs".to_owned()], + }) + .unwrap() +} + +fn broker(path: &Path) -> SharedHostAdmissionBroker { + let (runtime, _) = HostAdmissionRuntime::open(path, SpoolBounds::default()).unwrap(); + Arc::new(HostAdmissionBroker::new(runtime)) +} + +#[tokio::test] +async fn pending_source_is_exactly_routed_across_linked_worktrees_and_branch_changes() { + let (graph, project, _authority) = init_indexed_repo().await; + let linked_parent = tempfile::tempdir().unwrap(); + let linked = linked_parent.path().join("linked"); + git( + project.path(), + &[ + "worktree", + "add", + "-q", + "-b", + "linked", + linked.to_str().unwrap(), + ], + ); + let project_id = graph + .store_layout() + .identity + .project_id + .as_ref() + .unwrap() + .clone() + .try_into() + .unwrap(); + let main_scope = + tracedecay_code_index_runtime::resolved_scope_for_project(project.path(), &project_id) + .unwrap(); + let linked_scope = + tracedecay_code_index_runtime::resolved_scope_for_project(&linked, &project_id).unwrap(); + let spool = tempfile::tempdir().unwrap(); + let broker = broker(spool.path()); + let main = reader(&broker, project.path(), &main_scope); + let other = reader(&broker, &linked, &linked_scope); + broker + .admit("linked", &edit(&linked, linked_scope)) + .await + .unwrap(); + assert!(!main.read_pending().unwrap()); + assert!(other.read_pending().unwrap()); + git(project.path(), &["checkout", "-q", "-b", "next"]); + let current = + tracedecay_code_index_runtime::resolved_scope_for_project(project.path(), &project_id) + .unwrap(); + assert_ne!(current.reference, main_scope.reference); + broker + .admit("main", &edit(project.path(), main_scope)) + .await + .unwrap(); + assert!( + reader(&broker, project.path(), ¤t) + .read_pending() + .unwrap() + ); + let replay = broker.begin_replay().await.unwrap(); + let first = replay.lease_next().await.unwrap().unwrap(); + let second = replay.lease_next().await.unwrap().unwrap(); + assert!( + main.read_pending().unwrap(), + "leased debt still fences source currency" + ); + replay.commit(second.seq).await.unwrap(); + assert!( + !main.read_pending().unwrap(), + "completed debt behind a prefix is settled" + ); + assert!(other.read_pending().unwrap()); + replay.commit(first.seq).await.unwrap(); + assert!(!other.read_pending().unwrap()); +} + +#[tokio::test] +async fn unscoped_legacy_edits_remain_pending_and_invalid_plans_fail_closed() { + let (graph, project, _authority) = init_indexed_repo().await; + let project_id = graph + .store_layout() + .identity + .project_id + .as_ref() + .unwrap() + .clone() + .try_into() + .unwrap(); + let scope = + tracedecay_code_index_runtime::resolved_scope_for_project(project.path(), &project_id) + .unwrap(); + for (payload, expected) in [ + ( + encode_durable_hook_event_plan(&HookEventPlan::SyncFiles(vec!["src/a.rs".to_owned()])) + .unwrap(), + Ok(true), + ), + ( + encode_durable_hook_event_plan(&HookEventPlan::Noop).unwrap(), + Ok(false), + ), + ( + b"invalid".to_vec(), + Err(CodeIndexFreshnessReadFailureV1::ReadFailed), + ), + ( + br#"{"version":999,"plan":{"kind":"noop"}}"#.to_vec(), + Err(CodeIndexFreshnessReadFailureV1::ReadFailed), + ), + ] { + let spool = tempfile::tempdir().unwrap(); + let broker = broker(spool.path()); + broker.admit("source", &payload).await.unwrap(); + assert_eq!( + reader(&broker, project.path(), &scope).read_pending(), + expected + ); + } + let mut corrupt: serde_json::Value = + serde_json::from_slice(&edit(project.path(), scope.clone())).unwrap(); + corrupt["plan"]["scope"]["scope_digest"] = + serde_json::json!(format!("sha256:{}", "0".repeat(64))); + let spool = tempfile::tempdir().unwrap(); + let broker = broker(spool.path()); + broker + .admit("source", &serde_json::to_vec(&corrupt).unwrap()) + .await + .unwrap(); + assert_eq!( + reader(&broker, project.path(), &scope).read_pending(), + Err(CodeIndexFreshnessReadFailureV1::ReadFailed) + ); +} diff --git a/crates/tracedecay/src/daemon/project_open_owners/code_index_reads/ignored_dependency_admission_tests.rs b/crates/tracedecay/src/daemon/project_open_owners/code_index_reads/ignored_dependency_admission_tests.rs index e8a81338e7..af9b3851a5 100644 --- a/crates/tracedecay/src/daemon/project_open_owners/code_index_reads/ignored_dependency_admission_tests.rs +++ b/crates/tracedecay/src/daemon/project_open_owners/code_index_reads/ignored_dependency_admission_tests.rs @@ -111,7 +111,11 @@ export function generationAnchor() { return 1; } runtime.code_graph_seat_port(), database, CodeGraphActivationPolicyV1::Enabled, - tracedecay_code_index_runtime::registry_default_index_path_policy(), + tracedecay_code_index_runtime::code_index_scheduler::CodeIndexSourceInputsV1 { + path_policy: + tracedecay_code_index_runtime::registry_default_index_path_policy(), + pending_admission: None, + }, ) .await .expect("mount persistent graph-backed scheduler"); diff --git a/crates/tracedecay/src/daemon/tests.rs b/crates/tracedecay/src/daemon/tests.rs index 575dec1d97..fdef12b96b 100644 --- a/crates/tracedecay/src/daemon/tests.rs +++ b/crates/tracedecay/src/daemon/tests.rs @@ -301,7 +301,7 @@ fn enter_test_daemon_database_scope( .unwrap_or_else(|error| panic!("enter test daemon database scope ({label}): {error}")) } -async fn initialize_test_project( +pub(super) async fn initialize_test_project( project_root: &std::path::Path, client_identity: &DaemonClientIdentity, ) -> tracedecay_runtime_core::storage::StoreLayout { diff --git a/crates/tracedecay/src/daemon/tests/rmcp_route.rs b/crates/tracedecay/src/daemon/tests/rmcp_route.rs index ed5f434848..59d566a42f 100644 --- a/crates/tracedecay/src/daemon/tests/rmcp_route.rs +++ b/crates/tracedecay/src/daemon/tests/rmcp_route.rs @@ -17,6 +17,9 @@ use tracedecay_runtime_core::path_safety::canonical_existing_identity; use super::*; +#[cfg(unix)] +mod host_index_handoff; + const PHASE_TIMEOUT: Duration = Duration::from_secs(20); /// Wall-clock bound for the dedicated-runtime drain. Isolation `--exact` /// hangs sat in `Runtime::drop` after the two servers were already down; diff --git a/crates/tracedecay/src/daemon/tests/rmcp_route/host_index_handoff.rs b/crates/tracedecay/src/daemon/tests/rmcp_route/host_index_handoff.rs new file mode 100644 index 0000000000..d04a426d8d --- /dev/null +++ b/crates/tracedecay/src/daemon/tests/rmcp_route/host_index_handoff.rs @@ -0,0 +1,152 @@ +use tracedecay_contracts::code_index_freshness::{ + CodeIndexReadinessTargetV1, CodeIndexReadinessV1, CodeIndexReadinessWaitReadV1, +}; +use tracedecay_session_memory::event_lane::{self, ActivityFamilyV1}; + +use super::*; + +async fn ready_reading( + fixture: &RmcpRouteFixture, + project: &std::path::Path, +) -> tracedecay_contracts::code_index_freshness::CodeIndexWorktreeFreshnessV1 { + let reading = fixture + .engine + .invocation + .code_index_schedulers + .wait_for_readiness(project, CodeIndexReadinessTargetV1::Ready, PHASE_TIMEOUT) + .await + .unwrap(); + let CodeIndexReadinessWaitReadV1::Reached { reading } = reading else { + panic!("exact worktree did not become ready: {reading:?}"); + }; + *reading +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn committed_save_remains_noncurrent_until_its_broker_handoff() { + let fixture = rmcp_route_fixture_with_projects("held-hook-index-handoff", &[], true).await; + let project = + canonical_existing_identity(fixture.handshake.project_path.as_deref().unwrap()).unwrap(); + wait_for_host_ingest_publication(&fixture).await; + let initial = ready_reading(&fixture, &project).await; + let graph = fixture.server.cg_snapshot().await; + let project_id = graph.store_layout().identity.project_id.as_deref().unwrap(); + let scope = tracedecay_code_index_runtime::resolved_scope_for_project( + &project, + &project_id.to_owned().try_into().unwrap(), + ) + .unwrap(); + let database = fixture + .store_administration + .registered_project_session_database(graph.project_root(), graph.store_layout()) + .await + .unwrap(); + let broker = fixture + .store_administration + .host_admission_broker(&database) + .await + .unwrap(); + let registry = &fixture.engine.invocation.code_index_schedulers; + assert!( + registry + .latest_text_serving_freshness_for_scope(&scope) + .await + .unwrap() + .1 + ); + let _initial_graph_owner = if graph.get_config().native_graph_activation { + // A ready text owner can outlive its decoded generation. Admit the + // same exact-scope graph demand a real reader uses, under this phase's + // existing deadline, before proving the positive decoded census. + let owner = registry + .latest_complete_fresh_for_scope_awaiting_seat( + &scope, + std::time::Instant::now() + PHASE_TIMEOUT, + ) + .await; + assert!(owner.is_some(), "initial ready reading: {initial:?}"); + owner + } else { + None + }; + let replay = broker.begin_replay().await.unwrap(); + assert_eq!(broker.pending_replay_count().await.unwrap(), 0); + std::fs::write( + project.join("src/main.rs"), + "fn main() { let saved = 42; }\n", + ) + .unwrap(); + let request = stateless_request(json!({ + "jsonrpc": "2.0", "id": 51, + "method": tracedecay_hooks::core_events::HOOK_EVENT_METHOD, + "params": DaemonHookEvent::post_tool_use_edit( + HostIntegrationIdV1::Codex, vec!["src/main.rs".to_owned()], project.clone(), + ), + })); + let responses = unix_one_request(&fixture, &request).await; + assert_single_result(&responses, 51, "durable save while replay held"); + assert_eq!(broker.pending_replay_count().await.unwrap(), 1); + let owed = registry + .dashboard_freshness_read(&project) + .await + .unwrap() + .unwrap(); + assert_eq!(owed.latest_generation_id, initial.latest_generation_id); + assert_ne!( + owed.readiness(CodeIndexReadinessTargetV1::Ready), + CodeIndexReadinessV1::Reached + ); + assert_ne!( + owed.readiness(CodeIndexReadinessTargetV1::Fresh), + CodeIndexReadinessV1::Reached + ); + assert_eq!( + owed.hook_hint_count, + Some(0), + "broker debt is not a scheduler hint" + ); + assert!(!registry.has_current_ready_decoded_for_root_scope(&project, &scope)); + assert!( + registry + .latest_complete_ready_decoded_for_root_scope(&project, &scope) + .await + .is_none() + ); + let (_, current) = registry + .latest_text_serving_freshness_for_scope(&scope) + .await + .unwrap(); + assert!( + !current, + "the retained text owner must not claim source currency" + ); + drop(replay); + let saved = ready_reading(&fixture, &project).await; + assert_ne!(saved.latest_generation_id, initial.latest_generation_id); + assert_ne!( + saved.snapshot_content_identity, + initial.snapshot_content_identity + ); + assert_eq!(saved.source_reference, initial.source_reference); + assert!(initial.source_revision.is_some()); + assert_eq!(saved.source_revision, None); + assert_eq!(broker.pending_replay_count().await.unwrap(), 0); + let activity = event_lane::replay_after(&database, project_id, None) + .await + .unwrap() + .unwrap(); + assert_eq!( + activity + .records + .iter() + .filter(|record| record.pulse.family == ActivityFamilyV1::CodeIndex) + .count(), + 1 + ); + fixture.server.shutdown().await; + let shutdown = fixture.engine.shutdown_all().await; + assert!( + shutdown.project_servers.is_clean(), + "handoff shutdown: {shutdown:?}" + ); +} diff --git a/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test.rs b/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test.rs index d6fdc3580f..b00508831d 100644 --- a/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test.rs +++ b/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test.rs @@ -371,7 +371,10 @@ export function GenerationAnchor() { return 1; } graph_runtime.code_graph_seat_port(), project_database, CodeGraphActivationPolicyV1::Enabled, - tracedecay_code_index_runtime::registry_default_index_path_policy(), + tracedecay_code_index_runtime::code_index_scheduler::CodeIndexSourceInputsV1 { + path_policy: tracedecay_code_index_runtime::registry_default_index_path_policy(), + pending_admission: None, + }, ) .await .expect("mount persistent graph-backed scheduler"); diff --git a/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/branch_publication_tests.rs b/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/branch_publication_tests.rs index ef58b58589..57ef96ecd4 100644 --- a/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/branch_publication_tests.rs +++ b/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/branch_publication_tests.rs @@ -308,7 +308,9 @@ export function GenerationAnchor(value: PublicWidget) { return value; } graph_runtime.code_graph_seat_port(), Arc::clone(&project_database), tracedecay_code_index_runtime::code_index_scheduler::CodeGraphActivationPolicyV1::Enabled, - tracedecay_code_index_runtime::registry_default_index_path_policy(), +tracedecay_code_index_runtime::code_index_scheduler::CodeIndexSourceInputsV1 { + path_policy: tracedecay_code_index_runtime::registry_default_index_path_policy(), pending_admission: None, +}, ) .await .expect("mount persistent graph-backed scheduler"); diff --git a/crates/tracedecay/tests/daemon_suite/code_index_journey.rs b/crates/tracedecay/tests/daemon_suite/code_index_journey.rs index 822385a678..0704c56cbe 100644 --- a/crates/tracedecay/tests/daemon_suite/code_index_journey.rs +++ b/crates/tracedecay/tests/daemon_suite/code_index_journey.rs @@ -620,9 +620,9 @@ pub async fn deliver_save( project: &Path, paths: &[&str], ) { - // The acknowledgement is the event a following status read relies on, so - // wait for it within the journey's receipt bound rather than the host - // hook's latency bound. + // ACK commits broker admission; indexing follows in replay. A following + // readiness read must include that exact route's still-owed handoff. + // This journey bounds delivery with its existing receipt deadline. let outcome = tokio::time::timeout( RECEIPT_TIMEOUT, notify_hook_event( From d13fbc8882fd806e40a43a0ef985baecc4bb57d2 Mon Sep 17 00:00:00 2001 From: Zack Jackson Date: Sun, 18 Oct 2026 20:24:11 -0700 Subject: [PATCH 65/89] refactor(mcp): extract durable hook request dispatch --- crates/tracedecay/src/mcp/server/requests.rs | 53 ++++++++++++-------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/crates/tracedecay/src/mcp/server/requests.rs b/crates/tracedecay/src/mcp/server/requests.rs index 3445da757d..ba1fd4e6d6 100644 --- a/crates/tracedecay/src/mcp/server/requests.rs +++ b/crates/tracedecay/src/mcp/server/requests.rs @@ -342,29 +342,9 @@ impl McpServer { // survives the envelope move into `tools/call` preparation below. let method_name = request.method(); if matches!(method, McpMethod::HookEvent) { - let Some(id) = request.cloned_id() else { - Box::pin(self.handle_hook_event_notification( - request.raw_notification_params(), - &mut connection.route_cache, - )) + return self + .dispatch_hook_event(&request, &mut connection.route_cache) .await; - return None; - }; - let admitted = Box::pin(self.admit_hook_event_notification( - request.raw_notification_params(), - &mut connection.route_cache, - false, - )) - .await; - return Some(match admitted { - Ok(_) => JsonRpcResponse::success(id, json!({})), - Err(outcome) => JsonRpcResponse::error_with_data( - id, - ErrorCode::InternalError, - "hook event was not durably admitted".to_owned(), - Some(json!(outcome)), - ), - }); } if matches!(method, McpMethod::Cancelled) { self.record_request_accounting(method_name, false); @@ -454,6 +434,35 @@ impl McpServer { result } + async fn dispatch_hook_event( + &self, + request: &McpDispatchRequest<'_>, + route_cache: &mut HookProjectRouteCache, + ) -> Option { + let Some(id) = request.cloned_id() else { + Box::pin( + self.handle_hook_event_notification(request.raw_notification_params(), route_cache), + ) + .await; + return None; + }; + let admitted = Box::pin(self.admit_hook_event_notification( + request.raw_notification_params(), + route_cache, + false, + )) + .await; + Some(match admitted { + Ok(_) => JsonRpcResponse::success(id, json!({})), + Err(outcome) => JsonRpcResponse::error_with_data( + id, + ErrorCode::InternalError, + "hook event was not durably admitted".to_owned(), + Some(json!(outcome)), + ), + }) + } + pub(crate) async fn handle_hook_event_notification( &self, params: Option<&Value>, From d22446ace29c343791e3ebe9eb06aa364ab4ba53 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Mon, 19 Oct 2026 03:43:10 +0000 Subject: [PATCH 66/89] refactor(hooks): remove retired mountless admission route --- .../src/handlers/hook_runtime/admission.rs | 9 --------- .../tracedecay-mcp/src/handlers/hook_runtime/mod.rs | 2 +- .../tracedecay/src/daemon/hook_v2_replay_consumer.rs | 12 ++++++++++-- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/admission.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/admission.rs index 7b2b4ef1c0..29f29c207d 100644 --- a/crates/tracedecay-mcp/src/handlers/hook_runtime/admission.rs +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/admission.rs @@ -537,15 +537,6 @@ async fn admit_gated_native_lifecycle( .await } -pub async fn admit_hook_v2_envelope( - cg: &TraceDecay, - envelope: &tracedecay_hooks::HookEventEnvelopeV2, - native_session_id: Option, - now: UtcMicros, -) -> HookV2AdmissionOutcomeV1 { - admit_hook_v2_envelope_with_lifecycle(cg, envelope, native_session_id, None, false, now).await -} - pub async fn admit_hook_v2_replayed_envelope_with_lifecycle( cg: &TraceDecay, envelope: &tracedecay_hooks::HookEventEnvelopeV2, diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/mod.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/mod.rs index 3ba5220125..a2f708c0f7 100644 --- a/crates/tracedecay-mcp/src/handlers/hook_runtime/mod.rs +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/mod.rs @@ -23,7 +23,7 @@ mod entry_tests; mod test_support; pub use admission::{ - HookV2AdmissionLedgerUnavailable, HookV2AdmissionOutcomeV1, admit_hook_v2_envelope, + HookV2AdmissionLedgerUnavailable, HookV2AdmissionOutcomeV1, admit_hook_v2_replayed_envelope_with_lifecycle, hook_v2_native_lifecycle_enabled, hook_v2_pending_work_envelopes, }; diff --git a/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs b/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs index 856f47e20e..09136dab9d 100644 --- a/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs +++ b/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs @@ -644,7 +644,6 @@ async fn stop_replay_consumer( #[cfg(test)] mod tests { use super::*; - use tracedecay_mcp::handlers::hook_runtime::admit_hook_v2_envelope; #[test] fn only_an_admitted_redrive_settles_pending_work() { @@ -1002,7 +1001,16 @@ mod tests { ..envelope }; assert!(matches!( - admit_hook_v2_envelope(&graph, &disabled_edit, None, hook_replay_now()).await, + admit_hook_v2_replayed_envelope_with_lifecycle( + &graph, + &disabled_edit, + None, + None, + &project_sessions, + &background_cpu, + hook_replay_now(), + ) + .await, HookV2AdmissionOutcomeV1::Admitted { .. } )); let disabled_pending = From c1a6bbfe2e8a6590a93d1fa4e624083266cb7dab Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Mon, 19 Oct 2026 03:43:32 +0000 Subject: [PATCH 67/89] test(sessions): assert Git reads use the blocking pool --- .../git_correlation/backfill/bounded/tests.rs | 69 +++++++++---------- 1 file changed, 34 insertions(+), 35 deletions(-) diff --git a/crates/tracedecay-sessions/src/runtime/git_correlation/backfill/bounded/tests.rs b/crates/tracedecay-sessions/src/runtime/git_correlation/backfill/bounded/tests.rs index 9b378bc08c..2d1b5c4254 100644 --- a/crates/tracedecay-sessions/src/runtime/git_correlation/backfill/bounded/tests.rs +++ b/crates/tracedecay-sessions/src/runtime/git_correlation/backfill/bounded/tests.rs @@ -4,6 +4,7 @@ use std::process::{Command, Stdio}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::time::Duration; +use futures_util::FutureExt; use tracedecay_runtime_core::db::engine::{ Executor, QueryExecutor, ReadSnapshot, TestConnection, Transaction, TransactionBehavior, params, }; @@ -286,24 +287,34 @@ async fn touch(store: &TestStore, session_id: &str) -> i64 { } } -/// Real Git whose every read first takes two long polls, the way a large -/// reflog or many worktrees do. -struct SlowGit; +/// Real Git whose reads must execute on the native blocking pool. +#[derive(Default)] +struct BlockingPoolGit { + reads: AtomicUsize, +} -impl SlowGit { - fn stall() { - std::thread::sleep(native_runtime::LONG_POLL * 2); +impl BlockingPoolGit { + fn assert_blocking_pool(&self) { + // Nested blocking work runs inline on a pool thread. On a native + // worker it is queued elsewhere, so either readiness or identity + // differs without relying on host-dependent poll timing. + assert_eq!( + native_runtime::spawn_blocking(|| std::thread::current().id()).now_or_never(), + Some(Ok(std::thread::current().id())), + "Git reads must run on the native blocking pool" + ); + self.reads.fetch_add(1, Ordering::SeqCst); } } -impl GitReflogSource for SlowGit { +impl GitReflogSource for BlockingPoolGit { fn reflog(&self, worktree: &Path) -> Option { - Self::stall(); + self.assert_blocking_pool(); SystemGit.reflog(worktree) } fn current_branch(&self, worktree: &Path) -> Option { - Self::stall(); + self.assert_blocking_pool(); SystemGit.current_branch(worktree) } @@ -312,29 +323,16 @@ impl GitReflogSource for SlowGit { worktree: &Path, reference: &str, ) -> Result { - Self::stall(); + self.assert_blocking_pool(); SystemGit.commit_reference_exists(worktree, reference) } fn commit_log(&self, worktree: &Path, branch: &str, since: i64) -> Option { - Self::stall(); + self.assert_blocking_pool(); SystemGit.commit_log(worktree, branch, since) } } -#[track_caller] -fn spawned_here( - work: impl Future + Send + 'static, -) -> ( - native_runtime::NativeTask, - &'static std::panic::Location<'static>, -) { - ( - native_runtime::NativeTask::spawn(work).expect("native task"), - std::panic::Location::caller(), - ) -} - /// The session refresh worker runs this pass on a native task. Its Git reads /// wait on subprocess pipes, so they must wait on the blocking pool: on the /// worker they hold it for every read and stall every task queued behind it. @@ -342,26 +340,27 @@ fn spawned_here( async fn convergence_waits_for_git_off_the_native_worker() { let repository = repository_fixture(); let directory = tempfile::tempdir().unwrap(); - let store = prepare_store(&directory.path().join("sessions.db"), repository.path()).await; - - let (pass, site) = spawned_here(async move { - converge_git_evidence_pass(&store, Arc::new(SlowGit), None) + // Keep the fixture's writer teardown outside the native task. + let store = + Arc::new(prepare_store(&directory.path().join("sessions.db"), repository.path()).await); + let git = Arc::new(BlockingPoolGit::default()); + let task_store = Arc::clone(&store); + let task_git: Arc = git.clone(); + + let pass = native_runtime::NativeTask::spawn(async move { + converge_git_evidence_pass(task_store.as_ref(), task_git, None) .await .map(|outcome| outcome.pass.backfill) .map_err(|error| error.to_string()) - }); + }) + .expect("native task"); let native_runtime::NativeTaskOutcome::Completed(backfill) = pass.wait().await else { panic!("the convergence task did not complete"); }; let backfill = backfill.unwrap(); assert!(backfill.spans_written > 0, "{backfill:?}"); assert!(backfill.commits_attributed > 0, "{backfill:?}"); - assert_eq!( - native_runtime::long_native_polls().get(site), - None, - "a Git read held the native worker past {:?}", - native_runtime::LONG_POLL - ); + assert!(git.reads.load(Ordering::SeqCst) > 0, "real Git reads ran"); } /// The evidence rows, the attribution mark, and the history frontier commit From 5a5cb748b52f81756c872cc12ee5d156988f0c2f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 19 Oct 2026 04:44:47 +0000 Subject: [PATCH 68/89] fix(hooks): preserve ordered replay lifecycle evidence Co-authored-by: ScriptedAlchemy <25274700+ScriptedAlchemy@users.noreply.github.com> --- .../tracedecay-hooks/src/admission_ledger.rs | 32 +++++++++ crates/tracedecay-hooks/src/hook_v2_replay.rs | 4 ++ .../src/handlers/hook_runtime/admission.rs | 50 ++++++++++---- .../src/handlers/hook_runtime/mod.rs | 5 +- .../src/daemon/hook_v2_replay_consumer.rs | 65 ++++++++++++++----- 5 files changed, 124 insertions(+), 32 deletions(-) diff --git a/crates/tracedecay-hooks/src/admission_ledger.rs b/crates/tracedecay-hooks/src/admission_ledger.rs index ba77fff77f..7327329fd9 100644 --- a/crates/tracedecay-hooks/src/admission_ledger.rs +++ b/crates/tracedecay-hooks/src/admission_ledger.rs @@ -1350,6 +1350,38 @@ mod tests { assert_eq!(duplicate.order, first_order); } + #[test] + fn pending_work_order_survives_compaction_and_restart() { + let root = TestDir::new("ordered-pending-work"); + let older = envelope(21, 5); + let mut newer = envelope(22, 5); + newer.event = HookEventV2::SavedEdit { + file_id: [22; 16], + changed_range_count: 1, + }; + { + let mut ledger = open(root.path(), UtcMicros(1)); + for (envelope, at) in [(&older, 2), (&newer, 3)] { + let staged = ledger + .stage_admission(envelope, Some(envelope), UtcMicros(at)) + .unwrap(); + staged.commit.wait().unwrap(); + } + assert_eq!( + ledger.pending_work_envelopes(UtcMicros(4)).unwrap(), + vec![older.clone(), newer.clone()] + ); + ledger.rewrite().unwrap(); + } + + let mut reopened = open(root.path(), UtcMicros(5)); + assert_eq!( + reopened.pending_work_envelopes(UtcMicros(6)).unwrap(), + vec![older, newer], + "compaction and replay preserve producer admission order" + ); + } + #[test] fn pending_producer_work_redrives_until_completion_survives_reopen() { let root = TestDir::new("ledger-work-completion"); diff --git a/crates/tracedecay-hooks/src/hook_v2_replay.rs b/crates/tracedecay-hooks/src/hook_v2_replay.rs index c578800936..a9559a6068 100644 --- a/crates/tracedecay-hooks/src/hook_v2_replay.rs +++ b/crates/tracedecay-hooks/src/hook_v2_replay.rs @@ -67,6 +67,7 @@ pub enum HookReplayAdmissionOutcomeV1 { ExactDuplicate, Conflict, CatchupRequired, + Deferred, Backpressured, Unavailable, } @@ -264,6 +265,9 @@ where ))); break; } + HookReplayAdmissionOutcomeV1::Deferred => { + completions.push(ReplayCompletion::Retained(1)); + } } } } diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/admission.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/admission.rs index b5bd4023fb..4272a8c7d9 100644 --- a/crates/tracedecay-mcp/src/handlers/hook_runtime/admission.rs +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/admission.rs @@ -478,26 +478,42 @@ const fn intent_gated_lifecycle(producer: tracedecay_domain::NativeHostIdentityV ) } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum HookV2NativeLifecyclePolicyV1 { + Required, + Disabled, + Unavailable, +} + /// Whether this host's lifecycle is enabled under the same policy used by -/// admission and ledger replay. Claude omits lifecycle writes without intent. -pub async fn hook_v2_native_lifecycle_enabled( +/// admission and ledger replay. Claude omits lifecycle writes only when its +/// configuration explicitly disables intent; an unavailable authority is not +/// evidence that it is disabled. +pub async fn hook_v2_native_lifecycle_policy( cg: &TraceDecay, producer: tracedecay_domain::NativeHostIdentityV1, -) -> bool { +) -> HookV2NativeLifecyclePolicyV1 { if !intent_gated_lifecycle(producer) { - return true; + return HookV2NativeLifecyclePolicyV1::Required; } let Ok(pinned) = cg.configuration_runtime().client().current().await else { - return false; + return HookV2NativeLifecyclePolicyV1::Unavailable; }; - tracedecay_agent_hosts::agents::context_scout::address_registry::ContextScoutConfigurationPinV1::from_current( - &pinned.into_current_state(), - ) - .is_some_and(|pin| { - pin.intent_settings().is_some_and(|settings| settings.enabled) - && pin.control().state - != tracedecay_agent_hosts::agents::context_scout::ContextScoutServiceStateV1::Disabled - }) + let Some(pin) = + tracedecay_agent_hosts::agents::context_scout::address_registry::ContextScoutConfigurationPinV1::from_current( + &pinned.into_current_state(), + ) + else { + return HookV2NativeLifecyclePolicyV1::Unavailable; + }; + if pin.intent_settings().is_some_and(|settings| settings.enabled) + && pin.control().state + != tracedecay_agent_hosts::agents::context_scout::ContextScoutServiceStateV1::Disabled + { + HookV2NativeLifecyclePolicyV1::Required + } else { + HookV2NativeLifecyclePolicyV1::Disabled + } } async fn admit_gated_native_lifecycle( @@ -708,7 +724,13 @@ async fn admit_hook_v2_envelope_with_lifecycle_inner( if let Some(mount) = mount.as_ref() && let Some(native_lifecycle) = mount.lifecycle.as_ref() { - let lifecycle_enabled = hook_v2_native_lifecycle_enabled(cg, envelope.producer).await; + let lifecycle_enabled = match hook_v2_native_lifecycle_policy(cg, envelope.producer).await { + HookV2NativeLifecyclePolicyV1::Required => true, + HookV2NativeLifecyclePolicyV1::Disabled => false, + HookV2NativeLifecyclePolicyV1::Unavailable => { + return HookV2AdmissionOutcomeV1::Unavailable; + } + }; let Some(range) = hook_v2_lifecycle_range(envelope, receipt) else { return HookV2AdmissionOutcomeV1::Backpressured; }; diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/mod.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/mod.rs index 3ba5220125..5b85af0d28 100644 --- a/crates/tracedecay-mcp/src/handlers/hook_runtime/mod.rs +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/mod.rs @@ -23,8 +23,9 @@ mod entry_tests; mod test_support; pub use admission::{ - HookV2AdmissionLedgerUnavailable, HookV2AdmissionOutcomeV1, admit_hook_v2_envelope, - admit_hook_v2_replayed_envelope_with_lifecycle, hook_v2_native_lifecycle_enabled, + HookV2AdmissionLedgerUnavailable, HookV2AdmissionOutcomeV1, + HookV2NativeLifecyclePolicyV1, admit_hook_v2_envelope, + admit_hook_v2_replayed_envelope_with_lifecycle, hook_v2_native_lifecycle_policy, hook_v2_pending_work_envelopes, }; pub use envelope::daemon_mint_hook_v2_file_id; diff --git a/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs b/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs index 4935ec3b5f..716ae989c9 100644 --- a/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs +++ b/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs @@ -25,8 +25,8 @@ use tracedecay_hooks::{ }; use tracedecay_mcp::handlers::hook_runtime::{ - HookV2AdmissionOutcomeV1, admit_hook_v2_envelope, - admit_hook_v2_replayed_envelope_with_lifecycle, hook_v2_native_lifecycle_enabled, + HookV2AdmissionOutcomeV1, HookV2NativeLifecyclePolicyV1, admit_hook_v2_envelope, + admit_hook_v2_replayed_envelope_with_lifecycle, hook_v2_native_lifecycle_policy, hook_v2_pending_work_envelopes, }; @@ -210,6 +210,14 @@ async fn drain_all_hosts( Ok(more) => sweep.more_pending |= more, Err(cause) => sweep.fail(*host, StatusHookReplaySpoolV1::DeliveryReceipts, cause), } + let replay_fences = match Box::pin(redrive_pending_work( + graph, data_root, *host, project_id, now, &mut sweep, + )) + .await + { + Ok(fences) => fences, + Err(()) => continue, + }; let report = match Box::pin(drain_admitted_host_spool( *host, project_id, @@ -217,6 +225,7 @@ async fn drain_all_hosts( graph, project_sessions, background_cpu, + &replay_fences, )) .await { @@ -226,11 +235,6 @@ async fn drain_all_hosts( None } }; - // The host spool retains native lifecycle evidence the ledger cannot carry. - Box::pin(redrive_pending_work( - graph, data_root, *host, project_id, now, &mut sweep, - )) - .await; if let Some(report) = report && (report.committed > 0 || report.duplicates > 0 || report.tombstoned > 0) { @@ -245,10 +249,12 @@ async fn drain_all_hosts( ); // A pass settles a bounded batch; records behind it replay now, // not a full interval later. - match HookSpoolV1::has_records(&hook_v2_spool_root(data_root, *host)) { - Ok(more) => sweep.more_pending |= more, - Err(error) => { - sweep.fail(*host, StatusHookReplaySpoolV1::Records, error.to_string()); + if replay_fences.is_empty() { + match HookSpoolV1::has_records(&hook_v2_spool_root(data_root, *host)) { + Ok(more) => sweep.more_pending |= more, + Err(error) => { + sweep.fail(*host, StatusHookReplaySpoolV1::Records, error.to_string()); + } } } } @@ -265,7 +271,7 @@ async fn redrive_pending_work( project_id: Option<[u8; 16]>, now: UtcMicros, sweep: &mut HookReplaySweepV1, -) { +) -> std::result::Result, ()> { let pending_work = match hook_v2_pending_work_envelopes(data_root, host, now) { Ok(pending_work) => pending_work, Err(error) => { @@ -274,17 +280,29 @@ async fn redrive_pending_work( StatusHookReplaySpoolV1::PendingWork, error.to_string(), ); - Vec::new() + return Err(()); } }; + let mut replay_fences = BTreeMap::new(); + for envelope in &pending_work { + if project_id.is_none_or(|project_id| envelope.project_id == project_id) { + replay_fences + .entry(envelope.protected_session_id) + .or_insert(envelope.event_id); + } + } let mut owed = 0_usize; let mut owed_cause = None; for envelope in pending_work { if project_id.is_some_and(|project_id| envelope.project_id != project_id) { continue; } + if replay_fences.get(&envelope.protected_session_id) != Some(&envelope.event_id) { + continue; + } // Owed work the daemon cannot admit now stays in the ledger and - // is redriven by the next sweep. + // is redriven by the next sweep. The oldest event for a session fences + // later spooled events until its own producer work settles. let outcome = admit_replayed_envelope_with_authoritative_session( envelope, None, @@ -306,9 +324,16 @@ async fn redrive_pending_work( tracedecay_hooks::HookEventV2::ToolLifecycle { .. } | tracedecay_hooks::HookEventV2::SavedEdit { .. } ) - && hook_v2_native_lifecycle_enabled(graph, host).await { - return HookV2AdmissionOutcomeV1::Backpressured; + match hook_v2_native_lifecycle_policy(graph, host).await { + HookV2NativeLifecyclePolicyV1::Required => { + return HookV2AdmissionOutcomeV1::Backpressured; + } + HookV2NativeLifecyclePolicyV1::Unavailable => { + return HookV2AdmissionOutcomeV1::Unavailable; + } + HookV2NativeLifecyclePolicyV1::Disabled => {} + } } admit_hook_v2_envelope(graph, &envelope, native_session_id, hook_replay_now()) .await @@ -327,6 +352,7 @@ async fn redrive_pending_work( format!("{owed} pending producer work redrive(s) stayed owed; the first {cause}"), ); } + Ok(replay_fences) } /// Replays one host's spooled records, or `None` when it holds none. @@ -337,6 +363,7 @@ async fn drain_admitted_host_spool( graph: &tracedecay_project::project::TraceDecay, project_sessions: &tracedecay_global_db::RegisteredGlobalDb, background_cpu: &Arc, + replay_fences: &BTreeMap<[u8; 32], [u8; 16]>, ) -> Result, String> { let data_root = &graph.hook_store_layout().data_root; let root = hook_v2_spool_root(data_root, host); @@ -370,6 +397,12 @@ async fn drain_admitted_host_spool( &mut resolve_binding, now, |envelope, native_lifecycle| async move { + if replay_fences + .get(&envelope.protected_session_id) + .is_some_and(|event_id| *event_id != envelope.event_id) + { + return tracedecay_hooks::HookReplayAdmissionOutcomeV1::Deferred; + } let lifecycle_for_admission = native_lifecycle.clone(); replay_admission_outcome( admit_replayed_envelope_with_authoritative_session( From 5f5a19dd1b388fc44803a2cbc952d01132ee0a31 Mon Sep 17 00:00:00 2001 From: Zack Jackson Date: Sun, 18 Oct 2026 21:59:53 -0700 Subject: [PATCH 69/89] test(sessions): observe wake consumption before worker idleness --- .../src/session_temporal_refresh_scheduler/wake.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/wake.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/wake.rs index 9708b66190..8f37a1a1ed 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/wake.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/wake.rs @@ -689,7 +689,10 @@ impl SessionTemporalRefreshWakeState { #[cfg(any(test, feature = "test-helpers"))] pub fn is_idle(&self) -> bool { - !self.busy.load(Ordering::Acquire) && !self.has_pending_work() + // Consuming a wake follows marking the worker busy. Read the flags + // first so observing their consumption also makes that busy state + // visible, rather than combining an old idle read with cleared flags. + !self.has_pending_work() && !self.busy.load(Ordering::Acquire) } } From ab31e96dd4eda2362e5ac7ba693eec9345a02bc3 Mon Sep 17 00:00:00 2001 From: Zack Jackson Date: Sun, 18 Oct 2026 21:59:53 -0700 Subject: [PATCH 70/89] ci: use Ubuntu sources for the Windows cross-compiler --- .github/workflows/ci.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 77c8ed4556..e13ec8b452 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -619,8 +619,14 @@ jobs: - name: Install the Windows C cross-compiler run: | - sudo apt-get update - sudo apt-get install -y --no-install-recommends gcc-mingw-w64-x86-64 + # This compiler comes from Ubuntu, independently of third-party feeds + # preconfigured on the hosted image. + ubuntu_sources=( + -o Dir::Etc::sourcelist=sources.list.d/ubuntu.sources + -o Dir::Etc::sourceparts=- + ) + sudo apt-get "${ubuntu_sources[@]}" update + sudo apt-get "${ubuntu_sources[@]}" install -y --no-install-recommends gcc-mingw-w64-x86-64 - name: Check the Windows target run: >- From cc061b40e7ceea690b8c826aded5d0074ab70fe9 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Mon, 19 Oct 2026 05:00:52 +0000 Subject: [PATCH 71/89] fix(sessions): attach history before waking refresh workers --- .../registry.rs | 113 ++++++++++-------- .../registry_tests.rs | 41 +++++++ 2 files changed, 102 insertions(+), 52 deletions(-) diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/registry.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/registry.rs index a1f3bbc8e7..7a58d40ece 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/registry.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/registry.rs @@ -120,6 +120,28 @@ fn cancel_supervisor( } impl SessionTemporalRefreshSchedulerEntry { + fn wake_with_history(&self, history: Option) { + let Some(history) = history else { + self.wake.wake(); + return; + }; + let mut retained = self.history.write().unwrap_or_else(PoisonError::into_inner); + let installed = retained.is_none(); + if installed { + *retained = Some(history); + self.state.mark_history_pending(); + } + let blocked = !installed && self.state.history_is_blocked(); + drop(retained); + if !blocked { + // Publish both kinds of work before notifying the worker. + self.state.requeue_projection(); + self.state.wake_history(); + } else { + self.wake.wake(); + } + } + fn cancel(&self) { cancel_supervisor(&self.state, &self.history); } @@ -340,6 +362,10 @@ impl SessionTemporalRefreshSchedulerRegistry { history: Option, ) -> SessionTemporalRefreshSchedulerEntry { let state = Arc::new(SessionTemporalRefreshWakeState::default()); + if history.is_some() { + state.mark_history_pending(); + state.wake_history(); + } let history = Arc::new(std::sync::RwLock::new(history)); let wake = route.unwrap_or_else(|| state.handle()); wake.bind(&state); @@ -583,6 +609,16 @@ impl SessionTemporalRefreshSchedulerRegistry { &self, owner: StoreOwnerKey, database: RegisteredGlobalDbLeaseV1, + ) -> SessionTemporalRefreshWake { + self.ensure_project_with_retained_history(owner, database, None) + .await + } + + async fn ensure_project_with_retained_history( + &self, + owner: StoreOwnerKey, + database: RegisteredGlobalDbLeaseV1, + history: Option, ) -> SessionTemporalRefreshWake { if self.shutting_down.load(Ordering::Acquire) { return inert_session_temporal_refresh_wake(); @@ -613,7 +649,8 @@ impl SessionTemporalRefreshSchedulerRegistry { .history .read() .unwrap_or_else(PoisonError::into_inner) - .clone(); + .clone() + .or(history); self.retire(vec![(retired_owner.clone(), finished)]) .join() .await; @@ -631,11 +668,12 @@ impl SessionTemporalRefreshSchedulerRegistry { return wake; } if let Some(entry) = project.get(&owner) { - entry.wake.wake(); + // Install history before any wake can start a summary-only pass. + entry.wake_with_history(history); return entry.wake.clone(); } self.settle_retired(&retired_owner).await; - let entry = self.spawn_entry(database, None, None); + let entry = self.spawn_entry(database, None, history); let wake = entry.wake.clone(); project.insert(owner, entry); wake @@ -645,6 +683,16 @@ impl SessionTemporalRefreshSchedulerRegistry { &self, database_path: std::path::PathBuf, database: RegisteredGlobalDbLeaseV1, + ) -> SessionTemporalRefreshWake { + self.ensure_profile_with_retained_history(database_path, database, None) + .await + } + + async fn ensure_profile_with_retained_history( + &self, + database_path: std::path::PathBuf, + database: RegisteredGlobalDbLeaseV1, + history: Option, ) -> SessionTemporalRefreshWake { if self.shutting_down.load(Ordering::Acquire) { return inert_session_temporal_refresh_wake(); @@ -665,7 +713,8 @@ impl SessionTemporalRefreshSchedulerRegistry { .history .read() .unwrap_or_else(PoisonError::into_inner) - .clone(); + .clone() + .or(history); self.retire(vec![(retired_owner.clone(), finished)]) .join() .await; @@ -679,11 +728,11 @@ impl SessionTemporalRefreshSchedulerRegistry { return wake; } if let Some(entry) = profile.get(&database_path) { - entry.wake.wake(); + entry.wake_with_history(history); return entry.wake.clone(); } self.settle_retired(&retired_owner).await; - let entry = self.spawn_entry(database, None, None); + let entry = self.spawn_entry(database, None, history); let wake = entry.wake.clone(); profile.insert(database_path, entry); wake @@ -695,29 +744,9 @@ impl SessionTemporalRefreshSchedulerRegistry { database: RegisteredGlobalDbLeaseV1, history: SharedSessionHistoricalIngestor, ) -> SessionTemporalRefreshWake { - let wake = self.ensure_project(owner.clone(), database).await; - if session_ingest_disabled() { - // Dev/profiling switch: never install a history ingestor, so the - // project-open catch-up path cannot ingest either. - return wake; - } - if let Some(entry) = self.project.lock().await.get(&owner) { - let mut retained = entry - .history - .write() - .unwrap_or_else(PoisonError::into_inner); - let installed = retained.is_none(); - if installed { - *retained = Some(history); - entry.state.mark_history_pending(); - } - let blocked = !installed && entry.state.history_is_blocked(); - drop(retained); - if !blocked { - entry.state.wake_history(); - } - } - wake + let history = (!session_ingest_disabled()).then_some(history); + self.ensure_project_with_retained_history(owner, database, history) + .await } pub async fn ensure_profile_with_history( @@ -726,29 +755,9 @@ impl SessionTemporalRefreshSchedulerRegistry { database: RegisteredGlobalDbLeaseV1, history: SharedSessionHistoricalIngestor, ) -> SessionTemporalRefreshWake { - let wake = self.ensure_profile(database_path.clone(), database).await; - if session_ingest_disabled() { - // Dev/profiling switch: never install a history ingestor, so the - // project-open catch-up path cannot ingest either. - return wake; - } - if let Some(entry) = self.profile.lock().await.get(&database_path) { - let mut retained = entry - .history - .write() - .unwrap_or_else(PoisonError::into_inner); - let installed = retained.is_none(); - if installed { - *retained = Some(history); - entry.state.mark_history_pending(); - } - let blocked = !installed && entry.state.history_is_blocked(); - drop(retained); - if !blocked { - entry.state.wake_history(); - } - } - wake + let history = (!session_ingest_disabled()).then_some(history); + self.ensure_profile_with_retained_history(database_path, database, history) + .await } pub async fn rekey_project( diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/registry_tests.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/registry_tests.rs index 2d08dd1da0..ad4495ee87 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/registry_tests.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/registry_tests.rs @@ -6,6 +6,9 @@ use std::time::Duration; use futures_util::FutureExt; use tracedecay_global_db::tests::harness::RegisteredGlobalDbHarness; use tracedecay_runtime_core::native_runtime; +use tracedecay_sessions::serving::{ + SessionProjectionServingState, SessionProjectionServingStatusPort, SessionProjectionStaleReason, +}; use super::super::history::{ SessionHistoricalCapacityRelease, SessionHistoricalIngestOutcome, SessionHistoricalIngestPass, @@ -15,6 +18,44 @@ use super::super::wake::SessionTemporalRefreshWakeState; use super::SessionTemporalRefreshSchedulerRegistry; use crate::StoreOwnerKey; +/// A retained ingestor must own the first pass, even with every shared permit +/// occupied. A summary-only first pass would queue ahead of its history and +/// leave a caller waiting for the history refusal while holding those permits. +#[tokio::test] +async fn retained_history_precedes_summary_admission_on_worker_start() { + let harness = RegisteredGlobalDbHarness::open("refresh-history-before-summary").await; + let registry = SessionTemporalRefreshSchedulerRegistry::default(); + let admission = registry.historical_ingest_admission(); + let held = admission + .try_acquire_many(u32::try_from(admission.available_permits()).unwrap()) + .unwrap(); + let history = Arc::new(HeldHistory::default()); + let path = harness.registered.db_path().to_path_buf(); + let entry = registry.spawn_entry(harness.registered.clone(), None, Some(history.clone())); + let wake = entry.wake.clone(); + registry.profile.lock().await.insert(path.clone(), entry); + + tokio::time::timeout(Duration::from_secs(2), registry.wait_profile_idle(&path)) + .await + .expect("history refusal must not wait for summary admission"); + assert_eq!( + wake.serving_status().state, + SessionProjectionServingState::Stale { + reason: SessionProjectionStaleReason::HistoricalRetry { + reason_code: "history_admission_saturated".to_owned(), + }, + } + ); + assert_eq!(history.passes.load(Ordering::Acquire), 0); + drop(held); + tokio::time::timeout(Duration::from_secs(2), history.entered.notified()) + .await + .expect("the retained history gets the released permit"); + assert_eq!(history.passes.load(Ordering::Acquire), 1); + history.release.notify_one(); + registry.shutdown().await; +} + /// A history pass that ignores cancellation until the test releases it. The /// pass holds a clone of `held` for as long as the worker owns it, standing in /// for the lease clones a live worker retains. A `busy` pass is never released From bfb9aba2adca02b9bdf7ea90183e2265c3bd4fcc Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 19 Oct 2026 05:12:48 +0000 Subject: [PATCH 72/89] fix(hooks): fence replay across ledger and spool Co-authored-by: ScriptedAlchemy <25274700+ScriptedAlchemy@users.noreply.github.com> --- crates/tracedecay-hooks/src/hook_v2_replay.rs | 1 + .../src/daemon/hook_v2_replay_consumer.rs | 245 ++++++++++++++++-- 2 files changed, 228 insertions(+), 18 deletions(-) diff --git a/crates/tracedecay-hooks/src/hook_v2_replay.rs b/crates/tracedecay-hooks/src/hook_v2_replay.rs index a9559a6068..42463e4790 100644 --- a/crates/tracedecay-hooks/src/hook_v2_replay.rs +++ b/crates/tracedecay-hooks/src/hook_v2_replay.rs @@ -267,6 +267,7 @@ where } HookReplayAdmissionOutcomeV1::Deferred => { completions.push(ReplayCompletion::Retained(1)); + break; } } } diff --git a/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs b/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs index 716ae989c9..0a512ad561 100644 --- a/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs +++ b/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs @@ -4,6 +4,7 @@ //! admission. Host-spool replay itself lives in `tracedecay-hooks` and is //! invoked with an already-admitted spool plus the project's wire identity. +use std::cmp::Ordering; use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex as StdMutex, OnceLock, PoisonError, Weak}; @@ -210,8 +211,21 @@ async fn drain_all_hosts( Ok(more) => sweep.more_pending |= more, Err(cause) => sweep.fail(*host, StatusHookReplaySpoolV1::DeliveryReceipts, cause), } + let spool_records = match pending_host_spool_records(data_root, *host, now) { + Ok(records) => records, + Err(cause) => { + sweep.fail(*host, StatusHookReplaySpoolV1::Records, cause); + continue; + } + }; let replay_fences = match Box::pin(redrive_pending_work( - graph, data_root, *host, project_id, now, &mut sweep, + graph, + data_root, + *host, + project_id, + now, + &spool_records, + &mut sweep, )) .await { @@ -262,6 +276,69 @@ async fn drain_all_hosts( sweep } +fn pending_host_spool_records( + data_root: &Path, + host: NativeHostIdentityV1, + now: UtcMicros, +) -> Result, String> { + let root = hook_v2_spool_root(data_root, host); + if spool_root_absent(&root)? + || !HookSpoolV1::has_records(&root).map_err(|error| error.to_string())? + { + return Ok(Vec::new()); + } + let (mut spool, _) = HookSpoolV1::open(&root, HookSpoolConfigV1::stock(host), now) + .map_err(|error| error.to_string())?; + spool.import_staged(now).map_err(|error| error.to_string())?; + spool.pending_records().map_err(|error| error.to_string()) +} + +fn replay_event_order( + left: &tracedecay_hooks::HookEventEnvelopeV2, + right: &tracedecay_hooks::HookEventEnvelopeV2, +) -> Ordering { + match (left.ordering, right.ordering) { + ( + tracedecay_hooks::HookOrderingV1::ProviderSequence(left), + tracedecay_hooks::HookOrderingV1::ProviderSequence(right), + ) => left.cmp(&right), + _ => left.observed_at.cmp(&right.observed_at), + } +} + +fn replay_fences( + pending_work: &[tracedecay_hooks::HookEventEnvelopeV2], + spool_records: &[tracedecay_hooks::HookSpoolRecordV1], + project_id: Option<[u8; 16]>, + now: UtcMicros, +) -> BTreeMap<[u8; 32], [u8; 16]> { + let mut oldest = BTreeMap::<[u8; 32], tracedecay_hooks::HookEventEnvelopeV2>::new(); + for envelope in pending_work { + if project_id.is_none_or(|project_id| envelope.project_id == project_id) { + oldest + .entry(envelope.protected_session_id) + .or_insert_with(|| envelope.clone()); + } + } + for record in spool_records.iter().filter(|record| { + project_id.is_none_or(|project_id| record.envelope.project_id == project_id) + && now.0.saturating_sub(record.queued_at.0) + <= tracedecay_hooks::MAX_SPOOL_AGE_MICROS + }) { + let session = record.protected_session_id; + if oldest + .get(&session) + .is_none_or(|current| replay_event_order(&record.envelope, current).is_lt()) + { + oldest.insert(session, record.envelope.clone()); + } + } + oldest + .into_iter() + .map(|(session, envelope)| (session, envelope.event_id)) + .collect() +} + /// Redrives the producer work `host`'s admission ledger still owes. Work a /// redrive cannot admit stays owed, and the sweep names it. async fn redrive_pending_work( @@ -270,6 +347,7 @@ async fn redrive_pending_work( host: NativeHostIdentityV1, project_id: Option<[u8; 16]>, now: UtcMicros, + spool_records: &[tracedecay_hooks::HookSpoolRecordV1], sweep: &mut HookReplaySweepV1, ) -> std::result::Result, ()> { let pending_work = match hook_v2_pending_work_envelopes(data_root, host, now) { @@ -283,14 +361,7 @@ async fn redrive_pending_work( return Err(()); } }; - let mut replay_fences = BTreeMap::new(); - for envelope in &pending_work { - if project_id.is_none_or(|project_id| envelope.project_id == project_id) { - replay_fences - .entry(envelope.protected_session_id) - .or_insert(envelope.event_id); - } - } + let replay_fences = replay_fences(&pending_work, spool_records, project_id, now); let mut owed = 0_usize; let mut owed_cause = None; for envelope in pending_work { @@ -785,7 +856,10 @@ mod tests { 1, move |request: tracedecay_daemon_service::HookOrchestrationRequestV1, _| { let ran = ran.clone(); - async move { ran.send(request.lifecycle).unwrap() } + async move { + ran.send((request.hook.envelope().event_id, request.lifecycle)) + .unwrap() + } }, ) .unwrap(); @@ -828,11 +902,13 @@ mod tests { &background_cpu, ) .await; - let mounted = tokio::time::timeout(Duration::from_secs(10), runs.recv()) + let (mounted_event_id, mounted) = tokio::time::timeout(Duration::from_secs(10), runs.recv()) .await .unwrap() .unwrap() - .expect("spool replay must mount lifecycle"); + ; + assert_eq!(mounted_event_id, envelope.event_id); + let mounted = mounted.expect("spool replay must mount lifecycle"); assert_eq!(mounted.session_id, lifecycle.session_id); tokio::time::timeout(Duration::from_secs(10), async { while !pending().is_empty() { @@ -919,11 +995,12 @@ mod tests { &background_cpu, ) .await; - let second_mounted = tokio::time::timeout(Duration::from_secs(10), runs.recv()) + let (second_event_id, second_mounted) = tokio::time::timeout(Duration::from_secs(10), runs.recv()) .await .unwrap() - .unwrap() - .expect("second spool record must mount its own lifecycle"); + .unwrap(); + assert_eq!(second_event_id, second_edit.event_id); + let second_mounted = second_mounted.expect("second spool record must mount its own lifecycle"); assert_eq!(second_mounted.session_id, second_lifecycle.session_id); tokio::time::timeout(Duration::from_secs(10), async { while !pending().is_empty() { @@ -936,6 +1013,136 @@ mod tests { runs.try_recv().is_err(), "second edit executes exactly once" ); + + tracedecay_daemon_service::unregister_hook_orchestration_runtime( + project_id, + binding.worktree_id, + &runtime, + ); + let stop = tracedecay_hooks::HookEventEnvelopeV2 { + event_id: [12; 16], + observed_at: UtcMicros(now.0.saturating_add(10)), + event: tracedecay_hooks::HookEventV2::SessionBoundary { + boundary: tracedecay_hooks::HookBoundaryV1::TurnComplete, + }, + ..envelope.clone() + }; + let later_lifecycle = tracedecay_hooks::NativeContextScoutLifecycleV1::new( + "session.replay.edit", + "call.replay.after-stop", + [13; 16], + ) + .unwrap(); + let later_edit = tracedecay_hooks::HookEventEnvelopeV2 { + event_id: later_lifecycle.event_id, + observed_at: UtcMicros(now.0.saturating_add(20)), + event: tracedecay_hooks::HookEventV2::SavedEdit { + file_id: [13; 16], + changed_range_count: 1, + }, + ..envelope.clone() + }; + assert!(matches!( + admit_hook_v2_envelope(&graph, &stop, None, hook_replay_now()).await, + HookV2AdmissionOutcomeV1::Admitted { .. } + )); + assert!(matches!( + admit_hook_v2_envelope(&graph, &later_edit, None, hook_replay_now()).await, + HookV2AdmissionOutcomeV1::Admitted { .. } + )); + let (mut stop_spool, _) = + HookSpoolV1::open(&root, HookSpoolConfigV1::stock(host), hook_replay_now()).unwrap(); + stop_spool + .append_with_native_lifecycle( + later_edit.clone(), + Some(later_lifecycle.clone()), + &binding, + hook_replay_now(), + ) + .unwrap(); + drop(stop_spool); + assert_eq!(pending(), vec![stop.clone(), later_edit.clone()]); + assert!( + tracedecay_daemon_service::register_hook_orchestration_runtime( + project_id, + binding.worktree_id, + &runtime, + ) + ); + drain_all_hosts( + &graph, + data_root, + &fixture.settlements, + &project_sessions, + &background_cpu, + ) + .await; + let (stop_id, stop_lifecycle) = tokio::time::timeout(Duration::from_secs(10), runs.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!(stop_id, stop.event_id); + assert!(stop_lifecycle.is_some()); + assert!( + runs.try_recv().is_err(), + "the newer edit stays behind the ledger-only stop" + ); + tokio::time::timeout(Duration::from_secs(10), async { + while pending().contains(&stop) { + tokio::task::yield_now().await; + } + }) + .await + .expect("the older stop settles before the later edit"); + drain_all_hosts( + &graph, + data_root, + &fixture.settlements, + &project_sessions, + &background_cpu, + ) + .await; + let (later_event_id, later_mounted) = + tokio::time::timeout(Duration::from_secs(10), runs.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!(later_event_id, later_edit.event_id); + assert_eq!( + later_mounted.expect("the newer edit mounts its own lifecycle").session_id, + later_lifecycle.session_id + ); + tokio::time::timeout(Duration::from_secs(10), async { + while pending().contains(&later_edit) { + tokio::task::yield_now().await; + } + }) + .await + .expect("later edit settles after the stop"); + let (mut duplicate_spool, _) = + HookSpoolV1::open(&root, HookSpoolConfigV1::stock(host), hook_replay_now()).unwrap(); + duplicate_spool + .append_with_native_lifecycle( + later_edit.clone(), + Some(later_lifecycle), + &binding, + hook_replay_now(), + ) + .unwrap(); + drop(duplicate_spool); + drain_all_hosts( + &graph, + data_root, + &fixture.settlements, + &project_sessions, + &background_cpu, + ) + .await; + assert!( + runs.try_recv().is_err(), + "a late exact duplicate does not remount completed lifecycle work" + ); + // Claude deliberately omits lifecycle writes while intent is disabled; // an acknowledged edit can therefore need ledger-only recovery. tracedecay_daemon_service::unregister_hook_orchestration_runtime( @@ -982,12 +1189,14 @@ mod tests { &background_cpu, ) .await; - assert!( + let (disabled_event_id, disabled_lifecycle) = tokio::time::timeout(Duration::from_secs(10), runs.recv()) .await .unwrap() - .unwrap() - .is_none(), + .unwrap(); + assert_eq!(disabled_event_id, disabled_edit.event_id); + assert!( + disabled_lifecycle.is_none(), "disabled Claude edits settle without mounting a lifecycle" ); tokio::time::timeout(Duration::from_secs(10), async { From 4e6d11aba2dfacdaad88311087fa9733da01a190 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Mon, 19 Oct 2026 05:15:50 +0000 Subject: [PATCH 73/89] fix(hooks): settle cooperatively superseded producer work --- .../src/invocation/types.rs | 30 +++-- .../invocation/types/context_scout_tests.rs | 116 +++++++++++++----- 2 files changed, 101 insertions(+), 45 deletions(-) diff --git a/crates/tracedecay-daemon-service/src/invocation/types.rs b/crates/tracedecay-daemon-service/src/invocation/types.rs index 1574e8177f..c230e0b6da 100644 --- a/crates/tracedecay-daemon-service/src/invocation/types.rs +++ b/crates/tracedecay-daemon-service/src/invocation/types.rs @@ -267,7 +267,7 @@ impl BoundedHookOrchestratorV1 { emit_terminal: bool, permit: Option, ) { - let completions = { + let (completions, superseded) = { let mut in_flight = in_flight .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); @@ -298,20 +298,24 @@ impl BoundedHookOrchestratorV1 { // free while it still looks in flight. Its terminals, the host's // cue to send the session's next boundary, follow. drop(permit); - completions + // A cooperative producer can return a retryable result after the + // outer select polled cancellation. The same incumbent lock owns + // supersession and removal, so its proof remains terminal even + // when the work result itself does not report completion. + ( + completions, + operation + .superseded + .load(std::sync::atomic::Ordering::Acquire), + ) }; - if emit_terminal { + if emit_terminal || superseded { for completion in completions { - completion( - if operation - .superseded - .load(std::sync::atomic::Ordering::Acquire) - { - HookOrchestrationSettlementV1::Superseded - } else { - HookOrchestrationSettlementV1::Completed - }, - ); + completion(if superseded { + HookOrchestrationSettlementV1::Superseded + } else { + HookOrchestrationSettlementV1::Completed + }); } } } diff --git a/crates/tracedecay-daemon-service/src/invocation/types/context_scout_tests.rs b/crates/tracedecay-daemon-service/src/invocation/types/context_scout_tests.rs index 315b7cc58d..4d4051a856 100644 --- a/crates/tracedecay-daemon-service/src/invocation/types/context_scout_tests.rs +++ b/crates/tracedecay-daemon-service/src/invocation/types/context_scout_tests.rs @@ -1,5 +1,6 @@ use super::*; use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{OnceLock, Weak}; use tracedecay_hooks::{HookEventFamily, HookEventSupportV1}; use tracedecay_runtime_core::cancellation::{CancellationToken, MonotonicDeadline}; @@ -843,41 +844,42 @@ impl OrderedHookFixture { } fn request(&self, order: u8) -> HookOrchestrationRequestV1 { - let template = native_request(111); - let mut envelope = template.hook.envelope().clone(); - envelope.event_id = [order; 16]; - let binding = HookScopeBindingV1 { - host: envelope.producer, - project_id: envelope.project_id, - repository_id: envelope.repository_id, - worktree_id: envelope.worktree_id, - worktree_epoch: envelope.worktree_epoch, - binding_token: envelope.binding_token, - capabilities: vec![tracedecay_hooks::HookCapabilityV1 { - family: tracedecay_hooks::HookEventFamily::SavedEdit, - support: tracedecay_hooks::stock_event_support( - envelope.producer, - tracedecay_hooks::HookEventFamily::SavedEdit, - ), - }], - }; - let mut request = HookOrchestrationRequestV1::from_envelope( - envelope, - &binding, - template.lifecycle, - 1, - false, - ) - .unwrap(); - request.admission_order = Some(u64::from(order)); - let settled = self.settled.clone(); - request.completion = Some(Arc::new(move |terminal| { - settled.send((u64::from(order), terminal)).unwrap(); - })); - request + ordered_request(order, self.settled.clone()) } } +fn ordered_request( + order: u8, + settled: tokio::sync::mpsc::UnboundedSender<(u64, HookOrchestrationSettlementV1)>, +) -> HookOrchestrationRequestV1 { + let template = native_request(111); + let mut envelope = template.hook.envelope().clone(); + envelope.event_id = [order; 16]; + let binding = HookScopeBindingV1 { + host: envelope.producer, + project_id: envelope.project_id, + repository_id: envelope.repository_id, + worktree_id: envelope.worktree_id, + worktree_epoch: envelope.worktree_epoch, + binding_token: envelope.binding_token, + capabilities: vec![tracedecay_hooks::HookCapabilityV1 { + family: tracedecay_hooks::HookEventFamily::SavedEdit, + support: tracedecay_hooks::stock_event_support( + envelope.producer, + tracedecay_hooks::HookEventFamily::SavedEdit, + ), + }], + }; + let mut request = + HookOrchestrationRequestV1::from_envelope(envelope, &binding, template.lifecycle, 1, false) + .unwrap(); + request.admission_order = Some(u64::from(order)); + request.completion = Some(Arc::new(move |terminal| { + settled.send((u64::from(order), terminal)).unwrap(); + })); + request +} + #[tokio::test] async fn delayed_replay_preserves_newer_work_and_its_typed_terminal() { let mut fixture = OrderedHookFixture::new(); @@ -961,3 +963,53 @@ async fn concurrent_live_admission_and_delayed_replay_keep_newest_boundary() { } assert!(fixture.runtime.shutdown().await); } + +/// The outer biased select has already polled cancellation when this work +/// admits its successor. Its ready retryable result must still settle the +/// incumbent's canonically proven supersession. +#[tokio::test] +async fn cooperative_failure_during_supersession_emits_the_owed_terminal() { + let (settled, mut terminals) = tokio::sync::mpsc::unbounded_channel(); + let successor = Arc::new(StdMutex::new(Some(ordered_request(2, settled.clone())))); + let owner = Arc::new(OnceLock::>::new()); + let runtime = BoundedHookOrchestratorV1::new(1, { + let successor = Arc::clone(&successor); + let owner = Arc::clone(&owner); + move |request: HookOrchestrationRequestV1, cancellation: CancellationToken| { + let successor = Arc::clone(&successor); + let owner = Arc::clone(&owner); + async move { + if request.admission_order == Some(1) { + assert!(!cancellation.is_cancelled()); + let runtime = owner.get().unwrap().upgrade().unwrap(); + assert_eq!( + runtime.admit(successor.lock().unwrap().take().unwrap()), + HookOrchestrationAdmissionV1::Enqueued + ); + assert!(cancellation.is_cancelled()); + HookOrchestrationWorkOutcomeV1::RetryableFailure + } else { + HookOrchestrationWorkOutcomeV1::Completed + } + } + } + }) + .unwrap(); + assert!(owner.set(Arc::downgrade(&runtime)).is_ok()); + assert_eq!( + runtime.admit(ordered_request(1, settled)), + HookOrchestrationAdmissionV1::Enqueued + ); + let observed = tokio::time::timeout(std::time::Duration::from_secs(5), async { + [ + terminals.recv().await.unwrap(), + terminals.recv().await.unwrap(), + ] + }) + .await + .expect("both admitted events retain their typed terminal"); + assert!(observed.contains(&(1, HookOrchestrationSettlementV1::Superseded))); + assert!(observed.contains(&(2, HookOrchestrationSettlementV1::Completed))); + assert!(terminals.try_recv().is_err()); + assert!(runtime.shutdown().await); +} From fd7c693c00b24a8558d877f4a884c72c058ae6e3 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Mon, 19 Oct 2026 05:15:56 +0000 Subject: [PATCH 74/89] test(hooks): isolate replay identities and clarify assertions --- crates/tracedecay-hooks/src/admission_ledger.rs | 2 +- .../admission/tests/legacy_continuity.rs | 9 ++++++--- .../src/handlers/hook_runtime/context_scout.rs | 6 +----- .../src/daemon/hook_v2_replay_consumer.rs | 4 ++-- .../src/mcp/server/host_admission_tests.rs | 15 ++++++++++++--- 5 files changed, 22 insertions(+), 14 deletions(-) diff --git a/crates/tracedecay-hooks/src/admission_ledger.rs b/crates/tracedecay-hooks/src/admission_ledger.rs index 03b9276234..a913d8fa13 100644 --- a/crates/tracedecay-hooks/src/admission_ledger.rs +++ b/crates/tracedecay-hooks/src/admission_ledger.rs @@ -1545,7 +1545,7 @@ mod tests { assert!(first_receipt.work_settlement.is_some()); assert_eq!( ledger.pending_work_envelopes(UtcMicros(4)).unwrap(), - [second.clone()] + std::slice::from_ref(&second) ); ledger.forget(&first.event_id); ledger.rewrite().unwrap(); diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests/legacy_continuity.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests/legacy_continuity.rs index 12ad1a62ff..4d31a78506 100644 --- a/crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests/legacy_continuity.rs +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests/legacy_continuity.rs @@ -397,7 +397,7 @@ async fn retained_legacy_receipt_keeps_producer_work_owed_until_execution() { .commit_lifecycle(fixture.lifecycle.call_id.as_str(), 99) .await; fixture.write_shipped_ledger(true, true); - assert_eq!(fixture.pending(), [fixture.envelope.clone()]); + assert_eq!(fixture.pending(), std::slice::from_ref(&fixture.envelope)); fixture.restart(); let effects = Arc::new(AtomicUsize::new(0)); let producer = fixture.producer(Arc::clone(&effects)); @@ -442,7 +442,7 @@ async fn missing_or_conflicting_legacy_receipt_stays_owed_through_rewrite_and_re fixture.admit().await, HookV2AdmissionOutcomeV1::Backpressured )); - assert_eq!(fixture.pending(), [fixture.envelope.clone()]); + assert_eq!(fixture.pending(), std::slice::from_ref(&fixture.envelope)); fixture.restart(); } } @@ -462,7 +462,10 @@ async fn a_later_canonical_receipt_truthfully_supersedes_retained_legacy_work() fixture.admit().await, HookV2AdmissionOutcomeV1::ExactDuplicate { .. } )); - assert!(fixture.pending().is_empty()); + assert_eq!( + fixture.pending(), + Vec::::new() + ); assert_eq!(effects.load(Ordering::SeqCst), 0); let canonical = daemon_mint_hook_v2_envelope(&fixture.envelope); let receipt = stage_hook_v2_admission( diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/context_scout.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/context_scout.rs index ec6d6e6b4a..16e260859d 100644 --- a/crates/tracedecay-mcp/src/handlers/hook_runtime/context_scout.rs +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/context_scout.rs @@ -223,7 +223,7 @@ pub(super) async fn recovered_native_lifecycle_order( std::cmp::Ordering::Less => NativeLifecycleAdmissionV1::Unavailable, } } - Ok(None) | Ok(Some(_)) | Err(_) => NativeLifecycleAdmissionV1::Unavailable, + Ok(None | Some(_)) | Err(_) => NativeLifecycleAdmissionV1::Unavailable, } } @@ -260,10 +260,6 @@ pub(super) async fn native_lifecycle_order( } #[tracing::instrument(name = "mcp.hook_runtime.scout_lifecycle", level = "trace", skip_all)] -#[expect( - clippy::too_many_lines, - reason = "Context-scout admission is one native lifecycle bind for the scout claim." -)] pub(super) async fn admit_native_context_scout_lifecycle( sessions: &RegisteredGlobalDb, background_cpu: Option<&std::sync::Arc>, diff --git a/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs b/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs index 09136dab9d..26336d63a2 100644 --- a/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs +++ b/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs @@ -1076,7 +1076,7 @@ mod tests { ) { let data_root = &graph.hook_store_layout().data_root; let mut older = template.clone(); - older.event_id = [11; 16]; + older.event_id = [13; 16]; let older_lifecycle = tracedecay_hooks::NativeContextScoutLifecycleV1::new( session.session_id.as_str(), "call.replay.older", @@ -1097,7 +1097,7 @@ mod tests { HookV2AdmissionOutcomeV1::Backpressured, )); let mut newer = template.clone(); - newer.event_id = [12; 16]; + newer.event_id = [14; 16]; let newer_lifecycle = tracedecay_hooks::NativeContextScoutLifecycleV1::new( session.session_id.as_str(), "call.replay.newer", diff --git a/crates/tracedecay/src/mcp/server/host_admission_tests.rs b/crates/tracedecay/src/mcp/server/host_admission_tests.rs index afd6295aae..f6c689c697 100644 --- a/crates/tracedecay/src/mcp/server/host_admission_tests.rs +++ b/crates/tracedecay/src/mcp/server/host_admission_tests.rs @@ -184,7 +184,10 @@ async fn file_edit_commit_failure_queues_no_sync() { assert_eq!(outcome.reason_code, Some("spool_recovery_required")); assert_eq!(effects.load(Ordering::SeqCst), 0); assert_eq!(broker.pending_replay_count().await.unwrap(), 0); - assert!(code_index_activity(&server).await.is_empty()); + assert_eq!( + code_index_activity(&server).await, + Vec::::new() + ); server.shutdown().await; } @@ -245,7 +248,10 @@ async fn file_edit_request_acks_durable_work_while_effect_is_held() { entered.notified().await; assert_eq!(broker.pending_count().await, 1); assert_eq!(effects.load(Ordering::SeqCst), 0); - assert!(code_index_activity(&server).await.is_empty()); + assert_eq!( + code_index_activity(&server).await, + Vec::::new() + ); release.notify_one(); server.wait_project_host_admission_replay_idle().await; assert_eq!(broker.pending_count().await, 0); @@ -373,7 +379,10 @@ async fn retained_file_edits_reconcile_branch_drift_without_blocking_the_source( ); assert_eq!(broker.pending_replay_count().await.unwrap(), 2); assert_eq!(effects.load(Ordering::SeqCst), 0); - assert!(code_index_activity(&server).await.is_empty()); + assert_eq!( + code_index_activity(&server).await, + Vec::::new() + ); available.store(true, Ordering::SeqCst); assert_eq!( server.replay_host_admission(None).await, From b40eca6642ac83be11b4958525a36182a9b43239 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 18 Oct 2026 22:22:29 -0700 Subject: [PATCH 75/89] ci: use Ubuntu sources for the Windows cross-compiler --- .github/workflows/ci.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 77c8ed4556..e13ec8b452 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -619,8 +619,14 @@ jobs: - name: Install the Windows C cross-compiler run: | - sudo apt-get update - sudo apt-get install -y --no-install-recommends gcc-mingw-w64-x86-64 + # This compiler comes from Ubuntu, independently of third-party feeds + # preconfigured on the hosted image. + ubuntu_sources=( + -o Dir::Etc::sourcelist=sources.list.d/ubuntu.sources + -o Dir::Etc::sourceparts=- + ) + sudo apt-get "${ubuntu_sources[@]}" update + sudo apt-get "${ubuntu_sources[@]}" install -y --no-install-recommends gcc-mingw-w64-x86-64 - name: Check the Windows target run: >- From 0f189b3f880f44dfd3305448170b6155087394d7 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Mon, 19 Oct 2026 05:29:24 +0000 Subject: [PATCH 76/89] test(hooks): preserve owed work when lifecycle policy fails --- .../admission/tests/legacy_continuity.rs | 93 ++++++++++++++++++- 1 file changed, 92 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests/legacy_continuity.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests/legacy_continuity.rs index 4d31a78506..5ac7a794fd 100644 --- a/crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests/legacy_continuity.rs +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests/legacy_continuity.rs @@ -490,7 +490,10 @@ async fn disabled_claude_stop_follows_the_prior_session_cursor_without_minting_l let mut fixture = LegacyLifecycleFixture::open_for_host(tracedecay_domain::NativeHostIdentityV1::ClaudeCode) .await; - assert!(!hook_v2_native_lifecycle_enabled(&fixture.graph, fixture.envelope.producer).await); + assert_eq!( + hook_v2_native_lifecycle_policy(&fixture.graph, fixture.envelope.producer).await, + HookV2NativeLifecyclePolicyV1::Disabled, + ); fixture .commit_lifecycle("call.prior-real-session", 99) .await; @@ -555,3 +558,91 @@ async fn disabled_claude_stop_follows_the_prior_session_cursor_without_minting_l ); assert!(producer.shutdown().await); } + +#[tokio::test] +async fn unavailable_claude_policy_preserves_owed_work_without_lifecycle_or_producer_effects() { + let fixture = + LegacyLifecycleFixture::open_for_host(tracedecay_domain::NativeHostIdentityV1::ClaudeCode) + .await; + fixture + .commit_lifecycle("call.prior-real-session", 99) + .await; + let canonical = daemon_mint_hook_v2_envelope(&fixture.envelope); + let mut native_lifecycle = fixture.lifecycle.clone(); + native_lifecycle.event_id = canonical.event_id; + let staged = stage_hook_v2_admission( + &fixture.graph.hook_store_layout().data_root, + &canonical, + Some(&fixture.envelope), + Some(&native_lifecycle), + Some(99), + hook_now(), + ) + .unwrap(); + staged.commit.wait().unwrap(); + let effects = Arc::new(AtomicUsize::new(0)); + let producer = fixture.producer(Arc::clone(&effects)); + // The registered configuration reader refuses an ambiguous revision head. + // Existing immutable revisions remain untouched by this fault injection. + let database = fixture.graph.configuration_runtime().registered_database(); + let transaction = database.begin_write_transaction().await.unwrap(); + assert_eq!( + transaction + .execute( + "INSERT INTO configuration_revisions + (revision_id, parent_revision_id, snapshot_id, effective_behavior_digest, + resolution_provenance_digest, actor_id, operation_kind, created_at) + VALUES ('configuration.fault.ambiguous', NULL, 'unavailable', 'unavailable', + 'unavailable', 'actor.fixture', 'fault-injection', 0)", + (), + ) + .await + .unwrap(), + 1, + ); + transaction.commit().await.unwrap(); + assert!( + fixture + .graph + .configuration_runtime() + .client() + .current() + .await + .is_err() + ); + assert_eq!( + hook_v2_native_lifecycle_policy(&fixture.graph, fixture.envelope.producer).await, + HookV2NativeLifecyclePolicyV1::Unavailable, + ); + assert!(matches!( + fixture.admit().await, + HookV2AdmissionOutcomeV1::Unavailable + )); + assert_eq!(effects.load(Ordering::SeqCst), 0); + assert_eq!(fixture.pending(), std::slice::from_ref(&fixture.envelope)); + let staged = stage_hook_v2_admission( + &fixture.graph.hook_store_layout().data_root, + &canonical, + None, + None, + None, + hook_now(), + ) + .unwrap(); + assert_eq!(staged.receipt.order, 99); + assert_eq!(staged.receipt.work_settlement, None); + staged.commit.wait().unwrap(); + assert_eq!( + super::super::super::context_scout::native_lifecycle_order_floor( + &fixture.sessions, + ProviderId::new(fixture.envelope.producer.hook_key()).unwrap(), + fixture.lifecycle.session_id.clone(), + ) + .await, + Some(99), + "unavailable policy creates no lifecycle or cursor advance", + ); + fixture.restart(); + assert_eq!(fixture.pending(), std::slice::from_ref(&fixture.envelope)); + assert!(producer.shutdown().await); +} From 97d4a7135102e87d6e3a18fed046a7e610dbb7b2 Mon Sep 17 00:00:00 2001 From: Zack Jackson Date: Mon, 19 Oct 2026 00:45:13 -0700 Subject: [PATCH 77/89] fix(hooks): retain read availability when admission spool fails --- crates/tracedecay/src/daemon/branch_admin.rs | 58 ++++++++++++++++++- crates/tracedecay/src/daemon/pr_autotrack.rs | 2 +- .../src/daemon/project_composition.rs | 10 ++-- .../hook_source_pending.rs | 33 +++++++---- .../hook_source_pending/tests.rs | 21 ++++--- .../src/daemon/tests/scheduler_config.rs | 47 ++++++++++++++- 6 files changed, 143 insertions(+), 28 deletions(-) diff --git a/crates/tracedecay/src/daemon/branch_admin.rs b/crates/tracedecay/src/daemon/branch_admin.rs index c0d59b67ce..4d5f9cf3d9 100644 --- a/crates/tracedecay/src/daemon/branch_admin.rs +++ b/crates/tracedecay/src/daemon/branch_admin.rs @@ -50,9 +50,44 @@ mod remote_deletion_lifecycle; pub(in crate::daemon) mod remote_recovery_lifecycle; mod session_runtime_shutdown; -type HostAdmissionBrokerSlot = - Arc>; -type HostAdmissionBrokers = Arc>>; +#[derive(Default)] +pub(super) struct HostAdmissionBrokerSlot { + broker: tokio::sync::OnceCell, + initialized: tokio::sync::Notify, +} + +impl HostAdmissionBrokerSlot { + pub(super) async fn get_or_try_init( + &self, + initialize: F, + ) -> Result<&tracedecay_host_admission::SharedHostAdmissionBroker> + where + F: FnOnce() -> Fut, + Fut: Future>, + { + let broker = self.broker.get_or_try_init(initialize).await?; + self.initialized.notify_waiters(); + Ok(broker) + } + + pub(super) fn get(&self) -> Option<&tracedecay_host_admission::SharedHostAdmissionBroker> { + self.broker.get() + } + + pub(super) async fn wait_initialized(&self) { + loop { + let notified = self.initialized.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + if self.get().is_some() { + return; + } + notified.await; + } + } +} + +type HostAdmissionBrokers = Arc>>>; /// One profile session refresh service per profile session store. The service /// owns the opaque refresh handles it issued, so every route that reaches the /// same store (project MCP servers and the projectless client) must share the @@ -1206,6 +1241,23 @@ impl StoreAdministration { .await } + /// Borrow the canonical slot without making spool availability a prerequisite + /// for unrelated project reads. A later successful open fills this same slot. + pub(super) async fn host_admission_broker_slot( + &self, + database: &tracedecay_global_db::RegisteredGlobalDbLeaseV1, + ) -> Result> { + self.refuse_remotely_deleted_profile(database).await?; + let path = authority::canonical_identity_path(database.db_path())?; + Ok(Arc::clone( + self.host_admission_brokers + .lock() + .await + .entry(path) + .or_default(), + )) + } + #[tracing::instrument( name = "daemon.branch_admin.host_admission_broker", level = "trace", diff --git a/crates/tracedecay/src/daemon/pr_autotrack.rs b/crates/tracedecay/src/daemon/pr_autotrack.rs index b0d2ad4d3f..22cf713ca5 100644 --- a/crates/tracedecay/src/daemon/pr_autotrack.rs +++ b/crates/tracedecay/src/daemon/pr_autotrack.rs @@ -909,7 +909,7 @@ async fn activate_linked_worktree( let scope = tracedecay_code_index_runtime::resolved_scope_for_project(worktree, &project_id) .map_err(|error| scheduler_unavailable(&error.to_string()))?; - let (_, reader) = super::project_composition::project_hook_source_pending( + let reader = super::project_composition::project_hook_source_pending( stores, graph, worktree, &scope, ) .await diff --git a/crates/tracedecay/src/daemon/project_composition.rs b/crates/tracedecay/src/daemon/project_composition.rs index 379addffae..efa3c21625 100644 --- a/crates/tracedecay/src/daemon/project_composition.rs +++ b/crates/tracedecay/src/daemon/project_composition.rs @@ -576,7 +576,6 @@ struct OpenedProjectGraph { /// Ports both of this route's MCP servers publish. Built once before the core /// server; the full server that replaces the core publishes the same ports. struct ProjectRoutePorts { - host_admission_broker: tracedecay_host_admission::SharedHostAdmissionBroker, code_index: ProjectCodeIndexAuthorities, dashboard_code_index_freshness_reader: tracedecay_contracts::code_index_freshness::CodeIndexFreshnessReader, @@ -940,7 +939,7 @@ impl ProjectOpenInputs<'_> { &route_registered, *project_database_is_read_only, )?; - let (host_admission_broker, pending_source) = project_hook_source_pending( + let pending_source = project_hook_source_pending( self.store_administration, cg, self.canonical_project_path, @@ -1034,7 +1033,6 @@ impl ProjectOpenInputs<'_> { transcript_source_profile, code_index_activation, ports: ProjectRoutePorts { - host_admission_broker, code_index, dashboard_code_index_freshness_reader, code_index_readiness_waiter, @@ -1383,7 +1381,11 @@ impl ProjectOpenInputs<'_> { &delivery_access, ) .await?; - let host_admission_broker = Some(Arc::clone(&core.ports.host_admission_broker)); + let host_admission_broker = Some( + self.store_administration + .host_admission_broker(&session_db) + .await?, + ); let refresh_schedulers = self .store_administration .session_temporal_refresh_schedulers(); diff --git a/crates/tracedecay/src/daemon/project_composition/hook_source_pending.rs b/crates/tracedecay/src/daemon/project_composition/hook_source_pending.rs index 7510b881e6..99f51b8729 100644 --- a/crates/tracedecay/src/daemon/project_composition/hook_source_pending.rs +++ b/crates/tracedecay/src/daemon/project_composition/hook_source_pending.rs @@ -8,7 +8,6 @@ use tracedecay_code_index_runtime::code_index_scheduler::{ use tracedecay_contracts::ResolvedScope; use tracedecay_contracts::code_index_freshness::CodeIndexFreshnessReadFailureV1; use tracedecay_domain::errors::{Result, TraceDecayError}; -use tracedecay_host_admission::SharedHostAdmissionBroker; use tracedecay_mcp::hook_events::{ DurableHookEventDecodeError, HookEventPlan, decode_durable_hook_event_plan, }; @@ -16,13 +15,13 @@ use tracedecay_project::project::TraceDecay; use tracedecay_sessions::admission::HostAdmissionOutcome; use tracedecay_store::StoreShardScopeV1; -use crate::daemon::branch_admin::StoreAdministration; +use crate::daemon::branch_admin::{HostAdmissionBrokerSlot, StoreAdministration}; #[cfg(test)] mod tests; struct HookSourcePending { - broker: SharedHostAdmissionBroker, + broker: Arc, root: PathBuf, scope: ResolvedScope, } @@ -30,6 +29,8 @@ struct HookSourcePending { impl CodeIndexSourcePendingReaderV1 for HookSourcePending { fn read_pending(&self) -> std::result::Result { self.broker + .get() + .ok_or(CodeIndexFreshnessReadFailureV1::ReadFailed)? .pending_replay_matches(|payload| { let plan = decode_durable_hook_event_plan(payload).map_err(|error| { HostAdmissionOutcome::degraded(match error { @@ -65,11 +66,16 @@ impl CodeIndexSourcePendingReaderV1 for HookSourcePending { } fn revision(&self) -> u64 { - self.broker.progress_revision() + self.broker + .get() + .map_or(0, |broker| broker.progress_revision()) } fn changed(&self, observed: u64) -> CodeIndexSourcePendingChangedFutureV1<'_> { - Box::pin(self.broker.wait_for_progress(observed)) + match self.broker.get() { + Some(broker) => Box::pin(broker.wait_for_progress(observed)), + None => Box::pin(self.broker.wait_initialized()), + } } } @@ -81,10 +87,7 @@ pub(in crate::daemon) async fn project_hook_source_pending( graph: &TraceDecay, root: &Path, expected_scope: &ResolvedScope, -) -> Result<( - SharedHostAdmissionBroker, - Arc, -)> { +) -> Result> { let registry = stores.registered_profile_database().await?; let owner = registry .project_registry_context_by_id(expected_scope.project_id.as_str()) @@ -122,11 +125,17 @@ pub(in crate::daemon) async fn project_hook_source_pending( "hook source writer belongs to a different project", )); } - let broker = stores.host_admission_broker(&database).await?; + let broker = stores.host_admission_broker_slot(&database).await?; + if let Err(error) = stores.host_admission_broker(&database).await { + if error.hook_runtime_context().is_none() { + return Err(error); + } + tracing::warn!(%error, "hook source currency unavailable until admission spool opens"); + } let reader: Arc = Arc::new(HookSourcePending { - broker: Arc::clone(&broker), + broker, root, scope, }); - Ok((broker, reader)) + Ok(reader) } diff --git a/crates/tracedecay/src/daemon/project_composition/hook_source_pending/tests.rs b/crates/tracedecay/src/daemon/project_composition/hook_source_pending/tests.rs index 04e5da2c4c..a6824048eb 100644 --- a/crates/tracedecay/src/daemon/project_composition/hook_source_pending/tests.rs +++ b/crates/tracedecay/src/daemon/project_composition/hook_source_pending/tests.rs @@ -1,18 +1,24 @@ use tracedecay_domain::HostIntegrationIdV1; -use tracedecay_host_admission::{HostAdmissionBroker, HostAdmissionRuntime, SpoolBounds}; +use tracedecay_host_admission::{ + HostAdmissionBroker, HostAdmissionRuntime, SharedHostAdmissionBroker, SpoolBounds, +}; use tracedecay_mcp::hook_events::{HookEventKind, encode_durable_hook_event_plan}; use super::*; use crate::mcp::server::writer_test_support::init_indexed_repo; use crate::test_support::git::git; -fn reader( +async fn reader( broker: &SharedHostAdmissionBroker, root: &Path, scope: &ResolvedScope, ) -> HookSourcePending { + let slot = Arc::new(HostAdmissionBrokerSlot::default()); + slot.get_or_try_init(|| async { Ok(Arc::clone(broker)) }) + .await + .unwrap(); HookSourcePending { - broker: Arc::clone(broker), + broker: slot, root: root.to_owned(), scope: scope.clone(), } @@ -68,8 +74,8 @@ async fn pending_source_is_exactly_routed_across_linked_worktrees_and_branch_cha tracedecay_code_index_runtime::resolved_scope_for_project(&linked, &project_id).unwrap(); let spool = tempfile::tempdir().unwrap(); let broker = broker(spool.path()); - let main = reader(&broker, project.path(), &main_scope); - let other = reader(&broker, &linked, &linked_scope); + let main = reader(&broker, project.path(), &main_scope).await; + let other = reader(&broker, &linked, &linked_scope).await; broker .admit("linked", &edit(&linked, linked_scope)) .await @@ -87,6 +93,7 @@ async fn pending_source_is_exactly_routed_across_linked_worktrees_and_branch_cha .unwrap(); assert!( reader(&broker, project.path(), ¤t) + .await .read_pending() .unwrap() ); @@ -145,7 +152,7 @@ async fn unscoped_legacy_edits_remain_pending_and_invalid_plans_fail_closed() { let broker = broker(spool.path()); broker.admit("source", &payload).await.unwrap(); assert_eq!( - reader(&broker, project.path(), &scope).read_pending(), + reader(&broker, project.path(), &scope).await.read_pending(), expected ); } @@ -160,7 +167,7 @@ async fn unscoped_legacy_edits_remain_pending_and_invalid_plans_fail_closed() { .await .unwrap(); assert_eq!( - reader(&broker, project.path(), &scope).read_pending(), + reader(&broker, project.path(), &scope).await.read_pending(), Err(CodeIndexFreshnessReadFailureV1::ReadFailed) ); } diff --git a/crates/tracedecay/src/daemon/tests/scheduler_config.rs b/crates/tracedecay/src/daemon/tests/scheduler_config.rs index 96f33b0a0a..842de1c5f5 100644 --- a/crates/tracedecay/src/daemon/tests/scheduler_config.rs +++ b/crates/tracedecay/src/daemon/tests/scheduler_config.rs @@ -499,10 +499,55 @@ async fn unavailable_host_admission_spool_does_not_block_project_server_open() { "unavailable-host-admission-test", ); - engine + let server = engine .project_server(&handshake) .await .expect("read/query server must open when admission spool is unavailable"); + let graph = server.cg().await; + let scope = tracedecay_code_index_runtime::resolved_scope_for_project( + &project, + &graph + .store_layout() + .identity + .project_id + .as_ref() + .unwrap() + .clone() + .try_into() + .unwrap(), + ) + .unwrap(); + let reader = super::super::project_composition::project_hook_source_pending( + &engine.store_administration, + &graph, + &project, + &scope, + ) + .await + .unwrap(); + assert_eq!( + reader.read_pending(), + Err( + tracedecay_contracts::code_index_freshness::CodeIndexFreshnessReadFailureV1::ReadFailed + ), + "an unreadable spool cannot prove current source currency" + ); + let recovered = reader.changed(reader.revision()); + std::fs::remove_file(&spool_path).unwrap(); + let database = engine + .store_administration + .registered_project_session_database(graph.project_root(), graph.store_layout()) + .await + .unwrap(); + engine + .store_administration + .host_admission_broker(&database) + .await + .unwrap(); + tokio::time::timeout(Duration::from_secs(5), recovered) + .await + .expect("the original reader must observe the recovered canonical broker"); + assert_eq!(reader.read_pending(), Ok(false)); engine.shutdown_all().await; } From b721f91dfb9f350fb2d892ddb3adc2baf8b74825 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Mon, 19 Oct 2026 08:15:46 +0000 Subject: [PATCH 78/89] fix(hooks): recover intent delivery while feedback is pending --- .../src/handlers/hook_runtime/admission.rs | 193 ++++----- .../handlers/hook_runtime/context_scout.rs | 19 - .../hook_runtime/context_scout/tests.rs | 29 -- .../advisory_runtime/scout_journey_tests.rs | 123 ++++-- .../scout_journey_tests/replay.rs | 401 ++++++++++++++++++ .../replay/configuration.rs | 70 +++ .../scout_journey_tests/replay/pending.rs | 359 ++++++++++++++++ .../intent_pass_journey_test.rs | 129 ++++-- 8 files changed, 1079 insertions(+), 244 deletions(-) create mode 100644 crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests/replay.rs create mode 100644 crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests/replay/configuration.rs create mode 100644 crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests/replay/pending.rs diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/admission.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/admission.rs index 382067bb89..1610c0eb86 100644 --- a/crates/tracedecay-mcp/src/handlers/hook_runtime/admission.rs +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/admission.rs @@ -13,9 +13,8 @@ use tracedecay_runtime_core::background_cpu::ProcessBackgroundCpuV1; use super::context_scout::{ NativeLifecycleAdmissionV1, admit_native_context_scout_lifecycle, hook_v2_context_scout_lifecycle_for_session, hook_v2_native_context_scout_lifecycle, - lookup_hook_v2_delivery_claim, lookup_hook_v2_delivery_claim_for_event, native_lifecycle_order, - native_lifecycle_order_floor, recovered_native_lifecycle_order, remove_hook_v2_delivery_claim, - retain_hook_v2_delivery_claim, + lookup_hook_v2_delivery_claim, native_lifecycle_order, native_lifecycle_order_floor, + recovered_native_lifecycle_order, remove_hook_v2_delivery_claim, retain_hook_v2_delivery_claim, }; use super::envelope::{ daemon_mint_hook_v2_envelope, hook_now, hook_v2_envelope, hook_v2_family_label, @@ -440,32 +439,47 @@ async fn retain_ready_guidance( } } -fn ready_guidance_from_retained_claim( - hook: &tracedecay_hooks::HookEventEnvelopeV2, - claim: tracedecay_contracts::context_scout::ContextScoutDurableClaimV1, +async fn claim_exact_ready_guidance( + cg: &TraceDecay, + envelope: &tracedecay_hooks::HookEventEnvelopeV2, + claim_authority: Option<( + tracedecay_contracts::context_scout::ContextScoutAddressV1, + [u8; 32], + )>, configuration_revision: u64, + response_event_id: [u8; 16], now: UtcMicros, -) -> Option<( - tracedecay_hooks::HookReadyGuidanceV1, - tracedecay_contracts::context_scout::ContextScoutDurableClaimV1, -)> { - (claim.lease.lease_id == hook.event_id - && claim.lease.expires_at.0 > now.0 - && claim.entry.work.address.project_id == hook.project_id - && claim.entry.work.address.protected_session_id == hook.protected_session_id - && claim.entry.envelope.candidate.expires_at.0 > now.0) - .then(|| { - ( - tracedecay_hooks::HookReadyGuidanceV1 { - guidance_id: claim.entry.envelope.envelope_id, - event_id: hook.event_id, - configuration_revision, - expires_at: claim.entry.envelope.candidate.expires_at, - text: claim.entry.envelope.candidate.suggestion_text.clone(), - }, +) -> Option { + let (Some(owner), Some((address, input_watermark))) = + (cg.context_scout_owner(), claim_authority) + else { + return None; + }; + // Same-event leases are reclaimed through the owner's current configuration + // check, including when a prior host response was lost after claiming. + match owner + .claim_ready_guidance_exact( + envelope, + address, + input_watermark, + configuration_revision, + now, + ) + .await + { + Some((guidance, claim)) => Some( + retain_ready_guidance( + owner.as_ref(), + envelope.project_id, + guidance, claim, + response_event_id, + now, ) - }) + .await, + ), + None => None, + } } fn cursor_stack_wakeup_allowed( @@ -793,17 +807,7 @@ async fn admit_hook_v2_envelope_with_lifecycle_inner( if receipt.decision == tracedecay_hooks::HookAdmissionDecisionV1::ExactDuplicate && receipt.work_settlement.is_some() { - let retained_claim = host_response_available - .then(|| { - lookup_hook_v2_delivery_claim_for_event(envelope.project_id, envelope.event_id, now) - }) - .flatten(); - let retained_claim_authority = retained_claim - .as_ref() - .map(|claim| (claim.entry.work.address, claim.entry.work.input_watermark)); - let claim_authority = if retained_claim_authority.is_some() { - retained_claim_authority - } else if host_response_available { + let claim_authority = if host_response_available { let lifecycle = hook_v2_context_scout_lifecycle_for_session(envelope, native_session_id).await; match lifecycle.as_ref() { @@ -833,48 +837,16 @@ async fn admit_hook_v2_envelope_with_lifecycle_inner( let context_scout_address = claim_authority .as_ref() .map(|(address, _)| Box::new(*address)); - let ready_guidance = match (cg.context_scout_owner(), retained_claim, claim_authority) { - (Some(owner), Some(claim), _) => { - match ready_guidance_from_retained_claim(envelope, claim, snapshot.revision, now) { - Some((guidance, claim)) => { - retain_ready_guidance( - owner.as_ref(), - envelope.project_id, - guidance, - claim, - provider_envelope.event_id, - now, - ) - .await - } - None => Value::Null, - } - } - (Some(owner), None, Some((address, input_watermark))) => match owner - .claim_ready_guidance_exact( - envelope, - address, - input_watermark, - snapshot.revision, - now, - ) - .await - { - Some((guidance, claim)) => { - retain_ready_guidance( - owner.as_ref(), - envelope.project_id, - guidance, - claim, - provider_envelope.event_id, - now, - ) - .await - } - None => Value::Null, - }, - _ => Value::Null, - }; + let ready_guidance = claim_exact_ready_guidance( + cg, + envelope, + claim_authority, + snapshot.revision, + provider_envelope.event_id, + now, + ) + .await + .unwrap_or(Value::Null); return HookV2AdmissionOutcomeV1::ExactDuplicate { context_scout_address, ready_guidance, @@ -953,43 +925,36 @@ async fn admit_hook_v2_envelope_with_lifecycle_inner( let mut context_scout_address = claim_authority .as_ref() .map(|(address, _)| Box::new(*address)); - let ready_guidance = match (first_admission, cg.context_scout_owner(), claim_authority) { - (true, Some(owner), Some((address, input_watermark))) => match owner - .claim_ready_guidance_exact(envelope, address, input_watermark, snapshot.revision, now) + let exact_guidance = claim_exact_ready_guidance( + cg, + envelope, + claim_authority, + snapshot.revision, + provider_envelope.event_id, + now, + ) + .await; + let exact_claim_missed = exact_guidance.is_none(); + let mut ready_guidance = exact_guidance.unwrap_or(Value::Null); + if first_admission + && exact_claim_missed + && claim_authority.is_some() + && let Some(owner) = cg.context_scout_owner() + && let Some((guidance, claim)) = owner + .claim_ready_guidance(envelope, snapshot.revision, now) .await - { - Some((guidance, claim)) => { - retain_ready_guidance( - owner.as_ref(), - envelope.project_id, - guidance, - claim, - provider_envelope.event_id, - now, - ) - .await - } - None => match owner - .claim_ready_guidance(envelope, snapshot.revision, now) - .await - { - Some((guidance, claim)) => { - context_scout_address = Some(Box::new(claim.entry.work.address)); - retain_ready_guidance( - owner.as_ref(), - envelope.project_id, - guidance, - claim, - provider_envelope.event_id, - now, - ) - .await - } - None => Value::Null, - }, - }, - _ => Value::Null, - }; + { + context_scout_address = Some(Box::new(claim.entry.work.address)); + ready_guidance = retain_ready_guidance( + owner.as_ref(), + envelope.project_id, + guidance, + claim, + provider_envelope.event_id, + now, + ) + .await; + } let orchestration = tracedecay_daemon_service::admit_registered_hook_orchestration( envelope.clone(), snapshot.binding.clone(), diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/context_scout.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/context_scout.rs index 16e260859d..0198d8953b 100644 --- a/crates/tracedecay-mcp/src/handlers/hook_runtime/context_scout.rs +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/context_scout.rs @@ -399,25 +399,6 @@ pub(super) fn lookup_hook_v2_delivery_claim( .cloned() } -pub(super) fn lookup_hook_v2_delivery_claim_for_event( - project_id: [u8; 16], - event_id: [u8; 16], - now: UtcMicros, -) -> Option { - let claims = retained_hook_v2_delivery_claims().lock().ok()?; - let mut matching = claims - .iter() - .filter(|((candidate_project_id, _), claim)| { - *candidate_project_id == project_id && claim.lease.lease_id == event_id - }) - .map(|(_, claim)| claim); - let claim = matching.next()?; - if matching.next().is_some() || claim.lease.expires_at.0 <= now.0 { - return None; - } - Some(claim.clone()) -} - pub(super) fn remove_hook_v2_delivery_claim(project_id: [u8; 16], envelope_id: [u8; 16]) { if let Ok(mut claims) = retained_hook_v2_delivery_claims().lock() { claims.remove(&(project_id, envelope_id)); diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/context_scout/tests.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/context_scout/tests.rs index f47b7ca903..f253e8d9fe 100644 --- a/crates/tracedecay-mcp/src/handlers/hook_runtime/context_scout/tests.rs +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/context_scout/tests.rs @@ -35,35 +35,6 @@ fn exact_retained_claim_lookup_commits_beyond_thirty_two_entries() { } } -#[test] -fn exact_event_claim_lookup_is_unique_and_unexpired() { - let _guard = RETAINED_CLAIM_TEST_LOCK - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let project_id = [201; 16]; - let now = UtcMicros(999); - let claim = retained_claim(41); - let event_id = claim.lease.lease_id; - - assert!(lookup_hook_v2_delivery_claim_for_event(project_id, event_id, now).is_none()); - assert!(retain_hook_v2_delivery_claim(project_id, claim.clone(), now).is_ok()); - assert_eq!( - lookup_hook_v2_delivery_claim_for_event(project_id, event_id, now), - Some(claim.clone()) - ); - - let mut ambiguous = retained_claim(42); - ambiguous.lease.lease_id = event_id; - assert!(retain_hook_v2_delivery_claim(project_id, ambiguous.clone(), now).is_ok()); - assert!(lookup_hook_v2_delivery_claim_for_event(project_id, event_id, now).is_none()); - remove_hook_v2_delivery_claim(project_id, ambiguous.entry.envelope.envelope_id); - assert!( - lookup_hook_v2_delivery_claim_for_event(project_id, event_id, claim.lease.expires_at) - .is_none() - ); - remove_hook_v2_delivery_claim(project_id, claim.entry.envelope.envelope_id); -} - #[test] fn retained_claims_backpressure_at_a_deterministic_bound() { let _guard = RETAINED_CLAIM_TEST_LOCK diff --git a/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests.rs b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests.rs index 9e3eb41721..65bc3fa9a7 100644 --- a/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests.rs +++ b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests.rs @@ -936,6 +936,59 @@ impl IntentJourneyFixture { Some(typed_id::("refs/heads/main")), ) .expect("scope"); + let binding = tracedecay_hooks::HookScopeBindingV1 { + host: tracedecay_domain::NativeHostIdentityV1::ClaudeCode, + project_id: [8; 16], + repository_id: [61; 16], + worktree_id: [62; 16], + worktree_epoch: 1, + binding_token: [63; 32], + capabilities: [ + tracedecay_hooks::HookEventFamily::SessionBoundary, + tracedecay_hooks::HookEventFamily::ToolLifecycle, + ] + .into_iter() + .map(|family| tracedecay_hooks::HookCapabilityV1 { + family, + support: tracedecay_hooks::stock_event_support( + tracedecay_domain::NativeHostIdentityV1::ClaudeCode, + family, + ), + }) + .collect(), + }; + let lifecycle = + tracedecay_agent_hosts::agents::context_scout::address_registry::ContextScoutLifecycleAddressV1 { + profile_id: typed_id("profile.scout.intent"), + provider_id: typed_id("claude"), + project_id: scope.project_id.clone(), + worktree_id: scope.worktree_id.clone(), + session_id: typed_id(INTENT_SESSION), + thread_id: typed_id("thread.scout.intent"), + turn_id: typed_id("turn.scout.intent"), + agent_id: typed_id("agent.scout.intent"), + logical_message_id: typed_id("message.scout.intent"), + }; + Self::from_authorities( + temporary, + owner, + configuration, + scope, + binding, + lifecycle, + typed_id("commit.scout.intent"), + ) + } + + fn from_authorities( + temporary: tempfile::TempDir, + owner: Arc, + configuration: ContextScoutConfigurationPinV1, + scope: ResolvedScope, + binding: tracedecay_hooks::HookScopeBindingV1, + lifecycle: tracedecay_agent_hosts::agents::context_scout::address_registry::ContextScoutLifecycleAddressV1, + head_commit_id: CommitId, + ) -> Self { let issued_at = now_utc_micros(); let grant = CapabilityGrantSnapshot::new( typed_id("grant.scout.intent"), @@ -971,7 +1024,7 @@ impl IntentJourneyFixture { repository_id: scope.repository_id.clone(), worktree_id: scope.worktree_id.clone(), branch_ref: "refs/heads/main".to_owned(), - head_commit_id: typed_id("commit.scout.intent"), + head_commit_id, }, configuration, issued_at, @@ -982,39 +1035,6 @@ impl IntentJourneyFixture { scope.project_id.clone(), ) .expect("address registry"); - let binding = tracedecay_hooks::HookScopeBindingV1 { - host: tracedecay_domain::NativeHostIdentityV1::ClaudeCode, - project_id: [8; 16], - repository_id: [61; 16], - worktree_id: [62; 16], - worktree_epoch: 1, - binding_token: [63; 32], - capabilities: [ - tracedecay_hooks::HookEventFamily::SessionBoundary, - tracedecay_hooks::HookEventFamily::ToolLifecycle, - ] - .into_iter() - .map(|family| tracedecay_hooks::HookCapabilityV1 { - family, - support: tracedecay_hooks::stock_event_support( - tracedecay_domain::NativeHostIdentityV1::ClaudeCode, - family, - ), - }) - .collect(), - }; - let lifecycle = - tracedecay_agent_hosts::agents::context_scout::address_registry::ContextScoutLifecycleAddressV1 { - profile_id: typed_id("profile.scout.intent"), - provider_id: typed_id("claude"), - project_id: scope.project_id.clone(), - worktree_id: scope.worktree_id.clone(), - session_id: typed_id(INTENT_SESSION), - thread_id: typed_id("thread.scout.intent"), - turn_id: typed_id("turn.scout.intent"), - agent_id: typed_id("agent.scout.intent"), - logical_message_id: typed_id("message.scout.intent"), - }; Self { _temporary: temporary, owner, @@ -1141,19 +1161,10 @@ async fn seed_transcript_message( .expect("ingest transcript message"); } -/// The daemon intent pass at a completed tool call over a seeded session: one batched engine request -/// over the window, exactly one session-evidence suggestion at the session's -/// intent address, a claim at the next stop boundary, a delivery that writes -/// the `hint_emitted` row hint settlement resolves into `hint_outcome`, and a -/// later pass that withholds the delivered intent as a duplicate. -#[tokio::test] -async fn intent_pass_enqueues_one_session_suggestion_claimed_at_stop_and_settled() { - let fixture = IntentJourneyFixture::open(Some(intent_settings())).await; - let harness = - tracedecay_global_db::tests::harness::RegisteredGlobalDbHarness::open("intent-journey") - .await; - let sessions = harness.registered.as_ref(); - let project_key = "/project/intent-journey"; +async fn seed_intent_session( + sessions: &tracedecay_global_db::RegisteredGlobalDb, + project_key: &str, +) { assert!( sessions .upsert_session(&tracedecay_sessions::runtime::SessionRecord { @@ -1201,6 +1212,22 @@ async fn intent_pass_enqueues_one_session_suggestion_claimed_at_stop_and_settled now_secs - 20, ) .await; +} + +/// The daemon intent pass at a completed tool call over a seeded session: one batched engine request +/// over the window, exactly one session-evidence suggestion at the session's +/// intent address, a claim at the next stop boundary, a delivery that writes +/// the `hint_emitted` row hint settlement resolves into `hint_outcome`, and a +/// later pass that withholds the delivered intent as a duplicate. +#[tokio::test] +async fn intent_pass_enqueues_one_session_suggestion_claimed_at_stop_and_settled() { + let fixture = IntentJourneyFixture::open(Some(intent_settings())).await; + let harness = + tracedecay_global_db::tests::harness::RegisteredGlobalDbHarness::open("intent-journey") + .await; + let sessions = harness.registered.as_ref(); + let project_key = "/project/intent-journey"; + seed_intent_session(sessions, project_key).await; let engine = RecordingDecisionEngine::new(0.9); let boundary = fixture.hook(71, completed_tool()); @@ -1460,3 +1487,5 @@ async fn absent_or_disabled_intent_configuration_never_asks_the_engine() { } assert_eq!(engine.requests(), 0); } + +mod replay; diff --git a/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests/replay.rs b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests/replay.rs new file mode 100644 index 0000000000..79f5543371 --- /dev/null +++ b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests/replay.rs @@ -0,0 +1,401 @@ +use std::collections::BTreeMap; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Duration; + +use tracedecay_agent_hosts::agents::context_scout::address_registry::ContextScoutConfigurationPinV1; +use tracedecay_agent_hosts::agents::context_scout::intent_pass::ContextScoutIntentPassOutcomeV1; +use tracedecay_configuration::config::scope_control::daemon_owned_project_source_binding; +use tracedecay_contracts::context_scout::{ + ContextScoutDeliveryOutcomeV1, ContextScoutDeliveryReceiptV1, ContextScoutDurableQueueEntryV1, +}; +use tracedecay_contracts::retrieval::{ + ContextScoutStoreStatusV1, HookRuntimeResultV1, HookRuntimeSurfaceRequestV1, + HookV2AdmissionResultV1, HookV2AdmitRequestV1, +}; +use tracedecay_daemon_service::context_scout_lifecycle::{ + AuthorityRegistrationV1, lookup_registered_context_scout_lifecycle, + register_context_scout_lifecycle_authority, unregister_context_scout_lifecycle_authority, +}; +use tracedecay_daemon_service::{ + BoundedHookOrchestratorV1, HookOrchestrationRequestV1, HookOrchestrationTriggerV1, + register_hook_orchestration_runtime, unregister_hook_orchestration_runtime, +}; +use tracedecay_domain::configuration::{ + CONTEXT_SCOUT_SETTINGS_SETTING_KEY, ConfigurationLayerIdV1, ConfigurationRevisionId, + ConfigurationValueV1, ContextScoutConfigurationStateV1, ContextScoutSettingsV1, + SOURCE_BINDINGS_SETTING_KEY, +}; +use tracedecay_domain::{ProjectId, SessionId}; +use tracedecay_global_db::RegisteredGlobalDb; +use tracedecay_global_db::configuration::GlobalDbConfigurationControlStore; +use tracedecay_global_db::configuration::registry::ConfigurationRegistry; +use tracedecay_global_db::configuration::resolver::{ConfigurationLayerV1, resolve_configuration}; +use tracedecay_hooks::{ + HookBoundaryV1, HookEventEnvelopeV2, HookEventV2, HookOrderingV1, HookReadyGuidanceV1, + NativeContextScoutLifecycleV1, +}; +use tracedecay_mcp::handlers::SessionAuthorities; +use tracedecay_mcp::handlers::hook_runtime::{ + HookV2AdmissionOutcomeV1, admit_hook_v2_replayed_envelope_with_lifecycle, compute_hook_runtime, + hook_v2_pending_work_envelopes, +}; +use tracedecay_project::project::{TraceDecay, TraceDecayOpenOptions}; +use tracedecay_project::test_support::host_admission::HostAdmissionTestRuntimeV1; +use tracedecay_runtime_core::config::ProfileRoot; +use tracedecay_sessions::admission::HostAdmissionScope; + +use super::{ + INTENT_SESSION, IntentJourneyFixture, RecordingDecisionEngine, completed_tool, + context_scout_delivery_receipt_id, intent_settings, now_utc_micros, seed_intent_session, + typed_id, +}; +use crate::test_support::git::git; + +mod configuration; +mod pending; + +async fn registered_fixture() -> ( + IntentJourneyFixture, + TraceDecay, + Arc, +) { + let temporary = tempfile::tempdir().unwrap(); + let root = temporary.path().join("project"); + let profile = temporary.path().join("profile"); + std::fs::create_dir_all(root.join("src")).unwrap(); + std::fs::write(root.join("src/lib.rs"), "pub fn fixture() {}\n").unwrap(); + git(&root, &["init", "-q", "-b", "main"]); + git(&root, &["add", "."]); + git(&root, &["commit", "-q", "-m", "fixture"]); + let commit = typed_id(&git(&root, &["rev-parse", "HEAD"])); + let project = ProjectId::new(format!( + "project.scout.replay.{}", + temporary + .path() + .file_name() + .unwrap() + .to_str() + .unwrap() + .trim_start_matches('.') + )) + .unwrap(); + let runtime = Arc::new( + HostAdmissionTestRuntimeV1::project(&profile, &root, project.clone()) + .await + .unwrap(), + ); + let sessions = runtime + .registered_database_arc(HostAdmissionScope::Project) + .unwrap(); + let registry = ConfigurationRegistry::core().unwrap(); + let revision: ConfigurationRevisionId = typed_id("configuration.scout.replay.initial"); + let resolution = resolve_configuration( + ®istry, + &[ConfigurationLayerV1 { + layer: ConfigurationLayerIdV1::Project { + project_id: project.clone(), + }, + revision_id: revision.clone(), + entries: BTreeMap::from([ + ( + typed_id(SOURCE_BINDINGS_SETTING_KEY), + ConfigurationValueV1::SourceBindings(vec![ + daemon_owned_project_source_binding(&project).unwrap(), + ]), + ), + ( + typed_id(CONTEXT_SCOUT_SETTINGS_SETTING_KEY), + ConfigurationValueV1::ContextScoutSettings(ContextScoutSettingsV1 { + state: ContextScoutConfigurationStateV1::Active, + intent: Some(intent_settings()), + ..ContextScoutSettingsV1::disabled() + }), + ), + ]), + }], + ) + .unwrap(); + GlobalDbConfigurationControlStore::new_registered(&sessions) + .initialize_canonical(&revision, &resolution, now_utc_micros()) + .await + .unwrap(); + let graph = runtime + .initialize_project_graph_for_test( + &root, + TraceDecayOpenOptions { + profile_root: Some(profile.clone()), + global_db_path: None, + }, + ) + .await + .unwrap(); + let owner = graph.context_scout_owner().unwrap(); + let configuration = ContextScoutConfigurationPinV1::from_current( + &graph + .configuration_runtime() + .client() + .current() + .await + .unwrap() + .into_current_state(), + ) + .unwrap(); + owner + .install_configuration(configuration.clone(), None) + .await + .unwrap(); + let scope = tracedecay_code_index_runtime::resolved_scope_for_project(&root, &project).unwrap(); + let hook_runtime = crate::hook_runtime(ProfileRoot::new(profile)); + let worktree = tracedecay_agent_hosts::hooks::hook_worktree_id_for_layout( + &hook_runtime, + graph.hook_store_layout(), + ) + .unwrap(); + let binding = tracedecay_hooks::published_hook_scope_binding( + &graph.hook_store_layout().data_root, + worktree, + tracedecay_domain::NativeHostIdentityV1::ClaudeCode, + now_utc_micros(), + ) + .unwrap(); + assert_eq!( + register_context_scout_lifecycle_authority( + binding.project_id, + binding.worktree_id, + scope.project_id.clone(), + scope.worktree_id.clone(), + &sessions, + ), + AuthorityRegistrationV1::Registered + ); + let native = + NativeContextScoutLifecycleV1::new(INTENT_SESSION, "call.scout.prior", [69; 16]).unwrap(); + let prior = HookEventEnvelopeV2 { + schema_version: tracedecay_hooks::HOOK_EVENT_SCHEMA_VERSION, + event_id: native.event_id, + producer: binding.host, + protected_session_id: tracedecay_agent_hosts::hooks::protected_native_session_id( + INTENT_SESSION, + ), + project_id: binding.project_id, + repository_id: binding.repository_id, + worktree_id: binding.worktree_id, + worktree_epoch: binding.worktree_epoch, + binding_token: binding.binding_token, + ordering: HookOrderingV1::Unknown, + observed_at: now_utc_micros(), + event: completed_tool(), + }; + assert!(matches!( + admit_hook_v2_replayed_envelope_with_lifecycle( + &graph, + &prior, + Some(native.session_id.clone()), + Some(native), + &sessions, + &runtime.background_cpu(), + now_utc_micros(), + ) + .await, + HookV2AdmissionOutcomeV1::Admitted { .. } + )); + let lifecycle = lookup_registered_context_scout_lifecycle( + binding.project_id, + binding.worktree_id, + &SessionId::new(INTENT_SESSION).unwrap(), + ) + .await + .unwrap(); + let fixture = IntentJourneyFixture::from_authorities( + temporary, + owner, + configuration, + scope, + binding, + lifecycle, + commit, + ); + (fixture, graph, runtime) +} + +async fn mount_intent( + fixture: &IntentJourneyFixture, + graph: &TraceDecay, + sessions: &RegisteredGlobalDb, +) -> ( + Arc, + Box, +) { + let project_key = + tracedecay_global_db::RegisteredGlobalDb::canonical_project_key(graph.project_root()); + seed_intent_session(sessions, &project_key).await; + let engine = RecordingDecisionEngine::new(0.9); + let boundary = fixture.hook(71, completed_tool()); + let ContextScoutIntentPassOutcomeV1::Enqueued { entry, lifecycle } = fixture + .pass(&boundary, sessions, &project_key, &engine) + .await + else { + panic!("the registered session intent is enqueued"); + }; + assert!( + graph + .mount_current_context_scout_claim_authority( + Arc::clone(&fixture.registry), + &boundary, + 1, + fixture.pin.clone(), + fixture.context.clone(), + *lifecycle, + entry.work.address, + entry.work.input_watermark, + now_utc_micros(), + ) + .await + ); + (engine, entry) +} + +#[tokio::test] +async fn completed_stop_replay_claims_the_mounted_intent_once_without_reopening_producer_work() { + let (fixture, graph, runtime) = registered_fixture().await; + let sessions = runtime + .registered_database_arc(HostAdmissionScope::Project) + .unwrap(); + let (engine, entry) = mount_intent(&fixture, &graph, &sessions).await; + let effects = Arc::new(AtomicUsize::new(0)); + let producer = BoundedHookOrchestratorV1::new(1, { + let effects = Arc::clone(&effects); + move |request: HookOrchestrationRequestV1, _| { + let effects = Arc::clone(&effects); + async move { + if request.trigger == HookOrchestrationTriggerV1::IntentBoundary { + assert!(request.completion.is_none()); + return; + } + assert_eq!(request.trigger, HookOrchestrationTriggerV1::Stop); + assert!(request.lifecycle.is_some()); + effects.fetch_add(1, Ordering::SeqCst); + } + } + }) + .unwrap(); + assert!(register_hook_orchestration_runtime( + fixture.binding.project_id, + fixture.binding.worktree_id, + &producer + )); + let stop = fixture.hook( + 72, + HookEventV2::SessionBoundary { + boundary: HookBoundaryV1::TurnComplete, + }, + ); + let replay = admit_hook_v2_replayed_envelope_with_lifecycle( + &graph, + stop.envelope(), + Some(typed_id(INTENT_SESSION)), + None, + &sessions, + &runtime.background_cpu(), + now_utc_micros(), + ) + .await; + assert!( + matches!(replay, HookV2AdmissionOutcomeV1::Admitted { ready_guidance, .. } if ready_guidance.is_null()) + ); + tokio::time::timeout(Duration::from_secs(10), async { + while !hook_v2_pending_work_envelopes( + &graph.hook_store_layout().data_root, + fixture.binding.host, + now_utc_micros(), + ) + .unwrap() + .is_empty() + || producer.active_tasks() != 0 + { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!(effects.load(Ordering::SeqCst), 1); + let live_request = || { + HookRuntimeSurfaceRequestV1::HookV2Admit(HookV2AdmitRequestV1 { + envelope: serde_json::to_value(stop.envelope()).unwrap(), + native_session_id: Some(INTENT_SESSION.to_owned()), + native_lifecycle: None, + }) + }; + let authorities = || { + SessionAuthorities::new(Some(&sessions), None) + .with_background_cpu(Some(runtime.background_cpu())) + }; + let live = compute_hook_runtime(&graph, live_request(), None, None, authorities()) + .await + .unwrap(); + let HookRuntimeResultV1::HookV2Admit(HookV2AdmissionResultV1::ExactDuplicate { + context_scout_address: Some(address), + ready_guidance: Some(guidance), + .. + }) = live + else { + panic!("completed replay must retain delivery authority: {live:?}"); + }; + assert_eq!(address, entry.work.address); + let guidance: HookReadyGuidanceV1 = serde_json::from_value(guidance).unwrap(); + assert_eq!(guidance.event_id, stop.envelope().event_id); + assert_eq!(guidance.text, entry.envelope.candidate.suggestion_text); + let receipt = ContextScoutDeliveryReceiptV1 { + receipt_id: context_scout_delivery_receipt_id( + stop.envelope().event_id, + guidance.guidance_id, + ), + envelope_id: guidance.guidance_id, + delivered_at: now_utc_micros(), + outcome: ContextScoutDeliveryOutcomeV1::Attempted, + }; + assert!(matches!( + compute_hook_runtime( + &graph, + HookRuntimeSurfaceRequestV1::HookV2DeliveryReceipt { + receipt: serde_json::to_value(receipt).unwrap(), + }, + None, + None, + authorities() + ) + .await + .unwrap(), + HookRuntimeResultV1::HookV2DeliveryReceipt { + status: ContextScoutStoreStatusV1::Stored + } + )); + let duplicate = compute_hook_runtime(&graph, live_request(), None, None, authorities()) + .await + .unwrap(); + assert!(matches!( + duplicate, + HookRuntimeResultV1::HookV2Admit(HookV2AdmissionResultV1::ExactDuplicate { + ready_guidance: None, + .. + }) + )); + assert_eq!(producer.active_tasks(), 0); + assert_eq!(effects.load(Ordering::SeqCst), 1); + assert_eq!(engine.requests(), 1); + let recent = fixture.owner.recent_exact(address, 8).await.unwrap(); + assert!(recent.pending.is_empty()); + assert_eq!(recent.deliveries.len(), 1); + unregister_hook_orchestration_runtime( + fixture.binding.project_id, + fixture.binding.worktree_id, + &producer, + ); + assert!(producer.shutdown().await); + assert!(unregister_context_scout_lifecycle_authority( + fixture.binding.project_id, + fixture.binding.worktree_id, + &sessions, + )); +} diff --git a/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests/replay/configuration.rs b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests/replay/configuration.rs new file mode 100644 index 0000000000..e9b093e05f --- /dev/null +++ b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests/replay/configuration.rs @@ -0,0 +1,70 @@ +use tracedecay_agent_hosts::agents::context_scout::address_registry::ContextScoutConfigurationPinV1; +use tracedecay_domain::configuration::{ + ConfigurationLayerIdV1, ConfigurationMutationEffectV1, ConfigurationMutationGrantReceiptV1, + ConfigurationMutationOperationV1, ConfigurationMutationSinkV1, ConfigurationValueV1, + TELEMETRY_TIMINGS_SETTING_KEY, +}; +use tracedecay_domain::{AccessPolicyDigest, UtcMicros}; +use tracedecay_global_db::RegisteredGlobalDb; +use tracedecay_global_db::configuration::GlobalDbConfigurationControlStore; +use tracedecay_global_db::configuration::contracts::{ + ConfigurationControlStore, ConfigurationMutationAuthority, DirectConfigurationMutation, +}; +use tracedecay_global_db::configuration::store::commit_direct_in_transaction; + +use super::super::{now_utc_micros, typed_id}; + +pub(super) async fn advance_configuration( + sessions: &RegisteredGlobalDb, +) -> ContextScoutConfigurationPinV1 { + let store = GlobalDbConfigurationControlStore::new_registered(sessions); + let current = store.current().await.unwrap(); + let key = typed_id(TELEMETRY_TIMINGS_SETTING_KEY); + let ConfigurationValueV1::Boolean(previous) = current.snapshot.effective_values[&key] else { + panic!("canonical timings configuration must be boolean"); + }; + let mutation = DirectConfigurationMutation::Set { + layer: ConfigurationLayerIdV1::Project { + project_id: sessions + .binding() + .shard_id + .scope + .project_id() + .unwrap() + .clone(), + }, + key, + value: Box::new(ConfigurationValueV1::Boolean(!previous)), + }; + let now = now_utc_micros(); + let authority = ConfigurationMutationAuthority { + receipt: ConfigurationMutationGrantReceiptV1::issue( + typed_id("configuration.grant-receipt.scout-replay"), + typed_id("configuration.grant.scout-replay"), + typed_id("actor.scout-replay"), + ConfigurationMutationOperationV1::DirectMutation, + mutation.target_scope_digest().unwrap(), + current.revision_id.clone(), + 1, + AccessPolicyDigest::new(format!("sha256:{}", "b".repeat(64))).unwrap(), + ConfigurationMutationSinkV1::ConfigurationStore, + ConfigurationMutationEffectV1::CommitConfigurationRevision, + Some(typed_id("configuration.idempotency.scout-replay")), + now, + UtcMicros(now.0 + 10_000_000), + ) + .unwrap(), + }; + let transaction = sessions.begin_write_transaction().await.unwrap(); + let committed = + commit_direct_in_transaction(&transaction, &authority, &mutation, ¤t.revision_id) + .await + .unwrap(); + transaction.commit().await.unwrap(); + assert_ne!(committed.current.revision_id, current.revision_id); + assert_ne!( + committed.current.snapshot.effective_behavior_digest, + current.snapshot.effective_behavior_digest + ); + ContextScoutConfigurationPinV1::from_current(&committed.current).unwrap() +} diff --git a/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests/replay/pending.rs b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests/replay/pending.rs new file mode 100644 index 0000000000..4953b08851 --- /dev/null +++ b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests/replay/pending.rs @@ -0,0 +1,359 @@ +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use tracedecay_contracts::HookOrchestrationAdmissionV1; +use tracedecay_contracts::context_scout::{ + ContextScoutDeliveryOutcomeV1, ContextScoutDeliveryReceiptV1, +}; +use tracedecay_contracts::retrieval::{ + ContextScoutStoreStatusV1, HookRuntimeResultV1, HookRuntimeSurfaceRequestV1, + HookV2AdmissionResultV1, HookV2AdmitRequestV1, +}; +use tracedecay_daemon_service::context_scout_lifecycle::unregister_context_scout_lifecycle_authority; +use tracedecay_daemon_service::{ + BoundedHookOrchestratorV1, HookOrchestrationRequestV1, HookOrchestrationSettlementV1, + HookOrchestrationTriggerV1, register_hook_orchestration_runtime, + unregister_hook_orchestration_runtime, +}; +use tracedecay_hooks::{HookBoundaryV1, HookEventV2, HookReadyGuidanceV1}; +use tracedecay_mcp::handlers::SessionAuthorities; +use tracedecay_mcp::handlers::hook_runtime::{ + HookV2AdmissionOutcomeV1, admit_hook_v2_replayed_envelope_with_lifecycle, compute_hook_runtime, + hook_v2_pending_work_envelopes, +}; +use tracedecay_sessions::admission::HostAdmissionScope; + +use super::super::{INTENT_SESSION, context_scout_delivery_receipt_id, now_utc_micros, typed_id}; +use super::{mount_intent, registered_fixture}; + +#[tokio::test] +async fn pending_stop_replay_claims_its_mounted_intent_before_feedback_settles() { + pending_stop_delivers_once(false, None).await; +} + +#[tokio::test] +async fn pending_stop_retry_reuses_its_retained_claim_before_feedback_settles() { + pending_stop_delivers_once(true, None).await; +} + +#[tokio::test] +async fn pending_stop_rejects_a_retained_claim_after_configuration_advances() { + pending_stop_delivers_once(true, Some(false)).await; +} + +#[tokio::test] +async fn completed_stop_rejects_a_retained_claim_after_configuration_advances() { + pending_stop_delivers_once(true, Some(true)).await; +} + +async fn pending_stop_delivers_once( + first_response_available: bool, + advance_after_completion: Option, +) { + let (fixture, graph, runtime) = registered_fixture().await; + let sessions = runtime + .registered_database_arc(HostAdmissionScope::Project) + .unwrap(); + let (engine, entry) = mount_intent(&fixture, &graph, &sessions).await; + let effects = Arc::new(AtomicUsize::new(0)); + let captured = Arc::new(Mutex::new(None)); + let (entered, entered_rx) = tokio::sync::oneshot::channel(); + let entered = Arc::new(Mutex::new(Some(entered))); + let (release, released) = tokio::sync::oneshot::channel(); + let mut release = Some(release); + let released = Arc::new(tokio::sync::Mutex::new(Some(released))); + let producer = BoundedHookOrchestratorV1::new(1, { + let effects = Arc::clone(&effects); + let captured = Arc::clone(&captured); + move |request: HookOrchestrationRequestV1, _| { + let effects = Arc::clone(&effects); + let captured = Arc::clone(&captured); + let entered = Arc::clone(&entered); + let released = Arc::clone(&released); + async move { + if request.trigger == HookOrchestrationTriggerV1::IntentBoundary { + assert!(request.completion.is_none()); + return; + } + assert_eq!(request.trigger, HookOrchestrationTriggerV1::Stop); + effects.fetch_add(1, Ordering::SeqCst); + *captured.lock().unwrap() = Some(request); + entered.lock().unwrap().take().unwrap().send(()).unwrap(); + let released = released.lock().await.take().unwrap(); + released.await.unwrap(); + } + } + }) + .unwrap(); + assert!(register_hook_orchestration_runtime( + fixture.binding.project_id, + fixture.binding.worktree_id, + &producer + )); + let stop = fixture.hook( + 72, + HookEventV2::SessionBoundary { + boundary: HookBoundaryV1::TurnComplete, + }, + ); + let request = || { + HookRuntimeSurfaceRequestV1::HookV2Admit(HookV2AdmitRequestV1 { + envelope: serde_json::to_value(stop.envelope()).unwrap(), + native_session_id: Some(INTENT_SESSION.to_owned()), + native_lifecycle: None, + }) + }; + let authorities = || { + SessionAuthorities::new(Some(&sessions), None) + .with_background_cpu(Some(runtime.background_cpu())) + }; + let first_guidance = if first_response_available { + let first = compute_hook_runtime(&graph, request(), None, None, authorities()) + .await + .unwrap(); + let HookRuntimeResultV1::HookV2Admit(HookV2AdmissionResultV1::Accepted { + ready_guidance: Some(guidance), + .. + }) = first + else { + panic!("the first live admission must claim the mounted intent: {first:?}"); + }; + Some(guidance) + } else { + assert!(matches!(admit_hook_v2_replayed_envelope_with_lifecycle( + &graph, stop.envelope(), Some(typed_id(INTENT_SESSION)), None, + &sessions, &runtime.background_cpu(), now_utc_micros(), + ).await, HookV2AdmissionOutcomeV1::Admitted { ready_guidance, .. } if ready_guidance.is_null())); + None + }; + tokio::time::timeout(Duration::from_secs(10), entered_rx) + .await + .unwrap() + .unwrap(); + let pending = || { + hook_v2_pending_work_envelopes( + &graph.hook_store_layout().data_root, + fixture.binding.host, + now_utc_micros(), + ) + .unwrap() + }; + assert_eq!(pending(), vec![stop.envelope().clone()]); + assert_eq!(effects.load(Ordering::SeqCst), 1); + let (terminal, terminal_rx) = tokio::sync::oneshot::channel(); + let terminal = Mutex::new(Some(terminal)); + let mut terminal_rx = Some(terminal_rx); + let settlements = Arc::new(AtomicUsize::new(0)); + let mut observer = captured.lock().unwrap().clone().unwrap(); + observer.completion = Some(Arc::new({ + let settlements = Arc::clone(&settlements); + move |settlement| { + settlements.fetch_add(1, Ordering::SeqCst); + terminal + .lock() + .unwrap() + .take() + .unwrap() + .send(settlement) + .unwrap(); + } + })); + assert_eq!( + producer.admit(observer), + HookOrchestrationAdmissionV1::Enqueued + ); + if let Some(completed) = advance_after_completion { + if completed { + release.take().unwrap().send(()).unwrap(); + assert_eq!( + tokio::time::timeout(Duration::from_secs(10), terminal_rx.take().unwrap()) + .await + .unwrap() + .unwrap(), + HookOrchestrationSettlementV1::Completed + ); + wait_until_settled(&pending, &producer).await; + } + let next = super::configuration::advance_configuration(&sessions).await; + assert_ne!( + next.control().configuration_revision, + entry.envelope.configuration_revision + ); + // The durable pin must reject the old claim even before the owner has + // observed the committed configuration change. + for owner_updated in [false, true] { + if owner_updated { + fixture + .owner + .install_configuration(next.clone(), None) + .await + .unwrap(); + } + let retry = compute_hook_runtime(&graph, request(), None, None, authorities()) + .await + .unwrap(); + match retry { + HookRuntimeResultV1::HookV2Admit(HookV2AdmissionResultV1::Accepted { + ready_guidance: None, + .. + }) => assert!(!completed), + HookRuntimeResultV1::HookV2Admit(HookV2AdmissionResultV1::ExactDuplicate { + ready_guidance: None, + .. + }) => assert!(completed), + other => panic!("a retained claim cannot cross configuration revisions: {other:?}"), + } + assert_eq!(effects.load(Ordering::SeqCst), 1); + assert_eq!(pending().is_empty(), completed); + } + if !completed { + release.take().unwrap().send(()).unwrap(); + assert_eq!( + tokio::time::timeout(Duration::from_secs(10), terminal_rx.take().unwrap()) + .await + .unwrap() + .unwrap(), + HookOrchestrationSettlementV1::Completed + ); + wait_until_settled(&pending, &producer).await; + } + assert_eq!(settlements.load(Ordering::SeqCst), 1); + assert_eq!(effects.load(Ordering::SeqCst), 1); + assert_eq!(engine.requests(), 1); + let recent = fixture + .owner + .recent_exact(entry.work.address, 8) + .await + .unwrap(); + assert!(recent.deliveries.is_empty()); + unregister_hook_orchestration_runtime( + fixture.binding.project_id, + fixture.binding.worktree_id, + &producer, + ); + assert!(producer.shutdown().await); + assert!(unregister_context_scout_lifecycle_authority( + fixture.binding.project_id, + fixture.binding.worktree_id, + &sessions + )); + return; + } + let retry = compute_hook_runtime(&graph, request(), None, None, authorities()) + .await + .unwrap(); + let HookRuntimeResultV1::HookV2Admit(HookV2AdmissionResultV1::Accepted { + context_scout_address: Some(address), + ready_guidance: Some(guidance), + .. + }) = retry + else { + panic!("pending feedback must not suppress live retry delivery: {retry:?}"); + }; + if let Some(first_guidance) = first_guidance { + assert_eq!( + guidance, first_guidance, + "the lost response's exact claim is reused" + ); + } + let guidance: HookReadyGuidanceV1 = serde_json::from_value(guidance).unwrap(); + assert_eq!(address, entry.work.address); + assert_eq!(guidance.event_id, stop.envelope().event_id); + assert_eq!(guidance.text, entry.envelope.candidate.suggestion_text); + assert_eq!(pending(), vec![stop.envelope().clone()]); + let receipt = ContextScoutDeliveryReceiptV1 { + receipt_id: context_scout_delivery_receipt_id( + stop.envelope().event_id, + guidance.guidance_id, + ), + envelope_id: guidance.guidance_id, + delivered_at: now_utc_micros(), + outcome: ContextScoutDeliveryOutcomeV1::Attempted, + }; + assert!(matches!( + compute_hook_runtime( + &graph, + HookRuntimeSurfaceRequestV1::HookV2DeliveryReceipt { + receipt: serde_json::to_value(receipt).unwrap(), + }, + None, + None, + authorities() + ) + .await + .unwrap(), + HookRuntimeResultV1::HookV2DeliveryReceipt { + status: ContextScoutStoreStatusV1::Stored + } + )); + let duplicate = compute_hook_runtime(&graph, request(), None, None, authorities()) + .await + .unwrap(); + assert!(matches!( + duplicate, + HookRuntimeResultV1::HookV2Admit(HookV2AdmissionResultV1::Accepted { + ready_guidance: None, + .. + }) + )); + assert_eq!(pending(), vec![stop.envelope().clone()]); + assert_eq!(effects.load(Ordering::SeqCst), 1); + assert_eq!(engine.requests(), 1); + release.take().unwrap().send(()).unwrap(); + assert_eq!( + tokio::time::timeout(Duration::from_secs(10), terminal_rx.take().unwrap()) + .await + .unwrap() + .unwrap(), + HookOrchestrationSettlementV1::Completed + ); + tokio::time::timeout(Duration::from_secs(10), async { + while !pending().is_empty() || producer.active_tasks() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!(settlements.load(Ordering::SeqCst), 1); + assert_eq!(effects.load(Ordering::SeqCst), 1); + let completed_duplicate = compute_hook_runtime(&graph, request(), None, None, authorities()) + .await + .unwrap(); + assert!(matches!( + completed_duplicate, + HookRuntimeResultV1::HookV2Admit(HookV2AdmissionResultV1::ExactDuplicate { + ready_guidance: None, + .. + }) + )); + assert_eq!(effects.load(Ordering::SeqCst), 1); + assert_eq!(engine.requests(), 1); + let recent = fixture.owner.recent_exact(address, 8).await.unwrap(); + assert!(recent.pending.is_empty()); + assert_eq!(recent.deliveries.len(), 1); + unregister_hook_orchestration_runtime( + fixture.binding.project_id, + fixture.binding.worktree_id, + &producer, + ); + assert!(producer.shutdown().await); + assert!(unregister_context_scout_lifecycle_authority( + fixture.binding.project_id, + fixture.binding.worktree_id, + &sessions + )); +} + +async fn wait_until_settled( + pending: &impl Fn() -> Vec, + producer: &BoundedHookOrchestratorV1, +) { + tokio::time::timeout(Duration::from_secs(10), async { + while !pending().is_empty() || producer.active_tasks() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); +} diff --git a/crates/tracedecay/tests/hooks_lsp_suite/intent_pass_journey_test.rs b/crates/tracedecay/tests/hooks_lsp_suite/intent_pass_journey_test.rs index 5f4763fd17..5fbe014fda 100644 --- a/crates/tracedecay/tests/hooks_lsp_suite/intent_pass_journey_test.rs +++ b/crates/tracedecay/tests/hooks_lsp_suite/intent_pass_journey_test.rs @@ -10,6 +10,7 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use serde_json::{Value, json}; +use tracedecay_contracts::context_scout::ContextScoutAddressV1; use tracedecay_contracts::retrieval::StatusHookReplayV1; use tracedecay_domain::configuration::{ CONTEXT_SCOUT_SETTINGS_SETTING_KEY, ContextScoutConfigurationStateV1, ContextScoutIntentKindV1, @@ -212,6 +213,37 @@ fn run_tool(home: &Path, project: &Path, name: &str, args: &Value) -> Value { tool_json_structured_content(&output.stdout) } +fn delivery_evidence(home: &Path, project: &Path, contexts: &[String]) -> Value { + let profile = home.join(".tracedecay"); + let enrolled = + tracedecay_runtime_core::storage::default_profile_sharded_layout(project, &profile) + .unwrap() + .data_root; + let address = contexts + .iter() + .rev() + .flat_map(|context| context.lines()) + .find_map(|line| { + line.strip_prefix("TraceDecay Context Scout address for authorized operations: ") + .and_then(|encoded| serde_json::from_str::(encoded).ok()) + }); + let recent = address.map(|address| { + run_tool( + home, + project, + "tracedecay_context_scout_recent", + &json!({"address": address, "limit": 8}), + ) + }); + json!({ + "enrolled_hook_receipts": std::fs::read_to_string(enrolled.join("hook_analytics.jsonl")) + .unwrap_or_default(), + "profile_hook_receipts": std::fs::read_to_string(profile.join("hook_analytics.jsonl")) + .unwrap_or_default(), + "recent": recent, + }) +} + fn post_tool_use(project: &Path, session: &str, transcript: &Path) -> Value { json!({ "session_id": session, @@ -289,10 +321,14 @@ fn intent_outcomes(log: &Path, session: &str) -> Vec { .collect() } -fn wait_for(what: &str, mut ready: impl FnMut() -> bool) { +fn wait_for(what: &str, log: &Path, mut ready: impl FnMut() -> bool) { let deadline = Instant::now() + WAIT; while !ready() { - assert!(Instant::now() < deadline, "timed out waiting for {what}"); + assert!( + Instant::now() < deadline, + "timed out waiting for {what}\ndaemon: {}", + std::fs::read_to_string(log).unwrap_or_default(), + ); std::thread::sleep(Duration::from_millis(100)); } } @@ -403,16 +439,20 @@ fn claude_stop_delivers_the_intent_suggestion_the_daemon_judged_at_the_tool_boun "tracedecay_status", &json!({"wait_for": {"state": "ready", "timeout_ms": 90_000}}), ); - wait_for("the advisory owner that runs hook work to mount", || { - std::fs::read_to_string(&log) - .unwrap_or_default() - .lines() - .any(|line| { - line.contains("event=advisory_deferred_attempt ") - && line.contains(" phase=mounted ") - }) - }); - wait_for("both Claude transcripts to be ingested", || { + wait_for( + "the advisory owner that runs hook work to mount", + &log, + || { + std::fs::read_to_string(&log) + .unwrap_or_default() + .lines() + .any(|line| { + line.contains("event=advisory_deferred_attempt ") + && line.contains(" phase=mounted ") + }) + }, + ); + wait_for("both Claude transcripts to be ingested", &log, || { ingested_messages(home, &project, ABSENT_SESSION) == 3 && ingested_messages(home, &project, ENABLED_SESSION) == 3 }); @@ -423,7 +463,7 @@ fn claude_stop_delivers_the_intent_suggestion_the_daemon_judged_at_the_tool_boun "hook-claude-post-tool-use", &post_tool_use(&project, ABSENT_SESSION, &absent_transcript), ); - wait_for("the absent-block tool boundary's intent pass", || { + wait_for("the absent-block tool boundary's intent pass", &log, || { !intent_outcomes(&log, ABSENT_SESSION).is_empty() }); let absent_stop = hook_context(&run_hook( @@ -432,7 +472,7 @@ fn claude_stop_delivers_the_intent_suggestion_the_daemon_judged_at_the_tool_boun "hook-stop", &stop(&project, ABSENT_SESSION, &absent_transcript), )); - wait_for("the absent-block stop boundary's intent pass", || { + wait_for("the absent-block stop boundary's intent pass", &log, || { intent_outcomes(&log, ABSENT_SESSION).len() >= 2 }); assert!( @@ -486,15 +526,17 @@ fn claude_stop_delivers_the_intent_suggestion_the_daemon_judged_at_the_tool_boun ); let before = intent_outcomes(&log, ENABLED_SESSION).len(); - run_hook( + let enabled_tool = run_hook( home, &project, "hook-claude-post-tool-use", &post_tool_use(&project, ENABLED_SESSION, &enabled_transcript), ); - wait_for("the enabled tool boundary's intent pass", || { - intent_outcomes(&log, ENABLED_SESSION).len() > before - }); + wait_for( + &format!("the enabled tool boundary's intent pass after {enabled_tool:?}"), + &log, + || intent_outcomes(&log, ENABLED_SESSION).len() > before, + ); assert_eq!( intent_outcomes(&log, ENABLED_SESSION)[before..], ["mounted"], @@ -513,13 +555,18 @@ fn claude_stop_delivers_the_intent_suggestion_the_daemon_judged_at_the_tool_boun let mut stop_contexts = vec![hook_context(&first_stop)]; let mut stop_outputs = vec![first_stop]; if !stop_contexts.last().is_some_and(delivers) { - wait_for("the timed-out stop callback's replay to drain", || { - let status = run_tool(home, &project, "tracedecay_status", &json!({})); - let replay: StatusHookReplayV1 = serde_json::from_value(status["hook_replay"].clone()) - .unwrap_or_else(|error| panic!("hook replay status ({error}): {status}")); - replay == StatusHookReplayV1::Drained - && intent_outcomes(&log, ENABLED_SESSION).len() > before + 1 - }); + wait_for( + "the timed-out stop callback's replay to drain", + &log, + || { + let status = run_tool(home, &project, "tracedecay_status", &json!({})); + let replay: StatusHookReplayV1 = + serde_json::from_value(status["hook_replay"].clone()) + .unwrap_or_else(|error| panic!("hook replay status ({error}): {status}")); + replay == StatusHookReplayV1::Drained + && intent_outcomes(&log, ENABLED_SESSION).len() > before + 1 + }, + ); let next_stop = run_hook(home, &project, "hook-stop", &stop_event); stop_contexts.push(hook_context(&next_stop)); stop_outputs.push(next_stop); @@ -536,16 +583,14 @@ fn claude_stop_delivers_the_intent_suggestion_the_daemon_judged_at_the_tool_boun )) .collect::>(), std::fs::read_to_string(&log).unwrap_or_default(), - std::fs::read_to_string( - tracedecay_runtime_core::storage::default_profile_sharded_layout( - &project, - &home.join(".tracedecay"), - ) - .unwrap() - .data_root - .join("hook_analytics.jsonl") - ) - .unwrap_or_default(), + delivery_evidence(home, &project, &stop_contexts), + ); + assert_eq!( + stop_contexts + .iter() + .filter(|context| delivers(context)) + .count(), + 1 ); wait_for_hints(home, &project, &log, |(emitted, suppressed)| { emitted == 1 && suppressed >= 1 @@ -561,4 +606,18 @@ fn claude_stop_delivers_the_intent_suggestion_the_daemon_judged_at_the_tool_boun 1, "the session receives the suggestion exactly once" ); + let decisions_before_duplicate = engine.requests(); + let duplicate = run_hook(home, &project, "hook-stop", &stop_event); + assert!( + !delivers(&hook_context(&duplicate)), + "a completed retry delivers no second suggestion: {duplicate:?}" + ); + assert_eq!( + engine.requests(), + decisions_before_duplicate, + "completed retries do not rerun the intent producer: {duplicate:?}\ndaemon: {}\nrecent: {}", + std::fs::read_to_string(&log).unwrap_or_default(), + delivery_evidence(home, &project, &stop_contexts), + ); + assert_eq!(hint_counts(home, &project, YES_INTENT).0, 1); } From cbcb484476cd56f7ec27c43ba807647b6f80df1a Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Mon, 19 Oct 2026 08:16:48 +0000 Subject: [PATCH 79/89] fix(hooks): keep reads available through broker recovery --- .../src/code_index_scheduler.rs | 4 + .../src/code_index_scheduler/reconcile.rs | 9 + .../registry/owner_signals.rs | 19 ++ .../registry/serving_reads.rs | 2 +- crates/tracedecay-host-admission/src/lib.rs | 12 ++ crates/tracedecay/src/daemon/branch_admin.rs | 184 ++++++++++++++++-- crates/tracedecay/src/daemon/pr_autotrack.rs | 2 +- .../src/daemon/pr_autotrack/tests.rs | 138 +++++++++++++ .../src/daemon/project_composition.rs | 10 +- .../hook_source_pending.rs | 92 +++++---- .../hook_source_pending/tests.rs | 60 +++++- .../tests/rmcp_route/host_index_handoff.rs | 103 +++++++++- 12 files changed, 557 insertions(+), 78 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs index 05db158734..baca65a33f 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs @@ -36,6 +36,10 @@ pub trait CodeIndexSourcePendingReaderV1: Send + Sync { fn read_pending( &self, ) -> Result; + /// May wait for the canonical writer; call only from a blocking worker. + fn read_pending_blocking( + &self, + ) -> Result; fn revision(&self) -> u64; fn changed(&self, observed: u64) -> CodeIndexSourcePendingChangedFutureV1<'_>; } diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs index 10ca1218db..14e34cf58d 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs @@ -587,6 +587,15 @@ impl SourceFreshnessFenceV1 { .map_or(Ok(false), |reader| reader.read_pending()) } + pub(super) fn read_pending_source_blocking( + &self, + ) -> Result + { + self.pending_source + .as_ref() + .map_or(Ok(false), |reader| reader.read_pending_blocking()) + } + fn snapshot(&self) -> SourceFreshnessFenceStateV1 { self.state .lock() diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/owner_signals.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/owner_signals.rs index 6bc68025ea..a8d3976150 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/owner_signals.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/owner_signals.rs @@ -497,6 +497,25 @@ impl CodeIndexSchedulerRegistryV1 { budget: Duration, ) -> Result { let deadline = Instant::now() + budget; + // Every probe, including its blocking source read, shares the caller's + // one deadline. A timed-out blocking task cannot publish readiness. + match native_runtime::timeout_at( + deadline, + self.wait_for_readiness_until(project_root, target, deadline), + ) + .await + { + Ok(reading) => reading, + Err(_) => Ok(CodeIndexReadinessWaitReadV1::TimedOut { last: None }), + } + } + + async fn wait_for_readiness_until( + &self, + project_root: &Path, + target: CodeIndexReadinessTargetV1, + deadline: Instant, + ) -> Result { if target != CodeIndexReadinessTargetV1::Fresh && let Some(reading) = self .dashboard_freshness_read_with_source_admission( diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/serving_reads.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/serving_reads.rs index d2ddd3076c..07b55e53b8 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/serving_reads.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/serving_reads.rs @@ -315,7 +315,7 @@ impl CodeIndexSchedulerRegistryV1 { native_runtime::spawn_blocking(move || { // Sample the broker before any generation/fence value. A record // disappears only after the scheduler accepted the source hint. - let admission_pending = include_source_admission && source_freshness.read_pending_source()?; + let admission_pending = include_source_admission && source_freshness.read_pending_source_blocking()?; let mut progress = { let _span = tracing::trace_span!("daemon.code_index.dashboard.progress").entered(); build_progress diff --git a/crates/tracedecay-host-admission/src/lib.rs b/crates/tracedecay-host-admission/src/lib.rs index 6bb42c1477..d17fcd9ccd 100644 --- a/crates/tracedecay-host-admission/src/lib.rs +++ b/crates/tracedecay-host-admission/src/lib.rs @@ -240,6 +240,18 @@ impl HostAdmissionBroker { .pending_matches(predicate) } + /// Inspect the same canonical records from an existing blocking worker. + /// Synchronous serving probes use the nonblocking variant instead. + pub fn pending_replay_matches_blocking( + &self, + predicate: impl FnMut(&[u8]) -> Result, + ) -> Result { + self.runtime + .lock() + .map_err(|_| spool_runtime_unavailable())? + .pending_matches(predicate) + } + pub async fn pending_replay_count(&self) -> Result { self.with_runtime(|runtime| Ok(runtime.pending_count())) .await diff --git a/crates/tracedecay/src/daemon/branch_admin.rs b/crates/tracedecay/src/daemon/branch_admin.rs index c0d59b67ce..4592123821 100644 --- a/crates/tracedecay/src/daemon/branch_admin.rs +++ b/crates/tracedecay/src/daemon/branch_admin.rs @@ -50,9 +50,48 @@ mod remote_deletion_lifecycle; pub(in crate::daemon) mod remote_recovery_lifecycle; mod session_runtime_shutdown; -type HostAdmissionBrokerSlot = - Arc>; -type HostAdmissionBrokers = Arc>>; +#[derive(Default)] +pub(super) struct HostAdmissionBrokerSlot { + broker: tokio::sync::OnceCell, + available: tokio::sync::Notify, +} + +impl HostAdmissionBrokerSlot { + pub(super) fn get(&self) -> Option<&tracedecay_host_admission::SharedHostAdmissionBroker> { + self.broker.get() + } + + pub(super) fn progress_revision(&self) -> u64 { + self.get() + .map_or(0, |broker| broker.progress_revision().wrapping_add(1)) + } + + pub(super) async fn wait_for_progress(&self, observed: u64) { + let available = self.available.notified(); + tokio::pin!(available); + available.as_mut().enable(); + match self.get() { + Some(broker) => { + if self.progress_revision() == observed { + broker.wait_for_progress(observed.wrapping_sub(1)).await; + } + } + None => available.await, + } + } +} + +#[cfg(test)] +impl From for HostAdmissionBrokerSlot { + fn from(broker: tracedecay_host_admission::SharedHostAdmissionBroker) -> Self { + Self { + broker: tokio::sync::OnceCell::new_with(Some(broker)), + available: tokio::sync::Notify::new(), + } + } +} + +type HostAdmissionBrokers = Arc>>>; /// One profile session refresh service per profile session store. The service /// owns the opaque refresh handles it issued, so every route that reaches the /// same store (project MCP servers and the projectless client) must share the @@ -413,25 +452,29 @@ async fn open_host_admission_broker( open: impl FnOnce(&Path) -> Result<(HostAdmissionRuntime, SpoolOpenReport)> + Send + 'static, ) -> Result { let slot = Arc::clone(brokers.lock().await.entry(path.to_path_buf()).or_default()); - slot.get_or_try_init(|| async { - let open_path = path.to_path_buf(); - let (runtime, _) = tracedecay_runtime_core::native_runtime::spawn_blocking(move || { - let _span = - tracing::trace_span!("daemon.branch_admin.host_admission_runtime.open").entered(); - open(&open_path) + let broker = slot + .broker + .get_or_try_init(|| async { + let open_path = path.to_path_buf(); + let (runtime, _) = tracedecay_runtime_core::native_runtime::spawn_blocking(move || { + let _span = tracing::trace_span!("daemon.branch_admin.host_admission_runtime.open") + .entered(); + open(&open_path) + }) + .await + .map_err(|_| { + TraceDecayError::project_route( + "spool_runtime_unavailable", + true, + "host-admission spool runtime task failed", + ) + })??; + Ok::<_, TraceDecayError>(Arc::new(HostAdmissionBroker::new(runtime))) }) .await - .map_err(|_| { - TraceDecayError::project_route( - "spool_runtime_unavailable", - true, - "host-admission spool runtime task failed", - ) - })??; - Ok(Arc::new(HostAdmissionBroker::new(runtime))) - }) - .await - .cloned() + .cloned()?; + slot.available.notify_waiters(); + Ok(broker) } /// Coordinates every daemon operation that can create, rekey, or remove a @@ -1206,6 +1249,34 @@ impl StoreAdministration { .await } + /// Bind source currency to the existing broker slot even when its local + /// spool cannot open. Read/query composition can retain that typed + /// unavailability without granting host admission or losing recovery. + pub(super) async fn host_admission_source_broker( + &self, + database: &tracedecay_global_db::RegisteredGlobalDbLeaseV1, + ) -> Result> { + self.refuse_remotely_deleted_profile(database).await?; + let path = authority::canonical_identity_path(database.db_path())?; + let slot = Arc::clone( + self.host_admission_brokers + .lock() + .await + .entry(path.clone()) + .or_default(), + ); + if let Err(error) = open_host_admission_broker( + &self.host_admission_brokers, + &path, + HostAdmissionRuntime::open_for_database, + ) + .await + { + tracing::warn!(%error, "host-admission source currency is unavailable"); + } + Ok(slot) + } + #[tracing::instrument( name = "daemon.branch_admin.host_admission_broker", level = "trace", @@ -2195,6 +2266,79 @@ mod tests { administration.shutdown_host_admission_replay().await; } + #[tokio::test] + async fn unavailable_broker_slot_recovers_and_wakes_every_progress_observer() { + let temp = tempfile::tempdir().unwrap(); + let database = temp.path().join("sessions.db"); + let spool = temp.path().join(".sessions.db.host-admission"); + std::fs::write(&spool, "blocked").unwrap(); + let brokers: HostAdmissionBrokers = Arc::default(); + assert!( + open_host_admission_broker( + &brokers, + &database, + HostAdmissionRuntime::open_for_database + ) + .await + .is_err() + ); + let slot = Arc::clone(brokers.lock().await.get(&database).unwrap()); + assert!(slot.get().is_none()); + let unavailable = slot.progress_revision(); + let first = slot.wait_for_progress(unavailable); + let second = slot.wait_for_progress(unavailable); + tokio::pin!(first, second); + assert!( + std::future::poll_fn(|cx| std::task::Poll::Ready(first.as_mut().poll(cx).is_pending())) + .await + ); + assert!( + std::future::poll_fn(|cx| std::task::Poll::Ready( + second.as_mut().poll(cx).is_pending() + )) + .await + ); + std::fs::remove_file(&spool).unwrap(); + let broker = open_host_admission_broker( + &brokers, + &database, + HostAdmissionRuntime::open_for_database, + ) + .await + .unwrap(); + assert!(Arc::ptr_eq(slot.get().unwrap(), &broker)); + assert_ne!(slot.progress_revision(), unavailable); + tokio::time::timeout(Duration::from_secs(1), async { + tokio::join!(first, second); + // Opening between revision capture and waiter registration is + // observable even when the ready broker has made no progress yet. + slot.wait_for_progress(unavailable).await; + }) + .await + .unwrap(); + let ready = slot.progress_revision(); + let first = slot.wait_for_progress(ready); + let second = slot.wait_for_progress(ready); + tokio::pin!(first, second); + assert!( + std::future::poll_fn(|cx| std::task::Poll::Ready(first.as_mut().poll(cx).is_pending())) + .await + ); + assert!( + std::future::poll_fn(|cx| std::task::Poll::Ready( + second.as_mut().poll(cx).is_pending() + )) + .await + ); + broker.admit("source", b"edit").await.unwrap(); + tokio::time::timeout(Duration::from_secs(1), async { + tokio::join!(first, second); + }) + .await + .unwrap(); + assert_ne!(slot.progress_revision(), ready); + } + /// A spool open can stall for seconds on fsync under load. A first open of /// one database must not park the first open of an unrelated database, /// while a second opener of the same database shares the in-flight open. diff --git a/crates/tracedecay/src/daemon/pr_autotrack.rs b/crates/tracedecay/src/daemon/pr_autotrack.rs index b0d2ad4d3f..22cf713ca5 100644 --- a/crates/tracedecay/src/daemon/pr_autotrack.rs +++ b/crates/tracedecay/src/daemon/pr_autotrack.rs @@ -909,7 +909,7 @@ async fn activate_linked_worktree( let scope = tracedecay_code_index_runtime::resolved_scope_for_project(worktree, &project_id) .map_err(|error| scheduler_unavailable(&error.to_string()))?; - let (_, reader) = super::project_composition::project_hook_source_pending( + let reader = super::project_composition::project_hook_source_pending( stores, graph, worktree, &scope, ) .await diff --git a/crates/tracedecay/src/daemon/pr_autotrack/tests.rs b/crates/tracedecay/src/daemon/pr_autotrack/tests.rs index bac3d68667..a4c4f1c411 100644 --- a/crates/tracedecay/src/daemon/pr_autotrack/tests.rs +++ b/crates/tracedecay/src/daemon/pr_autotrack/tests.rs @@ -1,8 +1,14 @@ +#[cfg(unix)] +use std::future::Future; use std::sync::Arc; use std::time::Duration; use super::*; use crate::test_support::git::git; +#[cfg(unix)] +use tracedecay_contracts::code_index_freshness::{ + CodeIndexFreshnessReadFailureV1, CodeIndexReadinessTargetV1, CodeIndexReadinessWaitReadV1, +}; use tracedecay_runtime_core::native_runtime; #[test] @@ -1620,6 +1626,138 @@ struct HostedManualBranchFixture { _scope: tracedecay_runtime_core::db::DaemonDatabaseScope, } +#[cfg(unix)] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_pr_first_source_reader_recovers_through_the_canonical_broker_slot() { + let repo = tempfile::tempdir().unwrap(); + init_manual_branch_repo(repo.path(), "feature/recovery"); + let hosted = HostedManualBranchFixture::open(repo.path()).await; + let stores = &hosted.engine.store_administration; + let graph = &hosted.graph; + let schedulers = &hosted.engine.invocation.code_index_schedulers; + let database_path = &graph.store_layout().sessions_db_path; + let spool = database_path.parent().unwrap().join(format!( + ".{}.host-admission", + database_path.file_name().unwrap().to_str().unwrap() + )); + std::fs::write(&spool, "blocked").unwrap(); + let mounted = hosted.activate(schedulers, "feature/recovery").await; + assert!(schedulers.is_worktree_mounted(&mounted.worktree).await); + assert!(matches!( + schedulers.dashboard_freshness_read(&mounted.worktree).await, + Err(CodeIndexFreshnessReadFailureV1::ReadFailed) + )); + let database = stores + .registered_project_configuration_database(graph.project_root(), graph.store_layout()) + .await + .unwrap(); + let slot = stores + .host_admission_source_broker(&database) + .await + .unwrap(); + assert!(slot.get().is_none()); + let project_id = graph + .store_layout() + .identity + .project_id + .as_ref() + .unwrap() + .clone() + .try_into() + .unwrap(); + let scope = + tracedecay_code_index_runtime::resolved_scope_for_project(&mounted.worktree, &project_id) + .unwrap(); + // Independent Core/PR factories borrow the same canonical slot. Existing + // source-fence copies must recover without replacing the mounted reader. + let first = super::super::project_composition::project_hook_source_pending( + stores, + graph, + &mounted.worktree, + &scope, + ) + .await + .unwrap(); + let second = super::super::project_composition::project_hook_source_pending( + stores, + graph, + &mounted.worktree, + &scope, + ) + .await + .unwrap(); + assert!(Arc::ptr_eq( + &slot, + &stores + .host_admission_source_broker(&database) + .await + .unwrap() + )); + assert_eq!( + first.read_pending(), + Err(CodeIndexFreshnessReadFailureV1::ReadFailed) + ); + assert_eq!( + first.read_pending_blocking(), + Err(CodeIndexFreshnessReadFailureV1::ReadFailed) + ); + assert_eq!( + second.read_pending(), + Err(CodeIndexFreshnessReadFailureV1::ReadFailed) + ); + let observed = first.revision(); + let changed_first = first.changed(observed); + let changed_second = second.changed(observed); + tokio::pin!(changed_first, changed_second); + assert!( + std::future::poll_fn(|cx| std::task::Poll::Ready( + changed_first.as_mut().poll(cx).is_pending() + )) + .await + ); + assert!( + std::future::poll_fn(|cx| std::task::Poll::Ready( + changed_second.as_mut().poll(cx).is_pending() + )) + .await + ); + std::fs::remove_file(&spool).unwrap(); + let broker = stores.host_admission_broker(&database).await.unwrap(); + assert!(Arc::ptr_eq(slot.get().unwrap(), &broker)); + tokio::time::timeout(Duration::from_secs(1), async { + tokio::join!(changed_first, changed_second); + }) + .await + .unwrap(); + assert_eq!(first.read_pending(), Ok(false)); + assert_eq!(second.read_pending(), Ok(false)); + assert_ne!(first.revision(), observed); + activate_linked_worktree(Some(stores), schedulers, graph, &mounted.worktree) + .await + .unwrap(); + let reading = schedulers + .wait_for_readiness( + &mounted.worktree, + CodeIndexReadinessTargetV1::Ready, + Duration::from_secs(5), + ) + .await + .unwrap(); + assert!( + matches!(reading, CodeIndexReadinessWaitReadV1::Reached { .. }), + "{reading:?}" + ); + schedulers.shutdown().await; + assert!( + hosted + .engine + .shutdown_all() + .await + .project_servers + .is_clean() + ); +} + #[cfg(unix)] impl HostedManualBranchFixture { async fn open(repo: &Path) -> Self { diff --git a/crates/tracedecay/src/daemon/project_composition.rs b/crates/tracedecay/src/daemon/project_composition.rs index 379addffae..efa3c21625 100644 --- a/crates/tracedecay/src/daemon/project_composition.rs +++ b/crates/tracedecay/src/daemon/project_composition.rs @@ -576,7 +576,6 @@ struct OpenedProjectGraph { /// Ports both of this route's MCP servers publish. Built once before the core /// server; the full server that replaces the core publishes the same ports. struct ProjectRoutePorts { - host_admission_broker: tracedecay_host_admission::SharedHostAdmissionBroker, code_index: ProjectCodeIndexAuthorities, dashboard_code_index_freshness_reader: tracedecay_contracts::code_index_freshness::CodeIndexFreshnessReader, @@ -940,7 +939,7 @@ impl ProjectOpenInputs<'_> { &route_registered, *project_database_is_read_only, )?; - let (host_admission_broker, pending_source) = project_hook_source_pending( + let pending_source = project_hook_source_pending( self.store_administration, cg, self.canonical_project_path, @@ -1034,7 +1033,6 @@ impl ProjectOpenInputs<'_> { transcript_source_profile, code_index_activation, ports: ProjectRoutePorts { - host_admission_broker, code_index, dashboard_code_index_freshness_reader, code_index_readiness_waiter, @@ -1383,7 +1381,11 @@ impl ProjectOpenInputs<'_> { &delivery_access, ) .await?; - let host_admission_broker = Some(Arc::clone(&core.ports.host_admission_broker)); + let host_admission_broker = Some( + self.store_administration + .host_admission_broker(&session_db) + .await?, + ); let refresh_schedulers = self .store_administration .session_temporal_refresh_schedulers(); diff --git a/crates/tracedecay/src/daemon/project_composition/hook_source_pending.rs b/crates/tracedecay/src/daemon/project_composition/hook_source_pending.rs index 7510b881e6..0ec3620c54 100644 --- a/crates/tracedecay/src/daemon/project_composition/hook_source_pending.rs +++ b/crates/tracedecay/src/daemon/project_composition/hook_source_pending.rs @@ -8,7 +8,6 @@ use tracedecay_code_index_runtime::code_index_scheduler::{ use tracedecay_contracts::ResolvedScope; use tracedecay_contracts::code_index_freshness::CodeIndexFreshnessReadFailureV1; use tracedecay_domain::errors::{Result, TraceDecayError}; -use tracedecay_host_admission::SharedHostAdmissionBroker; use tracedecay_mcp::hook_events::{ DurableHookEventDecodeError, HookEventPlan, decode_durable_hook_event_plan, }; @@ -16,51 +15,63 @@ use tracedecay_project::project::TraceDecay; use tracedecay_sessions::admission::HostAdmissionOutcome; use tracedecay_store::StoreShardScopeV1; -use crate::daemon::branch_admin::StoreAdministration; +use crate::daemon::branch_admin::{HostAdmissionBrokerSlot, StoreAdministration}; #[cfg(test)] mod tests; struct HookSourcePending { - broker: SharedHostAdmissionBroker, + broker: Arc, root: PathBuf, scope: ResolvedScope, } +impl HookSourcePending { + fn matches_pending(&self, payload: &[u8]) -> std::result::Result { + let plan = decode_durable_hook_event_plan(payload).map_err(|error| { + HostAdmissionOutcome::degraded(match error { + DurableHookEventDecodeError::Malformed => "malformed_host_event_plan", + DurableHookEventDecodeError::UnsupportedVersion => "unsupported_host_event_plan", + }) + })?; + match plan { + HookEventPlan::SyncFilesAt { root, scope, .. } => { + // Reference is mutable attribution. The canonical route's + // project, repository, worktree and byte-exact locator own + // the event through checkout changes. + Ok(root.as_os_str() == self.root.as_os_str() + && scope.project_id == self.scope.project_id + && scope.repository_id == self.scope.repository_id + && scope.worktree_id == self.scope.worktree_id) + } + // Shipped unscoped file edits cannot prove a different route. + // Keep their source currency unresolved until replay settles. + HookEventPlan::SyncFiles(_) => Ok(true), + HookEventPlan::AddBranch(_) + | HookEventPlan::AddBranchAt { .. } + | HookEventPlan::SyncCurrentBranch { .. } + | HookEventPlan::DebouncedIncrementalSync(_) + | HookEventPlan::RecordTerminalReceipt { .. } + | HookEventPlan::MarkTurnIngested { .. } + | HookEventPlan::Noop => Ok(false), + } + } +} + impl CodeIndexSourcePendingReaderV1 for HookSourcePending { fn read_pending(&self) -> std::result::Result { self.broker - .pending_replay_matches(|payload| { - let plan = decode_durable_hook_event_plan(payload).map_err(|error| { - HostAdmissionOutcome::degraded(match error { - DurableHookEventDecodeError::Malformed => "malformed_host_event_plan", - DurableHookEventDecodeError::UnsupportedVersion => { - "unsupported_host_event_plan" - } - }) - })?; - match plan { - HookEventPlan::SyncFilesAt { root, scope, .. } => { - // Reference is mutable attribution. The canonical route's - // project, repository, worktree and byte-exact locator own - // the event through checkout changes. - Ok(root.as_os_str() == self.root.as_os_str() - && scope.project_id == self.scope.project_id - && scope.repository_id == self.scope.repository_id - && scope.worktree_id == self.scope.worktree_id) - } - // Shipped unscoped file edits cannot prove a different route. - // Keep their source currency unresolved until replay settles. - HookEventPlan::SyncFiles(_) => Ok(true), - HookEventPlan::AddBranch(_) - | HookEventPlan::AddBranchAt { .. } - | HookEventPlan::SyncCurrentBranch { .. } - | HookEventPlan::DebouncedIncrementalSync(_) - | HookEventPlan::RecordTerminalReceipt { .. } - | HookEventPlan::MarkTurnIngested { .. } - | HookEventPlan::Noop => Ok(false), - } - }) + .get() + .ok_or(CodeIndexFreshnessReadFailureV1::ReadFailed)? + .pending_replay_matches(|payload| self.matches_pending(payload)) + .map_err(|_| CodeIndexFreshnessReadFailureV1::ReadFailed) + } + + fn read_pending_blocking(&self) -> std::result::Result { + self.broker + .get() + .ok_or(CodeIndexFreshnessReadFailureV1::ReadFailed)? + .pending_replay_matches_blocking(|payload| self.matches_pending(payload)) .map_err(|_| CodeIndexFreshnessReadFailureV1::ReadFailed) } @@ -81,10 +92,7 @@ pub(in crate::daemon) async fn project_hook_source_pending( graph: &TraceDecay, root: &Path, expected_scope: &ResolvedScope, -) -> Result<( - SharedHostAdmissionBroker, - Arc, -)> { +) -> Result> { let registry = stores.registered_profile_database().await?; let owner = registry .project_registry_context_by_id(expected_scope.project_id.as_str()) @@ -112,7 +120,7 @@ pub(in crate::daemon) async fn project_hook_source_pending( )); } let database = stores - .registered_project_session_database(graph.project_root(), graph.store_layout()) + .registered_project_configuration_database(graph.project_root(), graph.store_layout()) .await?; if !matches!(&database.binding().shard_id.scope, StoreShardScopeV1::ProjectSessions { project_id } if project_id == &scope.project_id) { @@ -122,11 +130,11 @@ pub(in crate::daemon) async fn project_hook_source_pending( "hook source writer belongs to a different project", )); } - let broker = stores.host_admission_broker(&database).await?; + let broker = stores.host_admission_source_broker(&database).await?; let reader: Arc = Arc::new(HookSourcePending { - broker: Arc::clone(&broker), + broker, root, scope, }); - Ok((broker, reader)) + Ok(reader) } diff --git a/crates/tracedecay/src/daemon/project_composition/hook_source_pending/tests.rs b/crates/tracedecay/src/daemon/project_composition/hook_source_pending/tests.rs index 04e5da2c4c..2f27fe6b1e 100644 --- a/crates/tracedecay/src/daemon/project_composition/hook_source_pending/tests.rs +++ b/crates/tracedecay/src/daemon/project_composition/hook_source_pending/tests.rs @@ -1,5 +1,7 @@ use tracedecay_domain::HostIntegrationIdV1; -use tracedecay_host_admission::{HostAdmissionBroker, HostAdmissionRuntime, SpoolBounds}; +use tracedecay_host_admission::{ + HostAdmissionBroker, HostAdmissionRuntime, SharedHostAdmissionBroker, SpoolBounds, +}; use tracedecay_mcp::hook_events::{HookEventKind, encode_durable_hook_event_plan}; use super::*; @@ -12,7 +14,7 @@ fn reader( scope: &ResolvedScope, ) -> HookSourcePending { HookSourcePending { - broker: Arc::clone(broker), + broker: Arc::new(HostAdmissionBrokerSlot::from(Arc::clone(broker))), root: root.to_owned(), scope: scope.clone(), } @@ -144,10 +146,9 @@ async fn unscoped_legacy_edits_remain_pending_and_invalid_plans_fail_closed() { let spool = tempfile::tempdir().unwrap(); let broker = broker(spool.path()); broker.admit("source", &payload).await.unwrap(); - assert_eq!( - reader(&broker, project.path(), &scope).read_pending(), - expected - ); + let reader = reader(&broker, project.path(), &scope); + assert_eq!(reader.read_pending(), expected); + assert_eq!(reader.read_pending_blocking(), expected); } let mut corrupt: serde_json::Value = serde_json::from_slice(&edit(project.path(), scope.clone())).unwrap(); @@ -159,8 +160,53 @@ async fn unscoped_legacy_edits_remain_pending_and_invalid_plans_fail_closed() { .admit("source", &serde_json::to_vec(&corrupt).unwrap()) .await .unwrap(); + let reader = reader(&broker, project.path(), &scope); assert_eq!( - reader(&broker, project.path(), &scope).read_pending(), + reader.read_pending(), + Err(CodeIndexFreshnessReadFailureV1::ReadFailed) + ); + assert_eq!( + reader.read_pending_blocking(), + Err(CodeIndexFreshnessReadFailureV1::ReadFailed) + ); +} + +#[tokio::test] +async fn poisoned_pending_source_remains_a_typed_read_failure() { + let (graph, project, _authority) = init_indexed_repo().await; + let project_id = graph + .store_layout() + .identity + .project_id + .as_ref() + .unwrap() + .clone() + .try_into() + .unwrap(); + let scope = + tracedecay_code_index_runtime::resolved_scope_for_project(project.path(), &project_id) + .unwrap(); + let spool = tempfile::tempdir().unwrap(); + let broker = broker(spool.path()); + broker + .admit( + "source", + &encode_durable_hook_event_plan(&HookEventPlan::Noop).unwrap(), + ) + .await + .unwrap(); + let poison = std::thread::spawn({ + let broker = Arc::clone(&broker); + move || broker.pending_replay_matches(|_| panic!("poison the owned test broker runtime")) + }); + assert!(poison.join().is_err()); + let reader = reader(&broker, project.path(), &scope); + assert_eq!( + reader.read_pending(), + Err(CodeIndexFreshnessReadFailureV1::ReadFailed) + ); + assert_eq!( + reader.read_pending_blocking(), Err(CodeIndexFreshnessReadFailureV1::ReadFailed) ); } diff --git a/crates/tracedecay/src/daemon/tests/rmcp_route/host_index_handoff.rs b/crates/tracedecay/src/daemon/tests/rmcp_route/host_index_handoff.rs index d04a426d8d..06cc07fb88 100644 --- a/crates/tracedecay/src/daemon/tests/rmcp_route/host_index_handoff.rs +++ b/crates/tracedecay/src/daemon/tests/rmcp_route/host_index_handoff.rs @@ -1,6 +1,7 @@ use tracedecay_contracts::code_index_freshness::{ CodeIndexReadinessTargetV1, CodeIndexReadinessV1, CodeIndexReadinessWaitReadV1, }; +use tracedecay_runtime_core::native_runtime::{NativeTask, NativeTaskOutcome}; use tracedecay_session_memory::event_lane::{self, ActivityFamilyV1}; use super::*; @@ -8,6 +9,7 @@ use super::*; async fn ready_reading( fixture: &RmcpRouteFixture, project: &std::path::Path, + phase: &str, ) -> tracedecay_contracts::code_index_freshness::CodeIndexWorktreeFreshnessV1 { let reading = fixture .engine @@ -15,7 +17,7 @@ async fn ready_reading( .code_index_schedulers .wait_for_readiness(project, CodeIndexReadinessTargetV1::Ready, PHASE_TIMEOUT) .await - .unwrap(); + .unwrap_or_else(|error| panic!("{phase} readiness read failed: {error:?}")); let CodeIndexReadinessWaitReadV1::Reached { reading } = reading else { panic!("exact worktree did not become ready: {reading:?}"); }; @@ -28,7 +30,7 @@ async fn committed_save_remains_noncurrent_until_its_broker_handoff() { let project = canonical_existing_identity(fixture.handshake.project_path.as_deref().unwrap()).unwrap(); wait_for_host_ingest_publication(&fixture).await; - let initial = ready_reading(&fixture, &project).await; + let initial = ready_reading(&fixture, &project, "initial").await; let graph = fixture.server.cg_snapshot().await; let project_id = graph.store_layout().identity.project_id.as_deref().unwrap(); let scope = tracedecay_code_index_runtime::resolved_scope_for_project( @@ -121,7 +123,7 @@ async fn committed_save_remains_noncurrent_until_its_broker_handoff() { "the retained text owner must not claim source currency" ); drop(replay); - let saved = ready_reading(&fixture, &project).await; + let saved = ready_reading(&fixture, &project, "after broker release").await; assert_ne!(saved.latest_generation_id, initial.latest_generation_id); assert_ne!( saved.snapshot_content_identity, @@ -150,3 +152,98 @@ async fn committed_save_remains_noncurrent_until_its_broker_handoff() { "handoff shutdown: {shutdown:?}" ); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn readiness_budget_bounds_canonical_broker_runtime_contention() { + let fixture = rmcp_route_fixture_with_projects("contended-hook-source-read", &[], true).await; + let project = + canonical_existing_identity(fixture.handshake.project_path.as_deref().unwrap()).unwrap(); + wait_for_host_ingest_publication(&fixture).await; + let initial = ready_reading(&fixture, &project, "before runtime contention").await; + let graph = fixture.server.cg_snapshot().await; + let database = fixture + .store_administration + .registered_project_session_database(graph.project_root(), graph.store_layout()) + .await + .unwrap(); + let broker = fixture + .store_administration + .host_admission_broker(&database) + .await + .unwrap(); + let replay = broker.begin_replay().await.unwrap(); + let payload = tracedecay_mcp::hook_events::encode_durable_hook_event_plan( + &tracedecay_mcp::hook_events::HookEventPlan::Noop, + ) + .unwrap(); + broker + .admit("owned-runtime-contention", &payload) + .await + .unwrap(); + let (entered, entered_rx) = tokio::sync::oneshot::channel(); + let (release, released) = std::sync::mpsc::channel(); + let held = std::thread::spawn({ + let broker = Arc::clone(&broker); + move || { + let mut entered = Some(entered); + broker + .pending_replay_matches(|_| { + entered.take().unwrap().send(()).unwrap(); + let _ = released.recv(); + Ok(false) + }) + .unwrap() + } + }); + entered_rx.await.unwrap(); + assert_eq!( + broker + .pending_replay_matches(|_| Ok(false)) + .unwrap_err() + .reason_code, + Some("spool_runtime_unavailable") + ); + let (waiting, waiting_rx) = tokio::sync::oneshot::channel(); + let registry = fixture.engine.invocation.code_index_schedulers.clone(); + let waiting_root = project.clone(); + let readiness = NativeTask::spawn(async move { + waiting.send(()).unwrap(); + registry + .wait_for_readiness( + &waiting_root, + CodeIndexReadinessTargetV1::Ready, + PHASE_TIMEOUT, + ) + .await + }) + .unwrap(); + waiting_rx.await.unwrap(); + let progress = NativeTask::spawn(async { 1 }).unwrap(); + assert!(matches!( + tokio::time::timeout(PHASE_TIMEOUT, progress).await.unwrap(), + NativeTaskOutcome::Completed(1) + )); + let NativeTaskOutcome::Completed(reading) = readiness.await else { + panic!("the readiness wait must complete while the runtime remains held"); + }; + release.send(()).unwrap(); + assert!(!held.join().unwrap()); + assert!( + matches!(reading, Ok(CodeIndexReadinessWaitReadV1::TimedOut { .. })), + "runtime contention must remain within the original readiness budget: {reading:?}" + ); + let recovered = ready_reading(&fixture, &project, "after runtime contention").await; + assert_eq!( + recovered.snapshot_content_identity, + initial.snapshot_content_identity + ); + let record = replay.lease_next().await.unwrap().unwrap(); + replay.commit(record.seq).await.unwrap(); + drop(replay); + fixture.server.shutdown().await; + let shutdown = fixture.engine.shutdown_all().await; + assert!( + shutdown.project_servers.is_clean(), + "contention shutdown: {shutdown:?}" + ); +} From bdbaa9ae2b815a2724c857c19c3a35a68371eb56 Mon Sep 17 00:00:00 2001 From: Zack Jackson Date: Mon, 19 Oct 2026 01:34:31 -0700 Subject: [PATCH 80/89] test(hooks): import duration for spool recovery regression --- crates/tracedecay/src/daemon/tests/scheduler_config.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/tracedecay/src/daemon/tests/scheduler_config.rs b/crates/tracedecay/src/daemon/tests/scheduler_config.rs index 842de1c5f5..aefa7b6831 100644 --- a/crates/tracedecay/src/daemon/tests/scheduler_config.rs +++ b/crates/tracedecay/src/daemon/tests/scheduler_config.rs @@ -1,3 +1,6 @@ +#[cfg(unix)] +use std::time::Duration; + #[cfg(unix)] use super::*; From b88ffd47390cc38f11c68d44d509fa61cf896b0d Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Mon, 19 Oct 2026 08:41:33 +0000 Subject: [PATCH 81/89] fix(hooks): retain intent boundary work through replay --- .../handlers/hook_runtime/admission/tests.rs | 37 ++-- .../admission/tests/tool_dispatch.rs | 188 ++++++++++++++++++ .../admission/tests/tool_dispatch/fixture.rs | 169 ++++++++++++++++ .../src/handlers/hook_runtime/envelope.rs | 6 + .../src/daemon/hook_v2_replay_consumer.rs | 57 ++++-- .../project_open_owners/advisory_runtime.rs | 90 +++++++-- .../advisory_runtime/scout_journey_tests.rs | 47 +++++ .../scout_journey_tests/replay.rs | 37 ++++ .../advisory_runtime/tests.rs | 81 ++++++++ 9 files changed, 668 insertions(+), 44 deletions(-) create mode 100644 crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests/tool_dispatch.rs create mode 100644 crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests/tool_dispatch/fixture.rs diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests.rs index 2e813c384c..4a46ed4dee 100644 --- a/crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests.rs +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests.rs @@ -7,6 +7,7 @@ use super::super::*; use super::*; mod legacy_continuity; +mod tool_dispatch; #[test] fn daemon_admission_is_idempotent_per_identity_and_conflicts_on_different_bytes() { @@ -127,11 +128,10 @@ fn envelope_is_prompt(event: &tracedecay_hooks::HookEventV2) -> bool { matches!(event, tracedecay_hooks::HookEventV2::PromptBoundary) } -/// Prompt and completed-tool boundaries owe no durable producer work, yet -/// each admission enqueues the session intent pass on the project's -/// registered orchestrator; a started tool call is not a boundary. +/// Prompt and completed-tool boundaries retain their own intent work until +/// the registered orchestrator settles it; a started call is not a boundary. #[tokio::test] -async fn prompt_and_tool_admissions_enqueue_an_intent_orchestration_without_owed_work() { +async fn prompt_and_tool_admissions_settle_their_durable_intent_work() { let data_root = tempfile::tempdir().unwrap(); let binding = admission_test_binding(7); // Kiro admits prompt boundaries natively today; Claude gains them when @@ -183,17 +183,18 @@ async fn prompt_and_tool_admissions_enqueue_an_intent_orchestration_without_owed if prompt { envelope.producer = kiro; } - assert!(!hook_v2_requires_producer_work(&envelope)); + assert!(hook_v2_requires_producer_work(&envelope)); let staged = stage_hook_v2_admission( data_root.path(), &envelope, - None, + Some(&envelope), None, Some(0), UtcMicros(1_000), ) .expect("ledger available"); staged.commit.wait().unwrap(); + let completion = hook_v2_work_completion(data_root.path(), &envelope); assert_eq!( tracedecay_daemon_service::admit_registered_hook_orchestration( envelope, @@ -205,8 +206,8 @@ async fn prompt_and_tool_admissions_enqueue_an_intent_orchestration_without_owed None, 1, false, - None, - None, + Some(staged.receipt.order), + Some(completion), ), tracedecay_contracts::HookOrchestrationAdmissionV1::Enqueued ); @@ -218,11 +219,23 @@ async fn prompt_and_tool_admissions_enqueue_an_intent_orchestration_without_owed false )) ); + tokio::time::timeout(std::time::Duration::from_secs(10), async { + while runtime.active_tasks() != 0 + || !hook_v2_pending_work_envelopes( + data_root.path(), + if prompt { kiro } else { binding.host }, + UtcMicros(1_000), + ) + .unwrap() + .is_empty() + { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); } - assert!( - pending_work(data_root.path()).is_empty(), - "an intent pass is best-effort and owes no redriven producer work" - ); + assert!(pending_work(data_root.path()).is_empty()); let mut started = admission_test_envelope(44, 7); started.event = tool(tracedecay_hooks::HookLifecyclePhaseV1::Started); assert_eq!( diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests/tool_dispatch.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests/tool_dispatch.rs new file mode 100644 index 0000000000..c9ec3391e6 --- /dev/null +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests/tool_dispatch.rs @@ -0,0 +1,188 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Duration; + +use tracedecay_contracts::HookOrchestrationAdmissionV1; +use tracedecay_daemon_service::{ + BoundedHookOrchestratorV1, HookOrchestrationRequestV1, HookOrchestrationTriggerV1, + HookOrchestrationWorkOutcomeV1, register_hook_orchestration_runtime, + unregister_hook_orchestration_runtime, +}; +use tracedecay_hooks::{HookAdmissionDecisionV1, HookAdmissionWorkSettlementV1}; + +use super::super::{ + HookV2AdmissionOutcomeV1, admit_hook_v2_replayed_envelope_with_lifecycle, + hook_v2_pending_work_envelopes, stage_hook_v2_admission, +}; +use crate::handlers::hook_runtime::envelope::{ + daemon_mint_hook_v2_envelope, hook_now, hook_v2_requires_producer_work, +}; + +mod fixture; + +#[tokio::test] +async fn committed_claude_tool_boundary_replays_its_intent_once_after_dispatch_cutoff() { + prove_dispatch_cutoff(false).await; +} + +#[tokio::test] +async fn retryable_tool_intent_retains_debt_until_recovery_then_settles_once() { + prove_dispatch_cutoff(true).await; +} + +async fn prove_dispatch_cutoff(refuse_first: bool) { + let fixture = fixture::open().await; + let envelope = &fixture.envelope; + let effects = Arc::new(AtomicUsize::new(0)); + let attempts = Arc::new(AtomicUsize::new(0)); + let (ran, mut runs) = tokio::sync::mpsc::unbounded_channel(); + let producer = BoundedHookOrchestratorV1::new(1, { + let effects = Arc::clone(&effects); + let attempts = Arc::clone(&attempts); + let expected_event = daemon_mint_hook_v2_envelope(envelope); + let expected_session = fixture.lifecycle.session_id.clone(); + move |request: HookOrchestrationRequestV1, _| { + let effects = Arc::clone(&effects); + let attempts = Arc::clone(&attempts); + let ran = ran.clone(); + let expected_event = expected_event.clone(); + let expected_session = expected_session.clone(); + async move { + assert_eq!(request.trigger, HookOrchestrationTriggerV1::IntentBoundary); + assert_eq!(request.hook.envelope(), &expected_event); + let lifecycle = request.lifecycle.unwrap(); + assert_eq!(lifecycle.session_id, expected_session); + if attempts.fetch_add(1, Ordering::SeqCst) == 0 && refuse_first { + ran.send(false).unwrap(); + return HookOrchestrationWorkOutcomeV1::RetryableFailure; + } + effects.fetch_add(1, Ordering::SeqCst); + ran.send(true).unwrap(); + HookOrchestrationWorkOutcomeV1::Completed + } + } + }) + .unwrap(); + assert!(register_hook_orchestration_runtime( + envelope.project_id, + envelope.worktree_id, + &producer + )); + let canonical = daemon_mint_hook_v2_envelope(envelope); + let mut lifecycle = fixture.lifecycle.clone(); + lifecycle.event_id = canonical.event_id; + let staged = stage_hook_v2_admission( + &fixture.graph.hook_store_layout().data_root, + &canonical, + hook_v2_requires_producer_work(envelope).then_some(envelope), + Some(&lifecycle), + Some(0), + hook_now(), + ) + .unwrap(); + assert_eq!(staged.receipt.decision, HookAdmissionDecisionV1::Admitted); + staged.commit.wait().unwrap(); + assert_eq!( + effects.load(Ordering::SeqCst), + 0, + "cutoff precedes dispatch" + ); + let replay = || { + admit_hook_v2_replayed_envelope_with_lifecycle( + &fixture.graph, + envelope, + Some(fixture.lifecycle.session_id.clone()), + Some(fixture.lifecycle.clone()), + &fixture.sessions, + &fixture.background_cpu, + hook_now(), + ) + }; + assert!(matches!( + replay().await, + HookV2AdmissionOutcomeV1::Admitted { + orchestration: HookOrchestrationAdmissionV1::Enqueued, + .. + } | HookV2AdmissionOutcomeV1::ExactDuplicate { .. } + )); + let pending = || { + hook_v2_pending_work_envelopes( + &fixture.graph.hook_store_layout().data_root, + envelope.producer, + hook_now(), + ) + .unwrap() + }; + assert!( + effects.load(Ordering::SeqCst) == 1 || !pending().is_empty(), + "a committed Tool boundary cannot lose both its intent execution and durable debt" + ); + if refuse_first { + assert_eq!( + tokio::time::timeout(Duration::from_secs(10), runs.recv()) + .await + .unwrap(), + Some(false) + ); + tokio::time::timeout(Duration::from_secs(10), async { + while producer.active_tasks() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!(pending(), vec![envelope.clone()]); + assert_eq!(effects.load(Ordering::SeqCst), 0); + assert!(matches!( + replay().await, + HookV2AdmissionOutcomeV1::Admitted { .. } + | HookV2AdmissionOutcomeV1::ExactDuplicate { .. } + )); + } + assert_eq!( + tokio::time::timeout(Duration::from_secs(10), runs.recv()) + .await + .unwrap(), + Some(true) + ); + tokio::time::timeout(Duration::from_secs(10), async { + while producer.active_tasks() != 0 || !pending().is_empty() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!(effects.load(Ordering::SeqCst), 1); + let settled = stage_hook_v2_admission( + &fixture.graph.hook_store_layout().data_root, + &canonical, + None, + None, + None, + hook_now(), + ) + .unwrap(); + assert_eq!( + settled.receipt.decision, + HookAdmissionDecisionV1::ExactDuplicate + ); + assert_eq!( + settled.receipt.work_settlement, + Some(HookAdmissionWorkSettlementV1::Completed) + ); + settled.commit.wait().unwrap(); + assert!(matches!( + replay().await, + HookV2AdmissionOutcomeV1::ExactDuplicate { .. } + )); + assert_eq!(producer.active_tasks(), 0); + assert_eq!(effects.load(Ordering::SeqCst), 1); + assert_eq!( + attempts.load(Ordering::SeqCst), + if refuse_first { 2 } else { 1 } + ); + assert!(runs.try_recv().is_err()); + unregister_hook_orchestration_runtime(envelope.project_id, envelope.worktree_id, &producer); + assert!(producer.shutdown().await); + assert!(tracedecay_daemon_service::context_scout_lifecycle::unregister_context_scout_lifecycle_authority(envelope.project_id, envelope.worktree_id, &fixture.sessions)); +} diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests/tool_dispatch/fixture.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests/tool_dispatch/fixture.rs new file mode 100644 index 0000000000..b899ca60dc --- /dev/null +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests/tool_dispatch/fixture.rs @@ -0,0 +1,169 @@ +use std::collections::BTreeMap; +use std::sync::Arc; + +use tracedecay_configuration::config::scope_control::daemon_owned_project_source_binding; +use tracedecay_domain::configuration::{ + CONTEXT_SCOUT_SETTINGS_SETTING_KEY, ConfigurationLayerIdV1, ConfigurationRevisionId, + ConfigurationValueV1, ContextScoutConfigurationStateV1, ContextScoutIntentSettingsV1, + ContextScoutSettingsV1, DecisionEngineSettingsV1, IntentWindowSettingsV1, + SOURCE_BINDINGS_SETTING_KEY, SettingKey, +}; +use tracedecay_domain::{NativeHostIdentityV1, ProjectId}; +use tracedecay_global_db::configuration::GlobalDbConfigurationControlStore; +use tracedecay_global_db::configuration::registry::ConfigurationRegistry; +use tracedecay_global_db::configuration::resolver::{ConfigurationLayerV1, resolve_configuration}; +use tracedecay_hooks::{HookEventEnvelopeV2, HookEventV2, HookLifecyclePhaseV1, HookOrderingV1}; +use tracedecay_project::project::TraceDecayOpenOptions; +use tracedecay_project::test_support::host_admission::HostAdmissionTestRuntimeV1; +use tracedecay_runtime_core::background_cpu::ProcessBackgroundCpuV1; +use tracedecay_sessions::admission::HostAdmissionScope; + +use crate::handlers::hook_runtime::envelope::hook_now; +use tracedecay_global_db::RegisteredGlobalDb; +use tracedecay_hooks::NativeContextScoutLifecycleV1; +use tracedecay_project::project::TraceDecay; + +pub(super) struct ToolDispatchFixture { + _temporary: tempfile::TempDir, + _runtime: Arc, + pub(super) graph: TraceDecay, + pub(super) sessions: tracedecay_global_db::RegisteredGlobalDbLeaseV1, + pub(super) background_cpu: Arc, + pub(super) envelope: HookEventEnvelopeV2, + pub(super) lifecycle: NativeContextScoutLifecycleV1, +} + +pub(super) async fn open() -> ToolDispatchFixture { + let temporary = tempfile::tempdir().unwrap(); + let profile = temporary.path().join("profile"); + let root = temporary.path().join("project"); + std::fs::create_dir_all(root.join("src")).unwrap(); + std::fs::write(root.join("src/lib.rs"), "pub fn boundary() {}\n").unwrap(); + let project = ProjectId::new("project.tool-dispatch-cutoff").unwrap(); + let runtime = Arc::new( + HostAdmissionTestRuntimeV1::project(&profile, &root, project.clone()) + .await + .unwrap(), + ); + let sessions = runtime + .registered_database_arc(HostAdmissionScope::Project) + .unwrap(); + initialize_configuration(&sessions, &project).await; + let graph = runtime + .initialize_project_graph_for_test( + &root, + TraceDecayOpenOptions { + profile_root: Some(profile.clone()), + global_db_path: None, + }, + ) + .await + .unwrap(); + let hook_runtime = tracedecay_project::runtime_ports::hook_runtime( + tracedecay_runtime_core::config::ProfileRoot::new(profile), + ) + .unwrap(); + let worktree = tracedecay_agent_hosts::hooks::hook_worktree_id_for_layout( + &hook_runtime, + graph.hook_store_layout(), + ) + .unwrap(); + let binding = tracedecay_hooks::published_hook_scope_binding( + &graph.hook_store_layout().data_root, + worktree, + NativeHostIdentityV1::ClaudeCode, + hook_now(), + ) + .unwrap(); + let scope = (hook_runtime.scope_resolver)(&root, &project).unwrap(); + assert_eq!(tracedecay_daemon_service::context_scout_lifecycle::register_context_scout_lifecycle_authority(binding.project_id, binding.worktree_id, scope.project_id, scope.worktree_id, &sessions), tracedecay_daemon_service::context_scout_lifecycle::AuthorityRegistrationV1::Registered); + let lifecycle = + NativeContextScoutLifecycleV1::new("session.tool-dispatch", "call.tool-dispatch", [73; 16]) + .unwrap(); + let envelope = HookEventEnvelopeV2 { + schema_version: tracedecay_hooks::HOOK_EVENT_SCHEMA_VERSION, + event_id: lifecycle.event_id, + producer: binding.host, + protected_session_id: tracedecay_agent_hosts::hooks::protected_native_session_id( + lifecycle.session_id.as_str(), + ), + project_id: binding.project_id, + repository_id: binding.repository_id, + worktree_id: binding.worktree_id, + worktree_epoch: binding.worktree_epoch, + binding_token: binding.binding_token, + ordering: HookOrderingV1::Unknown, + observed_at: hook_now(), + event: HookEventV2::ToolLifecycle { + tool_id: [74; 16], + phase: HookLifecyclePhaseV1::Completed, + effect_receipt_id: None, + }, + }; + let background_cpu = runtime.background_cpu(); + ToolDispatchFixture { + _temporary: temporary, + _runtime: runtime, + graph, + sessions, + background_cpu, + envelope, + lifecycle, + } +} + +async fn initialize_configuration(sessions: &RegisteredGlobalDb, project: &ProjectId) { + let intent = ContextScoutIntentSettingsV1 { + enabled: true, + engine: DecisionEngineSettingsV1 { + base_url: "http://127.0.0.1:9".to_owned(), + model: Some("kev-4b".to_owned()), + timeout_ms: 1_000, + max_concurrent: 1, + max_state_bytes: 8_192, + }, + window: IntentWindowSettingsV1 { + max_user_turns: 3, + max_messages: 12, + max_bytes: 8_192, + max_age_secs: 600, + freshen_budget_ms: 0, + }, + questions: ContextScoutIntentSettingsV1::reference_questions(), + action_fact_score_permille: 700, + ttl_secs: 600, + min_delivery_gap_secs: 30, + prompt_lookup_deadline_ms: 200, + }; + let revision = ConfigurationRevisionId::new("configuration.tool-dispatch").unwrap(); + let resolution = resolve_configuration( + &ConfigurationRegistry::core().unwrap(), + &[ConfigurationLayerV1 { + layer: ConfigurationLayerIdV1::Project { + project_id: project.clone(), + }, + revision_id: revision.clone(), + entries: BTreeMap::from([ + ( + SettingKey::new(SOURCE_BINDINGS_SETTING_KEY).unwrap(), + ConfigurationValueV1::SourceBindings(vec![ + daemon_owned_project_source_binding(project).unwrap(), + ]), + ), + ( + SettingKey::new(CONTEXT_SCOUT_SETTINGS_SETTING_KEY).unwrap(), + ConfigurationValueV1::ContextScoutSettings(ContextScoutSettingsV1 { + state: ContextScoutConfigurationStateV1::Active, + intent: Some(intent), + ..ContextScoutSettingsV1::disabled() + }), + ), + ]), + }], + ) + .unwrap(); + GlobalDbConfigurationControlStore::new_registered(sessions) + .initialize_canonical(&revision, &resolution, hook_now()) + .await + .unwrap(); +} diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/envelope.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/envelope.rs index 419710bee6..995c73531f 100644 --- a/crates/tracedecay-mcp/src/handlers/hook_runtime/envelope.rs +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/envelope.rs @@ -29,6 +29,12 @@ pub(super) fn hook_v2_requires_producer_work( matches!( &envelope.event, tracedecay_hooks::HookEventV2::SavedEdit { .. } + | tracedecay_hooks::HookEventV2::PromptBoundary + | tracedecay_hooks::HookEventV2::ToolLifecycle { + phase: tracedecay_hooks::HookLifecyclePhaseV1::Completed + | tracedecay_hooks::HookLifecyclePhaseV1::Failed, + .. + } | tracedecay_hooks::HookEventV2::SessionBoundary { boundary: tracedecay_hooks::HookBoundaryV1::End | tracedecay_hooks::HookBoundaryV1::TurnComplete diff --git a/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs b/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs index 26336d63a2..88d16ff07a 100644 --- a/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs +++ b/crates/tracedecay/src/daemon/hook_v2_replay_consumer.rs @@ -736,6 +736,34 @@ mod tests { ), tracedecay_daemon_service::context_scout_lifecycle::AuthorityRegistrationV1::Registered ); + let (ran, mut runs) = tokio::sync::mpsc::unbounded_channel(); + let runtime = tracedecay_daemon_service::BoundedHookOrchestratorV1::new( + 1, + move |request: tracedecay_daemon_service::HookOrchestrationRequestV1, _| { + let ran = ran.clone(); + async move { + if request.trigger + == tracedecay_daemon_service::HookOrchestrationTriggerV1::IntentBoundary + { + assert!(request.lifecycle.is_some()); + } else { + assert_eq!( + request.trigger, + tracedecay_daemon_service::HookOrchestrationTriggerV1::SavedEdit + ); + ran.send(request.lifecycle).unwrap(); + } + } + }, + ) + .unwrap(); + assert!( + tracedecay_daemon_service::register_hook_orchestration_runtime( + project_id, + binding.worktree_id, + &runtime, + ) + ); // An earlier call establishes the session, but cannot supply this // edit's own native call identity when its spool copy is delayed. let mut prior_envelope = envelope.clone(); @@ -770,7 +798,18 @@ mod tests { ).await, Some(prior_lifecycle.session_id), ); - // Commit admission while no orchestrator is mounted, reproducing the + tokio::time::timeout(std::time::Duration::from_secs(10), async { + while runtime.active_tasks() != 0 + || !hook_v2_pending_work_envelopes(data_root, host, hook_replay_now()) + .unwrap() + .is_empty() + { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + // Commit the edit before its own lifecycle arrives, reproducing the // durable state left by a caller cancelled before lifecycle persistence. assert!(matches!( admit_hook_v2_replayed_envelope_with_lifecycle( @@ -788,22 +827,6 @@ mod tests { let pending = || hook_v2_pending_work_envelopes(data_root, host, hook_replay_now()).unwrap(); assert_eq!(pending(), vec![envelope.clone()]); - let (ran, mut runs) = tokio::sync::mpsc::unbounded_channel(); - let runtime = tracedecay_daemon_service::BoundedHookOrchestratorV1::new( - 1, - move |request: tracedecay_daemon_service::HookOrchestrationRequestV1, _| { - let ran = ran.clone(); - async move { ran.send(request.lifecycle).unwrap() } - }, - ) - .unwrap(); - assert!( - tracedecay_daemon_service::register_hook_orchestration_runtime( - project_id, - binding.worktree_id, - &runtime, - ) - ); let sweep = drain_all_hosts( &graph, data_root, diff --git a/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime.rs b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime.rs index 46e9014f60..41cbbadf10 100644 --- a/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime.rs +++ b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime.rs @@ -48,7 +48,9 @@ use tracedecay_application::lsp_runtime::DaemonLspSessionFactory; use tracedecay_application::operation_stream::OperationKind; use tracedecay_code_index_runtime::code_index_scheduler::CodeIndexSchedulerRegistryV1; use tracedecay_code_index_runtime::project_reads::code_index_disabled_for_scope; -use tracedecay_contracts::context_scout::ContextScoutDeliveryOutcomeV1; +use tracedecay_contracts::context_scout::{ + ContextScoutDeliveryOutcomeV1, ContextScoutModelOutcomeV1, +}; use tracedecay_contracts::feedback::observations::{ FeedbackDeliveryRouteV1, FeedbackOperationV1, FeedbackOutcomeV1, FeedbackSourceEventV1, }; @@ -96,7 +98,7 @@ use tracedecay_agent_hosts::agents::context_scout::intent_pass::{ use tracedecay_agent_hosts::agents::context_scout::owner::ProjectContextScoutOwnerV1; use tracedecay_agent_hosts::agents::context_scout::{ ContextScoutDeliverySelectionInputV1, ContextScoutRuntimeOutcomeV1, ContextScoutServiceStateV1, - ContextScoutTriggerV1, + ContextScoutSuppressionV1, ContextScoutTriggerV1, }; use tracedecay_daemon_service::RegisteredDeliveryReadAuthorityV1; use tracedecay_daemon_service::context_scout_lifecycle::{ @@ -1353,9 +1355,62 @@ fn log_scout_intent_outcome( ); } -/// One admitted boundary's session intent pass. Intent work is best-effort: -/// it owes no durable producer work, so every end settles the boundary, and -/// a newer boundary of the session supersedes it at its own work address. +// Evidence that is not caught up or could not be evaluated is still owed. +// Hard policy and token limits are terminal refusals, not produced suggestions. +fn intent_pass_work_outcome( + outcome: &ContextScoutIntentPassOutcomeV1, +) -> HookOrchestrationWorkOutcomeV1 { + let terminal = match outcome { + ContextScoutIntentPassOutcomeV1::Disabled + | ContextScoutIntentPassOutcomeV1::ProviderUnsupported => true, + ContextScoutIntentPassOutcomeV1::CodeGenerationUnavailable + | ContextScoutIntentPassOutcomeV1::AddressUnavailable + | ContextScoutIntentPassOutcomeV1::WindowUnavailable + | ContextScoutIntentPassOutcomeV1::WindowEmpty + | ContextScoutIntentPassOutcomeV1::WindowStale + | ContextScoutIntentPassOutcomeV1::Cancelled + | ContextScoutIntentPassOutcomeV1::Unavailable + // Enqueue alone is insufficient: the caller must mount its exact claim. + | ContextScoutIntentPassOutcomeV1::Enqueued { .. } => false, + ContextScoutIntentPassOutcomeV1::NoCandidate { model_outcome } => match model_outcome { + ContextScoutModelOutcomeV1::Succeeded + | ContextScoutModelOutcomeV1::NotRequested + | ContextScoutModelOutcomeV1::Disabled + | ContextScoutModelOutcomeV1::Denied + | ContextScoutModelOutcomeV1::TokenBudgetExceeded => true, + ContextScoutModelOutcomeV1::Unavailable + | ContextScoutModelOutcomeV1::Disconnected + | ContextScoutModelOutcomeV1::Cancelled + | ContextScoutModelOutcomeV1::DeadlineExceeded + | ContextScoutModelOutcomeV1::Backpressured + | ContextScoutModelOutcomeV1::InvalidOutput => false, + }, + ContextScoutIntentPassOutcomeV1::Suppressed { reason, .. } => match reason { + ContextScoutSuppressionV1::Disabled + | ContextScoutSuppressionV1::Paused + | ContextScoutSuppressionV1::QuietOrUnreceptive + | ContextScoutSuppressionV1::Duplicate + | ContextScoutSuppressionV1::PendingSibling + | ContextScoutSuppressionV1::NoEligibleCandidate => true, + ContextScoutSuppressionV1::DirtyOverlay + | ContextScoutSuppressionV1::Expired + | ContextScoutSuppressionV1::EvidenceUnavailable + | ContextScoutSuppressionV1::EvidenceStale + | ContextScoutSuppressionV1::EvidencePartial + | ContextScoutSuppressionV1::Cancelled + | ContextScoutSuppressionV1::ModelOutputInvalid => false, + }, + }; + if terminal { + HookOrchestrationWorkOutcomeV1::Completed + } else { + HookOrchestrationWorkOutcomeV1::RetryableFailure + } +} + +/// One admitted boundary's session intent pass. A Tool or Prompt boundary +/// retains its durable debt until the pass evaluates or truthfully refuses it. +/// Stop companions use this same pass without a durable completion callback. /// The pass pins the current configuration, mints the project's hook-runtime /// authority, reads the session window, asks the installed decision engine, /// and mounts the claim authority of the suggestion it enqueued. @@ -1372,7 +1427,7 @@ async fn run_production_intent_cycle( let project_root = producer.project_root.as_path(); if work_cancellation.is_cancelled() { log_scout_intent_outcome(project_root, &request, "cancelled"); - return HookOrchestrationWorkOutcomeV1::Completed; + return HookOrchestrationWorkOutcomeV1::RetryableFailure; } let Ok(pinned_configuration) = producer .graph @@ -1382,7 +1437,7 @@ async fn run_production_intent_cycle( .await else { log_scout_intent_outcome(project_root, &request, "configuration_unavailable"); - return HookOrchestrationWorkOutcomeV1::Completed; + return HookOrchestrationWorkOutcomeV1::RetryableFailure; }; let current_configuration = ConfigurationCurrentStateV1 { revision_id: pinned_configuration.revision_id().clone(), @@ -1392,7 +1447,7 @@ async fn run_production_intent_cycle( ContextScoutConfigurationPinV1::from_current(¤t_configuration) else { log_scout_intent_outcome(project_root, &request, "configuration_pin_unavailable"); - return HookOrchestrationWorkOutcomeV1::Completed; + return HookOrchestrationWorkOutcomeV1::RetryableFailure; }; // Disabled or absent: no window read, no engine request, no authority. if !scout_configuration @@ -1413,7 +1468,7 @@ async fn run_production_intent_cycle( ) else { log_scout_intent_outcome(project_root, &request, "model_configuration_unavailable"); - return HookOrchestrationWorkOutcomeV1::Completed; + return HookOrchestrationWorkOutcomeV1::RetryableFailure; }; // Installing the pinned configuration also builds the decision engine // when its settings digest changed since the last install. @@ -1429,7 +1484,7 @@ async fn run_production_intent_cycle( .is_err() { log_scout_intent_outcome(project_root, &request, "configuration_install_failed"); - return HookOrchestrationWorkOutcomeV1::Completed; + return HookOrchestrationWorkOutcomeV1::RetryableFailure; } let envelope = request.hook.envelope(); let lifecycle = match request.lifecycle.clone() { @@ -1454,7 +1509,7 @@ async fn run_production_intent_cycle( }; let Some(lifecycle) = lifecycle else { log_scout_intent_outcome(project_root, &request, "lifecycle_absent"); - return HookOrchestrationWorkOutcomeV1::Completed; + return HookOrchestrationWorkOutcomeV1::RetryableFailure; }; let observed_at = now_micros(); let Some(context) = daemon_owned_project_source_access_at( @@ -1468,7 +1523,7 @@ async fn run_production_intent_cycle( intent_authorization_context(&access, &cycle.feedback_scope, envelope, observed_at) }) else { log_scout_intent_outcome(project_root, &request, "authority_unavailable"); - return HookOrchestrationWorkOutcomeV1::Completed; + return HookOrchestrationWorkOutcomeV1::RetryableFailure; }; let Some(pin) = ContextScoutAuthorityPinV1::new( &context, @@ -1477,7 +1532,7 @@ async fn run_production_intent_cycle( observed_at, ) else { log_scout_intent_outcome(project_root, &request, "authority_pin_unavailable"); - return HookOrchestrationWorkOutcomeV1::Completed; + return HookOrchestrationWorkOutcomeV1::RetryableFailure; }; let code_generation_id = producer .code_index_schedulers @@ -1514,13 +1569,14 @@ async fn run_production_intent_cycle( cancellation: work_cancellation, }) .await; + let settlement = intent_pass_work_outcome(&outcome); let ContextScoutIntentPassOutcomeV1::Enqueued { entry, lifecycle: intent_lifecycle, } = outcome else { log_scout_intent_outcome(project_root, &request, outcome.as_str()); - return HookOrchestrationWorkOutcomeV1::Completed; + return settlement; }; let mounted = producer .graph @@ -1545,7 +1601,11 @@ async fn run_production_intent_cycle( "claim_mount_refused" }, ); - HookOrchestrationWorkOutcomeV1::Completed + if mounted { + HookOrchestrationWorkOutcomeV1::Completed + } else { + HookOrchestrationWorkOutcomeV1::RetryableFailure + } } /// Store reads and writes the intent pass performs around its freshen wait diff --git a/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests.rs b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests.rs index 65bc3fa9a7..fd99a0d2d4 100644 --- a/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests.rs +++ b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests.rs @@ -1489,3 +1489,50 @@ async fn absent_or_disabled_intent_configuration_never_asks_the_engine() { } mod replay; + +#[tokio::test] +async fn intent_waits_for_its_transcript_before_evaluation_and_disabled_never_asks() { + let fixture = IntentJourneyFixture::open(Some(intent_settings())).await; + let harness = tracedecay_global_db::tests::harness::RegisteredGlobalDbHarness::open( + "intent-window-recovery", + ) + .await; + let sessions = harness.registered.as_ref(); + let project_key = "/project/intent-window-recovery"; + let engine = RecordingDecisionEngine::new(0.9); + let boundary = fixture.hook(75, completed_tool()); + let outcome = fixture + .pass(&boundary, sessions, project_key, &engine) + .await; + assert_eq!(outcome, ContextScoutIntentPassOutcomeV1::WindowEmpty); + assert_eq!( + intent_pass_work_outcome(&outcome), + HookOrchestrationWorkOutcomeV1::RetryableFailure + ); + assert_eq!(engine.requests(), 0); + seed_intent_session(sessions, project_key).await; + let outcome = fixture + .pass(&boundary, sessions, project_key, &engine) + .await; + assert!(matches!( + outcome, + ContextScoutIntentPassOutcomeV1::Enqueued { .. } + )); + assert_eq!(engine.requests(), 1); + + let disabled = IntentJourneyFixture::open(None).await; + let outcome = disabled + .pass( + &disabled.hook(76, completed_tool()), + sessions, + project_key, + &engine, + ) + .await; + assert_eq!(outcome, ContextScoutIntentPassOutcomeV1::Disabled); + assert_eq!( + intent_pass_work_outcome(&outcome), + HookOrchestrationWorkOutcomeV1::Completed + ); + assert_eq!(engine.requests(), 1); +} diff --git a/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests/replay.rs b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests/replay.rs index 79f5543371..6a46ea3149 100644 --- a/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests/replay.rs +++ b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests/replay.rs @@ -187,6 +187,22 @@ async fn registered_fixture() -> ( observed_at: now_utc_micros(), event: completed_tool(), }; + let (ran, mut runs) = tokio::sync::mpsc::unbounded_channel(); + let prior_producer = + BoundedHookOrchestratorV1::new(1, move |request: HookOrchestrationRequestV1, _| { + let ran = ran.clone(); + async move { + assert_eq!(request.trigger, HookOrchestrationTriggerV1::IntentBoundary); + assert!(request.lifecycle.is_some()); + ran.send(()).unwrap(); + } + }) + .unwrap(); + assert!(register_hook_orchestration_runtime( + binding.project_id, + binding.worktree_id, + &prior_producer + )); assert!(matches!( admit_hook_v2_replayed_envelope_with_lifecycle( &graph, @@ -200,6 +216,27 @@ async fn registered_fixture() -> ( .await, HookV2AdmissionOutcomeV1::Admitted { .. } )); + tokio::time::timeout(Duration::from_secs(10), runs.recv()) + .await + .unwrap() + .unwrap(); + tokio::time::timeout(Duration::from_secs(10), async { + while prior_producer.active_tasks() != 0 + || !hook_v2_pending_work_envelopes( + &graph.hook_store_layout().data_root, + binding.host, + now_utc_micros(), + ) + .unwrap() + .is_empty() + { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + unregister_hook_orchestration_runtime(binding.project_id, binding.worktree_id, &prior_producer); + assert!(prior_producer.shutdown().await); let lifecycle = lookup_registered_context_scout_lifecycle( binding.project_id, binding.worktree_id, diff --git a/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/tests.rs b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/tests.rs index fd51b58453..a6ebe54fee 100644 --- a/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/tests.rs +++ b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/tests.rs @@ -163,3 +163,84 @@ fn stop_boundary_runs_over_the_first_analyzable_document() { assert_eq!(stop_boundary_document(&files), Some("src/lib.rs")); assert_eq!(stop_boundary_document(&files[1..3]), None); } + +#[test] +fn intent_work_distinguishes_evaluation_refusal_from_unavailable_evidence() { + use ContextScoutIntentPassOutcomeV1 as Pass; + use ContextScoutModelOutcomeV1 as Model; + use ContextScoutSuppressionV1 as Suppression; + use HookOrchestrationWorkOutcomeV1::{Completed, RetryableFailure}; + + for outcome in [Pass::Disabled, Pass::ProviderUnsupported] { + assert_eq!(intent_pass_work_outcome(&outcome), Completed); + } + for outcome in [ + Pass::CodeGenerationUnavailable, + Pass::AddressUnavailable, + Pass::WindowUnavailable, + Pass::WindowEmpty, + Pass::WindowStale, + Pass::Cancelled, + Pass::Unavailable, + ] { + assert_eq!(intent_pass_work_outcome(&outcome), RetryableFailure); + } + for model_outcome in [ + Model::Succeeded, + Model::NotRequested, + Model::Disabled, + Model::Denied, + Model::TokenBudgetExceeded, + ] { + assert_eq!( + intent_pass_work_outcome(&Pass::NoCandidate { model_outcome }), + Completed + ); + } + for model_outcome in [ + Model::Unavailable, + Model::Disconnected, + Model::Cancelled, + Model::DeadlineExceeded, + Model::Backpressured, + Model::InvalidOutput, + ] { + assert_eq!( + intent_pass_work_outcome(&Pass::NoCandidate { model_outcome }), + RetryableFailure + ); + } + for reason in [ + Suppression::Disabled, + Suppression::Paused, + Suppression::QuietOrUnreceptive, + Suppression::Duplicate, + Suppression::PendingSibling, + Suppression::NoEligibleCandidate, + ] { + assert_eq!( + intent_pass_work_outcome(&Pass::Suppressed { + reason, + model_outcome: Model::NotRequested + }), + Completed + ); + } + for reason in [ + Suppression::DirtyOverlay, + Suppression::Expired, + Suppression::EvidenceUnavailable, + Suppression::EvidenceStale, + Suppression::EvidencePartial, + Suppression::Cancelled, + Suppression::ModelOutputInvalid, + ] { + assert_eq!( + intent_pass_work_outcome(&Pass::Suppressed { + reason, + model_outcome: Model::NotRequested + }), + RetryableFailure + ); + } +} From e9e7aa188db2244cdb48c7adbf2354aa06f1b42c Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Mon, 19 Oct 2026 09:06:29 +0000 Subject: [PATCH 82/89] fix(hooks): recover committed intent before claim mounting --- .../src/agents/context_scout/intent_pass.rs | 76 +++++++++++--- .../scout_journey_tests/replay.rs | 99 ++++++++++++++++++- 2 files changed, 159 insertions(+), 16 deletions(-) diff --git a/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs b/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs index aec433daac..74d6f9047d 100644 --- a/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs +++ b/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs @@ -18,8 +18,8 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use tracedecay_contracts::context_scout::{ - ContextScoutCandidateV1, ContextScoutCategoryV1, ContextScoutDeliveryReceiptV1, - ContextScoutDeliveryWindowV1, ContextScoutDurableQueueEntryV1, + ContextScoutAddressV1, ContextScoutCandidateV1, ContextScoutCategoryV1, + ContextScoutDeliveryReceiptV1, ContextScoutDeliveryWindowV1, ContextScoutDurableQueueEntryV1, ContextScoutEvidenceAvailabilityV1, ContextScoutIntentAnswerV1, ContextScoutIntentReceiptV1, ContextScoutModelOutcomeV1, ContextScoutRedactionReceiptV1, ContextScoutSessionEvidenceV1, ContextScoutSessionWindowV1, DecisionEngineIdentityV1 as ReceiptEngineIdentityV1, @@ -228,6 +228,40 @@ pub fn intent_hint_emitted_event( }) } +/// Recover only this boundary's original queue commit. The caller must still +/// mount it through the current durable configuration and address authority. +async fn pending_intent_entry( + pass: &ContextScoutIntentPassV1<'_>, + address: ContextScoutAddressV1, +) -> Result>, ContextScoutIntentPassOutcomeV1> { + let recent = tokio::select! { + biased; + () = pass.cancellation.cancelled() => return Err(ContextScoutIntentPassOutcomeV1::Cancelled), + recent = native_runtime::timeout_at(pass.deadline.instant(), pass.owner.recent_exact(address, 1)) => { + match recent { + Ok(Ok(recent)) => recent, + _ => return Err(ContextScoutIntentPassOutcomeV1::Unavailable), + } + } + }; + let revision = pass.pin.configuration().control().configuration_revision; + if recent.configuration_revision != revision { + return Err(ContextScoutIntentPassOutcomeV1::Unavailable); + } + let envelope_id = + intent_envelope_id(pass.hook).ok_or(ContextScoutIntentPassOutcomeV1::Unavailable)?; + Ok(recent + .pending + .into_iter() + .find(|entry| { + entry.work.address == address + && entry.envelope.envelope_id == envelope_id + && entry.envelope.configuration_revision == revision + && entry.envelope.candidate.expires_at.0 > pass.observed_at.0 + }) + .map(Box::new)) +} + /// Runs one intent pass. The engine is asked at most once. #[tracing::instrument( name = "hosts.agent.context_scout.intent_pass", @@ -283,6 +317,16 @@ pub async fn run_context_scout_intent_pass( return ContextScoutIntentPassOutcomeV1::AddressUnavailable; } }; + match pending_intent_entry(&pass, address).await { + Ok(Some(entry)) => { + return ContextScoutIntentPassOutcomeV1::Enqueued { + entry, + lifecycle: Box::new(intent_lifecycle), + }; + } + Ok(None) => {} + Err(outcome) => return outcome, + } let Ok(session) = pass .owner .intent_session_state( @@ -424,18 +468,7 @@ pub async fn run_context_scout_intent_pass( let prepared = pass .owner .prepare_intent(&input, pass.deadline, pass.cancellation.clone()) - .await - .map(|outcome| match outcome { - // The store kept the copy of this suggestion it already claimed - // or delivered to the session. - ContextScoutRuntimeOutcomeV1::Enqueued { - store_outcome: ContextScoutDurableStoreOutcomeV1::Duplicate, - .. - } => ContextScoutRuntimeOutcomeV1::Suppressed { - reason: ContextScoutSuppressionV1::Duplicate, - }, - outcome => outcome, - }); + .await; match prepared { Ok(ContextScoutRuntimeOutcomeV1::Enqueued { entry, @@ -444,6 +477,21 @@ pub async fn run_context_scout_intent_pass( entry, lifecycle: Box::new(intent_lifecycle), }, + // A concurrent queue commit can land after the initial read. Store + // duplication alone does not prove delivery or a mounted claim. + Ok(ContextScoutRuntimeOutcomeV1::Enqueued { + store_outcome: + ContextScoutDurableStoreOutcomeV1::Duplicate + | ContextScoutDurableStoreOutcomeV1::Superseded, + .. + }) => match pending_intent_entry(&pass, address).await { + Ok(Some(entry)) => ContextScoutIntentPassOutcomeV1::Enqueued { + entry, + lifecycle: Box::new(intent_lifecycle), + }, + Ok(None) => ContextScoutIntentPassOutcomeV1::Unavailable, + Err(outcome) => outcome, + }, Ok(ContextScoutRuntimeOutcomeV1::Suppressed { reason }) => { let withheld = match reason { ContextScoutSuppressionV1::Duplicate => Some("suppressed_duplicate"), diff --git a/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests/replay.rs b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests/replay.rs index 6a46ea3149..bdc5c4df66 100644 --- a/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests/replay.rs +++ b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests/replay.rs @@ -19,7 +19,8 @@ use tracedecay_daemon_service::context_scout_lifecycle::{ }; use tracedecay_daemon_service::{ BoundedHookOrchestratorV1, HookOrchestrationRequestV1, HookOrchestrationTriggerV1, - register_hook_orchestration_runtime, unregister_hook_orchestration_runtime, + HookOrchestrationWorkOutcomeV1, register_hook_orchestration_runtime, + unregister_hook_orchestration_runtime, }; use tracedecay_domain::configuration::{ CONTEXT_SCOUT_SETTINGS_SETTING_KEY, ConfigurationLayerIdV1, ConfigurationRevisionId, @@ -275,6 +276,23 @@ async fn mount_intent( else { panic!("the registered session intent is enqueued"); }; + // The queue commit survived, but no claim authority was mounted before + // this retry. Recovery must keep that exact suggestion and its evidence. + let recovered = fixture + .pass(&boundary, sessions, &project_key, &engine) + .await; + assert_eq!( + recovered, + ContextScoutIntentPassOutcomeV1::Enqueued { + entry: entry.clone(), + lifecycle: lifecycle.clone(), + } + ); + assert_eq!( + engine.requests(), + 1, + "recovery does not re-judge committed work" + ); assert!( graph .mount_current_context_scout_claim_authority( @@ -294,7 +312,7 @@ async fn mount_intent( } #[tokio::test] -async fn completed_stop_replay_claims_the_mounted_intent_once_without_reopening_producer_work() { +async fn enqueued_intent_replay_mounts_and_delivers_once_without_reopening_producer_work() { let (fixture, graph, runtime) = registered_fixture().await; let sessions = runtime .registered_database_arc(HostAdmissionScope::Project) @@ -436,3 +454,80 @@ async fn completed_stop_replay_claims_the_mounted_intent_once_without_reopening_ &sessions, )); } + +#[tokio::test] +async fn enqueued_intent_recovery_cannot_mount_past_durable_configuration_drift() { + let (fixture, graph, runtime) = registered_fixture().await; + let sessions = runtime + .registered_database_arc(HostAdmissionScope::Project) + .unwrap(); + let project_key = RegisteredGlobalDb::canonical_project_key(graph.project_root()); + seed_intent_session(&sessions, &project_key).await; + let engine = RecordingDecisionEngine::new(0.9); + let boundary = fixture.hook(71, completed_tool()); + let original = fixture + .pass(&boundary, &sessions, &project_key, &engine) + .await; + let next = configuration::advance_configuration(&sessions).await; + let recovered = fixture + .pass(&boundary, &sessions, &project_key, &engine) + .await; + assert_eq!( + recovered, original, + "recovery preserves the original evidence" + ); + assert_eq!( + super::intent_pass_work_outcome(&recovered), + HookOrchestrationWorkOutcomeV1::RetryableFailure + ); + let ContextScoutIntentPassOutcomeV1::Enqueued { entry, lifecycle } = recovered else { + panic!("the queue commit remains pending before the owner observes configuration drift"); + }; + assert!( + !graph + .mount_current_context_scout_claim_authority( + Arc::clone(&fixture.registry), + &boundary, + 1, + fixture.pin.clone(), + fixture.context.clone(), + *lifecycle, + entry.work.address, + entry.work.input_watermark, + now_utc_micros(), + ) + .await, + "durable configuration must reject the stale pin before owner refresh" + ); + fixture + .owner + .install_configuration(next, None) + .await + .unwrap(); + assert_eq!( + fixture + .pass(&boundary, &sessions, &project_key, &engine) + .await, + ContextScoutIntentPassOutcomeV1::Unavailable + ); + assert_eq!( + engine.requests(), + 1, + "stale recovery never re-judges the event" + ); + assert!( + graph + .resolve_current_context_scout_session_claim_authority( + &boundary, + &fixture.lifecycle, + now_utc_micros(), + ) + .await + .is_none() + ); + assert!(unregister_context_scout_lifecycle_authority( + fixture.binding.project_id, + fixture.binding.worktree_id, + &sessions + )); +} From 55488d1ef3f15a6a0e6fd3a09686943cb122ae7a Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Mon, 19 Oct 2026 09:29:33 +0000 Subject: [PATCH 83/89] test(sessions): join the replacement worker after rekey --- .../tests/session_suite/session_runtime/temporal_refresh.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay/tests/session_suite/session_runtime/temporal_refresh.rs b/crates/tracedecay/tests/session_suite/session_runtime/temporal_refresh.rs index 1e372428d7..aade4e52f4 100644 --- a/crates/tracedecay/tests/session_suite/session_runtime/temporal_refresh.rs +++ b/crates/tracedecay/tests/session_suite/session_runtime/temporal_refresh.rs @@ -1489,7 +1489,8 @@ async fn project_rekey_retires_old_owner_before_rebinding_wake() { new_authority .rekey_project(®istry, &old_owner, new_owner.clone()) .await; - wake.wake(); + // The replacement worker owns the dirty flag; join its actual pass. + assert!(wake.wake_and_wait_until_idle(Duration::from_secs(1)).await); assert!(old_state.is_cancelled()); assert!(registry.project_state(&old_owner).await.is_none()); @@ -1501,6 +1502,5 @@ async fn project_rekey_retires_old_owner_before_rebinding_wake() { assert!(!stale.wake()); assert!(registry.project_state(&old_owner).await.is_none()); assert_eq!(registry.project_worker_count().await, 1); - assert!(new_state.take_dirty()); registry.shutdown().await; } From f99b25b14193cd714b909ef141364b7a397091e0 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Mon, 19 Oct 2026 09:53:49 +0000 Subject: [PATCH 84/89] chore(hooks): clarify pending intent diagnostics --- .../src/handlers/hook_runtime/admission/tests.rs | 2 +- .../src/daemon/project_open_owners/advisory_runtime.rs | 7 +++++++ .../advisory_runtime/scout_journey_tests/replay.rs | 2 +- .../advisory_runtime/scout_journey_tests/replay/pending.rs | 4 ++-- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests.rs index 4a46ed4dee..ca97774614 100644 --- a/crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests.rs +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests.rs @@ -235,7 +235,7 @@ async fn prompt_and_tool_admissions_settle_their_durable_intent_work() { .await .unwrap(); } - assert!(pending_work(data_root.path()).is_empty()); + assert_eq!(pending_work(data_root.path()), Vec::new()); let mut started = admission_test_envelope(44, 7); started.event = tool(tracedecay_hooks::HookLifecyclePhaseV1::Started); assert_eq!( diff --git a/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime.rs b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime.rs index 41cbbadf10..36415b92bb 100644 --- a/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime.rs +++ b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime.rs @@ -1575,6 +1575,13 @@ async fn run_production_intent_cycle( lifecycle: intent_lifecycle, } = outcome else { + if let ContextScoutIntentPassOutcomeV1::NoCandidate { model_outcome } = &outcome { + tracing::debug!( + event = "context_scout_intent_evaluation", + ?model_outcome, + "intent evaluation returned no candidate" + ); + } log_scout_intent_outcome(project_root, &request, outcome.as_str()); return settlement; }; diff --git a/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests/replay.rs b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests/replay.rs index bdc5c4df66..b022e81e08 100644 --- a/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests/replay.rs +++ b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests/replay.rs @@ -440,7 +440,7 @@ async fn enqueued_intent_replay_mounts_and_delivers_once_without_reopening_produ assert_eq!(effects.load(Ordering::SeqCst), 1); assert_eq!(engine.requests(), 1); let recent = fixture.owner.recent_exact(address, 8).await.unwrap(); - assert!(recent.pending.is_empty()); + assert_eq!(recent.pending, Vec::new()); assert_eq!(recent.deliveries.len(), 1); unregister_hook_orchestration_runtime( fixture.binding.project_id, diff --git a/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests/replay/pending.rs b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests/replay/pending.rs index 4953b08851..4a0037b9ca 100644 --- a/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests/replay/pending.rs +++ b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests/replay/pending.rs @@ -226,7 +226,7 @@ async fn pending_stop_delivers_once( .recent_exact(entry.work.address, 8) .await .unwrap(); - assert!(recent.deliveries.is_empty()); + assert_eq!(recent.deliveries, Vec::new()); unregister_hook_orchestration_runtime( fixture.binding.project_id, fixture.binding.worktree_id, @@ -330,7 +330,7 @@ async fn pending_stop_delivers_once( assert_eq!(effects.load(Ordering::SeqCst), 1); assert_eq!(engine.requests(), 1); let recent = fixture.owner.recent_exact(address, 8).await.unwrap(); - assert!(recent.pending.is_empty()); + assert_eq!(recent.pending, Vec::new()); assert_eq!(recent.deliveries.len(), 1); unregister_hook_orchestration_runtime( fixture.binding.project_id, From 90b2326f8e1fc238ff4f07fe3c9abcc4d501cbf6 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Mon, 19 Oct 2026 09:53:53 +0000 Subject: [PATCH 85/89] fix(index): retain completed readings at readiness deadline --- .../registry/owner_signals.rs | 51 +++++++++---------- 1 file changed, 24 insertions(+), 27 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/owner_signals.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/owner_signals.rs index a8d3976150..85bf11d081 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/owner_signals.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/owner_signals.rs @@ -11,7 +11,7 @@ use tracedecay_runtime_core::path_safety::canonical_existing_identity; use tracedecay_contracts::code_index_freshness::{ CodeIndexConvergenceParkedV1, CodeIndexFreshnessReadFailureV1, CodeIndexReadinessTargetV1, - CodeIndexReadinessV1, CodeIndexReadinessWaitReadV1, + CodeIndexReadinessV1, CodeIndexReadinessWaitReadV1, CodeIndexWorktreeFreshnessV1, }; use tracedecay_contracts::ResolvedScope; @@ -497,16 +497,20 @@ impl CodeIndexSchedulerRegistryV1 { budget: Duration, ) -> Result { let deadline = Instant::now() + budget; + let last = tokio::sync::Mutex::new(None); // Every probe, including its blocking source read, shares the caller's - // one deadline. A timed-out blocking task cannot publish readiness. + // one deadline. Retain only completed readings, so expiry needs no I/O + // and a timed-out blocking task cannot publish readiness. match native_runtime::timeout_at( deadline, - self.wait_for_readiness_until(project_root, target, deadline), + self.wait_for_readiness_until(project_root, target, deadline, &last), ) .await { Ok(reading) => reading, - Err(_) => Ok(CodeIndexReadinessWaitReadV1::TimedOut { last: None }), + Err(_) => Ok(CodeIndexReadinessWaitReadV1::TimedOut { + last: last.into_inner(), + }), } } @@ -515,6 +519,7 @@ impl CodeIndexSchedulerRegistryV1 { project_root: &Path, target: CodeIndexReadinessTargetV1, deadline: Instant, + last: &tokio::sync::Mutex>>, ) -> Result { if target != CodeIndexReadinessTargetV1::Fresh && let Some(reading) = self @@ -523,16 +528,19 @@ impl CodeIndexSchedulerRegistryV1 { target != CodeIndexReadinessTargetV1::GraphReady, ) .await? - .filter(|freshness| freshness.readiness(target) == CodeIndexReadinessV1::Reached) - && (target == CodeIndexReadinessTargetV1::GraphReady - || self - .subscribe_owner_activity(project_root) - .await - .is_some_and(|activity| activity.pass_finished())) { - return Ok(CodeIndexReadinessWaitReadV1::Reached { - reading: Box::new(reading), - }); + *last.lock().await = Some(Box::new(reading.clone())); + if reading.readiness(target) == CodeIndexReadinessV1::Reached + && (target == CodeIndexReadinessTargetV1::GraphReady + || self + .subscribe_owner_activity(project_root) + .await + .is_some_and(|activity| activity.pass_finished())) + { + return Ok(CodeIndexReadinessWaitReadV1::Reached { + reading: Box::new(reading), + }); + } } // The caller's budget bounds the sweep, and an unproven source cannot // be reported as reached. @@ -556,13 +564,7 @@ impl CodeIndexSchedulerRegistryV1 { Ok(Ok(())) => {} Err(_) => { return Ok(CodeIndexReadinessWaitReadV1::TimedOut { - last: self - .dashboard_freshness_read_with_source_admission( - project_root, - target != CodeIndexReadinessTargetV1::GraphReady, - ) - .await? - .map(Box::new), + last: last.lock().await.take(), }); } } @@ -577,6 +579,7 @@ impl CodeIndexSchedulerRegistryV1 { else { return Ok(None); }; + *last.lock().await = Some(Box::new(freshness.clone())); Ok(match freshness.readiness(target) { CodeIndexReadinessV1::Reached if owner_settled || target == CodeIndexReadinessTargetV1::GraphReady => @@ -606,13 +609,7 @@ impl CodeIndexSchedulerRegistryV1 { reason: "code_index_scheduler_registry_closed".to_owned(), }, CodeIndexSeatWaitV1::Deadline => CodeIndexReadinessWaitReadV1::TimedOut { - last: self - .dashboard_freshness_read_with_source_admission( - project_root, - target != CodeIndexReadinessTargetV1::GraphReady, - ) - .await? - .map(Box::new), + last: last.lock().await.take(), }, }) } From f7345368bf32b6062b1fae616aa69bba95e85fc6 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Mon, 19 Oct 2026 10:16:31 +0000 Subject: [PATCH 86/89] style(index): format integrated source inputs --- .../branch_publication_tests.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/branch_publication_tests.rs b/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/branch_publication_tests.rs index 57ef96ecd4..95d9e70253 100644 --- a/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/branch_publication_tests.rs +++ b/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/branch_publication_tests.rs @@ -308,9 +308,10 @@ export function GenerationAnchor(value: PublicWidget) { return value; } graph_runtime.code_graph_seat_port(), Arc::clone(&project_database), tracedecay_code_index_runtime::code_index_scheduler::CodeGraphActivationPolicyV1::Enabled, -tracedecay_code_index_runtime::code_index_scheduler::CodeIndexSourceInputsV1 { - path_policy: tracedecay_code_index_runtime::registry_default_index_path_policy(), pending_admission: None, -}, + tracedecay_code_index_runtime::code_index_scheduler::CodeIndexSourceInputsV1 { + path_policy: tracedecay_code_index_runtime::registry_default_index_path_policy(), + pending_admission: None, + }, ) .await .expect("mount persistent graph-backed scheduler"); From f395963d6daeeb64cb7de9d3d6916f9f1b25ce3e Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Mon, 19 Oct 2026 10:21:37 +0000 Subject: [PATCH 87/89] test(hooks): retain canonical evidence for hint timeouts --- .../intent_pass_journey_test.rs | 32 +++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/crates/tracedecay/tests/hooks_lsp_suite/intent_pass_journey_test.rs b/crates/tracedecay/tests/hooks_lsp_suite/intent_pass_journey_test.rs index 5fbe014fda..1d126368ec 100644 --- a/crates/tracedecay/tests/hooks_lsp_suite/intent_pass_journey_test.rs +++ b/crates/tracedecay/tests/hooks_lsp_suite/intent_pass_journey_test.rs @@ -215,10 +215,10 @@ fn run_tool(home: &Path, project: &Path, name: &str, args: &Value) -> Value { fn delivery_evidence(home: &Path, project: &Path, contexts: &[String]) -> Value { let profile = home.join(".tracedecay"); - let enrolled = - tracedecay_runtime_core::storage::default_profile_sharded_layout(project, &profile) - .unwrap() - .data_root; + let enrolled = tracedecay_runtime_core::storage::resolve_persisted_layout(project, &profile) + .unwrap() + .expect("enrolled hook store") + .data_root; let address = contexts .iter() .rev() @@ -333,7 +333,13 @@ fn wait_for(what: &str, log: &Path, mut ready: impl FnMut() -> bool) { } } -fn wait_for_hints(home: &Path, project: &Path, log: &Path, ready: impl Fn((i64, i64)) -> bool) { +fn wait_for_hints( + home: &Path, + project: &Path, + log: &Path, + contexts: &[String], + ready: impl Fn((i64, i64)) -> bool, +) { let deadline = Instant::now() + WAIT; loop { let counts = hint_counts(home, project, YES_INTENT); @@ -342,8 +348,10 @@ fn wait_for_hints(home: &Path, project: &Path, log: &Path, ready: impl Fn((i64, } assert!( Instant::now() < deadline, - "timed out at (emitted, suppressed) {counts:?} after intent outcomes {:?}", - intent_outcomes(log, ENABLED_SESSION) + "timed out at (emitted, suppressed) {counts:?} after intent outcomes {:?}\ndaemon: {}\nhook receipts: {}", + intent_outcomes(log, ENABLED_SESSION), + std::fs::read_to_string(log).unwrap_or_default(), + delivery_evidence(home, project, contexts), ); std::thread::sleep(Duration::from_millis(100)); } @@ -592,9 +600,13 @@ fn claude_stop_delivers_the_intent_suggestion_the_daemon_judged_at_the_tool_boun .count(), 1 ); - wait_for_hints(home, &project, &log, |(emitted, suppressed)| { - emitted == 1 && suppressed >= 1 - }); + wait_for_hints( + home, + &project, + &log, + &stop_contexts, + |(emitted, suppressed)| emitted == 1 && suppressed >= 1, + ); let after_mount = intent_outcomes(&log, ENABLED_SESSION)[before + 1..].to_vec(); assert!( !after_mount.is_empty() && after_mount.iter().all(|outcome| outcome == "suppressed"), From 77b0e76927d16653ab0187df071b887f2e60cb9e Mon Sep 17 00:00:00 2001 From: Zack Jackson Date: Mon, 19 Oct 2026 04:05:42 -0700 Subject: [PATCH 88/89] test(observability): leave a queue slot for the live owner claim --- .../application_suite/observability_runtime_contract.rs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/crates/tracedecay-application/tests/application_suite/observability_runtime_contract.rs b/crates/tracedecay-application/tests/application_suite/observability_runtime_contract.rs index f76f1bec62..94f53d0243 100644 --- a/crates/tracedecay-application/tests/application_suite/observability_runtime_contract.rs +++ b/crates/tracedecay-application/tests/application_suite/observability_runtime_contract.rs @@ -351,13 +351,8 @@ async fn live_queued_owner_claim_is_not_recovered_as_delayed_work() { .expect("first queue carrier"), ObservabilityEmissionOutcomeV1::Enqueued ); - tokio::task::yield_now().await; - assert_eq!( - producer - .try_emit(envelope(&scope, "caller", 73, 7_300_000)) - .expect("second queue carrier"), - ObservabilityEmissionOutcomeV1::Enqueued - ); + // The carrier and owner fit even before the worker dequeues anything. + // A scheduler yield cannot prove that a bounded queue slot was freed. let owner = envelope(&scope, "caller", 74, 7_400_000); let owner_admission = { let producer = Arc::clone(&producer); From 206248d27d1c9685ec0cabe5248e5225f4a206e8 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Mon, 19 Oct 2026 11:08:00 +0000 Subject: [PATCH 89/89] fix(hooks): reconcile intent duplicates after host claims --- .../src/agents/context_scout/intent_pass.rs | 125 +++++++++++++----- .../advisory_runtime/scout_journey_tests.rs | 29 ++++ .../scout_journey_tests/replay.rs | 93 +++++++++++++ 3 files changed, 212 insertions(+), 35 deletions(-) diff --git a/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs b/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs index 74d6f9047d..a6cadcb2e8 100644 --- a/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs +++ b/crates/tracedecay-agent-hosts/src/agents/context_scout/intent_pass.rs @@ -59,9 +59,9 @@ use super::session_window::{ }; use super::{ ContextScoutDeliverySelectionInputV1, ContextScoutDurableStoreOutcomeV1, - ContextScoutRuntimeOutcomeV1, ContextScoutSelectionInputV1, ContextScoutServiceStateV1, - ContextScoutSessionClaimInputV1, ContextScoutSessionEvidenceExt, ContextScoutSuppressionV1, - ContextScoutTriggerV1, select_context_scout_delivery_window, + ContextScoutRecentStateV1, ContextScoutRuntimeOutcomeV1, ContextScoutSelectionInputV1, + ContextScoutServiceStateV1, ContextScoutSessionClaimInputV1, ContextScoutSessionEvidenceExt, + ContextScoutSuppressionV1, ContextScoutTriggerV1, select_context_scout_delivery_window, }; const INTENT_CANDIDATE_DEDUPE_DOMAIN: &str = "tracedecay.context-scout.intent-candidate.v1"; @@ -228,12 +228,11 @@ pub fn intent_hint_emitted_event( }) } -/// Recover only this boundary's original queue commit. The caller must still -/// mount it through the current durable configuration and address authority. -async fn pending_intent_entry( +/// Read this exact address under the pass's current configuration and deadline. +async fn recent_intent_state( pass: &ContextScoutIntentPassV1<'_>, address: ContextScoutAddressV1, -) -> Result>, ContextScoutIntentPassOutcomeV1> { +) -> Result { let recent = tokio::select! { biased; () = pass.cancellation.cancelled() => return Err(ContextScoutIntentPassOutcomeV1::Cancelled), @@ -248,20 +247,80 @@ async fn pending_intent_entry( if recent.configuration_revision != revision { return Err(ContextScoutIntentPassOutcomeV1::Unavailable); } + Ok(recent) +} + +/// Recover only this boundary's original queue commit. The caller must still +/// mount it through the current durable configuration and address authority. +fn pending_intent_entry( + pass: &ContextScoutIntentPassV1<'_>, + address: ContextScoutAddressV1, + recent: &ContextScoutRecentStateV1, +) -> Result>, ContextScoutIntentPassOutcomeV1> { + let revision = pass.pin.configuration().control().configuration_revision; let envelope_id = intent_envelope_id(pass.hook).ok_or(ContextScoutIntentPassOutcomeV1::Unavailable)?; Ok(recent .pending - .into_iter() + .iter() .find(|entry| { entry.work.address == address && entry.envelope.envelope_id == envelope_id && entry.envelope.configuration_revision == revision && entry.envelope.candidate.expires_at.0 > pass.observed_at.0 }) + .cloned() .map(Box::new)) } +/// A concurrent queue commit or host claim may precede the enqueue result. +/// Preserve this boundary's owed entry; only a current matching commit can +/// establish duplicate suppression. Unrelated supersession stays unavailable. +async fn reconcile_intent_commit( + pass: &ContextScoutIntentPassV1<'_>, + address: ContextScoutAddressV1, + outcome: ContextScoutRuntimeOutcomeV1, +) -> Result { + match outcome { + ContextScoutRuntimeOutcomeV1::Enqueued { + entry: prepared_entry, + store_outcome: + store_outcome @ (ContextScoutDurableStoreOutcomeV1::Duplicate + | ContextScoutDurableStoreOutcomeV1::Superseded), + } => { + let recent = recent_intent_state(pass, address).await?; + if let Some(entry) = pending_intent_entry(pass, address, &recent)? { + return Ok(ContextScoutRuntimeOutcomeV1::Enqueued { + entry, + store_outcome, + }); + } + let matching_commit = recent + .pending + .iter() + .chain(recent.deliveries.iter().map(|delivery| &delivery.entry)) + .any(|entry| { + entry.work.address == address + && entry.envelope.configuration_revision + == pass.pin.configuration().control().configuration_revision + && entry.envelope.candidate.dedupe_key + == prepared_entry.envelope.candidate.dedupe_key + }); + if store_outcome == ContextScoutDurableStoreOutcomeV1::Duplicate && matching_commit { + pass.owner + .record_intent_outcome(ContextScoutSuppressionV1::Duplicate) + .await; + Ok(ContextScoutRuntimeOutcomeV1::Suppressed { + reason: ContextScoutSuppressionV1::Duplicate, + }) + } else { + Err(ContextScoutIntentPassOutcomeV1::Unavailable) + } + } + outcome => Ok(outcome), + } +} + /// Runs one intent pass. The engine is asked at most once. #[tracing::instrument( name = "hosts.agent.context_scout.intent_pass", @@ -317,7 +376,11 @@ pub async fn run_context_scout_intent_pass( return ContextScoutIntentPassOutcomeV1::AddressUnavailable; } }; - match pending_intent_entry(&pass, address).await { + let recent = match recent_intent_state(&pass, address).await { + Ok(recent) => recent, + Err(outcome) => return outcome, + }; + match pending_intent_entry(&pass, address, &recent) { Ok(Some(entry)) => { return ContextScoutIntentPassOutcomeV1::Enqueued { entry, @@ -465,34 +528,29 @@ pub async fn run_context_scout_intent_pass( delivered_dedupe_keys: session.delivered_dedupe_keys, candidates: vec![candidate], }; - let prepared = pass + let Ok(prepared) = pass .owner .prepare_intent(&input, pass.deadline, pass.cancellation.clone()) - .await; + .await + else { + return ContextScoutIntentPassOutcomeV1::Unavailable; + }; + let prepared = match reconcile_intent_commit(&pass, address, prepared).await { + Ok(prepared) => prepared, + Err(outcome) => return outcome, + }; match prepared { - Ok(ContextScoutRuntimeOutcomeV1::Enqueued { - entry, - store_outcome: ContextScoutDurableStoreOutcomeV1::Stored, - }) => ContextScoutIntentPassOutcomeV1::Enqueued { + ContextScoutRuntimeOutcomeV1::Enqueued { entry, - lifecycle: Box::new(intent_lifecycle), - }, - // A concurrent queue commit can land after the initial read. Store - // duplication alone does not prove delivery or a mounted claim. - Ok(ContextScoutRuntimeOutcomeV1::Enqueued { store_outcome: - ContextScoutDurableStoreOutcomeV1::Duplicate + ContextScoutDurableStoreOutcomeV1::Stored + | ContextScoutDurableStoreOutcomeV1::Duplicate | ContextScoutDurableStoreOutcomeV1::Superseded, - .. - }) => match pending_intent_entry(&pass, address).await { - Ok(Some(entry)) => ContextScoutIntentPassOutcomeV1::Enqueued { - entry, - lifecycle: Box::new(intent_lifecycle), - }, - Ok(None) => ContextScoutIntentPassOutcomeV1::Unavailable, - Err(outcome) => outcome, + } => ContextScoutIntentPassOutcomeV1::Enqueued { + entry, + lifecycle: Box::new(intent_lifecycle), }, - Ok(ContextScoutRuntimeOutcomeV1::Suppressed { reason }) => { + ContextScoutRuntimeOutcomeV1::Suppressed { reason } => { let withheld = match reason { ContextScoutSuppressionV1::Duplicate => Some("suppressed_duplicate"), ContextScoutSuppressionV1::PendingSibling => Some("suppressed_budget"), @@ -513,11 +571,8 @@ pub async fn run_context_scout_intent_pass( model_outcome, } } - Ok( - ContextScoutRuntimeOutcomeV1::Enqueued { .. } - | ContextScoutRuntimeOutcomeV1::Unavailable, - ) - | Err(_) => ContextScoutIntentPassOutcomeV1::Unavailable, + ContextScoutRuntimeOutcomeV1::Enqueued { .. } + | ContextScoutRuntimeOutcomeV1::Unavailable => ContextScoutIntentPassOutcomeV1::Unavailable, } } diff --git a/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests.rs b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests.rs index fd99a0d2d4..5b668a54d3 100644 --- a/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests.rs +++ b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests.rs @@ -782,6 +782,11 @@ async fn stock_disabled_configuration_produces_nothing() { ); } +type DecisionEngineRequestGate = ( + tokio::sync::oneshot::Sender<()>, + std::sync::mpsc::Receiver<()>, +); + /// In-process decision engine: answers every configured question, recalling /// the earlier session with `recall` and everything else with 0.05, and /// records each request it receives. @@ -789,6 +794,7 @@ struct RecordingDecisionEngine { recall: f64, requests: std::sync::atomic::AtomicUsize, states: std::sync::Mutex>, + request_gate: std::sync::Mutex>, } impl RecordingDecisionEngine { @@ -797,12 +803,31 @@ impl RecordingDecisionEngine { recall, requests: std::sync::atomic::AtomicUsize::new(0), states: std::sync::Mutex::new(Vec::new()), + request_gate: std::sync::Mutex::new(None), }) } fn requests(&self) -> usize { self.requests.load(std::sync::atomic::Ordering::SeqCst) } + + fn hold_next_request( + &self, + ) -> ( + tokio::sync::oneshot::Receiver<()>, + std::sync::mpsc::Sender<()>, + ) { + let (entered, started) = tokio::sync::oneshot::channel(); + let (release, held) = std::sync::mpsc::channel(); + assert!( + self.request_gate + .lock() + .unwrap() + .replace((entered, held)) + .is_none() + ); + (started, release) + } } impl DecisionEnginePortV1 for RecordingDecisionEngine { @@ -813,6 +838,10 @@ impl DecisionEnginePortV1 for RecordingDecisionEngine { .lock() .expect("recorded states") .push(request.state.clone()); + if let Some((entered, release)) = self.request_gate.lock().unwrap().take() { + entered.send(()).unwrap(); + release.recv_timeout(Duration::from_secs(5)).unwrap(); + } DecisionEngineOutcomeV1::Answered(DecisionAnswersV1 { answers: request .questions diff --git a/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests/replay.rs b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests/replay.rs index b022e81e08..a2c6aded37 100644 --- a/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests/replay.rs +++ b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests/replay.rs @@ -5,9 +5,13 @@ use std::time::Duration; use tracedecay_agent_hosts::agents::context_scout::address_registry::ContextScoutConfigurationPinV1; use tracedecay_agent_hosts::agents::context_scout::intent_pass::ContextScoutIntentPassOutcomeV1; +use tracedecay_agent_hosts::agents::context_scout::{ + ContextScoutDurableStoreOutcomeV1, ContextScoutSuppressionV1, +}; use tracedecay_configuration::config::scope_control::daemon_owned_project_source_binding; use tracedecay_contracts::context_scout::{ ContextScoutDeliveryOutcomeV1, ContextScoutDeliveryReceiptV1, ContextScoutDurableQueueEntryV1, + ContextScoutModelOutcomeV1, }; use tracedecay_contracts::retrieval::{ ContextScoutStoreStatusV1, HookRuntimeResultV1, HookRuntimeSurfaceRequestV1, @@ -311,6 +315,95 @@ async fn mount_intent( (engine, entry) } +async fn intent_pass_overlapping_a_claim_is_suppressed(record_delivery: bool) { + let (fixture, graph, runtime) = registered_fixture().await; + let sessions = runtime + .registered_database_arc(HostAdmissionScope::Project) + .unwrap(); + let (engine, entry) = mount_intent(&fixture, &graph, &sessions).await; + let stop = fixture.hook( + 73, + HookEventV2::SessionBoundary { + boundary: HookBoundaryV1::TurnComplete, + }, + ); + let project_key = RegisteredGlobalDb::canonical_project_key(graph.project_root()); + let (started, release) = engine.hold_next_request(); + let claim_during_evaluation = async { + started.await.unwrap(); + let claimed_at = now_utc_micros(); + let (guidance, claim) = fixture + .owner + .claim_ready_guidance_exact( + stop.envelope(), + entry.work.address, + entry.work.input_watermark, + 1, + claimed_at, + ) + .await + .unwrap(); + assert_eq!(guidance.text, entry.envelope.candidate.suggestion_text); + if record_delivery { + let receipt = ContextScoutDeliveryReceiptV1 { + receipt_id: context_scout_delivery_receipt_id( + stop.envelope().event_id, + guidance.guidance_id, + ), + envelope_id: guidance.guidance_id, + delivered_at: now_utc_micros(), + outcome: ContextScoutDeliveryOutcomeV1::Attempted, + }; + assert_eq!( + fixture.owner.record_delivery(&claim, &receipt).await, + ContextScoutDurableStoreOutcomeV1::Stored + ); + } + release.send(()).unwrap(); + }; + let (outcome, ()) = tokio::join!( + fixture.pass(&stop, &sessions, &project_key, &engine), + claim_during_evaluation, + ); + assert_eq!( + outcome, + ContextScoutIntentPassOutcomeV1::Suppressed { + reason: ContextScoutSuppressionV1::Duplicate, + model_outcome: ContextScoutModelOutcomeV1::Succeeded, + }, + "the original suggestion is already claimed; delivered={record_delivery}" + ); + assert_eq!(engine.requests(), 2); + let recent = fixture + .owner + .recent_exact(entry.work.address, 8) + .await + .unwrap(); + if record_delivery { + assert_eq!(recent.pending, Vec::new()); + assert_eq!(recent.deliveries.len(), 1); + assert_eq!(recent.deliveries[0].entry, *entry); + } else { + assert_eq!(recent.pending, vec![*entry]); + assert_eq!(recent.deliveries, Vec::new()); + } + assert!(unregister_context_scout_lifecycle_authority( + fixture.binding.project_id, + fixture.binding.worktree_id, + &sessions, + )); +} + +#[tokio::test] +async fn intent_pass_with_an_already_claimed_suggestion_is_suppressed() { + Box::pin(intent_pass_overlapping_a_claim_is_suppressed(false)).await; +} + +#[tokio::test] +async fn intent_pass_with_a_delivery_during_evaluation_is_suppressed() { + Box::pin(intent_pass_overlapping_a_claim_is_suppressed(true)).await; +} + #[tokio::test] async fn enqueued_intent_replay_mounts_and_delivers_once_without_reopening_producer_work() { let (fixture, graph, runtime) = registered_fixture().await;