diff --git a/.github/workflows/ci-ready.yml b/.github/workflows/ci-ready.yml index 4f84c3dac..aeda080ff 100644 --- a/.github/workflows/ci-ready.yml +++ b/.github/workflows/ci-ready.yml @@ -67,55 +67,3 @@ jobs: env: PR_BODY: ${{ github.event.pull_request.body }} run: python3 tools/check_pr_references.py - - kv-task-qualification: - # Bounded software-control qualification; not a branch-protection or efficacy gate. - if: github.event.pull_request.draft == false - runs-on: ubuntu-22.04 - timeout-minutes: 20 - permissions: - contents: read - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - with: - ref: ${{ github.sha }} - persist-credentials: false - - name: Install distribution bubblewrap without changing security policy - run: | - sudo apt-get update - sudo apt-get install --yes --no-install-recommends bubblewrap - - name: Synthetic CPU controls (no model, secret or submitted artifact) - shell: bash - run: | - env -i PATH=/usr/bin:/bin LANG=C.UTF-8 PYTHONPATH=src \ - /usr/bin/python3 -m tools.qualify_kv_reclamation \ - --plan evidence/kv-reclamation-20261030/plan/qualification-plan.json \ - --revision "$GITHUB_SHA" --output "$RUNNER_TEMP/kv-qualification" - - name: Retain privacy-safe qualification receipt including failures - if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: kv-qualification-${{ github.run_id }}-${{ github.run_attempt }} - path: | - ${{ runner.temp }}/kv-qualification/qualification.json - ${{ runner.temp }}/kv-qualification/raw-evidence.zip - if-no-files-found: error - retention-days: 14 - - name: Scripted actor and descendant qualification (no provider or credentials) - if: always() - shell: bash - run: | - env -i PATH=/usr/bin:/bin LANG=C.UTF-8 PYTHONPATH=src \ - /usr/bin/python3 -m tools.qualify_kv_edit_actor \ - --plan evidence/kv-actor-qualification-20261031/plan/qualification-plan.json \ - --revision "$GITHUB_SHA" --output "$RUNNER_TEMP/kv-actor-qualification" - - name: Retain separate scripted actor receipt including failures - if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: kv-actor-qualification-${{ github.run_id }}-${{ github.run_attempt }} - path: | - ${{ runner.temp }}/kv-actor-qualification/qualification.json - ${{ runner.temp }}/kv-actor-qualification/raw-evidence.zip - if-no-files-found: error - retention-days: 14 diff --git a/.github/workflows/kv-edit-pilot.yml b/.github/workflows/kv-edit-pilot.yml deleted file mode 100644 index 2433c35ef..000000000 --- a/.github/workflows/kv-edit-pilot.yml +++ /dev/null @@ -1,96 +0,0 @@ -name: Bounded KV editing feasibility (manual, one pair) - -on: - workflow_dispatch: - -permissions: {} - -concurrency: - group: kv-reclamation-one-pair - cancel-in-progress: false - -jobs: - execute: - if: github.repository == 'InferMatrixLab/InferRSI' && github.ref == 'refs/heads/main' && github.run_attempt == 1 && github.actor == 'Dong1017' && github.triggering_actor == 'Dong1017' - runs-on: ubuntu-22.04 - timeout-minutes: 30 - permissions: - contents: write - actions: read - pull-requests: read - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - with: - ref: ${{ github.sha }} - fetch-depth: 0 - persist-credentials: false - - name: Same qualified system Python and distribution sandbox, no credentials - run: | - sudo apt-get update - sudo apt-get install --yes --no-install-recommends bubblewrap - - name: Exact freeze and same-host preflight, before credentials - env: - PYTHONPATH: src - run: /usr/bin/python3 -m tools.kv_live_job prepare --output "$RUNNER_TEMP/kv-live" - - name: Fixed bounded execution after exact public freeze and preflight - env: - PYTHONPATH: src - GITHUB_TOKEN: ${{ github.token }} - SKILL_EVAL_YIBUAPI_KEY: ${{ secrets.SKILL_EVAL_YIBUAPI_KEY }} - # Existing credentials enter only this trusted controller step. Admission - # still requires the exact public locator, qualification and both ledgers. - run: /usr/bin/python3 -m tools.kv_live_job execute --output "$RUNNER_TEMP/kv-live" - - name: Data-only public projection - if: always() - run: | - env -i PATH=/usr/bin:/bin LANG=C.UTF-8 PYTHONPATH=src \ - /usr/bin/python3 -m tools.kv_live_job package --output "$RUNNER_TEMP/kv-live" - - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - if: always() - with: - name: kv-live-${{ github.run_id }}-${{ github.run_attempt }} - path: ${{ runner.temp }}/kv-live/public - if-no-files-found: error - retention-days: 30 - grade: - needs: execute - if: always() && github.repository == 'InferMatrixLab/InferRSI' && github.ref == 'refs/heads/main' && github.run_attempt == 1 && github.actor == 'Dong1017' && github.triggering_actor == 'Dong1017' - runs-on: ubuntu-22.04 - timeout-minutes: 10 - permissions: - contents: read - actions: read - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - with: - ref: ${{ github.sha }} - persist-credentials: false - - name: Install fixed secret-free grader dependencies - run: | - sudo apt-get update - sudo apt-get install --yes --no-install-recommends bubblewrap - - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # verified v4 - with: - name: kv-live-${{ github.run_id }}-${{ github.run_attempt }} - path: ${{ runner.temp }}/kv-final-input - - name: Final grader of the exact captured public authorization - if: always() - run: | - env -i PATH=/usr/bin:/bin LANG=C.UTF-8 PYTHONPATH=src \ - GITHUB_SHA="$GITHUB_SHA" GITHUB_RUN_ID="$GITHUB_RUN_ID" \ - /usr/bin/python3 -m tools.kv_live_job grade --bundle "$RUNNER_TEMP/kv-final-input" \ - --output "$RUNNER_TEMP/kv-grade" - - name: Retain first grader outputs even after interruption, never rerun - if: always() - run: | - env -i PATH=/usr/bin:/bin LANG=C.UTF-8 PYTHONPATH=src \ - GITHUB_SHA="$GITHUB_SHA" GITHUB_RUN_ID="$GITHUB_RUN_ID" \ - /usr/bin/python3 -m tools.kv_live_job package-grade \ - --bundle "$RUNNER_TEMP/kv-final-input" --output "$RUNNER_TEMP/kv-grade" - - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - if: always() - with: - name: kv-final-grade-${{ github.run_id }}-${{ github.run_attempt }} - path: ${{ runner.temp }}/kv-grade/public - if-no-files-found: error - retention-days: 30 diff --git a/docs/kv-edit-integration.md b/docs/kv-edit-integration.md deleted file mode 100644 index f16a3e896..000000000 --- a/docs/kv-edit-integration.md +++ /dev/null @@ -1,119 +0,0 @@ -# Bounded KV edit/check integration — model-free - -This slice connects the fixed edit protocol to the synthetic CPU task. It is not -a general coding agent, paid launcher, production KV implementation or efficacy -result. A2/A3 remain one prospective exploratory paired block, not a completed -model experiment. - -## Implemented boundaries - -- `kv_edit_workbench.VisibleWorkbench` accepts a frozen, extracted DEVELOPMENT - suite and public benchmark inputs/expected data. The full verifier collection, - optimized control and hidden grader inputs are not supplied to the worker. -- Candidate code runs only through `kv_reclamation.invoke` and its existing - network-free bubblewrap boundary. The visible benchmark composes the existing - `run_experiment`/`evaluate` AB/BA path with content-addressed source snapshots. - Host-observed whole-command elapsed time includes sandbox/interpreter startup; - full output must match trusted expected data before a metric is accepted. -- Every revision is retained outside the exposed workspace. Final hidden grading - takes the immutable returned source bytes and their exact digest, never a later - reopening of the mutable actor path. Its results do not feed another edit. -- `tools.kv_edit_pilot.prepare_scripted` freezes explicit actor files, prompt, - public suite, baseline, scripted responses and trusted dependency hashes. - `run_supervised` launches the exact fixed worker with `-I` through existing - `execute_command`, under the same inherited absolute deadline used by every - tool. A blocked worker is terminated, not granted a fresh callback deadline. -- `tools.kv_worker_supervisor.supervise` owns the shared private grant/completion - bridge. The scripted wrapper supplies only a fixed secret-free worker command - and environment. An injected live transport is not part of this slice. -- Existing `authorization_broker` is reused. Bounded private files outside the - actor workspace relay only the exact request grant and raw four-class usage. - The trusted parent reconciles completion before another grant. Missing, late, - invalid or unknown usage does not free capacity or permit another arm. -- An optional injected admission session exercises existing durable PR/day - claims and their live checks. The actual session and GitHub authority remain - in the parent; the worker receives only an ephemeral loopback gate capability. - See [pilot admission](kv-pilot-admission.md). - -The worker has **no provider transport, provider credential option, account -configuration or live-execution mode**. Its response file is JSON data, not an -executable script. Every result identifies zero real model calls and zero new -provider charges. Candidate source is never imported on the host. - -## Qualification and limits - -`tests/test_kv_edit_workbench.py` exercises edit/check/ABBA/rewrite/submission and -separate hidden grading with scripted sandbox outputs. `tests/test_kv_edit_pilot.py` -exercises the private request/usage bridge against fake durable stores, real -loopback authorization, missing usage, expired admission day, source/prompt drift -and a real hard timeout of a trusted blocking worker fixture. Scripted sandbox -outputs are **not** evidence that bubblewrap isolation worked. - -A real worker always probes isolation before its first simulated request. An -unavailable namespace or missing trusted boundary yields `INCONCLUSIVE`, without -host execution, a weaker sandbox, a different agent or a new paid attempt. -Qualified-host results must bind the exact implementation/plan/source hashes. -Artifact replay qualification alone does not certify the entire actor path or -arbitrary descendant cleanup. - -The actor/tool worker is trusted controller code, not an OS-sandboxed coding -agent. It interprets bounded response JSON as fixed tools. A2/A3 model-visible -files are constrained by the exact exposure manifest and read allowlist; -generated candidate execution alone is isolated by fresh workdirs, restricted -runtime mounts and unchanged candidate resource limits. - -`tools.qualify_kv_edit_actor` adds a separate, zero-provider software-control -receipt to the existing Ready Ubuntu 22.04 job. It requires all of: - -- Complete pinned K-owner exposure only in scripted A3, identical fixed tools, - two edit/check/measurement revisions, independent readback of retained source - bytes and receipts for revisions 0–2, and immutable final hidden grading. -- `tools.kv_edit_inputs` constructs identical common task/contract/starter, - workbench/development files and prompt for both arms; only A3 adds K. Public - development bytes contain IDs/events, not expected outputs or grader traces. - A per-arm common-contract digest binds exact files, protocol/client, limits, - check IDs, metric, suite and baseline independently of scripted responses. -- Production-profile denial of actual package/K/oracle/reference paths and - synthetic canaries. Runtime enumeration resolves bind roots but never follows - escaped descendant aliases; bounded size-indexed hashes detect renamed copies - including the complete hidden corpus. Escaped aliases must be unreadable in - the unchanged production sandbox; missing visibility is INCONCLUSIVE. -- A surviving observer around the actual outer-executor → trusted worker → - nested executor → sandbox chain. An authored fork/setsid probe binds its - namespace PID, starttime and PID-namespace identity to an already-owned host - descendant. Normal exit, outer-worker hard timeout and cancellation require - this observed chain and zero live survivors. Recovery kills never convert a - failed cleanup into PASS. Candidate execution still forbids fork. - -The separate `kv-actor-qualification--` artifact binds the exact -clean revision, dependency closure, plan, exposure, per-arm receipts and raw -archive digest. Prior task-only qualification is not this actor qualification. -Scripted actor identities include scripted responses; they do not identify a -future live actor or establish exact live exposure/transport qualification. -The final submitted tree still needs its actual hosted result; local fake -fixtures, inventory-only checks and green repository tests cannot substitute. - -## Future paid launcher — design only - -After independent integration/security review and exact-head qualification: - -1. Freeze the one legitimate execution PR, immutable campaign/attempt, actor/K - manifests, provider/model/group/rate evidence and all limits. Refresh expired - pricing and reread existing PR/day obligations; do not reuse consumed approval. -2. Reserve and claim the complete conservative block before disclosure. Preserve - the fixed eight requests per arm, sixteen total and no retry policy. Every send - must consume its preallocated grant; all uncertain outcomes retain their bound. -3. Supply the existing provider key only to trusted provider transport, never to - candidate sandboxes, exposed files, CLI arguments, artifacts or ordinary PR CI. - Replace scripted transport only through reviewed, exact-revision wiring; do - not treat this document or a response artifact as execution authority. -4. Keep the parent-owned admission/usage boundary and hard worker deadline. If - actual transport/worker failure leaves usage unresolved, stop subsequent calls - across both arms and retain the full reservation. -5. Freeze both final artifacts and finish independent hidden correctness/performance - evaluation under the prospectively frozen gates. Retain failures and missing - outcomes. One pair demonstrates execution feasibility only, not K efficacy or - production/service performance. - -This document does not supply credentials, choose a new execution environment, -alter repository settings, add a CI lane or authorize a paid launch. diff --git a/docs/kv-live-launcher.md b/docs/kv-live-launcher.md deleted file mode 100644 index 1cc1750de..000000000 --- a/docs/kv-live-launcher.md +++ /dev/null @@ -1,167 +0,0 @@ -# Bounded KV live launcher (one-pair gated execution) - -This is an Issue-scoped synthetic editing feasibility controller, not a default -CI lane, model benchmark, efficacy result or service qualification. The manual -workflow binds the existing provider key and GitHub token **only in the trusted -execute step**, after secret-free preparation. A disabled or incomplete public -locator still prevents paid admission. Ordinary tests use fake providers, fake -ledgers and synthetic keys only; no key values belong in the freeze or checkout. - -## Trust and execution boundaries - -1. `kv_live_policy` validates an exact owner-authorized public freeze, immutable - artifact/source digests, existing ledger anchors, pricing expiry and exact - main/run identity. No missing-ledger initialization or retry is supported. -2. `prepare` stages the fixed A2/A3 allowlists. It requires prior hosted actor - qualification plus same-host isolation, descendant cleanup and a scripted - read/edit/check/benchmark/submit replay of the exact staged inputs. Preparation - rejects a provider credential. Scripted performance failure is retained, not - relabelled as an optimization success. -3. `execute` reserves and claims the entire pair before disclosing either arm. - GitHub authority stays in the controller. The trusted child receives only the - provider key, ephemeral broker capability and explicit minimal environment. -4. `GrantedSender` consumes one outer authorization for one canonical request, - burns the attempt before sending, and invokes the existing transport once. - Unknown/lost/late usage, cancellation and ambiguous ledger writes stop the - pair without refund or relaunch. Each arm retains one absolute deadline. -5. Candidate code executes only in the existing network-free bubblewrap profile; - unavailable isolation has no host fallback. The trusted worker is not itself - claimed to be another OS sandbox. -6. `package` copies bounded final/revision sources and selected structured - feedback/usage records, never raw provider bodies, worker logs or environment - dumps. Unknown attempts and not-started/missing arms remain explicit. -7. A separate read-only, secret-free job downloads only this run's fixed artifact - name using a pinned action. It verifies actual source bytes and run/plan/actor - identities and current-revision feedback links before final hidden semantics - and fixed performance evaluation. The public authorization snapshot is checked - against the immutable locator offline; later revocation still stops spending - without deleting already retained no-spend grading evidence. - Missing results remain INCONCLUSIVE and never become actor feedback or retries. - -## First-run evidence, including interruptions - -The execution artifact contains the exact verified public authorization, a bounded -projection of preparation progress, staged workspace hashes and the actual -isolation/runtime/cleanup/replay receipts. Each preparation stage records its -start before running. Missing, failed or interrupted stages are explicit; no -exception text, process command, environment or arbitrary log is published. - -The execution bundle also projects closed worker diagnostics from fixed trusted -result paths: worker status, an allowlisted reason/error code, `provider_attempts` -and outer executor status/return code/elapsed seconds. `provider_attempts` counts -invocations of the transport function, **not confirmed HTTP delivery or charges**. -A present zero is distinct from a missing/malformed/interrupted record (`null`). -Only fixed reason enums or `http_100` through `http_599` are retained; unknown -exceptions become `unknown`, never arbitrary exception text. The transport code -is captured separately before the edit agent generalizes its failure. These -diagnostics do not settle costs, authorize retry or upgrade a verdict. Packaging -only reads existing first-run files, including when controller writing stopped. -This future retention correction cannot recover a reason omitted from an earlier -artifact; that earlier cause and billing remain unknown without separate evidence. - -The grader uploads only its dedicated `public/` projection, not its working -directory. `evidence.json` enumerates every assigned hidden trace and the fixed -alternating AB/BA timing trials plus quality validation. It binds the original -candidate, frozen trace/workload hashes, actual raw task-JSON hashes, measured -values, execution status/exit code and before/after source hashes where available. -The shared executor now records only status, return code and elapsed seconds -immediately after a candidate process returns, before any verifier early return -or JSON parsing. The projection accepts that exact three-field receipt. A process -killed before receipt persistence still has explicitly unavailable status; it is -never inferred from stdout or a PASS. Activation requires fresh model-free -qualification of this changed shared dependency, not the earlier receipt. - -Only exact raw JSON conforming to the fixed KV result schema is copied. Its -strings must be known task request IDs or the fixed `ValueError` marker. Each -output is at most 4 MiB, the whole two-arm projection at most 32 MiB, with at most -256 semantic traces per arm. Arbitrary/malformed candidate-only stdout is withheld -with observed byte count, content hash, parsing/schema failure and truncation -status; oversized/unsafe files remain unhashed/unavailable. These diagnostics are -not a sanitizer for private inputs or provider logs. No claim is made to preserve -all arbitrary raw bytes. Failed and missing trials are never upgraded to PASS. -Projection completion is not full-evidence acceptance: every arm's -`raw_outputs_complete` and unavailable execution fields must be inspected even -when a retained grading summary says PASS. - -`package-grade --bundle --output ` runs in -a separate `always()` environment-cleared workflow step. It validates the same -captured authorization and run identity, then packages **only already generated -first-run evidence**; it cannot execute candidates, call a model or retry grading. -This covers a grader process interrupted before its final cleanup. It also leaves -`projection.json` with started/completed/failed-or-interrupted state and final -manifest hashes. A workflow/host loss that prevents even the always-step or upload -cannot guarantee artifact retention; missing evidence remains INCONCLUSIVE, not -a reason to rerun the paid trial. Both Actions artifacts have 30-day retention; -long-term public archive publication remains a separately approved action. - -## Hosted qualification provenance - -The frozen `qualification` record contains `run_id`, `run_attempt`, `head` -(GitHub run head), `revision` (actual qualified checkout), `artifact_id`, the -bounded downloaded `archive`, exact `receipt` and `dependencies_sha256`. -`head` and `revision` are separate because PR workflow run identity and checkout -identity need not be the same commit. - -The archive must contain exactly `qualification.json` and `raw-evidence.zip`. -The launcher compares the receipt bytes and raw-evidence digest, verifies the -common actor dependency closure against this checkout, checks successful -isolation/cleanup/two-revision tool evidence and requires exact live exposure and -prompt hashes. A common-actor contract digest also binds client settings, input -ceiling, visible suite, baseline, edit limits, checks, metrics and exposed files; -only scripted callback/response-specific identity is excluded. Hosted and current -mount inventories need not have identical hashes: selective sensitive-file -hashing can differ. The canonical dependency closure binds the sandbox profile, -and a fresh current-host inventory/probe must independently PASS; both inventory -digests are retained. The live actor identity is deliberately distinct from the -scripted identity that also binds response fixtures. - -Immediately before admission and each request, live metadata must identify the -successful pinned Ready run and the exact nonexpired -`kv-actor-qualification--` artifact with matching archive -SHA-256. Generic green CI, a numeric task-only artifact or a locally authored -`PASS` JSON file cannot replace this receipt. - -## Required one-pair operational freeze - -- Accepted final common task/workbench/development bytes, limits and arm order; - exact A3 knowledge export audit and independent final-verifier inventory. -- Hosted actor qualification of the actual integrated dependency closure and - exact exposure/prompt; same-host probes must also pass at execution time. -- One legitimate charge PR/head and audited existing PR/day ledger histories. - Existing reservations/carryover remain obligations; a launcher PR does not - create another allowance. -- Refreshed price evidence, supported authorization expiry and enough full-pair - runway before expiry or Shanghai midnight. An expired plan is not renewed by - an offline test or by the larger task ceiling. -- Reviewed public freeze/locator with actual hashes and provenance, plus explicit - owner authorization for this existing-key binding and single manual execution. - -No deployment, credential or paid action is authorized merely by this template. -An operator must not enable the locator, add a secret binding or dispatch merely -because repository tests passed. These tests establish software behavior only; -real hosted isolation, provider delivery, measured model output and efficacy -remain separate evidence. - -## Explicit one-additional-attempt exception - -The default PR lifetime cap is unchanged. A v2 public plan may carry the reviewed -`budget_exception` object: exact repository/charge PR, trusted PR cap, fixed new -campaign/execution IDs, exact prior claimed binding and PR/day checkpoints. The -root validates it under the existing owner-comment/locator identity, then passes -an immutable key-bound limit only to the anchored PR reader. It is never a ledger -JSON field, environment/CLI setting, worker input or candidate-selected allowance. -Historical ledger bytes remain unchanged; an ordinary default reader still -rejects a cumulative balance exceeding its default cap. - -The additional admission requires the exact sole prior KV claim and retains it -in full. It rejects any other KV reservation/claim, including a partial uncertain -write. Once the new pair is claimed, every request requires exactly those two -claims. A third campaign, changed ID, new PR, different day or smaller requested -budget cannot reset this guard. Both caps are checked before the existing -daily-first CAS sequence; races, ambiguous writes and unknown usage still stop -without retries or refunds. The pilot total ceiling remains unchanged. - -Issue-scoped proposed values and remaining activation prerequisites live in -[`SECOND-ATTEMPT.md`](../evidence/kv-live-pilot-20261031/plan/SECOND-ATTEMPT.md). -The original v1 plan/binding remains historical; this software support alone -neither renews its expiry nor authorizes a model request. diff --git a/docs/kv-pilot-admission.md b/docs/kv-pilot-admission.md deleted file mode 100644 index 08d72a104..000000000 --- a/docs/kv-pilot-admission.md +++ /dev/null @@ -1,148 +0,0 @@ -# One-pair KV pilot admission seam (offline) - -`tools/kv_pilot_admission.py` is a narrow controller interface for the accepted -exploratory KV editing feasibility envelope: **one A2/A3 paired block**, at most -**8 provider attempts per arm / 16 total**, with a full-context worst-case -**USD 10.17 per request / USD 162.72 total**. These are spending bounds, not -scientific acceptance thresholds. One pair can establish implementation/execution -feasibility only; it cannot establish efficacy. No experiment is run by this module. - -The module introduces no CLI, default transport, credential read, workflow launch, -ledger initializer, new ledger schema or automatic retry. Tests use in-memory -stores and injected clocks/launch callbacks only. A real launcher remains a -separate reviewed integration, not authority implied by this offline seam. - -## Trust and immutable identity - -Only reviewed controller/root configuration can create `PilotPolicy`. Never load -it, its stores/anchors, or `check_current` from candidate code, PR artifacts, -author-written requests, mutable labels or model output. `check_current(policy)` -must return exactly `True` only after independently verifying owner authorization, -repository/legitimate execution PR and exact head, frozen source revision, visible -input/K/verifier hashes, model/provider/pricing identity, and qualified isolation. -The plan SHA-256 must bind those accepted inputs and the single-pair sample plan; -the per-arm actor hashes bind the complete editing protocol/manifests. The source -author model/effort provenance remains distinct from the evaluated provider model. - -`provider_json` is immutable canonical JSON and uses the separate editing protocol, -not the pre-existing read/submit agent. Existing `reservation()` recomputes the -full 1,000,000-token ceiling and 4,096-output bound; USD amounts are converted -without floating-point arithmetic. Plan and pricing expiration are enforced both -before admission and before every send. A frozen root-policy change must cause -`check_current` to deny; this API is not a way for candidate inputs to authorize -such a change. - -The **existing global daily ledger entry is the durable campaign-binding record**: - -- `Binding.run_id`: fixed `kv-reclamation-one-pair-v1:` identity. -- `Binding.plan_digest`: SHA-256 of the complete immutable policy, including - execution ID, repository/PR, exact head/source/plan, actors and pricing. -- `Binding.worst_case`: the entire USD 162.72 envelope. - -Admission rejects *any* existing entry in this one-pilot namespace, not only an -identical digest. Thus changing the campaign ID, digest, run/attempt, PR, date or -expiry cannot create a new allowance. The unchanged global CAS arbitrates racing -first admissions. A later legitimate experiment would need separately reviewed -scope and code; this seam deliberately offers no reset, takeover or resume path. - -## Durable ordering and accounting - -`admit(...)` reads both existing verified histories, checks their identities and -requires the selected PR's retained global obligations to equal its cumulative PR -charge. An incomplete old PR tip cannot erase global imported charges. It computes -both cap checks before writing, then performs exactly: - -```text -full daily reservation CAS → full PR reservation CAS - → daily one-shot claim → PR one-shot claim - → fresh verification of BOTH claims, live policy, price and Shanghai day - → process-local supervisor session -``` - -All earlier audited charges, reservations, retries and unresolved obligations must -already be present in verified history. Absent/corrupt/unknown history blocks; -there is no missing-file-zero fallback. One immutable campaign on one PR subject -to the USD 200 lifetime cap is already strictly below the USD 1,000 pilot-task -ceiling; this does not introduce a global lifetime cap. Unrelated settled past-day -campaigns are not new pilot spending. The USD 1,000 Shanghai-day cap still includes -all charges for that day and unresolved historical carryover. No accounting -meaning is inferred from a successful test or claimed actual usage. - -The synthetic regression fixture preserves USD 491.35 of unresolved historical -obligations, producing USD 654.07 retained after the complete reservation, USD -345.93 remaining that day, and USD 37.28 remaining on a fresh execution PR. The -new pilot itself moves from USD 0 reserved to USD 162.72 reserved, leaving USD -837.28 of its USD 1,000 task ceiling; the earlier USD 491.35 is global daily -carryover, not new pilot spend. These are fixture/audit amounts, **not a current -live ledger readback or invoice**. -Current verified refs/checkpoints and refreshed pricing remain launch prerequisites. - -A stale CAS, partial reservation, lost response or crash never grants execution. -Unknown claim outcomes cannot be reconciled into a new grant. A caller receiving -an uncertain first daily write must stop rather than blindly retry; an absent -observation is not proof that the old write cannot arrive. The durable no-relaunch -guarantee applies once a campaign entry persists, with CAS preventing competing -first admissions from both winning. Reservations and -claims stay in the existing ledgers after success, failure, cancellation or process -death; there is no refund or terminal-settlement writer here. Shanghai rollover -stops new requests, even if admitted earlier. Outstanding entries carry over; -midnight does not refund or relaunch the campaign. - -## Trusted supervisor interface - -`run_admitted(..., launch=callback)` invokes a trusted callback only after both -claims and closes the session in `finally`. For finer integration, `admit(...)` -returns the same process-local, non-pickleable session. Do not expose this object, -its stores, policy checker or completion method to model/candidate code. - -1. `start_arm(arm, absolute_deadline)` immediately **before disclosure/launch**. - The deadline is at most 600 seconds away, cannot be renewed, and arms cannot - overlap. The qualified external supervisor must enforce that same absolute - deadline; in-process checks alone cannot terminate blocked code. -2. `authorize(grant, absolute_deadline) -> True` immediately before **one send**. - The exact grant shape matches `anthropic_edit_agent.RequestSlots`: plan digest, - execution ID, arm, slot, payload SHA-256, actor SHA-256 and per-request bound. - Admission has independent counters, burns before live checks, verifies both - durable claims and day/price after reads, and allows no ninth slot or borrowing - from the other arm. The worker's local counters are an additional guard only. -3. `complete(same_grant, raw_usage)` after the trusted transport/worker has observed - the response for that exact request. Raw usage requires all four integer fields: - `input_tokens`, `output_tokens`, `cache_read_input_tokens`, - `cache_creation_input_tokens`. All input classes count toward the 1M bound. - Candidate-authored JSON, a narrative or unverified trace is not usage evidence. -4. `finish_arm(arm)` only when no request remains unresolved. Preserve failures - and missing artifacts in the experiment evidence rather than discarding them. -5. `close()` irreversibly stops this process-local session. No new process can - reacquire the already consumed durable claims. - -There may be only one pending request across both arms. A lost grant/reply, -transport failure, unknown/malformed/over-bound usage, mismatched completion, -revoked authorization or inconsistent claims stops both arms; failed/ambiguous -work cannot replenish slots. The full cost stays reserved. Report and reconcile -any observed over-bound exposure through the existing trusted quarantine/accounting -process before other paid work; this module does not claim to settle provider bills. - -`context(arm)` supplies an exact frozen context for the unchanged existing -`request_guard.authorization_broker`. No new network protocol is introduced. -The existing broker is a **trusted-client gate, not a candidate sandbox**. Real -wiring must keep provider credentials and GitHub authority outside candidate/test -execution, independently enforce exactly one send per grant/no transport retries, -and relay usage from trusted observation before the next authorization. Neither -this seam nor a green fake test proves that real isolation or billing is qualified. - -## Verification - -Run offline: - -```sh -pytest tests/test_kv_pilot_admission.py tests/test_skill_eval_budget.py \ - tests/test_skill_eval_daily_budget.py tests/test_skill_eval_budget_git.py \ - tests/test_request_guard.py tests/test_anthropic_edit_agent.py tests/test_anthropic_agent.py -ruff check tools/kv_pilot_admission.py tests/test_kv_pilot_admission.py -``` - -Coverage includes stale CAS, partial/unknown writes, death after every durable -step, duplicate claims and relaunch, new digest/campaign/PR/attempt, cap overflow, -Shanghai carryover/rollover, pricing expiry, final exact-claim verification, -revocation, unknown/mismatched usage across arms, frozen deadlines and the hard -8-per-arm/16-total limit. These are controller regressions, not paid-model results. diff --git a/docs/kv-reclamation-qualification.md b/docs/kv-reclamation-qualification.md index 6b8c3e1e7..50bab84b9 100644 --- a/docs/kv-reclamation-qualification.md +++ b/docs/kv-reclamation-qualification.md @@ -1,118 +1,47 @@ -# Synthetic CPU KV-block reclamation qualification +# Synthetic CPU KV-block reclamation qualification (retired) -This is a bounded **model-free mechanism-transfer task**, not vLLM conformance, -a natural historical-repository holdout, an accelerator result or agent efficacy. -The accepted slice is recorded in [the scoped execution record][scope]. It does -not replace the broader task-efficacy proposal, the serving-parser task or the -Indexer acceptance boundaries. +The tooling this page documented has been removed. It described a bounded, +model-free mechanism-transfer task built for the task-efficacy pilot in +[issue 248](https://github.com/InferMatrixLab/InferRSI/issues/248): a synthetic +CPU reclamation contract with an independently authored frozen oracle, an +isolated input-only replay and trusted event-by-event comparison through the +existing AB/BA runner. -This synthetic task was deliberately constructed around the nominated knowledge's -dual-readiness mechanism. Independent oracle authorship is **not independent task -selection**. Both arms' visible contract and correct starter already state the -readiness rules; a later comparison tests utility while preserving those rules, -not discovery of a hidden requirement. It is a selected synthetic near-transfer -probe, not an independently sampled case. +This page is kept because the retained evidence links to it, and that evidence +is hash-pinned and must not be edited. -The visible starter is a correct but deliberately repeated linear scan. Its -contract includes request finish, asynchronous save completion, a processed-step -write fence, deterministic reclamation order, generation identity, idempotent -callbacks and atomic invalid-input handling. Only `candidate.py` may change. -Shared blocks, refcounts, prefix matching, transport and real concurrency are -outside this task. +## What was removed -## Trust and evidence boundaries +`tools/qualify_kv_reclamation.py`, `tools/qualify_kv_edit_actor.py`, +`tools/qualify_kv_actor_cleanup.py`, the `tools/kv_*` live-launcher and +admission modules, `src/inferrsi/harness/kv_reclamation.py`, +`src/inferrsi/harness/kv_edit_workbench.py`, their tests and task fixtures, +the `kv-edit-pilot` workflow and the `kv-task-qualification` job in the ready +lane. None of the commands named on this page can be run from the repository +any more. -```mermaid -flowchart LR - Contract[Visible neutral contract] --> Starter[Editable candidate.py] - Contract --> Oracle[Independent frozen oracle] - Starter --> Sandbox[Isolated input-only replay] - Sandbox --> Compare[Trusted event-by-event comparison] - Oracle --> Compare - Compare --> Runner[Existing AB/BA runner and evaluator] -``` +## What was kept -- The independent verifier author freezes the semantic oracle and disjoint - development/grader traces without reading the nominated knowledge or target - implementation. Static trace observations are checked against that oracle. -- The candidate process receives its source, a trusted driver and input events, - never oracle code, expected answers, other trials, knowledge or reference code. - Return types, event counts, every reclamation list and pending state are checked. - Raw inputs necessarily reach the candidate during replay; expected results and - the grader fixture store do not. Every output byte remains untrusted, even if - the candidate tampers with its Python driver or output serializer. - In-process tuple/list assertions are ordinary compatibility checks, **not** - anti-tamper acceptance: malicious same-interpreter code can bypass them. The - externally enforced acceptance is frozen observable trace-envelope equality - and host whole-command timing, not immutable per-call driver/type behavior. -- `kv_reclamation.invoke` reuses the existing bubblewrap command and host - `execute_command` deadline. Network/PID namespaces, clear environment, - read-only mounts and hard process/memory/CPU/output limits are probed. There is - **no unisolated fallback**. This boundary confines artifacts, not a future actor. - Group-cleanup probes cover the tested host process group; the hard no-fork - boundary is checked separately. This is not arbitrary detached-tree or future - actor cleanup certification. -- Missing output, changed protected paths and incorrect artifacts are rejected. - Infrastructure failure, unavailable isolation and timeout remain unverified - (`INCONCLUSIVE`), distinct from a completed comparison refuting a claim. -- Workbench timing reuses `run_experiment` and `evaluate`. It measures host elapsed - time of the whole CPU task command, including sandbox/interpreter startup. The - candidate cannot supply a claimed timing metric. Inputs, repetition count and - numeric headroom gates live only in the frozen qualification plan. -- Correctness precedes timing. An optimized software control must pass the full - verifier and the headroom gate; premature save/fence release, generation errors, - cached answers/event skipping, missing output and protected edits are negative - controls. A/A and no-op comparisons must not meet the improvement gate. -- All runs, including failures, are retained. There is no retry-until-pass rule. - The CI receipt retains hashes, statuses, raw timing samples and control coverage. - Its `raw-evidence.zip` retains actual control outputs and AB/BA evidence with - labelled path redactions, without hostnames, credentials or full environment dumps. +The retained evidence stays in place and unmodified: -## Meaning of a successful qualification +- [`evidence/kv-reclamation-20261030/`](../evidence/kv-reclamation-20261030/README.md) +- `evidence/kv-actor-qualification-20261030/` +- `evidence/kv-actor-qualification-20261031/` +- `evidence/kv-live-pilot-20261031/` -Qualification can establish only that this exact host/task/verifier combination -has discriminating controls and measurable software headroom. CI success is not -an engine campaign `PASS`, a knowledge benefit, an actor isolation certificate or -permission to infer broad transfer. The original tests and historical outcomes -remain unchanged. +Their entries in `tools/evidence_inventory.json` are unchanged, so +`python -m tools.validate_evidence` still verifies them. -The prospective A2/A3 comparison holds W constant and changes only a complete, -pinned `audit-vllm-kv-lifetimes` owner bundle. Before any paid step it still needs a -frozen executable actor/tool protocol, demonstrated read isolation (including W -package data and host files), full K-to-target contamination audit, known charges, -conservative reservations, authorized plan identity and a final-artifact grader. -A provider `read_file`/`submit_answer` path is not an iterative editing agent. -No model calls or paid adapter are part of this delivery. +## What the evidence does and does not show -## Reproduce +The qualification receipts record **software and scripted-control execution** +only. Both live A2/A3 attempts on 2026-10-31 ended `INCONCLUSIVE`, the second +stopping at its first transport call on HTTP 403, so no completed arm, per-arm +outcome or A/A variance exists. The recorded dollar figures are reserved upper +bounds, not observed spend. Nothing here is a model-effect, knowledge-effect or +hardware result. -From a checkout with Linux system Python, Git and distribution bubblewrap: - -```sh -PYTHONPATH=src /usr/bin/python3 -m tools.qualify_kv_reclamation \ - --plan evidence/kv-reclamation-20261030/plan/qualification-plan.json \ - --revision "$(git rev-parse HEAD)" --output -``` - -The command never accepts arbitrary source or grader paths. The existing Ready -workflow runs a non-required Ubuntu job with no secrets and retains the sanitized -receipt. Its exit code gates software controls, **not** numerical performance: -always read `qualification_verdict` in the receipt. A green job with insufficient -headroom does not qualify the task. The original manual parser qualification, -test matrix, branch protection and default serving-parser adapter are unchanged. - -The initial local run is retained as `INCONCLUSIVE`: the cloud host refused the -required network namespace setup. It did not run any candidate without isolation. -Trusted direct-import controls are labelled separately and do not upgrade that -result. See the [frozen plan and retained evidence](../evidence/kv-reclamation-20261030/README.md). - -The independently frozen benchmark tests retain their original standalone layout. -In this integrated checkout, use the checked wrapper test or set the documented -oracle location without modifying frozen files: - -```sh -KV_FROZEN_ORACLE="$PWD/evidence/kv-reclamation-20261030/validator/independent/oracle.py" \ - pytest evidence/kv-reclamation-20261030/validator/test_benchmark.py -``` - -[scope]: https://github.com/InferMatrixLab/InferRSI/issues/248#issuecomment-6526000390 +The general SkillEval capability is unaffected: nothing in +`src/inferrsi/harness/skill_eval*.py`, the runner, the evaluator or +`tools/skill_eval_*.py` referenced the removed modules. The dependency ran one +way only, from this slice onto those shared parts. diff --git a/docs/operations.md b/docs/operations.md index 9614db4f2..bfafe399d 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -32,10 +32,6 @@ Effect classes used below: | Export a cloud-generated parser pair | [Generation-only protocol](skill-eval-generation-only.md), `python -m tools.skill_eval_cloud_export` | Local evidence-writing; no execution or publication | Validates frozen inputs and retained sources; writes one bounded inert JSON bundle, never executes parser source | | Run the fixed historical parser generation pilot | [Main-only historical job](skill-eval-historical-job.md), `python -m tools.skill_eval_historical_job` | Authorized paid generation / ledger-writing / evidence-writing | No dispatch inputs; reviewed main policy, pre-secret freeze, existing credentials, two-request cap; no generated-source execution or automatic retry | | Qualify an artifact verifier host | [Manual reviewer qualification](skill-eval-reviewer-qualification.md), `python -m tools.skill_eval_qualify` | Trusted-control execution / report-generating | Manual main-only workflow; fixed software controls, no model or external artifact input; unavailable isolation fails rather than skips | -| Qualify the synthetic CPU reclamation task | [Task qualification](kv-reclamation-qualification.md), `python -m tools.qualify_kv_reclamation` | Trusted-control execution / report-generating | Explicit frozen plan; independent verifier, isolated controls and existing AB/BA runner; no paid actor or efficacy claim | -| Qualify fixed KV editing mechanics | [Actor integration](kv-edit-integration.md), `python -m tools.qualify_kv_edit_actor` | Scripted trusted-control execution / report-generating | Existing Ready Ubuntu host; real isolated artifact replay, fixed edit/check/ABBA tools, immutable final grading and descendant cleanup; zero provider calls, no paid launcher or efficacy claim | -| Exercise pilot admission offline | [Admission seam](kv-pilot-admission.md), Python `tools.kv_pilot_admission.run_admitted` | Explicit injected-controller effect | Reuses audited PR/day ledger contracts; fake stores for tests, no credential discovery, ledger initialization, CLI or automatic paid launch | -| Run the one-pair gated KV launcher | [Live launcher boundaries](kv-live-launcher.md), `python -m tools.kv_live_job` (`prepare`, `execute`, `package`, `grade`, `package-grade`) | Manual workflow-only gated provider/ledger execution; separate grading and interruption-safe, data-only `package-grade` projection | Existing credentials bind only to the trusted execute step after secret-free preparation; exact public freeze, hosted/current-host qualification and durable full-pair admission required; `package-grade` never reruns candidates or models; no default CI spending | | Replay a cloud-generated parser pair | [Isolated cloud replay](skill-eval-cloud-replay.md), `python -m tools.skill_eval_cloud_replay` | Untrusted-artifact execution / report-generating | Manual pinned data ingress; same-host controls before bubblewrap-only replay; no models or credentials | | Validate one harness-improvement step | [Offline improvement](harness-improvement.md), Python `tools.skill_eval_improve.prepare` / `run_offline` | Snapshot-writing / offline experiment-executing | Supplied declarative INV/DOM edits, shared total limits, rerun and one holdout gate; trusted fixed-output clients only; no real-model or Store execution | | Plan and report skill PR evaluations | [Skill PR integration](skill-eval-pr.md), `python -m tools.skill_eval_pr` | Read-only planning / authorized experiment-executing / PR-publishing | Trusted mapping, exact operator approval, consumed execution identity and credential-separated publication from retained evidence; no automatic paid trials | diff --git a/src/inferrsi/harness/kv_edit_workbench.py b/src/inferrsi/harness/kv_edit_workbench.py deleted file mode 100644 index c887ea9f6..000000000 --- a/src/inferrsi/harness/kv_edit_workbench.py +++ /dev/null @@ -1,206 +0,0 @@ -"""Fixed visible KV workbench; submitted Python executes only through kv.invoke. - -The visible suite contains extracted DEVELOPMENT cases and a public benchmark, -never the full verifier collection. Candidate sandboxes receive events only. -Hidden grading is a separate operation on the final immutable source snapshot. -""" - -import hashlib -import json -import math -import os -import time -from dataclasses import dataclass -from pathlib import Path - -from ..infra.artifacts import atomic_write_json, digest -from ..model import anthropic_agent as provider -from ..model.anthropic_edit_agent import EditPlan -from ..runspec import RunSpec -from . import kv_reclamation as kv -from .runner import run_experiment - - -def _copy(value): - return json.loads(json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False)) - - -def _compare(trace, actual): - # Projection of frozen expected DATA, not a second state-machine oracle. - expected = {"observations": trace["expected"], - "pending": trace["expected"][-1]["pending"] if trace["expected"] else []} - return digest(expected) == digest(actual) - - -def _traces(traces, split): - if type(traces) is not list or not traces: - raise ValueError("Nonempty frozen trace set required") - ids = [] - for trace in traces: - if (type(trace) is not dict or set(trace) != {"id", "split", "events", "expected"} - or trace["split"] != split or type(trace["id"]) is not str - or type(trace["events"]) is not list or type(trace["expected"]) is not list - or len(trace["events"]) != len(trace["expected"])): - raise ValueError("Wrong trace split or frozen trace shape") - ids.append(trace["id"]) - if len(set(ids)) != len(ids): - raise ValueError("Duplicate trace identity") - return _copy(traces) - - -@dataclass(frozen=True) -class VisibleSuite: - """The whole byte string is hashed in the actor's callback/dependency freeze.""" - - serialized: bytes - - def __post_init__(self): - if type(self.serialized) is not bytes or not 0 < len(self.serialized) <= 16 * 1024 * 1024: - raise ValueError("Visible suite must be bounded immutable JSON bytes") - value = json.loads(self.serialized, object_pairs_hook=provider.unique_object) - if (type(value) is not dict or set(value) != {"schema", "development", "benchmark_events", - "benchmark_expected", "min_improvement_pct"} - or value["schema"] != "kv-visible-suite-v1" - or type(value["benchmark_events"]) is not list - or type(value["benchmark_expected"]) is not dict - or type(value["min_improvement_pct"]) not in (int, float) - or not math.isfinite(value["min_improvement_pct"]) - or value["min_improvement_pct"] < 0): - raise ValueError("Wrong visible suite shape") - _traces(value["development"], "development") - _copy(value) - - def read(self): - return json.loads(self.serialized) - - -class VisibleWorkbench: - """Fixed callbacks; no model-supplied paths, commands, traces or metric gates.""" - - def __init__(self, plan: EditPlan, suite: VisibleSuite, baseline: bytes, output: Path): - value = suite.read() - if tuple(t["id"] for t in value["development"]) != plan.check_ids: - raise ValueError("Actor check identities differ from the frozen visible suite") - if (type(baseline) is not bytes or not 0 < len(baseline) <= plan.limits.max_source_bytes - or plan.metric_key != "elapsed_seconds" or plan.metric_unit != "seconds" - or plan.direction != "minimize"): - raise ValueError("Wrong fixed KV source/metric contract") - baseline.decode("utf-8") - self.plan, self.suite, self.baseline = plan, suite, baseline - self.output = Path(output) - self.output.mkdir(parents=True, exist_ok=False) - self.check_count = self.benchmark_count = 0 - - def retain_revision(self, source, metadata, deadline): - if time.monotonic() >= deadline: - return False - if hashlib.sha256(source).hexdigest() != metadata["sha256"]: - raise ValueError("Revision digest does not match retained bytes") - path = self.output / "revisions" / f"revision-{metadata['revision']:04d}" - path.mkdir(parents=True, exist_ok=False) - with os.fdopen(os.open(path / "candidate.py", os.O_WRONLY | os.O_CREAT | os.O_EXCL, - 0o600), "wb") as stream: - stream.write(source) - stream.flush() - os.fsync(stream.fileno()) - atomic_write_json(path / "receipt.json", metadata) - return kv.source_bytes(path / "candidate.py") == source - - def visible_checks(self, source, deadline): - self.check_count += 1 - if self.check_count > self.plan.limits.max_checks: - raise ValueError("Frozen visible-check count exhausted") - traces = self.suite.read()["development"] - try: - result = kv.verify_source(source, traces, self.output / f"check-{self.check_count:03d}", - deadline, _compare) - except kv.SandboxUnavailable: - return {"verdict": "INCONCLUSIVE", "checks": {}, "reason": "sandbox_unavailable"} - reason = "none" if result["verdict"] == "PASS" else ( - "check_failed" if result["verdict"] == "FAIL" else - "deadline" if result.get("reason") in {"timeout", "deadline"} else "candidate_error") - return {"verdict": result["verdict"], "checks": result["checks"], "reason": reason} - - def visible_benchmark(self, source, deadline): - self.benchmark_count += 1 - if self.benchmark_count > self.plan.limits.max_benchmarks: - raise ValueError("Frozen visible-benchmark count exhausted") - empty = {"verdict": "INCONCLUSIVE", "baseline_values": [], "candidate_values": [], - "quality_verdict": "INCONCLUSIVE", "improvement_pct": None, - "reason": "deadline"} - left = deadline - time.monotonic() - if left <= 0: - return empty - root = self.output / f"benchmark-{self.benchmark_count:03d}" - sources = {"baseline": self.baseline, "candidate": bytes(source)} - for label, data in sources.items(): - directory = root / "sources" / label - directory.mkdir(parents=True) - (directory / "candidate.py").write_bytes(data) - suite = self.suite.read() - expected = digest(suite["benchmark_expected"]) - quality = [] - - def snapshot(repo, excluded): - data = kv.source_bytes(repo / "candidate.py") - return {"available": True, "snapshot_hash": hashlib.sha256(data).hexdigest(), - "kind": "immutable-candidate-bytes"} - - def execute(command, cwd, trial_dir, runner_deadline, metric_key): - if command not in (("fixed-kv", "baseline"), ("fixed-kv", "candidate")): - raise ValueError("Not a fixed KV workbench command") - label = command[1] - record, actual = kv.invoke(sources[label], {"mode": "benchmark", - "events": _copy(suite["benchmark_events"])}, trial_dir, - min(deadline, runner_deadline)) - valid = record["status"] == "completed" and digest(actual) == expected - elapsed = record.get("elapsed_seconds") - finite = type(elapsed) in (float, int) and math.isfinite(elapsed) and elapsed > 0 - record["metric"] = elapsed if valid and finite else None - record["metric_error"] = None if valid and finite else "unverified complete trace" - if record["status"] == "completed" and not valid: - record["status"] = "failed" - if metric_key is None: - quality.append("PASS" if valid else "FAIL" if record["status"] == "failed" - else "INCONCLUSIVE") - atomic_write_json(trial_dir / "execution.json", record) - return record - - spec = RunSpec( - repo=str(root / "sources/baseline"), candidate_repo=str(root / "sources/candidate"), - repo_type="synthetic-cpu-kv", backend="cpu", mode="code", - scenario="kv-visible-development", metric_key=self.plan.metric_key, - metric_unit=self.plan.metric_unit, direction=self.plan.direction, - comparison_scope="whole CPU task including sandbox and interpreter startup", - baseline_command=("fixed-kv", "baseline"), candidate_command=("fixed-kv", "candidate"), - validation_command=("fixed-kv", "candidate"), - repetitions=self.plan.limits.benchmark_repetitions, - max_minutes=left / 60, min_improvement_pct=suite["min_improvement_pct"], - output_dir=str(root / "runs"), - agent="bounded-edit-check-protocol") - try: - run = run_experiment(spec, {"scope": "synthetic visible development only"}, - execute=execute, snapshot=snapshot) - except kv.SandboxUnavailable: - return {**empty, "reason": "sandbox_unavailable"} - evaluation = json.loads((run / "evaluation.json").read_text()) - return {"verdict": evaluation["verdict"], - "baseline_values": (evaluation["baseline"] or {}).get("values", []), - "candidate_values": (evaluation["candidate"] or {}).get("values", []), - "quality_verdict": quality[-1] if quality else "INCONCLUSIVE", - "improvement_pct": evaluation["improvement_pct"], - "reason": "none" if evaluation["verdict"] == "PASS" else - "check_failed" if evaluation["verdict"] == "FAIL" else "measurement_incomplete"} - - -def grade_final(source, expected_sha256, hidden_traces, output, deadline): - """Separate post-submission grader; never return this data to the actor.""" - if type(source) is not bytes or hashlib.sha256(source).hexdigest() != expected_sha256: - raise ValueError("Final grader requires the exact immutable submitted source") - traces = _traces(hidden_traces, "grader") - try: - result = kv.verify_source(source, traces, output, deadline, _compare) - except kv.SandboxUnavailable: - result = {"verdict": "INCONCLUSIVE", "checks": {}, "reason": "sandbox_unavailable"} - return {**result, "artifact_sha256": expected_sha256, - "scope": "post-submission hidden correctness only; no search feedback"} diff --git a/src/inferrsi/harness/kv_reclamation.py b/src/inferrsi/harness/kv_reclamation.py deleted file mode 100644 index 261f5285c..000000000 --- a/src/inferrsi/harness/kv_reclamation.py +++ /dev/null @@ -1,214 +0,0 @@ -"""Bounded synthetic CPU artifact replay; no agent or model execution adapter. - -The oracle stays in the controller. Submitted code sees only inputs inside the -existing Linux bubblewrap boundary. Unavailable isolation has no host fallback. -""" - -import hashlib -import json -import os -import socket -import tempfile -import time -from pathlib import Path - -from ..infra.artifacts import atomic_write_json, write_json -from ..infra.executors.local import execute_command -from .skill_eval_engineering import SandboxUnavailable, read_regular, sandbox_command - -MAX_SOURCE_BYTES = 65536 - -# This driver contains no expected answers, verifier, knowledge or timing API. -# Timing is the existing host executor's elapsed time, not an artifact assertion. -DRIVER = '''import json, os, resource, socket, sys -resource.setrlimit(resource.RLIMIT_CPU, (16, 16)) -resource.setrlimit(resource.RLIMIT_AS, (268435456, 268435456)) -resource.setrlimit(resource.RLIMIT_FSIZE, (4194304, 4194304)) -resource.setrlimit(resource.RLIMIT_NOFILE, (32, 32)) -resource.setrlimit(resource.RLIMIT_NPROC, (0, 0)) -resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) -print("INFERRSI_KV_READY", file=sys.stderr, flush=True) -with open("/work/inputs.json", encoding="utf-8") as stream: - payload = json.load(stream) -if payload["mode"] == "probe": - inaccessible = [] - for path in payload["inaccessible"]: - try: - with open(path, "rb") as stream: - stream.read(1) - except OSError: - inaccessible.append(True) - else: - inaccessible.append(False) - try: - open("/work/write-probe", "w").close() - readonly = False - except OSError: - readonly = True - connection = socket.socket() - connection.settimeout(0.25) - try: - network_blocked = connection.connect_ex(("127.0.0.1", payload["host_port"])) != 0 - except OSError: - network_blocked = True - finally: - connection.close() - try: - child = os.fork() - except OSError: - fork_denied = True - else: - if child == 0: - os._exit(0) - os.waitpid(child, 0) - fork_denied = False - print(json.dumps({"inaccessible": inaccessible, "readonly": readonly, - "clean_env": "KV_QUALIFICATION_SECRET" not in os.environ, - "network_blocked": network_blocked, "fork_denied": fork_denied, - "limits": {name: list(resource.getrlimit(k)) for name, k in - (("memory", resource.RLIMIT_AS), ("processes", resource.RLIMIT_NPROC), - ("cpu", resource.RLIMIT_CPU), ("output", resource.RLIMIT_FSIZE), - ("files", resource.RLIMIT_NOFILE), ("core", resource.RLIMIT_CORE))}})) -else: - namespace = {"__name__": "submitted_candidate"} - with open("/work/candidate.py", encoding="utf-8") as stream: - exec(compile(stream.read(), "candidate.py", "exec"), namespace) - target = namespace["Reclaimer"]() - observations = [] - def pending(): - value = target.pending() - if (type(value) is not list or any(type(item) is not tuple or len(item) != 3 - or type(item[0]) is not str or type(item[1]) is not int - or type(item[2]) is not tuple - or any(type(block) is not int for block in item[2]) for item in value)): - raise TypeError("pending shape") - return list(value) - for event in payload["events"]: - try: - result = target.apply(event) - except ValueError: - observation = {"error": "ValueError"} - else: - if type(result) is not list or any(type(block) is not int for block in result): - raise TypeError("reclaimed shape") - observation = {"reclaimed": list(result)} - if payload["mode"] == "verify": - observation["pending"] = pending() - observations.append(observation) - print(json.dumps({"observations": observations, "pending": pending()}, allow_nan=False)) -''' - - -def source_bytes(path: Path) -> bytes: - source = read_regular(path, MAX_SOURCE_BYTES) - if not source: - raise ValueError("Candidate source is empty") - source.decode("utf-8") - return source - - -def workspace_hashes(workspace: Path) -> dict: - """Only candidate.py may differ; reject symlinks, special and extra nodes.""" - hashes = {} - for path in sorted(workspace.rglob("*")): - if path.is_symlink() or not (path.is_dir() or path.is_file()): - raise ValueError("Task workspace contains unsafe nodes") - if path.is_file(): - hashes[path.relative_to(workspace).as_posix()] = hashlib.sha256( - read_regular(path, 16 * 1024 * 1024)).hexdigest() - return hashes - - -def capture_artifact(workspace: Path, frozen: dict) -> bytes: - current = workspace_hashes(workspace) - if set(current) != set(frozen) or any(current[name] != value - for name, value in frozen.items() - if name != "candidate.py"): - raise ValueError("Protected paths changed or artifact missing") - return source_bytes(workspace / "candidate.py") - - -def invoke(source: bytes, payload: dict, output: Path, deadline: float) -> tuple[dict, dict | None]: - """One resource-bounded process; output/answer checking belongs to the parent.""" - if type(source) is not bytes or not 0 < len(source) <= MAX_SOURCE_BYTES: - raise ValueError("Candidate source must be bounded nonempty bytes") - source.decode("utf-8") - with tempfile.TemporaryDirectory(prefix="kv-replay-") as temporary: - work = Path(temporary) - (work / "candidate.py").write_bytes(source) - (work / "runner.py").write_text(DRIVER, encoding="utf-8", newline="\n") - write_json(work / "inputs.json", payload) - environment = {"PATH": "/usr/bin:/bin", "LANG": "C"} - if payload.get("mode") == "probe": - environment["KV_QUALIFICATION_SECRET"] = "synthetic-not-a-secret" - record = execute_command( - tuple(sandbox_command(work)), work, output, deadline, - environment=environment, - ) - # Retain the original outcome before readiness/output checks can return or raise. - # Closed projection only: no host paths, arbitrary error text or environment. - atomic_write_json(output / "execution.json", { - name: record.get(name) for name in ("status", "returncode", "elapsed_seconds") - }) - if record["status"] in {"timeout", "error", "cancelled"}: - return record, None - marker = Path(record["stderr_path"]).read_bytes()[:128] - if not marker.startswith(b"INFERRSI_KV_READY\n"): - raise SandboxUnavailable("KV replay did not establish its sandbox boundary") - if record["status"] != "completed": - return record, None - try: - data = read_regular(Path(record["stdout_path"]), 4 * 1024 * 1024) - return record, json.loads(data) - except (ValueError, UnicodeError, RecursionError): - return record, None - - -def process_group_gone(record: dict) -> bool: - group = record.get("process_group_id") - if os.name == "nt" or group is None: - return False - try: - os.killpg(group, 0) - except ProcessLookupError: - return True - return False - - -def probe_isolation(output: Path, deadline: float, forbidden: list[str]) -> dict: - forbidden = [*forbidden, f"/proc/{os.getpid()}/cmdline"] - with socket.socket() as listener: - listener.bind(("127.0.0.1", 0)) - listener.listen(1) - record, value = invoke(b"# no submitted code\n", { - "mode": "probe", "inaccessible": forbidden, - "host_port": listener.getsockname()[1], - }, output, deadline) - if (record["status"] != "completed" or not isinstance(value, dict) - or value.get("inaccessible") != [True] * len(forbidden) - or value.get("readonly") is not True or value.get("clean_env") is not True - or value.get("network_blocked") is not True or value.get("fork_denied") is not True - or value.get("limits") != {"memory": [268435456, 268435456], - "processes": [0, 0], "cpu": [16, 16], - "output": [4194304, 4194304], "files": [32, 32], - "core": [0, 0]} - or not process_group_gone(record)): - raise SandboxUnavailable("KV isolation probes failed; no host fallback") - value["normal_exit_process_group_gone"] = True - return value - - -def verify_source(source: bytes, traces: list[dict], output: Path, deadline: float, - compare) -> dict: - """Compare every event and pending state using an independently frozen oracle.""" - checks = {} - for index, trace in enumerate(traces): - if time.monotonic() >= deadline: - return {"verdict": "INCONCLUSIVE", "checks": checks, "reason": "deadline"} - record, value = invoke(source, {"mode": "verify", "events": trace["events"]}, - output / f"trace-{index:03d}", deadline) - if record["status"] in {"error", "timeout", "cancelled"}: - return {"verdict": "INCONCLUSIVE", "checks": checks, - "reason": record["status"]} - checks[trace["id"]] = compare(trace, value) - return {"verdict": "PASS" if all(checks.values()) else "FAIL", "checks": checks} diff --git a/tests/fixtures/kv_reclamation/candidate.py b/tests/fixtures/kv_reclamation/candidate.py deleted file mode 100644 index 467a8425b..000000000 --- a/tests/fixtures/kv_reclamation/candidate.py +++ /dev/null @@ -1,69 +0,0 @@ -"""Synthetic CPU task starter: a deliberately repeated linear scan, not vLLM.""" - - -class Reclaimer: - def __init__(self): - self._frontier = -1 - self._leases = [] - self._generations = {} - self._blocks = set() - - def apply(self, event): - if type(event) is not dict or type(event.get("op")) is not str: - raise ValueError("event must have an operation") - op = event["op"] - fields = { - "allocate": {"op", "request_id", "generation", "blocks", "write_step", - "save_required"}, - "finish": {"op", "request_id", "generation"}, - "save_complete": {"op", "request_id", "generation"}, - "advance": {"op", "processed_step"}, - } - if op not in fields or set(event) != fields[op]: - raise ValueError("unexpected operation or fields") - integers = ("processed_step",) if op == "advance" else ("generation",) - if op == "allocate": - integers += ("write_step",) - if any(type(event[key]) is not int or event[key] < 0 for key in integers): - raise ValueError("integer fields must be nonnegative integers, excluding bool") - if op != "advance" and type(event["request_id"]) is not str: - raise ValueError("request_id must be a string") - if op == "allocate": - request, generation, blocks = event["request_id"], event["generation"], event["blocks"] - if (type(blocks) is not list or not blocks - or any(type(block) is not int or block < 0 for block in blocks) - or len(set(blocks)) != len(blocks) - or type(event["save_required"]) is not bool): - raise ValueError("invalid allocation fields") - if (generation <= self._generations.get(request, -1) - or any(lease["request_id"] == request for lease in self._leases) - or self._blocks.intersection(blocks)): - raise ValueError("allocation conflicts with lifetime history") - self._leases.append({**event, "blocks": tuple(blocks), "finished": False, - "saved": False}) - self._generations[request] = generation - self._blocks.update(blocks) - elif op == "advance": - if event["processed_step"] < self._frontier: - raise ValueError("processed frontier cannot decrease") - self._frontier = event["processed_step"] - else: - for lease in self._leases: - if (lease["request_id"], lease["generation"]) == ( - event["request_id"], event["generation"]): - lease["finished" if op == "finish" else "saved"] = True - break - reclaimed, remaining = [], [] - for lease in self._leases: - if (lease["finished"] and (not lease["save_required"] or lease["saved"]) - and lease["write_step"] <= self._frontier): - reclaimed.extend(lease["blocks"]) - self._blocks.difference_update(lease["blocks"]) - else: - remaining.append(lease) - self._leases = remaining - return reclaimed - - def pending(self): - return [(lease["request_id"], lease["generation"], lease["blocks"]) - for lease in self._leases] diff --git a/tests/fixtures/kv_reclamation/reference.py b/tests/fixtures/kv_reclamation/reference.py deleted file mode 100644 index e529ff9e6..000000000 --- a/tests/fixtures/kv_reclamation/reference.py +++ /dev/null @@ -1,78 +0,0 @@ -"""Trusted authored headroom control, created after the independent oracle freeze. - -Never expose this implementation to a trial actor. It is not a model result. -""" - -import heapq - - -class Reclaimer: - def __init__(self): - self._frontier = -1 - self._leases = {} - self._generations = {} - self._blocks = set() - self._ready = [] - self._sequence = 0 - - def apply(self, event): - if type(event) is not dict or type(event.get("op")) is not str: - raise ValueError("event must have an operation") - op = event["op"] - fields = { - "allocate": {"op", "request_id", "generation", "blocks", "write_step", - "save_required"}, - "finish": {"op", "request_id", "generation"}, - "save_complete": {"op", "request_id", "generation"}, - "advance": {"op", "processed_step"}, - } - if op not in fields or set(event) != fields[op]: - raise ValueError("unexpected operation or fields") - integers = ("processed_step",) if op == "advance" else ("generation",) - if op == "allocate": - integers += ("write_step",) - if any(type(event[key]) is not int or event[key] < 0 for key in integers): - raise ValueError("integer fields must be nonnegative integers, excluding bool") - if op != "advance" and type(event["request_id"]) is not str: - raise ValueError("request_id must be a string") - if op == "allocate": - request, generation, blocks = event["request_id"], event["generation"], event["blocks"] - if (type(blocks) is not list or not blocks - or any(type(block) is not int or block < 0 for block in blocks) - or len(set(blocks)) != len(blocks) - or type(event["save_required"]) is not bool): - raise ValueError("invalid allocation fields") - if (generation <= self._generations.get(request, -1) - or request in self._leases or self._blocks.intersection(blocks)): - raise ValueError("allocation conflicts with lifetime history") - self._leases[request] = {**event, "blocks": tuple(blocks), "finished": False, - "saved": False, "queued": False, "sequence": self._sequence} - self._sequence += 1 - self._generations[request] = generation - self._blocks.update(blocks) - elif op == "advance": - if event["processed_step"] < self._frontier: - raise ValueError("processed frontier cannot decrease") - self._frontier = event["processed_step"] - else: - lease = self._leases.get(event["request_id"]) - if lease is not None and lease["generation"] == event["generation"]: - lease["finished" if op == "finish" else "saved"] = True - if (lease["finished"] and (not lease["save_required"] or lease["saved"]) - and not lease["queued"]): - heapq.heappush(self._ready, (lease["write_step"], lease["sequence"], - lease["request_id"])) - lease["queued"] = True - reclaimable = [] - while self._ready and self._ready[0][0] <= self._frontier: - _, sequence, request = heapq.heappop(self._ready) - reclaimable.append((sequence, self._leases.pop(request))) - reclaimed = [] - for _, lease in sorted(reclaimable): - reclaimed.extend(lease["blocks"]) - self._blocks.difference_update(lease["blocks"]) - return reclaimed - - def pending(self): - return [(lease["request_id"], lease["generation"], lease["blocks"]) - for lease in self._leases.values()] diff --git a/tests/fixtures/kv_reclamation/workload.py b/tests/fixtures/kv_reclamation/workload.py deleted file mode 100644 index fce20ce86..000000000 --- a/tests/fixtures/kv_reclamation/workload.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Protected deterministic input construction, independent of submitted code.""" - - -def workload(count): - if type(count) is not int or count < 1: - raise ValueError("count must be a positive integer") - events = [{"op": "allocate", "request_id": str(index), "generation": 0, - "blocks": [2 * index, 2 * index + 1], "write_step": index % 257, - "save_required": index % 3 != 0} for index in range(count)] - events += [{"op": "finish", "request_id": str(index), "generation": 0} - for index in reversed(range(count))] - events += [{"op": "save_complete", "request_id": str(index), "generation": 0} - for index in range(count)] - events += [{"op": "advance", "processed_step": frontier} for frontier in range(257)] - return events diff --git a/tests/test_kv_actor_cleanup.py b/tests/test_kv_actor_cleanup.py deleted file mode 100644 index 192d379e9..000000000 --- a/tests/test_kv_actor_cleanup.py +++ /dev/null @@ -1,163 +0,0 @@ -"""Observer and fail-closed controls; no submitted artifact executes on host.""" - -import json -import sys -import time - -import pytest - -from tools import qualify_kv_actor_cleanup as cleanup - - -def stat_file(root, pid, parent, start, state="S"): - path = root / str(pid) - path.mkdir() - fields = [state, str(parent)] + ["0"] * 17 + [str(start)] - (path / "stat").write_text(f"{pid} (trusted fixture) " + " ".join(fields)) - - -def test_observer_tracks_only_owned_tree_and_starttime_not_reused_pid(tmp_path): - stat_file(tmp_path, 10, 1, 100) - stat_file(tmp_path, 11, 10, 101) - stat_file(tmp_path, 12, 11, 102) - stat_file(tmp_path, 99, 1, 999) - owned, visible = cleanup._owned_snapshot(10, {}, proc=tmp_path) - assert visible and set(owned) == {11, 12} - # A formerly owned PID reused by an unrelated process is not a survivor. - owned, visible = cleanup._owned_snapshot(50, {99: 998}, proc=tmp_path) - assert visible and owned == {} - # An owned detached/reparented descendant remains tracked by PID+starttime. - owned, visible = cleanup._owned_snapshot(50, {12: 102}, proc=tmp_path) - assert visible and set(owned) == {12} - - -def test_unreadable_proc_is_not_empty_success(tmp_path): - assert cleanup._owned_snapshot(1, {}, proc=tmp_path / "missing") == ({}, False) - (tmp_path / "12").mkdir() - (tmp_path / "12/stat").write_text("malformed") - assert cleanup._owned_snapshot(1, {}, proc=tmp_path)[1] is False - - -def test_no_sandbox_never_runs_probe_on_host(tmp_path, monkeypatch): - monkeypatch.setattr(cleanup, "sandbox_command", lambda *_: (_ for _ in ()).throw( - cleanup.SandboxUnavailable("blocked"))) - monkeypatch.setattr(cleanup, "execute_command", - lambda *_args, **_kwargs: pytest.fail("no fallback")) - result = cleanup.probe_worker("normal", tmp_path / "out", time.monotonic() + 1) - assert result["verdict"] == "INCONCLUSIVE" - assert result["reason"] == "sandbox_unavailable" - - -def test_qualification_requires_all_three_modes_and_readable_worker_evidence(tmp_path, monkeypatch): - calls = [] - def execute(command, cwd, output, deadline, **kwargs): - calls.append(command[4]) - assert command[1] == "-I" - assert kwargs["environment"] == {"PATH": "/usr/bin:/bin", "LANG": "C"} - destination = tmp_path / "out" / command[4] - destination.mkdir(parents=True) - (destination / "result.json").write_text(json.dumps({"verdict": "PASS"})) - return {"status": "completed"} - monkeypatch.setattr(cleanup, "execute_command", execute) - result = cleanup.qualify(tmp_path / "out", time.monotonic() + 10, 1) - assert calls == ["normal", "timeout", "cancel"] and result["verdict"] == "PASS" - - -def test_probe_is_authored_fork_setsid_only_not_candidate_or_provider(): - assert "os.fork()" in cleanup.PROBE and "os.setsid()" in cleanup.PROBE - assert "candidate" not in cleanup.PROBE and "exec(" not in cleanup.PROBE - assert "SKILL_EVAL" not in cleanup.PROBE - - -def test_nested_worker_rejects_other_source_before_execution(tmp_path, monkeypatch): - (tmp_path / "runner.py").write_text("raise RuntimeError('not the trusted probe')\n") - monkeypatch.setattr(cleanup, "execute_command", lambda *a, **k: pytest.fail("must not run")) - with pytest.raises(ValueError, match="exact authored"): - cleanup.nested_worker("normal", tmp_path, tmp_path / "out", time.monotonic() + 1) - - -def test_detached_marker_requires_exact_owned_host_identity(tmp_path): - stat_file(tmp_path, 10, 1, 100) - stat_file(tmp_path, 11, 10, 101) - stat_file(tmp_path, 12, 11, 102) - marker = {"role": "detached", "own_session": True, "namespace_pid": 3, - "start": 102, "pid_namespace": "pid:[1234]"} - owned, _ = cleanup._owned_snapshot(1, {}, proc=tmp_path) - # A marker plus two ancestors never substitutes for observing the detached PID. - assert cleanup._bind_detached([marker], {p: owned[p] for p in (10, 11)}, proc=tmp_path) is None - (tmp_path / "12/status").write_text("Name:\ttrusted\nNSpid:\t12\t3\n") - (tmp_path / "12/ns").mkdir() - (tmp_path / "12/ns/pid").symlink_to("pid:[1234]") - detached = cleanup._bind_detached([marker], owned, proc=tmp_path) - assert detached == {"pid": 12, "start": 102} - assert cleanup._bind_chain([{"role": "worker", "pid": 10, "start": 100}], - detached, owned)["chain_length"] == 3 - for altered in ({"start": 999}, {"namespace_pid": 4}, {"pid_namespace": "pid:[4321]"}): - assert cleanup._bind_detached([{**marker, **altered}], owned, proc=tmp_path) is None - assert cleanup._bind_chain([{"role": "worker", "pid": 11, "start": 101}], - detached, owned) is None - - -@pytest.mark.skipif(sys.platform != "linux", reason="requires Linux proc identity observation") -@pytest.mark.parametrize("mode", ["timeout", "cancel"]) -def test_outer_worker_loss_detects_and_recovers_independent_session_survivors( - tmp_path, monkeypatch, mode -): - # Authored host-only fixture intentionally lacks bubblewrap die-with-parent. - # It exercises negative observer/supervisor behavior, never candidate code or - # qualification success. The detached descendants must be detected and killed. - output = tmp_path / "out" - events = output / "sandbox/stdout.log" - events.parent.mkdir(parents=True) - detached = '''import json, os, time -with open("/proc/self/stat") as stream: - start = int(stream.read().rsplit(")", 1)[1].split()[19]) -print(json.dumps({"role": "detached", "own_session": os.getsid(0) == os.getpid(), - "namespace_pid": os.getpid(), "start": start, - "pid_namespace": os.readlink("/proc/self/ns/pid")}), flush=True) -print(json.dumps({"role": "parent"}), flush=True) -time.sleep(30) -''' - middle = f'''import subprocess, sys, time -with open({str(events)!r}, "w") as stream: - subprocess.Popen([sys.executable, "-c", {detached!r}], stdout=stream, start_new_session=True) - time.sleep(30) -''' - worker = f'''import json, os, subprocess, sys, time -with open("/proc/self/stat") as stream: - start = int(stream.read().rsplit(")", 1)[1].split()[19]) -print(json.dumps({{"role": "worker", "pid": os.getpid(), "start": start}}), flush=True) -subprocess.Popen([sys.executable, "-c", {middle!r}], start_new_session=True) -time.sleep(30) -''' - execute = cleanup.execute_command - - def fixture_execute(command, cwd, trial, deadline, **kwargs): - assert "--nested-worker" in command - return execute((sys.executable, "-c", worker), cwd, trial, deadline, **kwargs) - - monkeypatch.setattr(cleanup, "sandbox_command", lambda _: ("never-run",)) - monkeypatch.setattr(cleanup, "execute_command", fixture_execute) - # probe_worker owns creation, not the fixture's precreated output parent. - events.parent.rmdir() - output.rmdir() - original_events = cleanup._events - - def read_events(path): - events.parent.mkdir(parents=True, exist_ok=True) - return original_events(path) - - monkeypatch.setattr(cleanup, "_events", read_events) - result = cleanup.probe_worker(mode, output, time.monotonic() + 2) - assert result["verdict"] == "FAIL" - assert result["outer_worker_supervised"] and result["bound_chain"]["chain_length"] >= 3 - assert result["surviving_descendants"] > 0 and result["recovery_kills"] > 0 - assert result["recovery_survivors"] == 0 and result["recovery_visibility_complete"] - assert result["explicit_cancel_sent"] == (mode == "cancel") - for role in ("worker", "detached"): - identity = result["bound_chain"][role] - try: - current = cleanup._stat(cleanup.Path("/proc") / str(identity["pid"]) / "stat") - except FileNotFoundError: - continue - assert current["start"] != identity["start"] or current["state"] == "Z" diff --git a/tests/test_kv_edit_inputs.py b/tests/test_kv_edit_inputs.py deleted file mode 100644 index b595e3f37..000000000 --- a/tests/test_kv_edit_inputs.py +++ /dev/null @@ -1,49 +0,0 @@ -import json -from copy import deepcopy - -import pytest - -from tools import kv_edit_inputs as inputs - - -def test_common_inputs_and_prompt_are_shared_without_expected_or_hidden_data(): - development = [{"id": "visible", "split": "development", "events": [{}], - "expected": [{"trusted_expected_canary": True}]}] - public, prompt, suite = inputs.build_inputs( - b"# starter\n", b"# neutral contract\n", development, [], {"benchmark_canary": True}, 0 - ) - assert set(public) == inputs.COMMON - assert prompt == inputs.PROMPT - assert json.loads(public["development.json"])["traces"] == [{"id": "visible", "events": [{}]}] - assert all(b"trusted_expected_canary" not in data and b"benchmark_canary" not in data - for data in public.values()) - assert suite.read()["development"] == development - assert inputs.build_inputs(b"# starter\n", b"# neutral contract\n", development, - [], {"benchmark_canary": True}, 0)[:2] == (public, prompt) - - -def test_hidden_split_is_never_accepted_as_public_input(): - with pytest.raises(ValueError, match="split"): - inputs.build_inputs(b"# starter\n", b"# contract\n", - [{"id": "secret", "split": "grader", "events": [], "expected": []}], - [], {}, 0) - - -@pytest.mark.parametrize("field", ["arm", "suite_sha256", "baseline_sha256", "client", - "model", "input_token_ceiling", "prompt", "files", "limits", - "check_ids", "metric_key", "metric_unit", "direction"]) -def test_common_contract_binds_all_non_transport_actor_inputs(field): - config = {k: k for k in ("arm", "model", "client", "input_token_ceiling", "prompt", - "suite_sha256", "baseline_sha256")} - config["edit_plan"] = {k: k for k in ("files", "limits", "check_ids", "metric_key", - "metric_unit", "direction", "callbacks_sha256")} - original = inputs.common_actor_contract_sha256(config) - changed = deepcopy(config) - (changed["edit_plan"] if field in config["edit_plan"] else changed)[field] = "changed" - assert inputs.common_actor_contract_sha256(changed) != original - scripted = deepcopy(config) - scripted["edit_plan"]["callbacks_sha256"] = "different-response-fixture-binding" - scripted["responses_sha256"] = "scripted-only" - scripted["plan_sha256"] = "separate-campaign" - scripted["actor_sha256"] = "different-transport-identity" - assert inputs.common_actor_contract_sha256(scripted) == original diff --git a/tests/test_kv_edit_pilot.py b/tests/test_kv_edit_pilot.py deleted file mode 100644 index ec7d82169..000000000 --- a/tests/test_kv_edit_pilot.py +++ /dev/null @@ -1,182 +0,0 @@ -"""Supervised scripted worker and fake-store admission: no model/provider calls.""" - -import json -import sys -import time -from datetime import timedelta -from pathlib import Path - -import pytest - -from inferrsi.harness import kv_reclamation as kv -from inferrsi.infra.executors.local import execute_command -from inferrsi.model import anthropic_agent as provider -from tests.test_anthropic_edit_agent import read_task, submit, work -from tests.test_kv_edit_workbench import BASELINE, CANDIDATE, fake_sandbox, stage, suite -from tests.test_kv_pilot_admission import World, policy -from tools import kv_edit_pilot as pilot -from tools import kv_pilot_admission as admission - - -def prepare(tmp_path, arm="A2", scripts=None): - parent = tmp_path / arm - parent.mkdir() - root, plan = stage(parent) - spec = {"kind": provider.EDIT_KIND, "provider": "yibuapi", "endpoint": provider.ENDPOINT, - "api_version": "2023-06-01", "max_requests": 8, "max_output_tokens": 4096, - "max_request_bytes": 262144, "max_read_bytes": 32768, "max_tool_calls": 32} - config = pilot.prepare_scripted(parent / "inputs", root, plan, suite(), BASELINE, - scripts or [read_task(), work(CANDIDATE), submit()], plan_sha256="1" * 64, - execution_id="example-run-1.1", arm=arm, model="claude-opus-5-5", spec=spec, - prompt="Optimize only the public task.", input_token_ceiling=1000000) - return parent / "inputs", root, config - - -def in_process_worker(command, cwd, trial_dir, deadline, *, environment): - """Test-only scripted executor: does not certify the subprocess/isolation boundary.""" - assert provider.KEY_ENV not in environment and "GITHUB_TOKEN" not in environment - assert "-I" in command and command[0] == sys.executable - assert "--offline-worker" in command - args = dict(zip(command[4::2], command[5::2])) - assert float(args["--deadline"]) == deadline - trial_dir.mkdir(parents=True) - with (trial_dir / "stdout.log").open("w") as stream: - code = pilot.scripted_worker(Path(args["--inputs"]), Path(args["--workspace"]), - Path(args["--output"]), deadline, environment=environment, - emit=lambda e: stream.write(provider.canonical(e) + "\n")) - (trial_dir / "stderr.log").write_text("") - return {"status": "completed" if code == 0 else "failed", "returncode": code, - "stdout_path": str(trial_dir / "stdout.log"), - "stderr_path": str(trial_dir / "stderr.log"), "elapsed_seconds": 0.01} - - -def fake_isolation(output, deadline, forbidden): - assert deadline > time.monotonic() and len(forbidden) == 4 - return {"scope": "scripted unit-test output; not isolation qualification"} - - -def test_supervised_scripted_pair_with_existing_claims_and_usage_bridge(tmp_path, monkeypatch): - a, b = prepare(tmp_path, "A2"), prepare(tmp_path, "A3") - monkeypatch.setattr(kv, "invoke", fake_sandbox) - monkeypatch.setattr(kv, "probe_isolation", fake_isolation) - world = World() - config = policy(actor_sha256=(a[2]["actor_sha256"], b[2]["actor_sha256"])) - session = admission.admit(config, world.daily, world.pr, check_current=lambda _: True, - wall_clock=lambda: world.now, monotonic_clock=time.monotonic) - assert len(world.events) == 4 # Whole plan reserved/claimed before either disclosure. - for inputs, root, _ in (a, b): - result = pilot.run_supervised(inputs, root, inputs.parent / "run", - time.monotonic() + 600, admission=session, execute=in_process_worker) - assert result["worker"]["status"] == "completed" - assert result["simulated_slots_consumed"] == 3 - assert result["request_completion_known"] - assert result["model_calls"] == 0 and result["provider_charges_usd"] == 0 - final = inputs.parent / "run/worker/final/candidate.py" - assert final.read_text() == CANDIDATE - assert world.pr.ledger.charged == 162720000 # No refund from simulated usage. - assert world.daily.ledger.charged_on("2026-10-30") == 654070000 - with pytest.raises(Exception): - session.start_arm("A2", time.monotonic() + 600) - session.close() - - -def test_unknown_usage_blocks_other_arm_and_retains_full_reservation(tmp_path, monkeypatch): - a = prepare(tmp_path, scripts=[{**read_task(), "usage": None}]) - b = prepare(tmp_path, "A3") - monkeypatch.setattr(kv, "invoke", fake_sandbox) - monkeypatch.setattr(kv, "probe_isolation", fake_isolation) - world = World() - session = admission.admit(policy(actor_sha256=(a[2]["actor_sha256"], b[2]["actor_sha256"])), - world.daily, world.pr, check_current=lambda _: True, wall_clock=lambda: world.now, - monotonic_clock=time.monotonic) - result = pilot.run_supervised(a[0], a[1], a[0].parent / "run", time.monotonic() + 600, - admission=session, execute=in_process_worker) - assert result["worker"]["status"] == "failed" - assert not result["request_completion_known"] and result["simulated_slots_consumed"] == 1 - with pytest.raises(Exception): - pilot.run_supervised(b[0], b[1], b[0].parent / "run", time.monotonic() + 600, - admission=session, execute=in_process_worker) - assert world.pr.ledger.charged == 162720000 - - -def test_missing_completion_after_grant_is_not_recovered_or_refunded(tmp_path, monkeypatch): - a = prepare(tmp_path) - monkeypatch.setattr(kv, "invoke", fake_sandbox) - monkeypatch.setattr(kv, "probe_isolation", fake_isolation) - original = pilot.atomic_write_json - def lose_completion(path, value): - if Path(path).name == "complete.json": - raise OSError("simulated loss") - return original(path, value) - monkeypatch.setattr(pilot, "atomic_write_json", lose_completion) - result = pilot.run_supervised(a[0], a[1], a[0].parent / "run", time.monotonic() + 600, - execute=in_process_worker) - assert result["simulated_slots_consumed"] == 1 - assert not result["request_completion_known"] - assert result["worker"]["reason"] == "provider_transport_error" - - -@pytest.mark.parametrize("name", ["suite.json", "baseline.py", "responses.json"]) -def test_changed_private_frozen_inputs_fail_before_worker_or_slots(tmp_path, name): - inputs, root, _ = prepare(tmp_path) - (inputs / name).write_text("changed") - with pytest.raises(ValueError): - pilot.run_supervised(inputs, root, inputs.parent / "run", time.monotonic() + 600, - execute=lambda *_args, **_kwargs: pytest.fail("must not launch")) - - -def test_changed_prompt_or_dependencies_fail_before_launch(tmp_path): - inputs, root, _ = prepare(tmp_path) - path = inputs / "worker.json" - config = json.loads(path.read_text()) - config["prompt"] = "changed prompt" - path.write_text(json.dumps(config)) - with pytest.raises(ValueError, match="binding changed"): - pilot.run_supervised(inputs, root, inputs.parent / "run", time.monotonic() + 600) - - -def test_sandbox_unavailable_precedes_every_simulated_request(tmp_path, monkeypatch): - inputs, root, _ = prepare(tmp_path) - def unavailable(*_): - raise kv.SandboxUnavailable("sandbox unavailable") - monkeypatch.setattr(kv, "probe_isolation", unavailable) - result = pilot.run_supervised(inputs, root, inputs.parent / "run", time.monotonic() + 30, - execute=in_process_worker) - assert result["worker"]["status"] == "inconclusive" - assert result["worker"]["reason"] == "sandbox_unavailable" - assert result["simulated_slots_consumed"] == 0 - - -def test_existing_executor_really_kills_blocked_trusted_worker_at_inherited_deadline(tmp_path): - inputs, root, _ = prepare(tmp_path) - def blocking_executor(command, cwd, output, deadline, *, environment): - # Trusted sleep fixture, NOT submitted Python or a candidate host fallback. - return execute_command((sys.executable, "-I", "-c", "import time; time.sleep(30)"), - cwd, output, deadline, environment=environment) - started = time.monotonic() - result = pilot.run_supervised(inputs, root, inputs.parent / "run", started + 0.2, - execute=blocking_executor) - assert result["execution"]["status"] == "timeout" - assert result["execution"]["elapsed_seconds"] < 2 - assert kv.process_group_gone(result["execution"]) - assert result["worker"]["status"] == "missing" - - -def test_shanghai_day_change_blocks_bridge_next_grant(tmp_path, monkeypatch): - a = prepare(tmp_path) - monkeypatch.setattr(kv, "invoke", fake_sandbox) - monkeypatch.setattr(kv, "probe_isolation", fake_isolation) - world = World() - session = admission.admit(policy(actor_sha256=(a[2]["actor_sha256"], "5" * 64)), - world.daily, world.pr, check_current=lambda _: True, wall_clock=lambda: world.now, - monotonic_clock=time.monotonic) - original = session.complete - def midnight(grant, usage): - original(grant, usage) - world.now += timedelta(hours=1) - session.complete = midnight - result = pilot.run_supervised(a[0], a[1], a[0].parent / "run", time.monotonic() + 600, - admission=session, execute=in_process_worker) - assert result["worker"]["status"] == "failed" - assert result["worker"]["reason"] == "request_authorization_denied" - assert world.pr.ledger.charged == 162720000 diff --git a/tests/test_kv_edit_workbench.py b/tests/test_kv_edit_workbench.py deleted file mode 100644 index 3017cfbbf..000000000 --- a/tests/test_kv_edit_workbench.py +++ /dev/null @@ -1,166 +0,0 @@ -"""Fixed workbench integration with scripted sandbox outputs; never host candidate exec.""" - -import copy -import json -import time -from pathlib import Path - -import pytest - -from inferrsi.harness import kv_edit_workbench as workbench -from inferrsi.harness import kv_reclamation as kv -from inferrsi.model import anthropic_edit_agent as edit -from tests.test_anthropic_edit_agent import call, read_task, response, submit, work - -BASELINE = b"# visible linear starter\n" -CANDIDATE = "raise AssertionError('must never run on host')\n" -TRACE = {"id": "dev_empty", "split": "development", "events": [], "expected": []} -EXPECTED = {"observations": [], "pending": []} - - -def suite(): - return workbench.VisibleSuite(json.dumps({ - "schema": "kv-visible-suite-v1", "development": [TRACE], "benchmark_events": [], - "benchmark_expected": EXPECTED, "min_improvement_pct": 0, - }).encode()) - - -def stage(tmp_path): - root = tmp_path / "workspace" - root.mkdir() - (root / "candidate.py").write_bytes(BASELINE) - (root / "task.json").write_text('{"request":"optimize"}') - plan = edit.EditPlan(edit.freeze_files(root, ["task.json", "candidate.py"]), - edit.EditLimits(65536, 4096, 8, 8, 4, 2, 600), ("dev_empty",), - "elapsed_seconds", "seconds", "minimize", "b" * 64) - return root, plan - - -def fake_sandbox(source, payload, output, deadline): - assert type(source) is bytes and deadline > time.monotonic() - assert set(payload) == {"mode", "events"} # Never expected answers or hidden data. - output.mkdir(parents=True) - return {"status": "completed", "elapsed_seconds": 2.0 if source == BASELINE else 1.0, - "metric": None, "metric_error": None}, copy.deepcopy(EXPECTED) - - -def test_scripted_edit_check_abba_rewrite_submit_hidden_grade(tmp_path, monkeypatch): - root, plan = stage(tmp_path) - monkeypatch.setattr(kv, "invoke", fake_sandbox) - bench = workbench.VisibleWorkbench(plan, suite(), BASELINE, tmp_path / "private") - scripts = [read_task(), work(CANDIDATE, "1"), work(CANDIDATE + "# final\n", "2"), submit()] - payloads, traces = [], [] - def send(payload, spec, remaining): - payloads.append(copy.deepcopy(payload)) - return scripts[len(payloads) - 1] - spec = {"kind": edit.KIND, "provider": "yibuapi", "endpoint": edit.provider.ENDPOINT, - "api_version": "2023-06-01", "max_requests": 8, "max_output_tokens": 4096, - "max_request_bytes": 1048576, "max_read_bytes": 65536, "max_tool_calls": 24} - answer = edit.run_edit_agent(root, plan, "Optimize", "claude-opus-5-5", spec, - time.monotonic() + 600, traces.append, arm="A2", - slots=edit.RequestSlots("c" * 64, {"A2": 8, "A3": 8}, 10170000, 162720000, - execution_id="offline-1"), - authorize=lambda *_: True, send=send, visible_checks=bench.visible_checks, - visible_benchmark=bench.visible_benchmark, retain_revision=bench.retain_revision, - input_token_ceiling=1000000) - assert answer["artifact_source"] == CANDIDATE + "# final\n" - assert len(list((tmp_path / "private/revisions").glob("*/candidate.py"))) == 3 - runs = sorted((tmp_path / "private").glob("benchmark-*/runs/*")) - assert len(runs) == 2 - for run in runs: - events = [json.loads(line) for line in (run / "events.jsonl").read_text().splitlines()] - assert [e["side"] for e in events if e["phase"] == "measurement"] == [ - "baseline", "candidate", "candidate", "baseline"] - evaluation = json.loads((run / "evaluation.json").read_text()) - assert evaluation["source_integrity"]["verified"] - assert evaluation["improvement_pct"] == 50 - # Freeze actual returned bytes before the separate hidden grader; changing the - # live actor path afterward must neither change grading input nor cause a retry. - (root / "candidate.py").write_text("# later unrelated mutation\n") - hidden = [{**TRACE, "split": "grader", "id": "hidden-id-not-for-actor"}] - result = workbench.grade_final(answer["artifact_source"].encode(), answer["sha256"], hidden, - tmp_path / "grading", time.monotonic() + 30) - assert result["verdict"] == "PASS" - assert "hidden-id-not-for-actor" not in edit.provider.canonical(payloads) - assert "private" not in edit.provider.canonical(payloads) - - -def test_benchmark_validates_entire_output_before_accepting_host_metric(tmp_path, monkeypatch): - _, plan = stage(tmp_path) - def bad(source, payload, output, deadline): - record, _ = fake_sandbox(source, payload, output, deadline) - return record, {"observations": [], "pending": [], "fake_latency": 0.00001} - monkeypatch.setattr(kv, "invoke", bad) - bench = workbench.VisibleWorkbench(plan, suite(), BASELINE, tmp_path / "private") - result = bench.visible_benchmark(CANDIDATE.encode(), time.monotonic() + 30) - assert result["verdict"] == "INCONCLUSIVE" - assert result["baseline_values"] == [] and result["candidate_values"] == [] - - -def test_same_absolute_deadline_reaches_every_sandbox_call(tmp_path, monkeypatch): - _, plan = stage(tmp_path) - seen = [] - def invoke(source, payload, output, deadline): - seen.append(deadline) - return fake_sandbox(source, payload, output, deadline) - monkeypatch.setattr(kv, "invoke", invoke) - bench = workbench.VisibleWorkbench(plan, suite(), BASELINE, tmp_path / "private") - deadline = time.monotonic() + 30 - bench.visible_checks(CANDIDATE.encode(), deadline) - bench.visible_benchmark(CANDIDATE.encode(), deadline) - assert len(seen) == 6 and all(value <= deadline for value in seen) - - -def test_unavailable_sandbox_is_inconclusive_without_fallback(tmp_path, monkeypatch): - _, plan = stage(tmp_path) - def unavailable(*_): - raise kv.SandboxUnavailable("local permission denied") - monkeypatch.setattr(kv, "invoke", unavailable) - bench = workbench.VisibleWorkbench(plan, suite(), BASELINE, tmp_path / "private") - for callback in (bench.visible_checks, bench.visible_benchmark): - result = callback(CANDIDATE.encode(), time.monotonic() + 30) - assert result["verdict"] == "INCONCLUSIVE" - assert result["reason"] == "sandbox_unavailable" - assert "permission" not in json.dumps(result) - - -def test_hidden_data_never_accepted_as_visible_suite(): - value = suite().read() - value["development"][0]["split"] = "grader" - with pytest.raises(ValueError): - workbench.VisibleSuite(json.dumps(value).encode()) - - -def test_hidden_grade_rejects_wrong_final_hash_before_execution(tmp_path, monkeypatch): - monkeypatch.setattr(kv, "invoke", lambda *_: pytest.fail("must not execute")) - with pytest.raises(ValueError, match="exact immutable"): - workbench.grade_final(BASELINE, "a" * 64, [], tmp_path, time.monotonic() + 1) - - -def test_retention_is_exclusive_and_exact(tmp_path): - _, plan = stage(tmp_path) - bench = workbench.VisibleWorkbench(plan, suite(), BASELINE, tmp_path / "private") - metadata = {"revision": 0, "sha256": edit.digest(BASELINE)} - assert bench.retain_revision(BASELINE, metadata, time.monotonic() + 30) - with pytest.raises(FileExistsError): - bench.retain_revision(BASELINE, metadata, time.monotonic() + 30) - with pytest.raises(ValueError): - bench.retain_revision(BASELINE, {"revision": 1, "sha256": "a" * 64}, - time.monotonic() + 30) - - -def test_does_not_call_candidate_import_or_trusted_observation_helpers(): - source = Path(workbench.__file__).read_text() - assert "exec(" not in source and "importlib" not in source and "observe_trusted" not in source - - -def test_malformed_tool_measurement_arguments_fail_before_callback(tmp_path, monkeypatch): - # Existing protocol strict tool validation is reused by the integration. - from tests.test_anthropic_edit_agent import run - root, plan = stage(tmp_path) - spec = {"kind": edit.KIND, "provider": "yibuapi", "endpoint": edit.provider.ENDPOINT, - "api_version": "2023-06-01", "max_requests": 8, "max_output_tokens": 4096, - "max_request_bytes": 1048576, "max_read_bytes": 65536, "max_tool_calls": 24} - with pytest.raises(edit.provider.AgentError, match="unsupported_tool"): - run((root, plan, spec), [read_task(), response(call("run_visible_benchmark", { - "command": "untrusted command", "workload": "hidden"}))]) diff --git a/tests/test_kv_live_job.py b/tests/test_kv_live_job.py deleted file mode 100644 index 778d5712e..000000000 --- a/tests/test_kv_live_job.py +++ /dev/null @@ -1,848 +0,0 @@ -"""Offline controller/workflow tests: synthetic identities, keys, stores and HTTP only.""" - -import copy -import json -import time -from datetime import datetime, timezone -from pathlib import Path -from types import SimpleNamespace -from unittest.mock import Mock - -import pytest -import yaml - -from inferrsi.harness import kv_reclamation as kv -from inferrsi.infra.artifacts import atomic_write_json -from inferrsi.model import anthropic_agent as provider -from inferrsi.model import anthropic_edit_agent as edit -from tests.test_anthropic_edit_agent import read_task, submit, work -from tests.test_kv_edit_workbench import BASELINE, CANDIDATE, TRACE, fake_sandbox, suite -from tests.test_kv_pilot_admission import World -from tests.test_kv_pilot_admission import policy as sample_policy -from tools import kv_live_job as job -from tools import kv_live_policy as policy -from tools import kv_live_worker as worker - - -def write(root, name, data): - if not isinstance(data, bytes): - data = provider.canonical(data).encode() - path = root / name - path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(data) - return {"path": name, "bytes": len(data), "sha256": policy.sha(data)} - - -@pytest.fixture -def staged(tmp_path, monkeypatch): - root, output = tmp_path / "checkout", tmp_path / "output" - root.mkdir() - output.mkdir() - monkeypatch.setattr(job, "ROOT", root) - monkeypatch.setattr(policy, "ROOT", root) - visible = suite().read() - common, prompt, _ = job.inputs.build_inputs(BASELINE, b"contract", visible["development"], - visible["benchmark_events"], visible["benchmark_expected"], visible["min_improvement_pct"]) - paths = {"candidate.py": "tests/fixtures/kv_reclamation/candidate.py", - "CONTRACT.md": "evidence/kv-reclamation-20261030/validator/independent/CONTRACT.md"} - sources = {name: (paths.get(name, "fixture/" + name), data) for name, data in common.items()} - sources.update({name: (job.KNOWLEDGE_ROOT + name[len("knowledge/"):], b"offline knowledge") - for name in job.KNOWLEDGE}) - records = {name: write(root, path, data) for name, (path, data) in sources.items()} - provider_config = json.loads(sample_policy().provider_json) - plan = {"actors": {}, "provider": provider_config, "execution_id": "offline-1", - "expires_at": int(time.time()) + 7200, "arm_order": ["A2", "A3"], - "suite": write(root, "fixture/suite.json", suite().serialized), - "final_suite": write(root, "fixture/final.json", suite().serialized), - "hidden_traces": write(root, "fixture/hidden.json", [{**TRACE, "split": "grader"}]), - "charge_pr": 999, "charge_head": "2" * 40, - "final_seconds": 180, "final_repetitions": 8} - configs = {} - for arm in ("A2", "A3"): - names = job.COMMON | (job.KNOWLEDGE if arm == "A3" else set()) - workspace = tmp_path / f"freeze-{arm}" - workspace.mkdir() - for name in names: - write(workspace, name, sources[name][1]) - ep = edit.EditPlan(edit.freeze_files(workspace, names), - edit.EditLimits(65536, 8192, 8, 8, 4, 2, 600), (TRACE["id"],), - "elapsed_seconds", "seconds", "minimize", "0" * 64) - configs[arm] = worker.configuration(workspace, ep, suite(), BASELINE, arm=arm, - model="claude-opus-5-5", spec=provider_config["client"], prompt=prompt) - plan["actors"][arm] = {"config": write(root, f"fixture/{arm}.json", configs[arm]), - "workspace": [{"path": n, "source": records[n]} - for n in sorted(names)]} - job._stage(plan, output) - atomic_write_json(output / "authorization.json", {"body": "offline frozen public fixture"}) - authorization = {"plan": plan, "plan_sha256": policy.fingerprint(plan)} - locator = {"offline": True} - prepared = {"schema": "kv-live-prepared-v1", "implementation_sha": "a" * 40, - "run_id": "123", "plan_sha256": authorization["plan_sha256"], - "locator_sha256": policy.fingerprint(locator), "cleanup": {"verdict": "PASS"}, - "runtime_profile": {"verdict": "PASS"}, - "replay": {a: {"verdict": "PASS", "model_calls": 0} for a in ("A2", "A3")}, - "inputs": kv.workspace_hashes(output)} - atomic_write_json(output / "prepared.json", prepared) - monkeypatch.setattr(policy, "frozen_authorization", lambda **_: (authorization, locator)) - monkeypatch.setattr(policy, "runtime", lambda: ("a" * 40, "123")) - monkeypatch.setattr(policy, "verify_live", lambda *_: None) - monkeypatch.setattr(job, "runway", lambda *_: None) - monkeypatch.setenv(provider.KEY_ENV, "OFFLINE-FAKE-PROVIDER-KEY") - monkeypatch.setenv("GITHUB_RUN_ID", "123") - monkeypatch.setenv("GITHUB_SHA", "a" * 40) - monkeypatch.delenv("GITHUB_TOKEN", raising=False) - world = World() - root_policy = sample_policy( - actor_sha256=tuple(configs[a]["actor_sha256"] for a in ("A2", "A3")), - plan_sha256=authorization["plan_sha256"], execution_id=plan["execution_id"]) - monkeypatch.setattr(policy, "policy", lambda *_: root_policy) - original_admit = job.admission.admit - monkeypatch.setattr(job.admission, "admit", lambda *args, **kwargs: - original_admit(*args, **({"wall_clock": lambda: world.now} | kwargs))) - monkeypatch.setattr(kv, "invoke", fake_sandbox) - monkeypatch.setattr(kv, "probe_isolation", lambda *_: {"offline": True}) - return SimpleNamespace(root=root, output=output, plan=plan, world=world, - configs=configs, authorization=authorization) - - -def launcher(scripts, seen, *, transport=None): - def execute(command, cwd, output, deadline, *, environment): - assert environment[provider.KEY_ENV] == "OFFLINE-FAKE-PROVIDER-KEY" - forbidden = {"GITHUB_TOKEN", "ACTIONS_RUNTIME_TOKEN", "HOME", "PYTHONPATH"} - assert not forbidden & set(environment) - args = dict(zip(command[3::2], command[4::2])) - assert float(args["--deadline"]) == deadline - arm = Path(args["--inputs"]).parent.name - seen.append(arm) - responses = iter(copy.deepcopy(scripts)) - code = worker.live_worker(args["--inputs"], args["--workspace"], args["--output"], - deadline, environment=environment, emit=lambda _: None, - transport=transport or (lambda *a, **kw: next(responses))) - return {"status": "completed" if code == 0 else "failed", "returncode": code} - - return lambda *args, **kwargs: worker.run_supervised(*args, **kwargs, execute=execute) - - -def test_fake_pair_reserves_before_disclosure_and_grades_immutable_bytes(staged, monkeypatch): - seen = [] - scripts = [read_task(), work(CANDIDATE), work(CANDIDATE + "# final\n", "2"), submit()] - launch = launcher(scripts, seen) - def checked_launch(*args, **kwargs): - assert len(staged.world.events) == 4 - return launch(*args, **kwargs) - result = job.execute(staged.output, api=SimpleNamespace(), - stores=(staged.world.daily, staged.world.pr), launch=checked_launch) - assert seen == ["A2", "A3"] - assert all(a["status"] == "completed" for a in result["arms"].values()) - assert staged.world.pr.ledger.charged == 162720000 - assert staged.world.daily.ledger.charged_on("2026-10-30") == 654070000 - summary = job.package(staged.output) - assert all(len(r) == 4 for r in summary["requests"].values()) - assert all(a["status"] == "submitted" for a in summary["artifacts"].values()) - assert all(len(v["revisions"]) == 3 and len(v["feedback"]) == 4 - for v in summary["evidence"].values()) - for arm in seen: - (staged.output / arm / "workspace/candidate.py").write_text("changed after submission") - monkeypatch.delenv(provider.KEY_ENV) - graded = job.grade(staged.output / "public", staged.output / "grade") - assert all(a["semantic"]["verdict"] == "PASS" for a in graded["arms"].values()) - public = b"".join(p.read_bytes() for p in (staged.output / "public").rglob("*") if p.is_file()) - assert b"OFFLINE-FAKE-PROVIDER-KEY" not in public - - -def test_unknown_usage_retains_attempt_and_does_not_disclose_other_arm(staged): - seen = [] - result = job.execute(staged.output, api=SimpleNamespace(), - stores=(staged.world.daily, staged.world.pr), - launch=launcher([{**read_task(), "usage": None}], seen)) - assert seen == ["A2"] - assert result["arms"]["A3"]["status"] == "not_started" - summary = job.package(staged.output) - assert summary["requests"]["A2"][0]["status"] == "unresolved" - assert summary["requests"]["A2"][0]["usage"] is None - assert staged.world.pr.ledger.charged == 162720000 - assert summary["diagnostics"]["A2"]["reason"] == "unknown_usage" - assert summary["diagnostics"]["A2"]["provider_attempts"] == 1 - assert summary["diagnostics"]["A3"]["provider_attempts"] is None - - -def test_closed_worker_failure_is_packaged_without_retry_or_arbitrary_text(staged): - seen = [] - transport = Mock(side_effect=provider.AgentError("http_429")) - job.execute(staged.output, api=SimpleNamespace(), - stores=(staged.world.daily, staged.world.pr), - launch=launcher([], seen, transport=transport)) - assert seen == ["A2"] and transport.call_count == 1 - assert staged.world.pr.ledger.charged == 162720000 - summary = job.package(staged.output) - diagnostic = summary["diagnostics"]["A2"] - assert diagnostic == {"worker_status": "failed", "reason": "provider_transport_error", - "provider_attempts": 1, "transport_error_code": "http_429", - "execution": {"status": "failed", "returncode": 1, "elapsed_seconds": None}} - assert summary["requests"]["A2"][0]["usage"] is None - assert summary["diagnostics"]["A3"]["worker_status"] == "unavailable" - public = b"".join(p.read_bytes() for p in (staged.output / "public").rglob("*") if p.is_file()) - assert b"OFFLINE-FAKE-PROVIDER-KEY" not in public - - -def test_failed_request_gate_consumes_slot_but_retains_zero_transport_invocations( - staged, monkeypatch): - monkeypatch.setattr(job.admission._Admission, "authorize", lambda *_: False) - seen, transport = [], Mock() - result = job.execute(staged.output, api=SimpleNamespace(), - stores=(staged.world.daily, staged.world.pr), - launch=launcher([], seen, transport=transport)) - assert seen == ["A2"] and transport.call_count == 0 - assert result["arms"]["A2"]["slots_consumed"] == 1 - assert result["arms"]["A2"]["request_completion_known"] is False - summary = job.package(staged.output) - assert summary["diagnostics"]["A2"]["provider_attempts"] == 0 - assert summary["diagnostics"]["A2"]["reason"] == "request_authorization_denied" - assert summary["diagnostics"]["A3"]["provider_attempts"] is None - assert staged.world.pr.ledger.charged == 162720000 - - -@pytest.mark.parametrize("attempts", [0, 1, 8, True, -1, 9, "1", None]) -def test_closed_outcome_recovers_fixed_files_even_without_execution_summary( - staged, attempts): - root = staged.output / "A2/run" - (root / "worker").mkdir(parents=True) - atomic_write_json(root / "worker/result.json", {"schema": worker.SCHEMA, - "status": "failed", "reason": "PRIVATE-CANARY arbitrary text", - "provider_attempts": attempts, "transport_error_code": "PRIVATE-CANARY", - "environment": {"secret": "PRIVATE-CANARY"}}) - atomic_write_json(root / "supervision.json", {"schema": "kv-worker-supervision-v1", - "execution": {"status": "failed", "returncode": -15, "elapsed_seconds": 1.5, - "metric": 99, "argv": ["PRIVATE-CANARY"], "error": "PRIVATE-CANARY"}}) - assert not (staged.output / "execution.json").exists() - summary = job.package(staged.output) - diagnostic = summary["diagnostics"]["A2"] - assert diagnostic["reason"] == diagnostic["transport_error_code"] == "unknown" - assert diagnostic["provider_attempts"] == ( - attempts if type(attempts) is int and 0 <= attempts <= 8 else None) - assert diagnostic["execution"] == { - "status": "failed", "returncode": -15, "elapsed_seconds": 1.5} - assert summary["diagnostics"]["A3"]["provider_attempts"] is None - assert "PRIVATE-CANARY" not in provider.canonical(summary) - assert summary["artifacts"]["A2"]["status"] == "missing_or_invalid" - - -@pytest.mark.parametrize("data", [b"not-json private", b"[]", b'{"schema":"wrong"}']) -def test_missing_or_malformed_worker_result_does_not_infer_zero(staged, data): - root = staged.output / "A2/run" - write(root, "worker/result.json", data) - diagnostic = job.package(staged.output)["diagnostics"]["A2"] - assert diagnostic == {"worker_status": "unavailable", "reason": None, - "provider_attempts": None, "transport_error_code": None, - "execution": {"status": "unavailable", "returncode": None, "elapsed_seconds": None}} - - -def test_prepared_input_mutation_blocks_before_reservation(staged): - (staged.output / "A2/workspace/task.json").write_text("changed") - with pytest.raises(ValueError, match="inputs changed"): - job.execute(staged.output, api=SimpleNamespace(), - stores=(staged.world.daily, staged.world.pr), launch=Mock()) - assert staged.world.events == [] - - -def test_stage_rejects_sibling_leak_or_nonidentical_common_bytes(staged, tmp_path): - value = copy.deepcopy(staged.plan) - value["actors"]["A2"]["workspace"].append({"path": "knowledge/SKILL.md"}) - with pytest.raises(ValueError, match="allowlist"): - job._stage(value, tmp_path / "bad-stage") - - -@pytest.mark.parametrize("credential", [provider.KEY_ENV, "GITHUB_TOKEN"]) -def test_final_grade_rejects_injected_authority(tmp_path, monkeypatch, credential): - monkeypatch.setenv(credential, "OFFLINE-FAKE") - with pytest.raises(ValueError, match="without injected"): - job.grade(tmp_path / "input", tmp_path / "output") - - -def test_immutable_final_bundle_rejects_wrong_run_before_grading(staged, monkeypatch): - job.execute(staged.output, api=SimpleNamespace(), stores=(staged.world.daily, staged.world.pr), - launch=launcher([read_task(), work(CANDIDATE), submit()], [])) - job.package(staged.output) - monkeypatch.delenv(provider.KEY_ENV) - monkeypatch.setenv("GITHUB_RUN_ID", "124") - result = job.grade(staged.output / "public", staged.output / "grade") - assert all(a["reason"] == "invalid_execution_identity" for a in result["arms"].values()) - - -@pytest.mark.parametrize("now,seconds", [("2026-10-30T15:59:00", 7200), - ("2026-10-30T12:00:00", 1319)]) -def test_no_partial_pair_before_expiry_or_shanghai_midnight(now, seconds): - now = datetime.fromisoformat(now).replace(tzinfo=timezone.utc) - with pytest.raises(ValueError, match="runway"): - job.runway(now.timestamp() + seconds, now=now) - - -@pytest.mark.parametrize("method,path,data", [("POST", "/git/refs", {}), - ("PATCH", "/git/refs/heads/main", {"sha": "a" * 40, "force": False}), - ("PATCH", "/git/refs/heads/skill-eval-budget/1409534281/999", - {"sha": "a" * 40, "force": True}), ("GET", "/pulls/1?x=1", None)]) -def test_controller_api_rejects_scope_before_http(method, path, data): - opener = Mock() - api = job.DeadlineGitHub("OFFLINE-FAKE", time.monotonic() + 30, opener=opener) - with pytest.raises(ValueError, match="scope"): - api.request(method, path, data) - opener.open.assert_not_called() - - -def test_workflow_manual_no_inputs_execute_only_credentials_and_separate_pinned_grade(): - path = Path(__file__).parents[1] / ".github/workflows/kv-edit-pilot.yml" - text = path.read_text() - value = yaml.load(text, Loader=yaml.BaseLoader) - assert value["on"] == {"workflow_dispatch": ""} - assert value["permissions"] == {} - assert value["concurrency"]["cancel-in-progress"] == "false" - assert text.count("secrets.") == 1 - assert "env" not in value - execution = value["jobs"]["execute"] - bound = [s for s in execution["steps"] if " execute --output " in s.get("run", "")] - assert len(bound) == 1 - assert bound[0]["env"] == { - "PYTHONPATH": "src", "GITHUB_TOKEN": "${{ github.token }}", - "SKILL_EVAL_YIBUAPI_KEY": "${{ secrets.SKILL_EVAL_YIBUAPI_KEY }}", - } - prepare = next(s for s in execution["steps"] if " prepare --output " in s.get("run", "")) - assert execution["steps"].index(prepare) < execution["steps"].index(bound[0]) - for name, job_value in value["jobs"].items(): - assert "env" not in job_value - condition = job_value["if"] - for guard in ("github.repository", "github.ref", "github.run_attempt", "github.actor", - "github.triggering_actor"): - assert guard in condition - for step in job_value["steps"]: - if step is not bound[0]: - assert not {"GITHUB_TOKEN", provider.KEY_ENV} & set(step.get("env", {})) - assert "secrets." not in provider.canonical(step) - if "uses" in step: - assert policy._hex(step["uses"].split("@")[1], 40) - if step.get("uses", "").startswith("actions/checkout@"): - assert step["with"]["persist-credentials"] == "false" - assert step["with"]["ref"] == "${{ github.sha }}" - grade = value["jobs"]["grade"] - assert grade["permissions"]["contents"] == "read" - download = next(s for s in grade["steps"] if s.get("uses", "").startswith("actions/download")) - assert set(download["with"]) == {"name", "path"} # Current-run only; no URL/run-id input. - assert download["with"]["name"] == "kv-live-${{ github.run_id }}-${{ github.run_attempt }}" - projection = next(s for s in grade["steps"] if "package-grade" in s.get("run", "")) - assert projection["if"] == "always()" and "env -i" in projection["run"] - upload = next(s for s in grade["steps"] if s.get("uses", "").startswith("actions/upload")) - assert upload["with"]["path"] == "${{ runner.temp }}/kv-grade/public" - operations = (path.parents[2] / "docs/operations.md").read_text() - assert "`package-grade` never reruns candidates or models" in operations - - -def test_same_host_preflight_uses_real_scoped_scripted_loop_with_fake_sandbox(staged, monkeypatch): - from tests.test_kv_edit_pilot import in_process_worker - original = worker.scripted.run_supervised - monkeypatch.setattr(worker.scripted, "run_supervised", lambda *a, **kw: - original(*a, **kw, execute=in_process_worker)) - monkeypatch.delenv(provider.KEY_ENV) - result = job.replay_preflight(staged.output, time.monotonic() + 120) - assert set(result) == {"A2", "A3"} - assert all(v["model_calls"] == 0 and v["verdict"] == "PASS" for v in result.values()) - - -def test_grade_rejects_symlink_source_as_inconclusive(staged, monkeypatch): - job.execute(staged.output, api=SimpleNamespace(), stores=(staged.world.daily, staged.world.pr), - launch=launcher([read_task(), work(CANDIDATE), submit()], [])) - job.package(staged.output) - candidate = staged.output / "public/A2/candidate.py" - candidate.unlink() - candidate.symlink_to(staged.output / "A2/run/worker/final/candidate.py") - monkeypatch.delenv(provider.KEY_ENV) - result = job.grade(staged.output / "public", staged.output / "grade") - assert result["arms"]["A2"]["verdict"] == "INCONCLUSIVE" - assert result["arms"]["A3"]["semantic"]["verdict"] == "PASS" - - -def test_missing_entire_artifact_bundle_retains_both_arms_as_inconclusive(staged, monkeypatch): - monkeypatch.delenv(provider.KEY_ENV) - result = job.grade(staged.output / "missing", staged.output / "grade") - assert set(result["arms"]) == {"A2", "A3"} - assert all(v["verdict"] == "INCONCLUSIVE" for v in result["arms"].values()) - assert (staged.output / "grade/grade.json").is_file() - - -def test_uncertain_admission_retains_identity_without_zero_cost_claim(staged, monkeypatch): - staged.world.daily.outcomes = ["unknown_commit"] - with pytest.raises(job.policy.budget.BudgetError): - job.execute(staged.output, api=SimpleNamespace(), - stores=(staged.world.daily, staged.world.pr), launch=Mock()) - assert staged.world.daily.ledger.charged_on("2026-10-30") == 654070000 - assert staged.world.pr.ledger.charged == 0 - result = job.package(staged.output) - assert result["execution"]["admission_outcome"].endswith("unknown_no_retry") - assert result["execution"]["reserved_usd"] is None - assert result["execution"]["actual_cost_usd"] is None - monkeypatch.delenv(provider.KEY_ENV) - grade = job.grade(staged.output / "public", staged.output / "grade") - assert all(a["verdict"] == "INCONCLUSIVE" for a in grade["arms"].values()) - - -@pytest.mark.parametrize("field,value", [("execution_id", "wrong-execution"), ("revision", 999), - ("artifact", "wrong.py"), ("check_receipt", "0" * 64), ("benchmark_receipt", "0" * 64)]) -def test_corrupt_final_receipt_cannot_be_packaged_or_graded(staged, monkeypatch, field, value): - job.execute(staged.output, api=SimpleNamespace(), stores=(staged.world.daily, staged.world.pr), - launch=launcher([read_task(), work(CANDIDATE), submit()], [])) - path = staged.output / "A2/run/worker/final/receipt.json" - receipt = policy.read_json(path) - receipt[field] = value - atomic_write_json(path, receipt) - summary = job.package(staged.output) - assert summary["artifacts"]["A2"]["status"] == "missing_or_invalid" - monkeypatch.delenv(provider.KEY_ENV) - result = job.grade(staged.output / "public", staged.output / "grade") - assert result["arms"]["A2"]["verdict"] == "INCONCLUSIVE" - - -def test_corrupted_feedback_bytes_are_not_current_revision_evidence(staged): - job.execute(staged.output, api=SimpleNamespace(), stores=(staged.world.daily, staged.world.pr), - launch=launcher([read_task(), work(CANDIDATE), submit()], [])) - path = staged.output / "A2/run/worker/feedback/0001.json" - receipt = policy.read_json(path) - receipt["feedback"]["reason"] = "check_failed" - atomic_write_json(path, receipt) - summary = job.package(staged.output) - assert summary["artifacts"]["A2"]["status"] == "missing_or_invalid" - - -def test_final_grade_uses_captured_public_freeze_not_revoked_live_comment(staged, monkeypatch): - job.execute(staged.output, api=SimpleNamespace(), stores=(staged.world.daily, staged.world.pr), - launch=launcher([read_task(), work(CANDIDATE), submit()], [])) - job.package(staged.output) - monkeypatch.delenv(provider.KEY_ENV) - def offline_only(*, fetch, now): - assert now == 0 - assert fetch(123)["body"] == "offline frozen public fixture" - return staged.authorization, {"offline": True} - monkeypatch.setattr(policy, "frozen_authorization", offline_only) - monkeypatch.setattr(policy, "fetch_comment", Mock(side_effect=AssertionError("no live fetch"))) - result = job.grade(staged.output / "public", staged.output / "grade") - assert all(a["semantic"]["verdict"] == "PASS" for a in result["arms"].values()) - - -@pytest.mark.parametrize("failure", ["exception", "duplicate_keys", "oversize", "late"]) -def test_controller_http_response_failure_is_bounded_redacted_and_never_retried(failure): - class Response: - def __enter__(self): - return self - def __exit__(self, *_): - return None - def read1(self, size): - if failure == "duplicate_keys": - raw, self.raw = getattr(self, "raw", b'{"x":1,"x":2}'), b"" - return raw - if failure == "late": - api.deadline = time.monotonic() - 1 - return b"{}" - return b"x" * size - opener = Mock() - opener.open.side_effect = (OSError("OFFLINE-FAKE never retain") - if failure == "exception" else None) - opener.open.return_value = Response() - api = job.DeadlineGitHub("OFFLINE-FAKE", time.monotonic() + 30, opener=opener) - with pytest.raises(ValueError, match="response unavailable") as error: - api.request("GET", "") - assert "OFFLINE-FAKE" not in str(error.value) - assert opener.open.call_count == 1 - - -def test_pre_execution_rejection_retains_inconclusive_grade(staged, monkeypatch): - summary = job.package(staged.output) # Successful prepare but execute was never admitted. - assert summary["execution"] == {} - monkeypatch.delenv(provider.KEY_ENV) - grader = Mock(side_effect=AssertionError("unbound candidates must not be graded")) - monkeypatch.setattr(job.workbench, "grade_final", grader) - result = job.grade(staged.output / "public", staged.output / "grade") - assert all(a["reason"] == "invalid_execution_identity" for a in result["arms"].values()) - assert (staged.output / "grade/grade.json").is_file() - grader.assert_not_called() - - -@pytest.mark.parametrize("change", ["none", "inventory", "mounts", "inconclusive"]) -def test_same_host_runtime_must_match_actual_qualified_profile(tmp_path, monkeypatch, change): - import sys - - import tools - expected = {"verdict": "PASS", "inventory_sha256": "a" * 64, "mount_count": 4} - actual = dict(expected) - if change == "inventory": - actual["inventory_sha256"] = "b" * 64 - elif change == "mounts": - actual["mount_count"] = 5 - elif change == "inconclusive": - actual["verdict"] = "INCONCLUSIVE" - runtime = SimpleNamespace(qualify=lambda *a, **kw: actual) - monkeypatch.setitem(sys.modules, "tools.kv_runtime_qualification", runtime) - monkeypatch.setattr(tools, "kv_runtime_qualification", runtime, raising=False) - monkeypatch.setattr(policy, "validate_qualification", lambda _: {"runtime_isolation": expected, - "dependencies": {"src/inferrsi/harness/skill_eval_engineering.py": "c" * 64}}) - if change in {"none", "inventory"}: - result = job.runtime_preflight(tmp_path, time.monotonic() + 30, {}) - assert result["verdict"] == "PASS" - assert result["inventory_sha256"] == actual["inventory_sha256"] - assert result["qualified_inventory_sha256"] == expected["inventory_sha256"] - assert result["profile_source_sha256"] == "c" * 64 - else: - with pytest.raises(ValueError, match="runtime/profile"): - job.runtime_preflight(tmp_path, time.monotonic() + 30, {}) - - -def retained_fake_sandbox(source, payload, output, deadline): - record, actual = fake_sandbox(source, payload, output, deadline) - # Mimic the real executor's fixed filenames, including deliberately excluded text. - (output / "stdout.log").write_bytes(provider.canonical(actual).encode()) - (output / "stderr.log").write_text("OFFLINE-PRIVATE-CANARY") - (output / "unrequested.json").write_text('"OFFLINE-PRIVATE-CANARY"') - atomic_write_json(output / "execution.json", {"status": record["status"], - "returncode": 0, "elapsed_seconds": record["elapsed_seconds"]}) - return {**record, "returncode": 0, "argv": ["OFFLINE-PRIVATE-CANARY"], - "cwd": "OFFLINE-PRIVATE-CANARY", "error": "OFFLINE-PRIVATE-CANARY"}, actual - - -def test_final_artifact_reconstructs_original_semantics_and_abba_samples(staged, monkeypatch): - job.execute(staged.output, api=SimpleNamespace(), stores=(staged.world.daily, staged.world.pr), - launch=launcher([read_task(), work(CANDIDATE), submit()], [])) - job.package(staged.output) - monkeypatch.delenv(provider.KEY_ENV) - monkeypatch.setattr(kv, "invoke", retained_fake_sandbox) - result = job.grade(staged.output / "public", staged.output / "grade") - public = staged.output / "grade/public" - manifest = policy.read_json(public / "evidence.json") - assert manifest["execution"]["run_id"] == "123" - assert policy.read_json(public / "authorization.json")["body"].startswith("offline") - assert policy.read_json(public / "grade.json") == result - for arm in ("A2", "A3"): - evidence = manifest["arms"][arm] - assert evidence["raw_outputs_complete"] - assert len(evidence["semantics"]) == 1 and len(evidence["performance"]) == 17 - assert evidence["semantics"][0]["check"] is True - assert evidence["semantics"][0]["execution"] == { - "status": "completed", "returncode": 0, "elapsed_seconds": 1.0} - for row in evidence["semantics"] + evidence["performance"]: - raw = (public / row["output"]["path"]).read_bytes() - assert policy.sha(raw) == row["output"]["sha256"] - assert len(raw) == row["output"]["bytes"] - assert policy.fingerprint(json.loads(raw)) == row["output"]["data_sha256"] - assert [r["side"] for r in evidence["performance"][:4]] == [ - "baseline", "candidate", "candidate", "baseline"] - for side in ("baseline", "candidate"): - observed = [r["execution"]["metric"] for r in evidence["performance"] - if r["side"] == side and r["phase"] == "measurement"] - assert observed == result["arms"][arm]["performance"][side + "_values"] - assert all(r["source_verified"] and r["execution"]["returncode"] == 0 - for r in evidence["performance"]) - assert all(r["output"]["data_sha256"] == r["expected_sha256"] - for r in evidence["performance"]) - published = b"".join(p.read_bytes() for p in public.rglob("*") if p.is_file()) - assert b"OFFLINE-PRIVATE-CANARY" not in published - assert b"OFFLINE-FAKE-PROVIDER-KEY" not in published - assert str(staged.output).encode() not in published - assert not list(public.rglob("*.log")) - - -@pytest.mark.parametrize("bad", [b'not JSON OFFLINE-PRIVATE-CANARY', - b'{"observations":[],"pending":[],"extra":"OFFLINE-PRIVATE-CANARY"}', - b'{"observations":[],"pending":[["OFFLINE-PRIVATE-CANARY",0,[]]]}', - b'{"observations":[],"pending":[],"pending":[]}', - b'{"observations":[],"pending":[[[],0,[]]]}']) -def test_raw_projection_never_uploads_arbitrary_or_malformed_output(tmp_path, bad): - source, target = tmp_path / "raw", tmp_path / "published.json" - source.write_bytes(bad) - result = job._raw_kv_output(source, target, [], verify=True, remaining=[1024]) - assert result["status"] == "redacted_invalid_output" - assert not target.exists() - assert result["sha256"] == policy.sha(bad) and result["bytes"] == len(bad) - assert result["truncated"] is False - assert "OFFLINE-PRIVATE-CANARY" not in provider.canonical(result) - - -@pytest.mark.parametrize("unsafe", ["symlink", "oversize", "whole_budget"]) -def test_raw_projection_file_and_total_limits(tmp_path, unsafe): - source, target = tmp_path / "raw", tmp_path / "published.json" - good = b'{"observations":[],"pending":[]}' - if unsafe == "symlink": - (tmp_path / "other").write_bytes(good) - source.symlink_to(tmp_path / "other") - else: - source.write_bytes(good if unsafe == "whole_budget" else b" " * (4 * policy.MAX_JSON + 1)) - result = job._raw_kv_output(source, target, [], verify=True, remaining=[1]) - assert result["status"] != "retained" and not target.exists() - - -def test_all_frozen_hidden_expected_outputs_fit_public_projection(tmp_path): - # Real frozen DATA only; this does not execute a candidate or re-author the oracle. - root = Path(__file__).parents[1] - frozen = policy.read_json(root / ( - "evidence/kv-reclamation-20261030/validator/independent/fixtures.json")) - traces = [t for t in frozen["traces"] if t["split"] == "grader"] - assert len(traces) == 77 - remaining = [32 * policy.MAX_JSON] - for index, trace in enumerate(traces): - expected = {"observations": trace["expected"], - "pending": trace["expected"][-1]["pending"] if trace["expected"] else []} - raw = provider.canonical(expected).encode() - source, target = tmp_path / "source", tmp_path / f"trace-{index:03d}.json" - source.write_bytes(raw) - result = job._raw_kv_output(source, target, trace["events"], - verify=True, remaining=remaining) - assert result["status"] == "retained", trace["id"] - assert target.read_bytes() == raw - - -def test_partial_preflight_retains_failed_stage_without_exception_text(staged, monkeypatch): - monkeypatch.delenv(provider.KEY_ENV) - monkeypatch.setattr(policy, "fetch_comment", lambda _: {"id": 123, "body": "offline", - "issue_url": "https://example.invalid/public", "user": {"login": "owner", "type": "User"}}) - def frozen(*, fetch=None, **kwargs): - if fetch is not None: - fetch(123) - return staged.authorization, {"offline": True} - monkeypatch.setattr(policy, "frozen_authorization", frozen) - monkeypatch.setattr(kv, "probe_isolation", - Mock(side_effect=ValueError("OFFLINE-PRIVATE-CANARY"))) - fresh = staged.output.parent / "failed-prepare" - with pytest.raises(ValueError): - job.prepare(fresh) - job.package(fresh) - proof = policy.read_json(fresh / "public/preflight.json") - assert proof["status"] == "partial" and proof["stages"]["stage"] == "completed" - assert proof["stages"]["isolation"] == "failed_or_interrupted" - assert proof["stages"]["cleanup"] == "not_started" - assert "OFFLINE-PRIVATE-CANARY" not in provider.canonical(proof) - - -def test_preflight_projection_excludes_arbitrary_nested_text(staged): - path = staged.output / "prepared.json" - value = policy.read_json(path) - value["runtime_profile"].update(inventory_sha256="a" * 64, - error="OFFLINE-PRIVATE-CANARY", environment={"KEY": "OFFLINE-PRIVATE-CANARY"}, - probe={"network_blocked": True, "inaccessible": [True, False], - "unexpected": "OFFLINE-PRIVATE-CANARY"}) - value["cleanup"]["cases"] = {mode: {"verdict": "FAIL", "surviving_descendants": 1, - "error": "OFFLINE-PRIVATE-CANARY"} for mode in ("normal", "timeout", "cancel")} - atomic_write_json(path, value) - job.package(staged.output) - proof = policy.read_json(staged.output / "public/preflight.json") - assert proof["proofs"]["runtime_profile"]["probe"]["inaccessible"] == [True, False] - assert proof["proofs"]["cleanup"]["cases"]["normal"]["surviving_descendants"] == 1 - assert "OFFLINE-PRIVATE-CANARY" not in provider.canonical(proof) - - -def test_interrupted_grader_keeps_first_outputs_and_explicit_unstarted_arm(staged, monkeypatch): - job.execute(staged.output, api=SimpleNamespace(), stores=(staged.world.daily, staged.world.pr), - launch=launcher([read_task(), work(CANDIDATE), submit()], [])) - job.package(staged.output) - monkeypatch.delenv(provider.KEY_ENV) - monkeypatch.setattr(kv, "invoke", retained_fake_sandbox) - original = job.workbench.grade_final - def interrupted(*args): - original(*args) - raise KeyboardInterrupt - monkeypatch.setattr(job.workbench, "grade_final", interrupted) - with pytest.raises(KeyboardInterrupt): - job.grade(staged.output / "public", staged.output / "grade") - public = staged.output / "grade/public" - result = policy.read_json(public / "grade.json") - assert result["arms"]["A2"]["reason"] == "grading_interrupted" - assert result["arms"]["A3"]["reason"] == "not_evaluated" - evidence = policy.read_json(public / "evidence.json")["arms"] - assert evidence["A2"]["semantics"][0]["output"]["status"] == "retained" - assert not evidence["A3"]["raw_outputs_complete"] - - -def test_always_projection_recovers_failed_projection_without_any_reexecution(staged, monkeypatch): - job.execute(staged.output, api=SimpleNamespace(), stores=(staged.world.daily, staged.world.pr), - launch=launcher([read_task(), work(CANDIDATE), submit()], [])) - job.package(staged.output) - monkeypatch.delenv(provider.KEY_ENV) - monkeypatch.setattr(kv, "invoke", retained_fake_sandbox) - original = job._raw_kv_output - monkeypatch.setattr(job, "_raw_kv_output", - Mock(side_effect=RuntimeError("projection interrupted"))) - with pytest.raises(RuntimeError, match="projection interrupted"): - job.grade(staged.output / "public", staged.output / "grade") - public = staged.output / "grade/public" - assert policy.read_json(public / "projection.json")["status"] == "failed_or_interrupted" - first = (staged.output / "grade/A2/semantics/trace-000/stdout.log").read_bytes() - monkeypatch.setattr(job, "_raw_kv_output", original) - monkeypatch.setattr(kv, "invoke", Mock(side_effect=AssertionError("no candidate rerun"))) - result = job.package_grade(staged.output / "public", staged.output / "grade") - assert all(arm["raw_outputs_complete"] for arm in result["arms"].values()) - assert (public / "A2/semantics/trace-000.json").read_bytes() == first - assert policy.read_json(public / "projection.json")["status"] == "completed" - grade = policy.read_json(public / "grade.json") - assert grade["arms"]["A2"]["semantic"]["verdict"] == "PASS" - assert "verdict" not in grade["arms"]["A2"] # No invented aggregate verdict. - - -def test_always_projection_keeps_raw_partial_output_without_finally(staged, monkeypatch): - job.execute(staged.output, api=SimpleNamespace(), stores=(staged.world.daily, staged.world.pr), - launch=launcher([read_task(), work(CANDIDATE), submit()], [])) - job.package(staged.output) - monkeypatch.delenv(provider.KEY_ENV) - grade = staged.output / "grade" - retained_fake_sandbox(BASELINE, {"mode": "verify", "events": []}, - grade / "A2/semantics/trace-000", time.monotonic() + 10) - monkeypatch.setattr(kv, "invoke", Mock(side_effect=AssertionError("no candidate rerun"))) - manifest = job.package_grade(staged.output / "public", grade) - assert manifest["arms"]["A2"]["semantics"][0]["output"]["status"] == "retained" - assert manifest["arms"]["A2"]["semantics"][0]["check"] is None - assert not manifest["arms"]["A2"]["raw_outputs_complete"] - assert all(arm["verdict"] == "INCONCLUSIVE" - for arm in policy.read_json(grade / "public/grade.json")["arms"].values()) - - -def test_grade_projection_retains_failed_and_skipped_timing_rows(staged, monkeypatch): - job.execute(staged.output, api=SimpleNamespace(), stores=(staged.world.daily, staged.world.pr), - launch=launcher([read_task(), work(CANDIDATE), submit()], [])) - job.package(staged.output) - monkeypatch.delenv(provider.KEY_ENV) - def failed(source, payload, output, deadline): - record, actual = retained_fake_sandbox(source, payload, output, deadline) - if payload["mode"] == "benchmark": - record.update(status="timeout", returncode=-9) - return record, actual - monkeypatch.setattr(kv, "invoke", failed) - result = job.grade(staged.output / "public", staged.output / "grade") - assert result["arms"]["A2"]["performance"]["verdict"] == "INCONCLUSIVE" - rows = policy.read_json(staged.output / "grade/public/evidence.json")["arms"]["A2"] - assert len(rows["performance"]) == 17 - assert rows["performance"][0]["execution"]["status"] == "timeout" - assert rows["performance"][0]["execution"]["returncode"] == -9 - assert all(row["execution"]["status"] == "unavailable" for row in rows["performance"][1:]) - assert not rows["raw_outputs_complete"] - - -@pytest.mark.parametrize("corruption", ["run", "locator", "workspace"]) -def test_preflight_projection_rejects_unbound_or_private_prepared_fields(staged, corruption): - job.execute(staged.output, api=SimpleNamespace(), stores=(staged.world.daily, staged.world.pr), - launch=launcher([read_task(), work(CANDIDATE), submit()], [])) - path = staged.output / "prepared.json" - value = policy.read_json(path) - if corruption == "run": - value["run_id"] = "9999" - elif corruption == "locator": - value["locator_sha256"] = "0" * 64 - else: - value["inputs"]["A2/workspace/task.json"] = "OFFLINE-PRIVATE-CANARY" - atomic_write_json(path, value) - job.package(staged.output) - projected = policy.read_json(staged.output / "public/preflight.json") - assert projected["status"] != "prepared" - assert "OFFLINE-PRIVATE-CANARY" not in provider.canonical(projected) - - -def test_always_projection_rejects_corrupt_grade_text_and_false_pass(staged, monkeypatch): - job.execute(staged.output, api=SimpleNamespace(), stores=(staged.world.daily, staged.world.pr), - launch=launcher([read_task(), work(CANDIDATE), submit()], [])) - job.package(staged.output) - monkeypatch.delenv(provider.KEY_ENV) - output = staged.output / "grade" - output.mkdir() - atomic_write_json(output / "grade.json", {"schema": "kv-live-final-grade-v1", - "plan_sha256": staged.authorization["plan_sha256"], "scope": "OFFLINE-PRIVATE-CANARY", - "arms": {a: {"reason": "OFFLINE-PRIVATE-CANARY", "semantic": { - "verdict": "PASS", "checks": {TRACE["id"]: False}, "artifact_sha256": "a" * 64, - "environment": "OFFLINE-PRIVATE-CANARY"}} for a in ("A2", "A3")}}) - monkeypatch.setattr(kv, "invoke", Mock(side_effect=AssertionError("no candidate rerun"))) - job.package_grade(staged.output / "public", output) - raw = (output / "public/grade.json").read_bytes() - assert b"OFFLINE-PRIVATE-CANARY" not in raw - assert all(a["verdict"] == "INCONCLUSIVE" for a in json.loads(raw)["arms"].values()) - - -@pytest.mark.parametrize("credential", [provider.KEY_ENV, "GITHUB_TOKEN"]) -def test_always_projection_rejects_injected_secrets(tmp_path, monkeypatch, credential): - monkeypatch.setenv(credential, "OFFLINE-FAKE") - with pytest.raises(ValueError, match="without injected"): - job.package_grade(tmp_path / "bundle", tmp_path / "output") - - -@pytest.mark.parametrize("receipt", ["failed", "absent", "unexpected_fields"]) -def test_trace_exit_failure_cannot_be_upgraded_by_correct_stdout(staged, monkeypatch, receipt): - job.execute(staged.output, api=SimpleNamespace(), stores=(staged.world.daily, staged.world.pr), - launch=launcher([read_task(), work(CANDIDATE), submit()], [])) - job.package(staged.output) - monkeypatch.delenv(provider.KEY_ENV) - def failure(source, payload, output, deadline): - record, actual = retained_fake_sandbox(source, payload, output, deadline) - if payload["mode"] != "verify": - return record, actual - record.update(status="failed", returncode=1) - path = output / "execution.json" - if receipt == "failed": - atomic_write_json(path, {k: record[k] for k in - ("status", "returncode", "elapsed_seconds")}) - elif receipt == "absent": - path.unlink() # Process loss before first receipt cannot imply successful exit. - else: - atomic_write_json(path, {**record, "secret": "OFFLINE-PRIVATE-CANARY"}) - return record, None # Real invoke rejects a nonzero exit even with valid JSON. - monkeypatch.setattr(kv, "invoke", failure) - result = job.grade(staged.output / "public", staged.output / "grade") - assert result["arms"]["A2"]["semantic"]["verdict"] == "FAIL" - row = policy.read_json(staged.output / "grade/public/evidence.json")[ - "arms"]["A2"]["semantics"][0] - assert row["output"]["status"] == "retained" and row["check"] is False - assert row["execution"]["status"] == ("failed" if receipt == "failed" else "unavailable") - if receipt == "failed": - assert row["execution"]["returncode"] == 1 - assert "OFFLINE-PRIVATE-CANARY" not in provider.canonical(row) - - -def test_fake_second_pair_preserves_first_claim_and_cannot_launch_third(staged, monkeypatch): - from dataclasses import replace - - from tests.test_kv_pilot_admission import additional_world - world, first, second = additional_world() - staged.world = world - extra = replace(second.additional_attempt, execution_id=staged.plan['execution_id']) - second = replace(second, execution_id=extra.execution_id, additional_attempt=extra, - plan_sha256=staged.authorization['plan_sha256'], - actor_sha256=tuple(staged.configs[a]['actor_sha256'] for a in ('A2', 'A3'))) - monkeypatch.setattr(policy, 'policy', lambda *_: second) - seen = [] - scripts = [read_task(), work(CANDIDATE), work(CANDIDATE + '# second\n', '2'), submit()] - launch = launcher(scripts, seen) - def after_both_claims(*args, **kwargs): - assert len(world.events) == 8 - assert world.pr.ledger.charged == 325_440_000 - assert world.pr.ledger.entries[0].binding == first.binding - return launch(*args, **kwargs) - result = job.execute(staged.output, api=SimpleNamespace(), - stores=(world.daily, world.pr), launch=after_both_claims) - assert seen == ['A2', 'A3'] - assert all(a['status'] == 'completed' for a in result['arms'].values()) - assert world.daily.ledger.charged_on('2026-10-30') == 816_790_000 - summary = job.package(staged.output) - assert all(len(r) == 4 for r in summary['requests'].values()) - assert all(a['status'] == 'submitted' for a in summary['artifacts'].values()) - with pytest.raises(policy.budget.BudgetError): - world.admit(second) - assert seen == ['A2', 'A3'] and len(world.events) == 8 - - -def test_controller_supplies_only_root_bound_limit_to_anchored_pr_store(staged, monkeypatch): - from tests.test_kv_pilot_admission import additional_world - world, _, second = additional_world() - monkeypatch.setattr(policy, 'policy', lambda *_: second) - staged.plan['ledger'] = {'ref': 'refs/heads/skill-eval-budget/123/999', - 'genesis_commit': 'a' * 40, 'checkpoint_commit': 'b' * 40} - staged.plan['daily_ledger'] = {'ref': 'refs/heads/skill-eval-budget/1409534281/daily', - 'genesis_commit': 'c' * 40, 'checkpoint_commit': 'd' * 40} - captured = [] - monkeypatch.setattr(job, 'GitHubDailyStore', lambda api, anchor: world.daily) - def pr_store(api, anchor, **kwargs): - captured.append((anchor, kwargs)) - raise ValueError('offline stop before admission') - monkeypatch.setattr(job, 'GitHubGitStore', pr_store) - with pytest.raises(ValueError, match='offline stop'): - job.execute(staged.output, api=SimpleNamespace()) - assert captured[0][0].key == second.key - assert captured[0][1] == {'trusted_limit': second.trusted_limit} - assert len(world.events) == 4 # Only the synthetic previous pair, no new write. diff --git a/tests/test_kv_live_policy.py b/tests/test_kv_live_policy.py deleted file mode 100644 index 00da1f98b..000000000 --- a/tests/test_kv_live_policy.py +++ /dev/null @@ -1,300 +0,0 @@ -"""All identities, files, API reads and clock values below are offline fixtures.""" - -import copy -import json -import zipfile -from pathlib import Path -from types import SimpleNamespace - -import pytest - -from tests.test_kv_pilot_admission import NOW -from tests.test_kv_pilot_admission import policy as sample_policy -from tools import kv_live_policy as live - - -def write(root, name, value): - data = json.dumps(value).encode() - path = root / name - path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(data) - return {"path": name, "bytes": len(data), "sha256": live.sha(data)} - - -@pytest.fixture -def case(tmp_path, monkeypatch): - monkeypatch.setattr(live, "ROOT", tmp_path) - monkeypatch.setattr(live, "LOCATOR", tmp_path / "locator.json") - monkeypatch.setattr(live, "DEPENDENCIES", ("code.py",)) - monkeypatch.setattr(live.scripted, "dependencies", lambda: {}) - monkeypatch.setattr(live, "QUALIFICATION_DEPENDENCIES", {"code.py"}) - (tmp_path / "code.py").write_text("# offline code fixture\n") - sources = {"code.py": live.sha((tmp_path / "code.py").read_bytes())} - monkeypatch.setattr(live, "qualification_dependencies", lambda: sources) - actor_config = {"actor_sha256": "4" * 64, "edit_plan": {"files": []}, - "prompt": "offline prompt", "arm": "A2", "model": "claude-opus-5-5", - "client": {}, "input_token_ceiling": 1000000, - "suite_sha256": "7" * 64, "baseline_sha256": "8" * 64} - actor = write(tmp_path, "actor.json", actor_config) - evidence = b"offline data-only archive fixture" - qualified = write(tmp_path, "qualified.json", { - "schema": "kv-scripted-actor-qualification-v1", "revision": "6" * 40, - "verdict": "PASS", "model_calls": 0, "provider_charges_usd": 0, - "dependencies": sources, "dependencies_sha256": live.fingerprint(sources), - "software_checks_passed": True, "source_unchanged": True, - "cleanup": {"verdict": "PASS"}, - "runtime_isolation": {"verdict": "PASS", "inventory_sha256": "9" * 64, - "mount_count": 4}, - "raw_evidence": {"sha256": live.sha(evidence)}, - "arms": {arm: {"executor_status": "completed", "worker_status": "completed", - "request_completion_known": True, - "simulated_slots_consumed": 4, "visible_workbench_verified": True, - "two_revision_receipts_retained": True, "hidden_correctness": {"verdict": "PASS"}, - "exposure": [], "prompt_sha256": live.sha(b"offline prompt"), - "common_actor_contract_sha256": live.inputs.common_actor_contract_sha256(actor_config)} - for arm in ("A2", "A3")}}) - with zipfile.ZipFile(tmp_path / "qualified.zip", "w") as archive: - archive.writestr("qualification.json", (tmp_path / "qualified.json").read_bytes()) - archive.writestr("raw-evidence.zip", evidence) - raw_archive = (tmp_path / "qualified.zip").read_bytes() - archive_record = {"path": "qualified.zip", "bytes": len(raw_archive), - "sha256": live.sha(raw_archive)} - suite = write(tmp_path, "suite.json", {}) - plan = {"schema": "kv-live-plan-v1", "repository_id": live.REPOSITORY_ID, "issue": 248, - "campaign_id": "offline-campaign", "execution_id": "offline-execution", - "charge_pr": 999, "charge_head": "2" * 40, "expires_at": int(NOW.timestamp()) + 7200, - "provider": json.loads(sample_policy().provider_json), "dependencies": sources, - "actors": {arm: {"config": actor, "workspace": [{"path": "candidate.py"}]} - for arm in ("A2", "A3")}, - "ledger": {"ref": f"refs/heads/skill-eval-budget/{live.REPOSITORY_ID}/999", - "genesis_commit": "3" * 40, "checkpoint_commit": "3" * 40}, - "daily_ledger": {"ref": f"refs/heads/skill-eval-budget/{live.REPOSITORY_ID}/daily", - "genesis_commit": "5" * 40, "checkpoint_commit": "5" * 40}, - "qualification": {"run_id": 123, "run_attempt": 1, "head": "6" * 40, - "revision": "6" * 40, "artifact_id": 456, - "archive": archive_record, "receipt": qualified, - "dependencies_sha256": live.fingerprint(sources)}, - "suite": suite, "final_suite": suite, "hidden_traces": suite, - "arm_order": ["A2", "A3"], "scope": "synthetic feasibility only", - "price_evidence": "offline synthetic price fixture", - "prior_owner_group_statement": "offline fixture, not live attestation", - "final_seconds": 180, "final_repetitions": 8} - authorization = {"schema": "kv-live-authorization-v1", "plan_sha256": live.fingerprint(plan), - "plan": plan} - body = live.PREFIX + json.dumps(authorization) + live.SUFFIX - locator = {"schema": "kv-live-authorization-locator-v1", "enabled": True, - "repository_id": live.REPOSITORY_ID, "issue": 248, "comment_id": 12345, - "author_login": live.OPERATOR, "body_sha256": live.sha(body.encode())} - write(tmp_path, "locator.json", locator) - comment = {"id": 12345, "body": body, "user": {"login": live.OPERATOR, "type": "User"}, - "issue_url": f"https://api.github.com/repos/{live.REPOSITORY}/issues/248"} - return SimpleNamespace(root=tmp_path, plan=plan, locator=locator, comment=comment, - authorization=authorization) - - -def test_exact_frozen_public_authorization_has_no_spending_side_effect(case): - actual = live.frozen_authorization(fetch=lambda n: case.comment, now=NOW.timestamp()) - assert actual == (case.authorization, case.locator) - assert live.policy(case.plan, "7" * 40).binding.worst_case == 162720000 - - -@pytest.mark.parametrize("field,value", [ - ("id", 12346), ("body", "changed"), ("issue_url", "https://example.invalid/other"), - ("user", {"login": "other", "type": "User"}), - ("user", {"login": live.OPERATOR, "type": "Bot"}), -]) -def test_comment_identity_hash_and_author_are_not_interchangeable(case, field, value): - case.comment[field] = value - with pytest.raises(ValueError, match="identity"): - live.frozen_authorization(fetch=lambda n: case.comment, now=NOW.timestamp()) - - -@pytest.mark.parametrize("change", ["expired", "source", "qualifier", "missing_dependency", - "amount", "requests", "charge_ref", "unknown_field"]) -def test_bad_frozen_plan_fails_before_any_request_or_ledger(case, change): - plan = copy.deepcopy(case.plan) - if change == "expired": - plan["expires_at"] = 1 - elif change == "source": - (case.root / "code.py").write_text("changed") - elif change == "qualifier": - plan["qualification"]["dependencies_sha256"] = "0" * 64 - elif change == "missing_dependency": - plan["dependencies"].clear() - elif change == "amount": - plan["provider"]["spend_limit"]["amount"] = "201" - elif change == "requests": - plan["provider"]["client"]["max_requests"] = 9 - elif change == "charge_ref": - plan["ledger"]["ref"] += "1" - else: - plan["secret"] = "offline-not-allowed" - with pytest.raises((ValueError, live.budget.BudgetError)): - live.validate_plan(plan, now=NOW.timestamp()) - - -def test_disabled_default_does_not_fetch_comment(tmp_path, monkeypatch): - monkeypatch.setattr(live, "LOCATOR", tmp_path / "locator.json") - write(tmp_path, "locator.json", {"schema": "kv-live-authorization-locator-v1", - "enabled": False}) - with pytest.raises(ValueError, match="disabled"): - live.frozen_authorization(fetch=lambda _: pytest.fail("must not fetch")) - - -@pytest.mark.parametrize("name,value", [("GITHUB_EVENT_NAME", "pull_request"), - ("GITHUB_REF", "refs/heads/other"), ("GITHUB_RUN_ATTEMPT", "2"), - ("GITHUB_REPOSITORY", "other/repo"), ("GITHUB_ACTOR", "other"), - ("GITHUB_TRIGGERING_ACTOR", "other")]) -def test_runtime_rejects_other_entry_points(name, value, monkeypatch): - env = {"GITHUB_EVENT_NAME": "workflow_dispatch", "GITHUB_REF": "refs/heads/main", - "GITHUB_REPOSITORY": live.REPOSITORY, "GITHUB_RUN_ATTEMPT": "1", - "GITHUB_ACTOR": live.OPERATOR, "GITHUB_TRIGGERING_ACTOR": live.OPERATOR, - "GITHUB_SHA": "a" * 40, "GITHUB_RUN_ID": "123"} - env[name] = value - with pytest.raises(ValueError): - live.runtime(env) - - -def test_live_current_main_charge_head_run_and_qualification(case): - repo = {"id": live.REPOSITORY_ID} - calls = {"": repo, "/git/ref/heads/main": {"ref": "refs/heads/main", - "object": {"type": "commit", "sha": "a" * 40}}, - "/pulls/999": {"number": 999, "head": {"sha": "2" * 40}, "draft": False, - "base": {"ref": "main", "repo": repo}, "merged": True}, - "/actions/runs/1": {"event": "workflow_dispatch", "head_branch": "main", - "head_sha": "a" * 40, "run_attempt": 1, "status": "in_progress", - "path": live.WORKFLOW, "repository": repo, - "actor": {"login": live.OPERATOR}, "triggering_actor": {"login": live.OPERATOR}}, - "/actions/runs/123": {"head_sha": "6" * 40, "path": ".github/workflows/ci-ready.yml", - "status": "completed", "conclusion": "success", "repository": repo, - "run_attempt": 1}, - "/actions/artifacts/456": {"id": 456, "name": "kv-actor-qualification-123-1", - "expired": False, "digest": "sha256:" + case.plan["qualification"]["archive"]["sha256"], - "workflow_run": {"id": 123, "head_sha": "6" * 40, - "head_repository_id": live.REPOSITORY_ID}}} - api = SimpleNamespace(request=lambda method, path: calls[path]) - live.verify_live(api, case.plan, "a" * 40, "1") - for path, field, value in [("/pulls/999", "merged", False), - ("/actions/runs/1", "run_attempt", 2), - ("/actions/runs/123", "conclusion", "failure"), - ("/actions/artifacts/456", "digest", "sha256:" + "0" * 64), - ("/actions/artifacts/456", "name", "kv-qualification-123-1")]: - previous = calls[path][field] - calls[path][field] = value - with pytest.raises(ValueError): - live.verify_live(api, case.plan, "a" * 40, "1") - calls[path][field] = previous - - -@pytest.mark.parametrize("change", ["exposure", "prompt", "suite", "client", "limits", - "receipt", "archive", "dependency"]) -def test_hosted_qualification_cannot_be_swapped_for_generic_ci_or_other_actor(case, change): - plan = copy.deepcopy(case.plan) - if change in {"exposure", "prompt", "suite", "client", "limits"}: - actor = live.read_json(case.root / "actor.json") - if change == "exposure": - actor["edit_plan"]["files"] = [{"path": "other-file"}] - elif change == "suite": - actor["suite_sha256"] = "9" * 64 - elif change == "client": - actor["client"] = {"max_requests": 9} - elif change == "limits": - actor["edit_plan"]["limits"] = {"max_checks": 9} - else: - actor["prompt"] = "different prompt" - for arm in ("A2", "A3"): - plan["actors"][arm]["config"] = write(case.root, "actor.json", actor) - elif change == "receipt": - plan["qualification"]["receipt"] = write(case.root, "other.json", {"verdict": "PASS"}) - elif change == "archive": - plan["qualification"]["archive"]["sha256"] = "0" * 64 - else: - (case.root / "code.py").write_text("changed dependency") - with pytest.raises(ValueError): - live.validate_qualification(plan) - - -def add_exception(case): - from tools import kv_pilot_admission as admission - value = {"schema": "kv-one-additional-attempt-v1", - "key": {"repository_id": live.REPOSITORY_ID, "pr_number": case.plan["charge_pr"]}, - "pr_lifetime_cap_microusd": 350_000_000, - "pilot_lifetime_cap_microusd": 1_000_000_000, "additional_attempts": 1, - "prior_binding": {"plan_digest": "8" * 64, - "run_id": admission.CAMPAIGN_NAMESPACE + "original-campaign", - "worst_case": admission.FULL_RESERVATION_MICROUSD}, - "pr_checkpoint": case.plan["ledger"]["checkpoint_commit"], - "daily_checkpoint": case.plan["daily_ledger"]["checkpoint_commit"], - "campaign_id": case.plan["campaign_id"], "execution_id": case.plan["execution_id"]} - case.plan.update(schema="kv-live-plan-v2", budget_exception=value) - return value - - -def test_only_exact_public_v2_freeze_can_resolve_scoped_cap(case): - value = add_exception(case) - # Changing local plan/exception cannot change the verified public bytes. - assert live.frozen_authorization(fetch=lambda _: case.comment, now=NOW.timestamp())[0][ - "plan"]["schema"] == "kv-live-plan-v1" - case.authorization.update(plan=case.plan, plan_sha256=live.fingerprint(case.plan)) - case.comment["body"] = live.PREFIX + json.dumps(case.authorization) + live.SUFFIX - with pytest.raises(ValueError, match="exact bytes"): - live.frozen_authorization(fetch=lambda _: case.comment, now=NOW.timestamp()) - case.locator["body_sha256"] = live.sha(case.comment["body"].encode()) - write(case.root, "locator.json", case.locator) # Simulated separately reviewed root binding. - authorization, _ = live.frozen_authorization(fetch=lambda _: case.comment, now=NOW.timestamp()) - resolved = live.policy(authorization["plan"], "7" * 40) - assert resolved.trusted_limit.key == live.budget.PRKey(live.REPOSITORY_ID, 999) - assert resolved.trusted_limit.limit_microusd == 350_000_000 - assert resolved.additional_attempt.prior_binding.run_id == value["prior_binding"]["run_id"] - assert resolved.additional_attempt.pr_checkpoint == case.plan["ledger"]["checkpoint_commit"] - assert resolved.reservation(now=NOW.timestamp())["amount"] == "162.72" - - -@pytest.mark.parametrize("field,value", [ - ("key", {"repository_id": 1409534281, "pr_number": 275}), - ("key", {"repository_id": 1, "pr_number": 999}), - ("pr_lifetime_cap_microusd", True), ("pr_lifetime_cap_microusd", 200_000_000), - ("pr_lifetime_cap_microusd", 1_000_000_001), - ("pilot_lifetime_cap_microusd", 1_000_000_001), - ("additional_attempts", True), ("additional_attempts", 2), - ("pr_checkpoint", "9" * 40), ("daily_checkpoint", "9" * 40), - ("campaign_id", "different"), ("execution_id", "different"), - ("prior_binding", None), ("prior_binding", {}), ("unexpected", 350_000_000), -]) -def test_v2_exception_cannot_change_key_identity_count_or_anchored_history(case, field, value): - add_exception(case)[field] = value - with pytest.raises(ValueError): - live.validate_plan(case.plan, now=NOW.timestamp()) - - -def test_legacy_plan_rejects_exception_without_versioned_freeze(case): - add_exception(case) - case.plan["schema"] = "kv-live-plan-v1" - with pytest.raises(ValueError): - live.validate_plan(case.plan, now=NOW.timestamp()) - with pytest.raises(ValueError): - live.policy(case.plan, "7" * 40) - - -def test_checked_in_second_attempt_is_exactly_owner_scoped_and_preserves_old_binding(): - root = Path(__file__).resolve().parents[1] - folder = root / 'evidence/kv-live-pilot-20261031/plan' - old = live.read_json(folder / 'plan.json') - old_binding = live.policy(old, '3ece458ceaa92c5dad8d8e0b1e782a6961400b96').binding - extra = live.read_json(folder / 'second-attempt-budget.json') - assert extra['key'] == {'repository_id': live.REPOSITORY_ID, 'pr_number': 275} - assert extra['pr_lifetime_cap_microusd'] == 350_000_000 - assert extra['pilot_lifetime_cap_microusd'] == 1_000_000_000 - assert extra['additional_attempts'] == 1 - assert extra['prior_binding'] == {'plan_digest': old_binding.plan_digest, - 'run_id': old_binding.run_id, 'worst_case': old_binding.worst_case} - new = {**old, 'schema': 'kv-live-plan-v2', 'budget_exception': extra, - 'campaign_id': extra['campaign_id'], 'execution_id': extra['execution_id'], - 'ledger': {**old['ledger'], 'checkpoint_commit': extra['pr_checkpoint']}, - 'daily_ledger': {**old['daily_ledger'], 'checkpoint_commit': extra['daily_checkpoint']}} - resolved = live.additional_attempt(new) - assert resolved.limit.key == live.budget.PRKey(live.REPOSITORY_ID, 275) - assert resolved.limit.limit_microusd == 350_000_000 - assert live.additional_attempt(old) is None - assert live.budget.PR_LIMIT_MICROUSD == 200_000_000 - assert live.budget.DAILY_LIMIT_MICROUSD == 1_000_000_000 diff --git a/tests/test_kv_live_worker.py b/tests/test_kv_live_worker.py deleted file mode 100644 index 802290b8b..000000000 --- a/tests/test_kv_live_worker.py +++ /dev/null @@ -1,166 +0,0 @@ -"""Synthetic-key/fake-HTTP controls only; never provider calls or live ledger access.""" - -import copy -import io -import json -import time -from unittest.mock import Mock -from urllib.error import HTTPError - -import pytest - -from inferrsi.model import anthropic_agent as provider -from tests.test_anthropic_edit_agent import read_task -from tests.test_kv_pilot_admission import policy as sample_policy -from tools import kv_live_worker as worker - -GATE = {"SKILL_EVAL_GATE_PORT": "1234", "SKILL_EVAL_GATE_TOKEN": "a" * 64, - "SKILL_EVAL_GATE_CONTEXT": "b" * 64} - - -def sender(tmp_path, *, transport=None): - config = {"model": "claude-opus-5-5", "input_token_ceiling": 1000000, - "client": {"max_requests": 8, "max_output_tokens": 4096}} - http = transport or Mock(return_value=read_task()) - ipc = Mock() - value = worker.GrantedSender(config, tmp_path, time.monotonic() + 600, - key="OFFLINE-FAKE-KEY", gate=GATE, transport=http, authorize_ipc=ipc) - return value, http, ipc - - -def grant(payload, slot=1): - return {"request_sha256": worker.edit.digest(provider.canonical(payload).encode()), - "slot": slot} - - -def test_exactly_one_gate_and_http_per_slot_with_no_implicit_second_gate(tmp_path): - value, http, ipc = sender(tmp_path) - payload = {"model": "claude-opus-5-5"} - for slot in range(1, 9): - assert value.authorize(grant(payload, slot), value.deadline) - value.send(payload, value.config["client"], 100) - assert value.attempts == ipc.call_count == http.call_count == 8 - assert http.call_args.kwargs == {"environment": {}, "require_authorization": False} - assert http.call_args.args[2] == "OFFLINE-FAKE-KEY" - with pytest.raises(provider.AgentError): - value.send(payload, value.config["client"], 100) - assert http.call_count == 8 - raw = worker.scripted._read_json(tmp_path / "complete.json") - assert raw["usage"] == read_task()["usage"] # RAW cache classes, not normalized totals. - assert "OFFLINE-FAKE-KEY" not in (tmp_path / "complete.json").read_text() - - -@pytest.mark.parametrize("failure", ["transport", "lost_gate", "hash", "spec", "deadline"]) -def test_no_retry_or_http_without_one_exact_valid_grant(tmp_path, failure): - value, http, ipc = sender(tmp_path) - payload = {"model": "claude-opus-5-5"} - if failure == "lost_gate": - ipc.side_effect = OSError("offline lost reply") - with pytest.raises(OSError): - value.authorize(grant(payload), value.deadline) - else: - value.authorize(grant(payload), value.deadline) - spec = value.config["client"] - if failure == "transport": - http.side_effect = OSError("OFFLINE-FAKE-KEY must not enter error") - elif failure == "hash": - payload["changed"] = True - elif failure == "spec": - spec = {**spec, "max_requests": 9} - elif failure == "deadline": - value.deadline = time.monotonic() - 1 - with pytest.raises(provider.AgentError) as error: - value.send(payload, spec, 100) - assert "OFFLINE-FAKE-KEY" not in str(error.value) - with pytest.raises(provider.AgentError): - value.send(payload, spec, 100) - assert http.call_count == (1 if failure == "transport" else 0) - - -@pytest.mark.parametrize("change", ["missing", "late", "wrong_model", "over_bound"]) -def test_unknown_response_usage_is_relayed_as_unresolved_not_refunded(tmp_path, change): - response = copy.deepcopy(read_task()) - if change == "missing": - response["usage"].pop("cache_creation_input_tokens") - elif change == "wrong_model": - response["model"] = "other" - elif change == "over_bound": - response["usage"]["output_tokens"] = 4097 - value, _, _ = sender(tmp_path, transport=Mock(return_value=response)) - if change == "late": - def late(*args, **kwargs): - value.deadline = time.monotonic() - 1 - return response - value.transport = late - payload = {"model": "claude-opus-5-5"} - value.authorize(grant(payload), value.deadline) - value.send(payload, value.config["client"], 100) - assert worker.scripted._read_json(tmp_path / "complete.json")["usage"] is None - - - -def test_ninth_authorized_send_cannot_make_a_ninth_http_attempt(tmp_path): - value, http, _ = sender(tmp_path) - payload = {"model": "claude-opus-5-5"} - for slot in range(1, 9): - value.authorize(grant(payload, slot), value.deadline) - value.send(payload, value.config["client"], 100) - value.authorize(grant(payload, 9), value.deadline) - with pytest.raises(provider.AgentError): - value.send(payload, value.config["client"], 100) - assert http.call_count == 8 - - -@pytest.mark.parametrize("failure,expected", [ - (401, "http_401"), (429, "http_429"), (500, "http_500"), (503, "http_503"), - ("bad_json", "invalid_response_json"), ("echo", "credential_echo"), - ("network", "transport_error"), -]) -def test_real_transport_closed_failure_code_survives_generic_failure( - tmp_path, monkeypatch, failure, expected): - value, _, _ = sender(tmp_path, transport=provider.send_message) - value.config["client"] = json.loads(sample_policy().provider_json)["client"] - opener = Mock() - monkeypatch.setattr(provider.urllib.request, "build_opener", Mock(return_value=opener)) - if type(failure) is int: - opener.open.side_effect = HTTPError("https://unused.invalid", failure, - "OFFLINE-FAKE-KEY private error", {}, io.BytesIO(b"private response body")) - elif failure == "network": - opener.open.side_effect = OSError("OFFLINE-FAKE-KEY private network detail") - else: - raw = b"not JSON: private response" if failure == "bad_json" else ( - provider.canonical({"text": "OFFLINE-FAKE-KEY"}).encode()) - opener.open.return_value = io.BytesIO(raw) - payload = {"model": "claude-opus-5-5"} - value.authorize(grant(payload), value.deadline) - with pytest.raises(provider.AgentError, match="^provider_transport_error$"): - value.send(payload, value.config["client"], 100) - assert value.transport_error_code == expected - assert value.attempts == opener.open.call_count == 1 - assert worker.scripted._read_json(tmp_path / "complete.json")["usage"] is None - with pytest.raises(provider.AgentError): - value.send(payload, value.config["client"], 100) - assert opener.open.call_count == 1 - - -def test_arbitrary_exception_and_nonclosed_codes_are_never_retained(tmp_path): - value, http, _ = sender(tmp_path, transport=Mock( - side_effect=provider.AgentError("OFFLINE-FAKE-KEY http_401 private payload"))) - payload = {"model": "claude-opus-5-5"} - value.authorize(grant(payload), value.deadline) - with pytest.raises(provider.AgentError, match="^provider_transport_error$"): - value.send(payload, value.config["client"], 100) - assert value.transport_error_code == "unknown" - assert value.attempts == http.call_count == 1 - for invalid in ("http_099", "http_600", "http_４０１", "http_401\n", {}, None, True): - assert worker.closed_reason(invalid) == "unknown" - - -def test_failed_presend_gate_has_known_zero_transport_invocations(tmp_path): - value, http, ipc = sender(tmp_path) - ipc.side_effect = OSError("OFFLINE-FAKE-KEY private gate detail") - with pytest.raises(OSError): - value.authorize(grant({"model": "claude-opus-5-5"}), value.deadline) - assert value.attempts == http.call_count == 0 - assert value.transport_error_code is None - assert not (tmp_path / "complete.json").exists() diff --git a/tests/test_kv_pilot_admission.py b/tests/test_kv_pilot_admission.py deleted file mode 100644 index 226f164d1..000000000 --- a/tests/test_kv_pilot_admission.py +++ /dev/null @@ -1,572 +0,0 @@ -"""Fake stores/clock/launch only: no credentials, network or provider calls.""" - -import json -import pickle -from dataclasses import replace -from datetime import datetime, timedelta, timezone - -import pytest - -from inferrsi.model import anthropic_agent as provider -from tools import kv_pilot_admission as pilot -from tools import skill_eval_budget as b - -NOW = datetime(2026, 10, 30, 15, tzinfo=timezone.utc) -KEY = b.PRKey(123, 999) -DAY = b.shanghai_day(NOW) -BOUND = pilot.FULL_RESERVATION_MICROUSD -USAGE = {"input_tokens": 10, "output_tokens": 4, - "cache_read_input_tokens": 5, "cache_creation_input_tokens": 6} - - -def policy(**changes): - config = { - "model": {"name": "claude-opus-5-5", "effort": "not_applicable"}, - "client": {"kind": provider.EDIT_KIND, "provider": "yibuapi", - "endpoint": provider.ENDPOINT, "api_version": "2023-06-01", - "max_requests": 8, "max_output_tokens": 4096, - "max_request_bytes": 262144, "max_read_bytes": 32768, - "max_tool_calls": 32}, - "pricing": {"currency": "USD", "unit": "per_million_tokens", - "model": "claude-opus-5-5", "group": "audited-example-group", - "rates": {"input": "8", "output": "40", "cache_read": "0.4", - "cache_write": "10"}, "request_fee": "0", - "max_input_tokens": 1_000_000, - "rate_evidence": "https://github.com/example/repo/issues/1#issuecomment-1", - "expires_at": int(NOW.timestamp()) + 7200}, - "spend_limit": {"currency": "USD", "amount": "200"}, - } - values = {"campaign_id": "example-feasibility", "key": KEY, - "execution_id": "example-run-1.1", "plan_sha256": "1" * 64, - "pr_head": "2" * 40, "source_revision": "3" * 40, - "actor_sha256": ("4" * 64, "5" * 64), - "provider_json": provider.canonical(config), - "expires_at": int(NOW.timestamp()) + 7200} - return pilot.PilotPolicy(**(values | changes)) - - -class FakeStore: - """Exercise the existing store transition/CAS contract, including ambiguity.""" - - def __init__(self, ledger, events): - self.ledger, self.events = ledger, events - self.daily = type(ledger) is b.DailyLedger - self.name = "daily" if self.daily else "pr" - self.token = 0 - self.outcomes = [] - self.before_cas = None - self.on_read = None - - def read_verified(self, key): - if self.on_read: - self.on_read(self) - assert key == self.ledger.key - cls = b.DailySnapshot if self.daily else b.Snapshot - return cls(str(self.token), self.ledger) - - def compare_and_swap(self, expected, successor): - validate = b.validate_daily_successor if self.daily else b.validate_successor - validate(expected.ledger, successor) - if self.before_cas: - callback, self.before_cas = self.before_cas, None - callback(self) - if expected != self.read_verified(self.ledger.key): - return b.WriteResult.CONFLICT - outcome = self.outcomes.pop(0) if self.outcomes else "commit" - if outcome == "conflict": - return b.WriteResult.CONFLICT - if outcome == "unknown": - return b.WriteResult.UNKNOWN - self.ledger = successor - self.token += 1 - self.events.append((self.name, successor.entries[-1].phase)) - if outcome == "crash": - raise KeyboardInterrupt("simulated process death after durable write") - return b.WriteResult.UNKNOWN if outcome == "unknown_commit" else b.WriteResult.COMMITTED - - -class World: - def __init__(self, *, carryover=True, legacy=0): - self.now, self.monotonic = NOW, 100.0 - self.events, self.live = [], True - entries = [] - if carryover: - # Synthetic ledger identities reproduce the audited $491.35 retained - # obligation, not invoices or a claim that old reservations settled. - for index, amount in enumerate((162_720_000, 122_720_000, 185_570_000, - 20_340_000), 1): - entries.append(b.DailyEntry(b.PRKey(KEY.repository_id, index), - b.Binding(f"{index:064x}", f"old-{index}.1", amount), - "2026-10-29", "claimed")) - if legacy: - entries.append(b.DailyEntry(KEY, b.Binding("9" * 64, "legacy.1", legacy), - "2026-10-29", "claimed")) - self.daily = FakeStore(b.DailyLedger(b.DailyKey(KEY.repository_id), "audited-import", - tuple(entries)), self.events) - self.pr = FakeStore(b.Ledger(KEY, b.Genesis("audited-pr-history", legacy)), self.events) - - def kwargs(self): - return {"check_current": lambda value: self.live, - "wall_clock": lambda: self.now, "monotonic_clock": lambda: self.monotonic} - - def admit(self, config=None): - return pilot.admit(config or policy(), self.daily, self.pr, **self.kwargs()) - - -def grant(session, arm="A2", slot=1, **changes): - return {"plan_sha256": session.policy.plan_sha256, - "execution_id": session.policy.execution_id, "arm": arm, "slot": slot, - "request_sha256": f"{slot:064x}", - "actor_sha256": session.policy.actor_sha256[pilot.ARMS.index(arm)], - "reserved_microusd": pilot.PER_REQUEST_MICROUSD} | changes - - -def started(world=None): - world = world or World() - session = world.admit() - session.start_arm("A2", 700.0) - return world, session - - -def test_full_reservation_and_both_claims_precede_launch_and_every_send(): - world = World() - launched = [] - - def launch(session): - assert world.events == [("daily", "reserved"), ("pr", "reserved"), - ("daily", "claimed"), ("pr", "claimed")] - assert world.daily.ledger.charged_on(DAY) == 654_070_000 - assert world.pr.ledger.remaining == 37_280_000 - session.start_arm("A2", 700.0) - request = grant(session) - assert session.authorize(request, 700.0) is True - session.complete(request, USAGE) - launched.append(session) - return "offline-result" - - assert pilot.run_admitted(policy(), world.daily, world.pr, launch=launch, - **world.kwargs()) == "offline-result" - with pytest.raises(b.BudgetError): - launched[0].authorize(grant(launched[0], slot=2), 700.0) - assert world.pr.ledger.charged == BOUND # No release after success either. - - -@pytest.mark.parametrize("changes", [{}, {"plan_sha256": "6" * 64}, - {"campaign_id": "new-id"}, - {"execution_id": "example-run-2.1"}, - {"expires_at": int(NOW.timestamp()) + 7000}]) -def test_duplicate_new_digest_new_id_replan_relaunch_cannot_reset(changes): - world = World() - world.admit() - with pytest.raises(b.BudgetError): - world.admit(policy(**changes)) - assert len(world.events) == 4 - assert world.pr.ledger.charged == BOUND - - -def test_moving_campaign_to_another_pr_cannot_reset(): - world = World() - world.admit() - other_key = b.PRKey(KEY.repository_id, KEY.pr_number + 1) - other_pr = FakeStore(b.Ledger(other_key, b.Genesis("new-pr-audit", 0)), world.events) - with pytest.raises(b.BudgetError, match="already bound"): - pilot.admit(policy(key=other_key, campaign_id="also-new-id"), world.daily, - other_pr, **world.kwargs()) - assert other_pr.ledger.charged == 0 - - -@pytest.mark.parametrize("store_name, outcomes", [ - ("daily", ["conflict"]), ("daily", ["unknown"]), ("daily", ["unknown_commit"]), - ("pr", ["conflict"]), ("pr", ["unknown"]), ("pr", ["unknown_commit"]), - ("daily", ["commit", "unknown_commit"]), ("pr", ["commit", "unknown_commit"]), -]) -def test_conflict_unknown_and_partial_reservation_never_launch(store_name, outcomes): - world = World() - getattr(world, store_name).outcomes = outcomes - launches = [] - with pytest.raises(b.BudgetError): - pilot.run_admitted(policy(), world.daily, world.pr, - launch=launches.append, **world.kwargs()) - assert not launches - charged = world.daily.ledger.charged_on(DAY) - if charged > 491_350_000: - with pytest.raises(b.BudgetError): - world.admit() - assert world.daily.ledger.charged_on(DAY) == charged - - -@pytest.mark.parametrize("store_name, outcomes", [ - ("daily", ["crash"]), ("pr", ["crash"]), - ("daily", ["commit", "crash"]), ("pr", ["commit", "crash"]), -]) -def test_crash_after_any_durable_step_has_no_takeover(store_name, outcomes): - world = World() - getattr(world, store_name).outcomes = outcomes - with pytest.raises(KeyboardInterrupt): - world.admit() - with pytest.raises(b.BudgetError): - world.admit() - assert world.daily.ledger.charged_on(DAY) == 654_070_000 - - -def test_stale_pr_cas_keeps_daily_reservation_no_launch(): - world = World() - world.pr.before_cas = lambda store: setattr(store, "token", store.token + 1) - with pytest.raises(b.BudgetError, match="conflicted"): - world.admit() - assert world.events == [("daily", "reserved")] - assert world.daily.ledger.charged_on(DAY) == 654_070_000 - assert world.pr.ledger.charged == 0 - - -def test_pr_cap_includes_legacy_and_prechecks_before_any_write(): - world = World(legacy=37_280_001) - with pytest.raises(b.BudgetError, match="PR cumulative"): - world.admit() - assert not world.events - assert World(legacy=37_280_000).admit() - - -def test_unrelated_settled_history_is_not_new_pilot_spend_or_current_day_charge(): - world = World(carryover=False) - entries = tuple(b.DailyEntry(b.PRKey(123, index), - b.Binding(f"{index:064x}", f"past-{index}.1", 200_000_000), - f"2026-10-{index:02}", "finished", f"2026-10-{index:02}", 1) - for index in range(1, 6)) - world.daily.ledger = b.DailyLedger(b.DailyKey(123), "audited-import", entries) - assert world.daily.ledger.charged_on(DAY) == 0 - world.admit() - assert world.pr.ledger.charged == BOUND - assert world.daily.ledger.charged_on(DAY) == BOUND - assert sum(e.binding.worst_case for e in world.daily.ledger.entries) > 1_000_000_000 - - -def test_unsettled_historical_charges_still_carry_and_enforce_daily_cap(): - world = World(carryover=False) - entries = tuple(b.DailyEntry(b.PRKey(123, index), - b.Binding(f"{index:064x}", f"past-{index}.1", 200_000_000), - f"2026-10-{index:02}", "claimed") for index in range(1, 6)) - world.daily.ledger = b.DailyLedger(b.DailyKey(123), "audited-import", entries) - assert world.daily.ledger.charged_on(DAY) == 1_000_000_000 - with pytest.raises(b.BudgetError, match="Global Shanghai daily budget"): - world.admit() - assert not world.events - - -def test_inconsistent_pr_tip_cannot_erase_imported_daily_obligations(): - world = World(legacy=1) - world.pr.ledger = b.Ledger(KEY, b.Genesis("incorrect-tip-subtotal", 0)) - with pytest.raises(b.BudgetError, match="Inconsistent"): - world.admit() - assert not world.events - - -def test_rollover_during_claim_burns_reservations_without_launch(): - world = World() - - def rollover(store): - world.now = NOW.replace(hour=16) - - world.pr.before_cas = rollover - with pytest.raises(b.BudgetError, match="admission day"): - world.admit() - assert world.pr.ledger.charged == BOUND - assert world.daily.ledger.charged_on("2026-10-31") == 654_070_000 - assert world.events == [("daily", "reserved"), ("pr", "reserved")] - - -def test_rollover_and_pricing_expiry_block_new_grants_without_refund(): - for change in (lambda w: setattr(w, "now", NOW.replace(hour=16)), - lambda w: setattr(w, "now", NOW + timedelta(hours=2))): - world, session = started() - change(world) - with pytest.raises(b.BudgetError): - session.authorize(grant(session), 700.0) - assert session.attempts() == {"A2": 1, "A3": 0} - assert world.pr.ledger.charged == BOUND - - -@pytest.mark.parametrize("which", ["daily", "pr"]) -def test_each_grant_rechecks_both_exact_durable_claims(which): - world, session = started() - store = getattr(world, which) - old = store.ledger.entries[-1] - store.ledger = replace(store.ledger, entries=store.ledger.entries[:-1] + ( - replace(old, phase="reserved"),), revision=store.ledger.revision - 1) - with pytest.raises(b.BudgetError, match="Both exact durable"): - session.authorize(grant(session), 700.0) - assert session.attempts()["A2"] == 1 - - -def test_revoked_authorization_burns_attempt_and_stops_both_arms(): - world, session = started() - world.live = False - with pytest.raises(b.BudgetError): - session.authorize(grant(session), 700.0) - world.live = True - with pytest.raises(b.BudgetError): - session.start_arm("A3", 700.0) - assert session.attempts()["A2"] == 1 - - -def test_known_usage_allows_exactly_eight_per_arm_sixteen_total(): - world = World() - session = world.admit() - for arm in pilot.ARMS: - session.start_arm(arm, 700.0) - for slot in range(1, 9): - request = grant(session, arm, slot) - assert session.authorize(request, 700.0) - session.complete(request, USAGE) - session.finish_arm(arm) - assert session.attempts() == {"A2": 8, "A3": 8} - with pytest.raises(b.BudgetError): - session.authorize(grant(session, "A3", 9), 700.0) - assert session.attempts() == {"A2": 8, "A3": 8} - assert world.pr.ledger.charged == BOUND - - -def test_ninth_slot_cannot_borrow_from_other_arm(): - _, session = started() - for slot in range(1, 9): - request = grant(session, slot=slot) - session.authorize(request, 700.0) - session.complete(request, USAGE) - with pytest.raises(b.BudgetError): - session.authorize(grant(session, slot=9), 700.0) - assert session.attempts() == {"A2": 8, "A3": 0} - - -@pytest.mark.parametrize("usage", [None, {}, {**USAGE, "output_tokens": True}, - {**USAGE, "output_tokens": 4097}, - {**USAGE, "cache_creation_input_tokens": 1_000_001}]) -def test_unknown_or_overbound_usage_stops_both_arms(usage): - world, session = started() - request = grant(session) - session.authorize(request, 700.0) - with pytest.raises((b.BudgetError, provider.AgentError)): - session.complete(request, usage) - for action in (lambda: session.authorize(grant(session, slot=2), 700.0), - lambda: session.finish_arm("A2"), - lambda: session.start_arm("A3", 700.0)): - with pytest.raises(b.BudgetError): - action() - assert world.pr.ledger.charged == BOUND - - -def test_missing_usage_lost_reply_blocks_next_request_and_restart(): - world, session = started() - session.authorize(grant(session), 700.0) - with pytest.raises(b.BudgetError): - session.authorize(grant(session, slot=2), 700.0) - with pytest.raises(b.BudgetError): - world.admit() - assert session.attempts() == {"A2": 1, "A3": 0} - - -@pytest.mark.parametrize("changes", [{"plan_sha256": "a" * 64}, {"slot": True}, - {"slot": 2}, {"execution_id": "other"}, - {"actor_sha256": "a" * 64}, - {"reserved_microusd": 1}]) -def test_candidate_grant_cannot_choose_plan_actor_slot_or_budget(changes): - _, session = started() - with pytest.raises(b.BudgetError): - session.authorize(grant(session, **changes), 700.0) - assert session.attempts()["A2"] == 1 - - -def test_repeated_completion_and_wrong_request_never_reopen_latch(): - for wrong in (False, True): - _, session = started() - request = grant(session) - session.authorize(request, 700.0) - if not wrong: - session.complete(request, USAGE) - with pytest.raises(b.BudgetError): - session.complete(grant(session, slot=2) if wrong else request, USAGE) - with pytest.raises(b.BudgetError): - session.authorize(grant(session, slot=2), 700.0) - - -def test_deadline_cannot_be_extended_and_session_cannot_be_serialized(): - for deadline in (701.0, 699.0): - _, session = started() - with pytest.raises(b.BudgetError): - session.authorize(grant(session), deadline) - world, session = started() - world.monotonic = 700 - with pytest.raises(b.BudgetError, match="deadline"): - session.authorize(grant(session), 700) - with pytest.raises(TypeError): - pickle.dumps(session) - - -def test_existing_full_context_read_submit_kind_is_not_editing_authority(): - config = json.loads(policy().provider_json) - config["client"].update(kind=provider.FULL_CONTEXT_KIND, max_requests=1) - with pytest.raises(b.BudgetError): - policy(provider_json=provider.canonical(config)) - - -def additional_world(): - world = World() - first = policy() - world.admit(first) - original_pr = b.encode(world.pr.ledger) - limit = b.TrustedPRLimit(KEY, 350_000_000) - world.pr.ledger = b.decode(original_pr, trusted_limit=limit) - extra = pilot.AdditionalAttempt(limit, first.binding, "one-fixed-additional", - "one-fixed-additional.1", "a" * 40, "b" * 40) - second = policy(campaign_id=extra.campaign_id, execution_id=extra.execution_id, - plan_sha256="6" * 64, additional_attempt=extra) - assert b.encode(world.pr.ledger) == original_pr - return world, first, second - - -def test_one_additional_pair_keeps_both_full_claims_and_every_ceiling(): - world, first, second = additional_world() - prior_pr, prior_daily = world.pr.ledger.entries, world.daily.ledger.entries - session = world.admit(second) - assert world.pr.ledger.entries[:1] == prior_pr - assert world.daily.ledger.entries[:-1] == prior_daily - assert world.pr.ledger.charged == 325_440_000 - assert world.pr.ledger.remaining == 24_560_000 - assert world.daily.ledger.charged_on(DAY) == 816_790_000 - assert b.DAILY_LIMIT_MICROUSD - world.daily.ledger.charged_on(DAY) == 183_210_000 - assert pilot.PILOT_LIMIT_MICROUSD - 2 * BOUND == 674_560_000 - assert all(e.phase == "claimed" and e.actual_cost is None for e in world.pr.ledger.entries) - session.start_arm("A2", 700.0) - session.authorize(grant(session), 700.0) - with pytest.raises(provider.AgentError, match="unknown_usage"): - session.complete(grant(session), None) - with pytest.raises(b.BudgetError): - session.start_arm("A3", 700.0) - assert world.pr.ledger.entries[0].binding == first.binding - assert len(world.events) == 8 - - -def test_historical_v1_binding_digest_is_unchanged(): - from dataclasses import asdict - old = policy() - fields = asdict(old) - fields.pop("additional_attempt") - assert old.binding.plan_digest == pilot._fingerprint(fields) - - -@pytest.mark.parametrize("change", ["same", "digest", "third", "day", "smaller"]) -def test_additional_attempt_cannot_be_reused_or_turned_into_third(change): - world, _, second = additional_world() - world.admit(second) - before = list(world.events) - with pytest.raises(b.BudgetError): - if change == "digest": - second = replace(second, plan_sha256="7" * 64) - elif change == "third": - extra = replace(second.additional_attempt, prior_binding=second.binding, - campaign_id="third", execution_id="third.1") - second = replace(second, campaign_id=extra.campaign_id, - execution_id=extra.execution_id, additional_attempt=extra) - elif change == "day": - world.now += timedelta(days=1) - elif change == "smaller": - config = json.loads(second.provider_json) - config["client"]["max_requests"] = 2 - second = replace(second, provider_json=provider.canonical(config)) - world.admit(second) - assert world.events == before - assert world.pr.ledger.charged == 325_440_000 - - -def test_other_pr_or_repository_cannot_borrow_exception_context(): - world, _, second = additional_world() - for key in (b.PRKey(KEY.repository_id, KEY.pr_number + 1), b.PRKey(9999, KEY.pr_number)): - with pytest.raises(b.BudgetError, match="reviewed scope"): - replace(second, key=key) - # Even a separately constructed context on another PR cannot move the old KV claim. - key = b.PRKey(KEY.repository_id, KEY.pr_number + 1) - extra = replace(second.additional_attempt, limit=b.TrustedPRLimit(key, 350_000_000)) - other = replace(second, key=key, additional_attempt=extra) - store = FakeStore(b.Ledger(key, b.Genesis("other audit", 0), - trusted_limit=extra.limit), world.events) - with pytest.raises(b.BudgetError, match="prior claim"): - pilot.admit(other, world.daily, store, **world.kwargs()) - assert store.ledger.charged == 0 and len(world.events) == 4 - - -@pytest.mark.parametrize("change", ["prior_binding", "prior_phase", "missing", "extra_pr"]) -def test_exact_original_claim_is_required_before_any_new_write(change): - world, _, second = additional_world() - if change == "prior_binding": - second = replace(second, additional_attempt=replace(second.additional_attempt, - prior_binding=replace(second.additional_attempt.prior_binding, plan_digest="8" * 64))) - elif change == "prior_phase": - old = world.pr.ledger.entries[0].binding - world.pr.ledger = b.advance(world.pr.ledger, old, "finished", actual_cost=0) - world.daily.ledger = b.daily_advance(world.daily.ledger, KEY, old, - "finished", DAY, actual_cost=0) - elif change == "missing": - world = World() - world.pr.ledger = replace(world.pr.ledger, trusted_limit=second.trusted_limit) - else: - other = b.Binding("8" * 64, "unrelated-pr-history", 1) - world.pr.ledger = b.reserve(world.pr.ledger, other) - world.daily.ledger = b.daily_reserve(world.daily.ledger, KEY, other, DAY) - before = list(world.events) - with pytest.raises(b.BudgetError): - world.admit(second) - assert world.events == before - - -@pytest.mark.parametrize("store_name,outcomes", [ - ("daily", ["conflict"]), ("daily", ["unknown"]), ("daily", ["unknown_commit"]), - ("pr", ["conflict"]), ("pr", ["unknown"]), ("pr", ["unknown_commit"]), - ("daily", ["commit", "unknown_commit"]), ("pr", ["commit", "unknown_commit"]), - ("daily", ["crash"]), ("pr", ["crash"]), -]) -def test_additional_partial_or_uncertain_cas_never_grants_or_refunds(store_name, outcomes): - world, _, second = additional_world() - getattr(world, store_name).outcomes = outcomes - launched = [] - with pytest.raises((b.BudgetError, KeyboardInterrupt)): - pilot.run_admitted(second, world.daily, world.pr, launch=launched.append, **world.kwargs()) - assert launched == [] - assert world.pr.ledger.charged >= BOUND - assert world.daily.ledger.charged_on(DAY) >= 654_070_000 - assert world.pr.ledger.entries[0].phase == "claimed" - if world.daily.ledger.charged_on(DAY) > 654_070_000: - with pytest.raises(b.BudgetError): - world.admit(second) - - -def test_two_racing_additional_admissions_have_only_one_durable_winner(): - world, _, second = additional_world() - winners = [] - world.daily.before_cas = lambda _: winners.append(world.admit(second)) - with pytest.raises(b.BudgetError): - world.admit(second) - assert len(winners) == 1 and len(world.events) == 8 - assert world.pr.ledger.charged == 325_440_000 - assert world.daily.ledger.charged_on(DAY) == 816_790_000 - - -def test_additional_pair_checks_full_daily_cap_before_any_write(): - world, _, second = additional_world() - extra = b.Binding('9' * 64, 'unrelated-new-obligation', 200_000_000) - world.daily.ledger = b.daily_reserve(world.daily.ledger, b.PRKey(KEY.repository_id, 50), - extra, DAY) - before = list(world.events) - with pytest.raises(b.BudgetError, match='daily'): - world.admit(second) - assert world.events == before - assert world.pr.ledger.charged == BOUND - - -def test_namespace_history_is_rechecked_before_each_additional_request(): - world, _, second = additional_world() - session = world.admit(second) - session.start_arm('A2', 700.0) - unrelated = b.Binding('9' * 64, pilot.CAMPAIGN_NAMESPACE + 'unapproved-third', 1) - world.daily.ledger = b.daily_reserve(world.daily.ledger, b.PRKey(KEY.repository_id, 50), - unrelated, DAY) - with pytest.raises(b.BudgetError, match='already bound'): - session.authorize(grant(session), 700.0) diff --git a/tests/test_kv_reclamation.py b/tests/test_kv_reclamation.py deleted file mode 100644 index edf0ee164..000000000 --- a/tests/test_kv_reclamation.py +++ /dev/null @@ -1,254 +0,0 @@ -"""Software contract tests; mocks do not establish artifact isolation or efficacy.""" - -import json -import subprocess -import sys -import time -from pathlib import Path -from unittest.mock import patch - -import pytest - -from inferrsi.harness import kv_reclamation as kv - - -def test_capture_only_allows_candidate_bytes(tmp_path): - (tmp_path / "candidate.py").write_text("# starter\n") - (tmp_path / "contract.md").write_text("contract\n") - frozen = kv.workspace_hashes(tmp_path) - (tmp_path / "candidate.py").write_text("# changed\n") - assert kv.capture_artifact(tmp_path, frozen) == b"# changed\n" - (tmp_path / "contract.md").write_text("tampered\n") - with pytest.raises(ValueError, match="Protected"): - kv.capture_artifact(tmp_path, frozen) - - -@pytest.mark.parametrize("change", ["missing", "extra", "symlink", "empty", "oversized"]) -def test_bad_artifact_and_workspace_fail_before_execution(tmp_path, change): - target = tmp_path / "candidate.py" - target.write_text("# starter\n") - frozen = kv.workspace_hashes(tmp_path) - if change == "missing": - target.unlink() - elif change == "extra": - (tmp_path / "other.py").write_text("# extra\n") - elif change == "symlink": - target.unlink() - target.symlink_to("other.py") - else: - target.write_bytes(b"" if change == "empty" else b"x" * 65537) - with pytest.raises(ValueError): - kv.capture_artifact(tmp_path, frozen) - - -def fake_executor(command, cwd, output, deadline, *, environment): - assert environment == {"PATH": "/usr/bin:/bin", "LANG": "C"} - assert set(p.name for p in cwd.iterdir()) == {"candidate.py", "runner.py", "inputs.json"} - assert "expected" not in (cwd / "inputs.json").read_text() - assert "oracle" not in (cwd / "inputs.json").read_text() - output.mkdir(parents=True) - (output / "stdout.log").write_text(json.dumps({"observations": [], "pending": []})) - (output / "stderr.log").write_text("INFERRSI_KV_READY\n") - return {"status": "completed", "returncode": 0, "elapsed_seconds": 0.5, - "stdout_path": str(output / "stdout.log"), - "stderr_path": str(output / "stderr.log")} - - -def test_candidate_receives_only_inputs_not_answers_or_knowledge(tmp_path): - with patch.object(kv, "sandbox_command", return_value=["sandbox"]), \ - patch.object(kv, "execute_command", side_effect=fake_executor): - record, value = kv.invoke(b"# source", {"mode": "verify", "events": []}, - tmp_path / "output", time.monotonic() + 1) - assert record["elapsed_seconds"] == 0.5 - assert value == {"observations": [], "pending": []} - assert json.loads((tmp_path / "output/execution.json").read_text()) == { - "status": "completed", "returncode": 0, "elapsed_seconds": 0.5} - - -@pytest.mark.parametrize("status,returncode,output_kind,expected", [ - ("completed", 0, "valid", "value"), - ("completed", 0, "invalid_json", None), - ("completed", 0, "empty", None), - ("failed", 1, "valid", None), - ("failed", -9, "valid", None), - ("timeout", -9, "no_logs", None), - ("cancelled", -15, "no_logs", None), - ("error", None, "no_logs", None), - ("completed", 0, "no_marker", kv.SandboxUnavailable), - ("completed", 0, "missing_stdout", None), - ("completed", 0, "missing_stderr", FileNotFoundError), -]) -def test_invoke_retains_closed_original_execution_before_all_output_paths( - tmp_path, status, returncode, output_kind, expected -): - output = tmp_path / "run" - actual = {"observations": [], "pending": []} - original = {"status": status, "returncode": returncode, "elapsed_seconds": 0.125, - "stdout_path": str(output / "stdout.log"), - "stderr_path": str(output / "stderr.log"), "metric": 123, - "argv": ["private-canary"], "cwd": "private-canary", - "error": "private-canary", "environment": {"KEY": "private-canary"}, - "pid": 12345, "extra": "private-canary"} - - def execute(*args, **kwargs): - output.mkdir() - if output_kind not in {"no_logs", "missing_stdout"}: - data = "invalid JSON" if output_kind == "invalid_json" else ( - "" if output_kind == "empty" else json.dumps(actual)) - (output / "stdout.log").write_text(data) - if output_kind not in {"no_logs", "missing_stderr"}: - (output / "stderr.log").write_text( - "" if output_kind == "no_marker" else "INFERRSI_KV_READY\n") - return original - - before = dict(original) - with patch.object(kv, "sandbox_command", return_value=["sandbox"]), \ - patch.object(kv, "execute_command", side_effect=execute) as launch: - if isinstance(expected, type): - with pytest.raises(expected): - kv.invoke(b"# source", {}, output, time.monotonic() + 1) - else: - record, value = kv.invoke(b"# source", {}, output, time.monotonic() + 1) - assert record is original - assert value == (actual if expected == "value" else None) - launch.assert_called_once() - assert original == before # Preserve elapsed/metric and all existing return semantics. - raw = (output / "execution.json").read_text() - assert json.loads(raw) == {"status": status, "returncode": returncode, - "elapsed_seconds": 0.125} - assert "private-canary" not in raw and str(tmp_path) not in raw - - -def test_execution_retention_failure_stops_before_output_check_without_retry(tmp_path): - with patch.object(kv, "sandbox_command", return_value=["sandbox"]), \ - patch.object(kv, "execute_command", side_effect=fake_executor) as launch, \ - patch.object(kv, "atomic_write_json", side_effect=OSError("storage unavailable")), \ - patch.object(kv, "read_regular") as read: - with pytest.raises(OSError, match="storage unavailable"): - kv.invoke(b"# source", {}, tmp_path / "run", time.monotonic() + 1) - launch.assert_called_once() - read.assert_not_called() - assert not (tmp_path / "run/execution.json").exists() - - -def test_executor_did_not_return_has_no_invented_execution_record(tmp_path): - with patch.object(kv, "sandbox_command", return_value=["sandbox"]), \ - patch.object(kv, "execute_command", side_effect=KeyboardInterrupt) as launch: - with pytest.raises(KeyboardInterrupt): - kv.invoke(b"# source", {}, tmp_path / "run", time.monotonic() + 1) - launch.assert_called_once() - assert not (tmp_path / "run/execution.json").exists() - - -def test_no_sandbox_has_no_host_fallback(tmp_path): - with patch.object(kv, "sandbox_command", side_effect=kv.SandboxUnavailable("blocked")), \ - patch.object(kv, "execute_command") as execute: - with pytest.raises(kv.SandboxUnavailable): - kv.invoke(b"# source", {}, tmp_path, time.monotonic() + 1) - execute.assert_not_called() - - -def test_missing_worker_marker_is_infrastructure_failure(tmp_path): - def execute(*args, **kwargs): - record = fake_executor(*args, **kwargs) - Path(record["stderr_path"]).write_bytes(b"") - return record - - with patch.object(kv, "sandbox_command", return_value=["sandbox"]), \ - patch.object(kv, "execute_command", side_effect=execute): - with pytest.raises(kv.SandboxUnavailable, match="boundary"): - kv.invoke(b"# source", {}, tmp_path / "run", time.monotonic() + 1) - - -def test_each_event_compared_by_trusted_oracle_and_missing_outputs_rejected(tmp_path): - traces = [{"id": "one", "events": []}, {"id": "two", "events": []}] - def compare(trace, value): - return value == trace["id"] - with patch.object(kv, "invoke", side_effect=[ - ({"status": "completed"}, "one"), ({"status": "completed"}, None)]): - result = kv.verify_source(b"# source", traces, tmp_path, time.monotonic() + 1, compare) - assert result == {"verdict": "FAIL", "checks": {"one": True, "two": False}} - - -def test_grader_failure_is_unverified_not_refuted(tmp_path): - with patch.object(kv, "invoke", return_value=({"status": "timeout"}, None)): - result = kv.verify_source(b"# source", [{"id": "one", "events": []}], tmp_path, - time.monotonic() + 1, lambda *_: True) - assert result["verdict"] == "INCONCLUSIVE" - assert result["checks"] == {} - - -def test_expired_deadline_never_launches(tmp_path): - with patch.object(kv, "invoke") as invoke: - result = kv.verify_source(b"# source", [{"id": "one", "events": []}], tmp_path, - time.monotonic() - 1, lambda *_: True) - invoke.assert_not_called() - assert result["verdict"] == "INCONCLUSIVE" - - -def test_real_executor_expired_before_launch_needs_no_output_files(tmp_path): - with patch.object(kv, "sandbox_command", return_value=["not-launched"]): - record, value = kv.invoke(b"# source", {"mode": "verify", "events": []}, - tmp_path / "run", time.monotonic() - 1) - assert record["status"] == "timeout" and value is None - assert not (tmp_path / "run/stderr.log").exists() - assert json.loads((tmp_path / "run/execution.json").read_text()) == { - "status": "timeout", "returncode": None, - "elapsed_seconds": record["elapsed_seconds"]} - - -def test_driver_snapshots_mutable_return_buffers(): - assert 'observation = {"reclaimed": list(result)}' in kv.DRIVER - assert "return list(value)" in kv.DRIVER - - -@pytest.mark.skipif(sys.platform != "linux", reason="Trusted Linux resource-limit driver") -def test_authored_driver_control_can_reuse_return_buffers(tmp_path): - # Trusted local software test, not a submitted artifact or sandbox fallback. - source = '''class Reclaimer: - def __init__(self): - self.result = [] - self.state = [] - def apply(self, event): - self.result[:] = [event["value"]] - return self.result - def pending(self): - self.state[:] = [(str(self.result[0]), 0, (self.result[0],))] - return self.state -''' - (tmp_path / "candidate.py").write_text(source) - (tmp_path / "inputs.json").write_text(json.dumps( - {"mode": "verify", "events": [{"value": 1}, {"value": 2}]})) - (tmp_path / "runner.py").write_text(kv.DRIVER.replace("/work/", str(tmp_path) + "/")) - run = subprocess.run([sys.executable, "-I", "-S", "-B", str(tmp_path / "runner.py")], - capture_output=True, text=True, timeout=5, check=False) - assert run.returncode == 0, run.stderr - value = json.loads(run.stdout) - assert value["observations"][0] == {"reclaimed": [1], "pending": [["1", 0, [1]]]} - assert value["observations"][1] == {"reclaimed": [2], "pending": [["2", 0, [2]]]} - - -@pytest.mark.parametrize("broken", [None, "inaccessible", "readonly", "clean_env", - "network_blocked", "fork_denied", "limits", "cleanup"]) -def test_preflight_requires_each_actual_probe(tmp_path, broken): - value = {"inaccessible": [True, True], "readonly": True, "clean_env": True, - "network_blocked": True, "fork_denied": True, - "limits": {"memory": [268435456, 268435456], "processes": [0, 0], "cpu": [16, 16], - "output": [4194304, 4194304], "files": [32, 32], "core": [0, 0]}} - if broken and broken != "cleanup": - value[broken] = False - with patch.object(kv, "invoke", return_value=({"status": "completed"}, value)), \ - patch.object(kv, "process_group_gone", return_value=broken != "cleanup"): - if broken: - with pytest.raises(kv.SandboxUnavailable): - kv.probe_isolation(tmp_path, time.monotonic() + 1, ["/synthetic-hidden"]) - else: - assert kv.probe_isolation(tmp_path, time.monotonic() + 1, - ["/synthetic-hidden"])["normal_exit_process_group_gone"] - - -def test_driver_has_no_expected_answers_or_candidate_timing_authority(): - assert "perf_counter" not in kv.DRIVER and "process_time" not in kv.DRIVER - assert "expected" not in kv.DRIVER and "oracle" not in kv.DRIVER - assert "resource.RLIMIT_NPROC, (0, 0)" in kv.DRIVER - assert "resource.RLIMIT_AS, (268435456, 268435456)" in kv.DRIVER diff --git a/tests/test_kv_runtime_qualification.py b/tests/test_kv_runtime_qualification.py deleted file mode 100644 index 7d4844f8c..000000000 --- a/tests/test_kv_runtime_qualification.py +++ /dev/null @@ -1,292 +0,0 @@ -"""Metadata/runtime probes only: no submitted artifact is run by these unit tests.""" - -import importlib.metadata -import json -import time -from pathlib import Path - -import pytest - -from tools import kv_runtime_qualification as runtime - - -def fixture(tmp_path): - mount = tmp_path / "runtime" - mount.mkdir() - (mount / "stdlib.py").write_text("# trusted standard runtime\n") - sensitive = tmp_path / "private/skill_library/SKILL.md" - sensitive.parent.mkdir(parents=True) - sensitive.write_text("Known sensitive knowledge bytes. " * 10) - return mount, sensitive - - -def test_runtime_inventory_keeps_only_digest_counts_and_no_absolute_paths(tmp_path): - mount, sensitive = fixture(tmp_path) - result, escapes = runtime.inventory( - [(mount, Path("/runtime"))], [sensitive], time.monotonic() + 5 - ) - assert result["verdict"] == "PASS" and result["nodes"] == 2 and escapes == [] - assert str(tmp_path) not in json.dumps(result) - - -@pytest.mark.parametrize("variant", ["named_package", "renamed_copy", "symlink_sensitive"]) -def test_runtime_inventory_rejects_sensitive_data_inside_mounted_tree(tmp_path, variant): - mount, sensitive = fixture(tmp_path) - if variant == "named_package": - (mount / "inferrsi").mkdir() - elif variant == "renamed_copy": - (mount / "unremarkable.txt").write_bytes(sensitive.read_bytes()) - else: - (mount / "alias").symlink_to(sensitive) - result, _ = runtime.inventory([(mount, Path("/runtime"))], [sensitive], time.monotonic() + 5) - assert result["verdict"] == "FAIL" - - -def test_escaped_symlink_requires_same_profile_probe_not_assumed_safe(tmp_path, monkeypatch): - mount, sensitive = fixture(tmp_path) - unrelated = tmp_path / "external.txt" - unrelated.write_text("outside\n") - (mount / "escape").symlink_to(unrelated) - monkeypatch.setattr(runtime, "sensitive_files", lambda *a: [sensitive]) - monkeypatch.setattr(runtime, "runtime_mounts", lambda *a: [(mount, Path("/runtime"))]) - forbidden = [] - - def probe(output, deadline, paths): - forbidden.extend(paths) - raise runtime.kv.SandboxUnavailable("unavailable") - - monkeypatch.setattr(runtime.kv, "probe_isolation", probe) - result = runtime.qualify( - tmp_path / "out", - time.monotonic() + 5, - root=tmp_path, - independent=tmp_path, - fixtures=tmp_path, - knowledge_root=tmp_path, - ) - assert "/runtime/escape" in forbidden - assert str(sensitive) in forbidden - assert result["verdict"] == "INCONCLUSIVE" - assert len([p for p in forbidden if p.endswith("-canary.txt")]) == 2 - - -def test_bind_source_symlink_inventories_actual_mounted_contents(tmp_path): - mount, sensitive = fixture(tmp_path) - alias = tmp_path / "bind-source" - alias.symlink_to(mount, target_is_directory=True) - (mount / "renamed.txt").write_bytes(sensitive.read_bytes()) - result, escapes = runtime.inventory( - [(alias, Path("/runtime"))], [sensitive], time.monotonic() + 5 - ) - assert result["verdict"] == "FAIL" and result["nodes"] == 3 - assert escapes == [] - - -def test_complete_renamed_large_hidden_corpus_is_rejected(tmp_path): - from tools import qualify_kv_reclamation as task - - mount, _ = fixture(tmp_path) - sensitive = task.INDEPENDENT / "fixtures.json" - assert sensitive.stat().st_size > 65536 - (mount / "large-data.bin").write_bytes(sensitive.read_bytes()) - result, _ = runtime.inventory([(mount, Path("/runtime"))], [sensitive], time.monotonic() + 5) - assert result["verdict"] == "FAIL" - - -def test_hash_byte_limit_fails_closed(tmp_path, monkeypatch): - mount, sensitive = fixture(tmp_path) - monkeypatch.setattr(runtime, "MAX_HASH_BYTES", 1) - with pytest.raises(ValueError, match="hash byte limit"): - runtime.inventory([(mount, Path("/runtime"))], [sensitive], time.monotonic() + 5) - - -def test_failed_inventory_does_not_probe_or_claim_success(tmp_path, monkeypatch): - mount, sensitive = fixture(tmp_path) - (mount / "inferrsi").mkdir() - monkeypatch.setattr(runtime, "sensitive_files", lambda *a: [sensitive]) - monkeypatch.setattr(runtime, "runtime_mounts", lambda *a: [(mount, Path("/runtime"))]) - monkeypatch.setattr(runtime.kv, "probe_isolation", lambda *a: pytest.fail("inventory rejected")) - result = runtime.qualify( - tmp_path / "out", - time.monotonic() + 5, - root=tmp_path, - independent=tmp_path, - fixtures=tmp_path, - knowledge_root=tmp_path, - ) - assert result["verdict"] == "FAIL" - - -def test_unreadable_enumeration_is_inconclusive(tmp_path, monkeypatch): - mount, sensitive = fixture(tmp_path) - monkeypatch.setattr(runtime, "sensitive_files", lambda *a: [sensitive]) - monkeypatch.setattr(runtime, "runtime_mounts", lambda *a: [(mount, Path("/runtime"))]) - - def denied(*a): - raise PermissionError("private-path-canary") - - monkeypatch.setattr(runtime.os, "scandir", denied) - result = runtime.qualify( - tmp_path / "out", - time.monotonic() + 5, - root=tmp_path, - independent=tmp_path, - fixtures=tmp_path, - knowledge_root=tmp_path, - ) - assert result["verdict"] == "INCONCLUSIVE" - assert "private-path-canary" not in json.dumps(result) - - -def test_actual_package_paths_and_oracle_are_regular_forbidden_files(): - from tools import qualify_kv_reclamation as task - - files = runtime.sensitive_files( - task.ROOT, - task.INDEPENDENT, - task.FIXTURES, - task.ROOT / "src/inferrsi/data/skill_library/repositories/vllm/audit-vllm-kv-lifetimes", - time.monotonic() + 10, - ) - assert task.INDEPENDENT / "oracle.py" in files - assert task.INDEPENDENT / "fixtures.json" in files - assert task.FIXTURES / "reference.py" in files - assert all(p.is_file() for p in files) - assert any("skill_library" in p.parts for p in files) - - -def test_missing_required_sensitive_anchor_is_not_silently_dropped(tmp_path): - with pytest.raises(ValueError, match="anchor missing"): - runtime.sensitive_files(tmp_path, tmp_path, tmp_path, tmp_path, time.monotonic() + 5) - - -def test_inventory_deadline_and_node_limit_fail_closed(tmp_path, monkeypatch): - mount, sensitive = fixture(tmp_path) - with pytest.raises(ValueError, match="inventory limit"): - runtime.inventory([(mount, Path("/runtime"))], [sensitive], time.monotonic() - 1) - monkeypatch.setattr(runtime, "MAX_NODES", 1) - with pytest.raises(ValueError, match="inventory limit"): - runtime.inventory([(mount, Path("/runtime"))], [sensitive], time.monotonic() + 5) - - -def package(directory): - directory.mkdir(parents=True) - (directory / "__init__.py").write_text("# synthetic package\n") - (directory / "data").mkdir() - (directory / "data/private.json").write_text('{"synthetic_sensitive_copy": true}\n') - return directory - - -def installed_distribution(metadata, *, record=None, source_list=None, direct_url=None): - metadata.mkdir(parents=True) - (metadata / ("PKG-INFO" if metadata.suffix == ".egg-info" else "METADATA")).write_text( - "Metadata-Version: 2.1\nName: inferrsi\nVersion: 0.1.0\n" - ) - for name, value in (("RECORD", record), ("SOURCES.txt", source_list), - ("direct_url.json", direct_url)): - if value is not None: - (metadata / name).write_text(value) - return importlib.metadata.PathDistribution(metadata) - - -def installed_files(distribution, monkeypatch): - from tools import qualify_kv_reclamation as task - - monkeypatch.setattr(runtime.importlib.metadata, "distributions", lambda: [distribution]) - return runtime.sensitive_files( - task.ROOT, task.INDEPENDENT, task.FIXTURES, - task.ROOT / "src/inferrsi/data/skill_library/repositories/vllm/audit-vllm-kv-lifetimes", - time.monotonic() + 10, - ) - - -def test_source_egg_info_inventories_actual_tree_not_source_archive_paths(tmp_path, monkeypatch): - actual = package(tmp_path / "project/src/inferrsi") - distribution = installed_distribution( - tmp_path / "project/src/inferrsi.egg-info", - source_list="src/inferrsi/__init__.py\nsrc/inferrsi/data/private.json\n", - ) - # This is the real PathDistribution mismatch seen on Python 3.10/3.11 CI. - assert not distribution.locate_file("src/inferrsi/__init__.py").exists() - files = installed_files(distribution, monkeypatch) - assert actual / "__init__.py" in files and actual / "data/private.json" in files - - -def test_installed_wheel_copy_includes_all_actual_package_nodes(tmp_path, monkeypatch): - actual = package(tmp_path / "site/inferrsi") - distribution = installed_distribution( - tmp_path / "site/inferrsi-0.1.0.dist-info", record="inferrsi/__init__.py,,\n" - ) - files = installed_files(distribution, monkeypatch) - assert actual / "__init__.py" in files and actual / "data/private.json" in files - - -@pytest.mark.parametrize("variant", ["missing", "symlink", "absolute", "traversal"]) -def test_raw_record_does_not_filter_missing_or_unsafe_installed_nodes( - tmp_path, monkeypatch, variant -): - actual = package(tmp_path / "site/inferrsi") - entry = "inferrsi/missing.py" - if variant == "symlink": - (actual / "alias.py").symlink_to(actual / "__init__.py") - entry = "inferrsi/alias.py" - elif variant == "absolute": - entry = str(actual / "missing.py") - elif variant == "traversal": - entry = "inferrsi/../missing.py" - distribution = installed_distribution( - tmp_path / "site/inferrsi-0.1.0.dist-info", record=f"inferrsi/__init__.py,,\n{entry},,\n" - ) - - class FilteredFilesDistribution: - # Newer importlib.metadata filters missing paths from .files; never use - # that lossy view to decide completeness of the authoritative RECORD. - files = [] - metadata = distribution.metadata - read_text = distribution.read_text - locate_file = distribution.locate_file - - with pytest.raises(ValueError, match="package (inventory incomplete|RECORD)"): - installed_files(FilteredFilesDistribution(), monkeypatch) - - -@pytest.mark.parametrize("layout", ["src", "flat"]) -def test_editable_record_maps_independent_source_copy(tmp_path, monkeypatch, layout): - source = tmp_path / "project" - actual = package(source / ("src/inferrsi" if layout == "src" else "inferrsi")) - distribution = installed_distribution( - tmp_path / "site/inferrsi-0.1.0.dist-info", - record="__editable__.inferrsi.pth,,\n", - direct_url=json.dumps({"url": source.as_uri(), "dir_info": {"editable": True}}), - ) - files = installed_files(distribution, monkeypatch) - assert actual / "__init__.py" in files and actual / "data/private.json" in files - - -@pytest.mark.parametrize("variant", ["missing", "ambiguous", "symlink", "remote"]) -def test_unknown_editable_layout_never_silently_drops_installed_copy( - tmp_path, monkeypatch, variant -): - source = tmp_path / "project" - source.mkdir() - if variant in {"ambiguous", "remote"}: - package(source / "src/inferrsi") - if variant == "ambiguous": - package(source / "inferrsi") - if variant == "symlink": - actual = package(tmp_path / "other/inferrsi") - (source / "inferrsi").symlink_to(actual, target_is_directory=True) - url = "https://example.invalid/source" if variant == "remote" else source.as_uri() - distribution = installed_distribution( - tmp_path / "site/inferrsi-0.1.0.dist-info", record="__editable__.inferrsi.pth,,\n", - direct_url=json.dumps({"url": url, "dir_info": {"editable": True}}), - ) - with pytest.raises(ValueError): - installed_files(distribution, monkeypatch) - - -def test_missing_installed_record_is_not_treated_as_source_egg_info(tmp_path, monkeypatch): - package(tmp_path / "site/inferrsi") - distribution = installed_distribution(tmp_path / "site/inferrsi-0.1.0.dist-info") - with pytest.raises(ValueError, match="inventory metadata"): - installed_files(distribution, monkeypatch) diff --git a/tests/test_kv_worker_supervisor.py b/tests/test_kv_worker_supervisor.py deleted file mode 100644 index ed2adc40e..000000000 --- a/tests/test_kv_worker_supervisor.py +++ /dev/null @@ -1,135 +0,0 @@ -"""Fake supervisor controls only; no providers or live ledger writes.""" - -import time -from contextlib import contextmanager -from unittest.mock import Mock - -import pytest - -from inferrsi.infra.artifacts import atomic_write_json -from tests.test_kv_edit_pilot import prepare -from tools import kv_edit_pilot as pilot -from tools import kv_worker_supervisor as supervisor - - -@pytest.mark.parametrize("extra", ["GITHUB_TOKEN", "ACTIONS_RUNTIME_TOKEN", "PYTHONPATH", "HOME"]) -def test_shared_supervisor_rejects_authority_and_path_environment_injection(tmp_path, extra): - inputs, workspace, config = prepare(tmp_path) - _, plan, *_ = pilot.checked_inputs(inputs) - admission = Mock() - with pytest.raises(ValueError, match="environment"): - supervisor.supervise(config, plan, inputs, workspace, tmp_path / "output", - time.monotonic() + 30, checkout=tmp_path, admission=admission, - command_factory=Mock(), environment_factory=lambda gate: {**gate, extra: "offline"}, - execute=Mock(side_effect=AssertionError("must not execute"))) - admission.close.assert_called_once() - - -def test_shared_supervisor_closes_admission_on_outer_cancellation(tmp_path): - inputs, workspace, config = prepare(tmp_path) - _, plan, *_ = pilot.checked_inputs(inputs) - admission = Mock() - with pytest.raises(KeyboardInterrupt): - supervisor.supervise(config, plan, inputs, workspace, tmp_path / "output", - time.monotonic() + 30, checkout=tmp_path, admission=admission, - command_factory=lambda *_: ("fixed",), environment_factory=lambda gate: gate, - execute=Mock(side_effect=KeyboardInterrupt)) - admission.close.assert_called_once() - - -@pytest.mark.parametrize("status", ["cancelled", "timeout", "error"]) -def test_returned_executor_failure_stops_session_without_finishing_arm(tmp_path, status): - inputs, workspace, config = prepare(tmp_path) - _, plan, *_ = pilot.checked_inputs(inputs) - admission = Mock() - result = supervisor.supervise(config, plan, inputs, workspace, tmp_path / "output", - time.monotonic() + 30, checkout=tmp_path, admission=admission, - command_factory=lambda *_: ("fixed",), environment_factory=lambda gate: gate, - execute=Mock(return_value={"status": status})) - assert not result["request_completion_known"] - admission.close.assert_called_once() - admission.finish_arm.assert_not_called() - - - -@pytest.mark.parametrize("returncode,has_result", [(-15, True), (-9, True), (0, False), - (1, False), (1, True)]) -def test_signal_death_or_missing_result_blocks_following_arm(tmp_path, returncode, has_result): - inputs, workspace, config = prepare(tmp_path) - _, plan, *_ = pilot.checked_inputs(inputs) - admission = Mock() - def execute(command, checkout, output, deadline, *, environment): - if has_result: - (output.parent / "worker").mkdir() - atomic_write_json(output.parent / "worker/result.json", {"status": "completed"}) - return {"status": "failed" if returncode else "completed", "returncode": returncode} - result = supervisor.supervise(config, plan, inputs, workspace, tmp_path / "output", - time.monotonic() + 30, checkout=tmp_path, admission=admission, - command_factory=lambda *_: ("fixed",), environment_factory=lambda gate: gate, - execute=execute) - assert not result["request_completion_known"] - admission.close.assert_called_once() - admission.finish_arm.assert_not_called() - - -@pytest.mark.parametrize("record,accepted", [ - ({"status": "completed", "returncode": 0}, True), - ({"status": "failed", "returncode": 1}, False), - ({"status": "failed", "returncode": 0}, False), - ({"status": "completed", "returncode": 1}, False), - ({"status": "completed", "returncode": False}, False), - ({"status": "completed", "returncode": True}, False), - ({"status": "completed", "returncode": 0.0}, False), - ({"status": "completed", "returncode": "0"}, False), - ({"status": "completed", "returncode": None}, False), - ({"status": "completed"}, False), - ({"returncode": 0}, False), - ({"status": "unknown", "returncode": 0}, False), - ({"status": "cancelled", "returncode": 0}, False), - ({"status": "timeout", "returncode": 0}, False), - ({"status": "error", "returncode": 0}, False), -]) -def test_known_request_and_worker_result_cannot_upgrade_executor_outcome( - tmp_path, monkeypatch, record, accepted -): - inputs, workspace, config = prepare(tmp_path) - _, plan, *_ = pilot.checked_inputs(inputs) - admission = Mock() - admission.authorize.return_value = True - broker_check = None - - @contextmanager - def broker(check, *args, **kwargs): - nonlocal broker_check - broker_check = check - yield {} - - monkeypatch.setattr(supervisor, "authorization_broker", broker) - - def execute(command, checkout, output, deadline, *, environment): - grant = {k: config[k] for k in ("plan_sha256", "execution_id", "arm", "actor_sha256")} - grant.update(slot=1, request_sha256="a" * 64, reserved_microusd=10170000) - bridge = output.parent / "bridge" - atomic_write_json(bridge / "pending.json", grant) - assert broker_check() is True - atomic_write_json(bridge / "complete.json", {"grant": grant, "usage": { - "input_tokens": 1, "output_tokens": 1, - "cache_read_input_tokens": 0, "cache_creation_input_tokens": 0}}) - (output.parent / "worker").mkdir() - atomic_write_json(output.parent / "worker/result.json", {"status": "completed"}) - return record - - result = supervisor.supervise(config, plan, inputs, workspace, tmp_path / "output", - time.monotonic() + 30, checkout=tmp_path, admission=admission, - command_factory=lambda *_: ("fixed",), environment_factory=lambda gate: gate, - execute=execute) - assert result["execution"] == record and result["worker"]["status"] == "completed" - assert result["slots_consumed"] == 1 and result["completions"][0]["status"] == "known" - assert result["request_completion_known"] is accepted - admission.complete.assert_called_once() - if accepted: - admission.finish_arm.assert_called_once_with(config["arm"]) - admission.close.assert_not_called() - else: - admission.close.assert_called_once() - admission.finish_arm.assert_not_called() diff --git a/tests/test_qualify_kv_edit_actor.py b/tests/test_qualify_kv_edit_actor.py deleted file mode 100644 index 5d77e5f31..000000000 --- a/tests/test_qualify_kv_edit_actor.py +++ /dev/null @@ -1,302 +0,0 @@ -"""Fixed actor qualifier controls; fake worker outputs are not host qualification.""" - -import json -import zipfile - -import pytest -import yaml - -from inferrsi.model import anthropic_edit_agent as edit -from tools import qualify_kv_edit_actor as qualify - -PLAN = qualify.ROOT / "evidence/kv-actor-qualification-20261031/plan/qualification-plan.json" -HISTORICAL = qualify.ROOT / "evidence/kv-actor-qualification-20261030" - - -def test_historical_qualification_bundle_is_retained_byte_for_byte(): - integrity = HISTORICAL / "plan/integrity.json" - assert qualify.task.file_hash(integrity) == ( - "d6c018da4cf1cde265d9dce36cb4368d75953d2291acb1ef48ddb67d09d89336" - ) - for name, digest in json.loads(integrity.read_text()).items(): - assert qualify.task.file_hash(HISTORICAL / name) == digest - - -def test_new_payload_plan_preserves_the_software_contract(): - current = qualify.checked_plan(PLAN) - historical = json.loads((HISTORICAL / "plan/qualification-plan.json").read_text()) - identity_fields = {"base_revision", "knowledge_revision", "knowledge_files_sha256"} - assert {k: v for k, v in current.items() if k not in identity_fields} == { - k: v for k, v in historical.items() if k not in identity_fields - } - assert current["knowledge_revision"] == "a04c599c49465b67ea40a1f0de622706ab3da955" - changed = { - name for name, digest in current["knowledge_files_sha256"].items() - if historical["knowledge_files_sha256"][name] != digest - } - assert changed == {"metadata.json"} - - -def test_current_plan_rejects_metadata_drift(tmp_path, monkeypatch): - plan = qualify.checked_plan(PLAN) - owner = tmp_path / plan["knowledge_root"] - for name in plan["knowledge_files_sha256"]: - source = qualify.ROOT / plan["knowledge_root"] / name - target = owner / name - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(source.read_bytes()) - monkeypatch.setattr(qualify, "ROOT", tmp_path) - qualify.checked_plan(PLAN) - metadata = owner / "metadata.json" - metadata.write_bytes(metadata.read_bytes() + b"\n") - with pytest.raises(ValueError, match="Pinned K bytes changed"): - qualify.checked_plan(PLAN) - - -def identity(): - return {"available": True, "head": "a" * 40, "snapshot_hash": "exact", "dirty": False} - - -def fake_worker(inputs, workspace, output, deadline): - config = json.loads((inputs / "worker.json").read_text()) - scripts = json.loads((inputs / "responses.json").read_text()) - source = scripts[-2]["content"][0]["input"]["source"].encode() - final = output / "worker/final" - final.mkdir(parents=True) - (final / "candidate.py").write_bytes(source) - # Parent final grader must use final bytes, not this later live-path mutation. - (workspace / "candidate.py").write_bytes(b"# later live actor path\n") - for i in (0, 1, 2): - directory = output / f"worker/workbench/revisions/revision-{i:04d}" - directory.mkdir(parents=True) - (directory / "candidate.py").write_bytes(source) - return { - "execution": {"status": "completed"}, - "worker": { - "status": "completed", - "artifact": {"sha256": edit.digest(source), "revision": 2}, - }, - "simulated_slots_consumed": 4, - "request_completion_known": True, - "config": config, - } - - -def test_fixed_actual_owner_exposure_and_separate_final_grade(tmp_path, monkeypatch): - monkeypatch.setattr(qualify, "repository_identity", lambda *_: identity()) - monkeypatch.setattr(qualify.runtime, "qualify", lambda *a, **k: {"verdict": "PASS"}) - monkeypatch.setattr(qualify.cleanup, "qualify", lambda *_: {"verdict": "PASS"}) - monkeypatch.setattr(qualify.pilot, "run_supervised", fake_worker) - monkeypatch.setattr(qualify, "visible_receipts", lambda *_: True) - seen = [] - - def grade(source, digest, hidden, output, deadline): - assert edit.digest(source) == digest - assert b"later live actor path" not in source - assert all(t["split"] == "grader" for t in hidden) - seen.append(source) - return {"verdict": "PASS"} - - monkeypatch.setattr(qualify.workbench, "grade_final", grade) - result = qualify.qualify(PLAN, "a" * 40, tmp_path / "out") - assert result["verdict"] == "PASS" and result["software_checks_passed"] - assert result["model_calls"] == 0 and len(seen) == 2 - assert not any(f["path"].startswith("knowledge/") for f in result["arms"]["A2"]["exposure"]) - names = {f["path"] for f in result["arms"]["A3"]["exposure"]} - assert names == { - "task.json", - "candidate.py", - "CONTRACT.md", - "workbench.md", - "development.json", - "knowledge/SKILL.md", - "knowledge/metadata.json", - "knowledge/references/source-notes.md", - } - common = qualify.shared_inputs.COMMON - a2 = {f["path"]: f for f in result["arms"]["A2"]["exposure"]} - a3 = {f["path"]: f for f in result["arms"]["A3"]["exposure"]} - assert {p: a2[p] for p in common} == {p: a3[p] for p in common} - assert result["arms"]["A2"]["prompt_sha256"] == result["arms"]["A3"]["prompt_sha256"] - assert all(f["bytes"] <= qualify.checked_plan(PLAN)["client"]["max_read_bytes"] - for f in a3.values()) - with zipfile.ZipFile(tmp_path / "out/raw-evidence.zip") as archive: - raw = "\n".join(archive.read(name).decode() for name in archive.namelist()) - assert str(tmp_path) not in raw - assert "" in raw or "scripted" in raw - assert any(name.endswith("candidate.py") for name in archive.namelist()) - - -def test_unavailable_worker_is_not_pass_and_does_not_grade(tmp_path, monkeypatch): - monkeypatch.setattr(qualify, "repository_identity", lambda *_: identity()) - monkeypatch.setattr(qualify.runtime, "qualify", lambda *a, **k: {"verdict": "PASS"}) - monkeypatch.setattr(qualify.cleanup, "qualify", lambda *_: {"verdict": "INCONCLUSIVE"}) - monkeypatch.setattr( - qualify.pilot, - "run_supervised", - lambda *_: { - "execution": {"status": "failed"}, - "worker": {"status": "inconclusive", "reason": "sandbox_unavailable"}, - "simulated_slots_consumed": 0, - "request_completion_known": True, - }, - ) - monkeypatch.setattr(qualify.workbench, "grade_final", lambda *_: pytest.fail("not submitted")) - result = qualify.qualify(PLAN, "a" * 40, tmp_path / "out") - assert result["verdict"] == "INCONCLUSIVE" and not result["software_checks_passed"] - assert all(a["simulated_slots_consumed"] == 0 for a in result["arms"].values()) - - -def test_worker_receipt_cannot_override_failed_outer_execution(tmp_path, monkeypatch): - monkeypatch.setattr(qualify, "repository_identity", lambda *_: identity()) - monkeypatch.setattr(qualify.runtime, "qualify", lambda *a, **k: {"verdict": "PASS"}) - monkeypatch.setattr(qualify.cleanup, "qualify", lambda *_: {"verdict": "PASS"}) - monkeypatch.setattr(qualify, "visible_receipts", lambda *_: True) - monkeypatch.setattr(qualify.workbench, "grade_final", lambda *_: {"verdict": "PASS"}) - - def failed_worker(*args): - return {**fake_worker(*args), "execution": {"status": "failed"}} - - monkeypatch.setattr(qualify.pilot, "run_supervised", failed_worker) - result = qualify.qualify(PLAN, "a" * 40, tmp_path / "out") - assert result["verdict"] == "INCONCLUSIVE" and not result["software_checks_passed"] - - -def test_current_clean_exact_revision_is_required(tmp_path, monkeypatch): - monkeypatch.setattr(qualify, "repository_identity", lambda *_: {**identity(), "dirty": True}) - with pytest.raises(ValueError, match="clean Git revision"): - qualify.qualify(PLAN, "a" * 40, tmp_path / "out") - assert not (tmp_path / "out").exists() - - -def test_outbound_owner_links_cannot_expand_allowlist(tmp_path): - plan = json.loads(PLAN.read_text()) - plan["knowledge_files_sha256"]["../SKILL.md"] = "a" * 64 - path = tmp_path / "plan.json" - path.write_text(json.dumps(plan)) - with pytest.raises(ValueError, match="complete pinned K owner"): - qualify.checked_plan(path) - - -def test_qualification_reuses_1800_workload_not_a_post_outcome_choice(): - plan = qualify.checked_plan(PLAN) - task_plan = qualify.task.checked_plan(qualify.TASK_PLAN) - assert plan["visible_workload_count"] == task_plan["timing"]["lease_count"] == 1800 - assert plan["actor_limits"]["max_seconds"] == 600 - - -def test_existing_ready_job_only_adds_secret_free_separate_receipt(): - workflow = yaml.safe_load((qualify.ROOT / ".github/workflows/ci-ready.yml").read_text()) - job = workflow["jobs"]["kv-task-qualification"] - assert job["runs-on"] == "ubuntu-22.04" and job["permissions"] == {"contents": "read"} - commands = "\n".join(step.get("run", "") for step in job["steps"]) - assert "tools.qualify_kv_edit_actor" in commands and "env -i" in commands - assert f"--plan {PLAN.relative_to(qualify.ROOT).as_posix()}" in commands - assert "secrets." not in commands and "send_message" not in commands - assert "--privileged" not in commands and "sysctl" not in commands - uploads = [s["with"] for s in job["steps"] if "upload-artifact" in s.get("uses", "")] - assert len(uploads) == 2 and uploads[0]["name"].startswith("kv-qualification-") - assert uploads[1]["name"].startswith("kv-actor-qualification-") - - -def receipt_fixture(tmp_path): - actor = "a" * 64 - revisions = tmp_path / "revisions" - sources = {r: f"# Scripted revision {r}\n".encode() for r in (0, 1, 2)} - for revision, source in sources.items(): - directory = revisions / f"revision-{revision:04d}" - directory.mkdir(parents=True) - (directory / "candidate.py").write_bytes(source) - artifact = { - "revision": 2, - "sha256": edit.digest(sources[2]), - "plan_sha256": "c" * 64, - "execution_id": "fixture", - "arm": "A2", - } - events = [] - for revision, source in sources.items(): - metadata = {"revision": revision, "sha256": edit.digest(source), - "actor_sha256": actor, "artifact": "candidate.py", - **{k: artifact[k] for k in ("plan_sha256", "execution_id", "arm")}} - (revisions / f"revision-{revision:04d}/receipt.json").write_text(json.dumps(metadata)) - events.append({"type": "edit.artifact", "bytes": len(source), **metadata}) - for revision in (1, 2): - for kind in ("checks", "benchmark"): - feedback = ( - {"verdict": "PASS"} - if kind == "checks" - else { - "verdict": "FAIL", - "quality_verdict": "PASS", - "baseline_values": [1, 1], - "candidate_values": [1, 1], - "improvement_pct": 0, - } - ) - receipt = { - "revision": revision, - "kind": kind, - "feedback": feedback, - "artifact_sha256": edit.digest(sources[revision]), - **{k: artifact[k] for k in ("plan_sha256", "execution_id", "arm")}, - } - receipt["receipt_sha256"] = edit.fingerprint({**receipt, "actor_sha256": actor}) - events.append({"type": "agent.tool", "receipt": receipt}) - if revision == 2: - artifact["check_receipt" if kind == "checks" else "benchmark_receipt"] = receipt[ - "receipt_sha256" - ] - path = tmp_path / "trace.jsonl" - path.write_text("\n".join(json.dumps(e) for e in events)) - return path, artifact, actor, revisions, events - - -def test_numeric_gain_is_not_required_by_receipt_readback(tmp_path): - path, artifact, actor, revisions, events = receipt_fixture(tmp_path) - assert qualify.visible_receipts(path, artifact, actor, 2, revisions) - events[-1]["receipt"]["feedback"]["quality_verdict"] = "FAIL" - path.write_text("\n".join(json.dumps(e) for e in events)) - assert not qualify.visible_receipts(path, artifact, actor, 2, revisions) - - -@pytest.mark.parametrize("variant", ["duplicate_final", "wrong_binding", "retained_drift", - "retained_receipt_drift", "initial_source_drift"]) -def test_receipt_readback_requires_both_bound_revisions_and_retained_bytes(tmp_path, variant): - path, artifact, actor, revisions, events = receipt_fixture(tmp_path) - if variant == "duplicate_final": - events[3:5] = events[5:] - elif variant == "wrong_binding": - receipt = events[3]["receipt"] - receipt["arm"] = "A3" - body = {k: v for k, v in receipt.items() if k != "receipt_sha256"} - receipt["receipt_sha256"] = edit.fingerprint({**body, "actor_sha256": actor}) - elif variant == "retained_drift": - (revisions / "revision-0001/candidate.py").write_text("# changed retained bytes\n") - elif variant == "retained_receipt_drift": - (revisions / "revision-0001/receipt.json").write_text("{}") - else: - (revisions / "revision-0000/candidate.py").write_text("# changed initial source\n") - path.write_text("\n".join(json.dumps(e) for e in events)) - assert not qualify.visible_receipts(path, artifact, actor, 2, revisions) - - -def test_runtime_visibility_failure_cannot_be_hidden_by_good_scripted_arms(tmp_path, monkeypatch): - monkeypatch.setattr(qualify, "repository_identity", lambda *_: identity()) - monkeypatch.setattr(qualify.cleanup, "qualify", lambda *_: {"verdict": "PASS"}) - monkeypatch.setattr(qualify.runtime, "qualify", lambda *a, **k: {"verdict": "FAIL"}) - monkeypatch.setattr(qualify.pilot, "run_supervised", fake_worker) - monkeypatch.setattr(qualify, "visible_receipts", lambda *_: True) - monkeypatch.setattr(qualify.workbench, "grade_final", lambda *_: {"verdict": "PASS"}) - result = qualify.qualify(PLAN, "a" * 40, tmp_path / "out") - assert result["verdict"] == "FAIL" and not result["software_checks_passed"] - assert result["dependencies_sha256"] == edit.fingerprint(result["dependencies"]) - - -def test_dependency_receipt_binds_transitive_identity_metrics_budget_and_initializers(): - assert { - "src/inferrsi/infra/targets/git.py", "src/inferrsi/infra/targets/__init__.py", - "src/inferrsi/model/protocols/metrics.py", "src/inferrsi/harness/scenarios/external.py", - "tools/skill_eval_budget.py", "tools/kv_edit_inputs.py", "tools/kv_worker_supervisor.py", - "src/inferrsi/__init__.py", - } <= qualify.dependencies().keys() diff --git a/tests/test_qualify_kv_reclamation.py b/tests/test_qualify_kv_reclamation.py deleted file mode 100644 index 2ebac1c2e..000000000 --- a/tests/test_qualify_kv_reclamation.py +++ /dev/null @@ -1,148 +0,0 @@ -"""Model-free controller tests; mocked timing values are not performance evidence.""" - -import json -import os -import subprocess -import sys -import zipfile -from types import SimpleNamespace -from unittest.mock import patch - -import pytest -import yaml - -from tools import qualify_kv_reclamation as qualify - -PLAN = qualify.EVIDENCE / "plan/qualification-plan.json" - - -def control_results(): - plan = qualify.checked_plan(PLAN) - return ([{"verdict": "PASS", "checks": {"fixture": True}} - for _ in plan["required_good"]] - + [{"verdict": "FAIL", "checks": {"fixture": False}} - for _ in plan["required_bad"]]) - - -def module(name, path): - if name == "kv_independent_verifier": - return SimpleNamespace(load_traces=lambda: [{"split": "development"}], compare=lambda: True) - if name == "kv_workload": - return SimpleNamespace(workload=lambda count: []) - return SimpleNamespace(benchmark_expected=lambda count: {"observations": [], "pending": []}) - - -def measurement(name, *args): - gain = 50 if name == "reference" else 0 - return {"evaluation": {"verdict": "PASS" if gain else "FAIL", "improvement_pct": gain}, - "records": []} - - -def test_full_qualification_requires_good_bad_integrity_deadline_and_headroom(tmp_path): - identity = {"available": True, "head": "a" * 40, "snapshot_hash": "frozen", "dirty": False} - with patch.object(qualify, "repository_identity", return_value=identity), \ - patch.object(qualify.kv, "probe_isolation", return_value={"readonly": True}), \ - patch.object(qualify.kv, "verify_source", side_effect=control_results()), \ - patch.object(qualify.kv, "invoke", return_value=( - {"status": "timeout", "elapsed_seconds": 0.3}, None)), \ - patch.object(qualify.kv, "process_group_gone", return_value=True), \ - patch.object(qualify, "load_module", side_effect=module), \ - patch.object(qualify, "file_hash", wraps=qualify.file_hash) as hashes, \ - patch.object(qualify, "measure", side_effect=measurement): - result = qualify.qualify(PLAN, "a" * 40, tmp_path / "output") - assert result["verdict"] == "PASS" - assert result["software_checks_passed"] - assert len(result["control_source_sha256"]) == 9 - assert hashes.call_count > 5 - - -def test_unavailable_is_retained_inconclusive_and_no_controls_run(tmp_path): - identity = {"available": True, "head": "a" * 40, "snapshot_hash": "frozen", "dirty": False} - with patch.object(qualify, "repository_identity", return_value=identity), \ - patch.object(qualify.kv, "probe_isolation", side_effect=qualify.kv.SandboxUnavailable( - "unavailable")), patch.object(qualify.kv, "verify_source") as verify: - result = qualify.qualify(PLAN, "a" * 40, tmp_path / "output") - verify.assert_not_called() - assert result["verdict"] == "INCONCLUSIVE" - assert not result["software_checks_passed"] - assert json.loads((tmp_path / "output/qualification.json").read_text()) == result - assert result["model_calls"] == 0 and result["provider_charges_usd"] == 0 - - -def test_wrong_revision_is_rejected_before_execution(tmp_path): - with pytest.raises(ValueError, match="exact"): - qualify.qualify(PLAN, "b" * 40, tmp_path / "output") - assert not (tmp_path / "output").exists() - - -def test_frozen_plan_changed_input_fails(tmp_path): - plan = json.loads(PLAN.read_text()) - plan["starter_sha256"] = "0" * 64 - path = tmp_path / "plan.json" - path.write_text(json.dumps(plan)) - with pytest.raises(ValueError, match="changed"): - qualify.checked_plan(path) - - -def test_cli_does_not_turn_numeric_gate_into_general_ci_gate(tmp_path): - result = {"verdict": "FAIL", "software_checks_passed": True} - with patch("sys.argv", ["qualify", "--plan", str(PLAN), "--revision", "a" * 40, - "--output", str(tmp_path / "output")]), \ - patch.object(qualify, "qualify", return_value=result): - assert qualify.main() == 0 - result["software_checks_passed"] = False - assert qualify.main() == 1 - - -def test_existing_ready_job_security_and_parser_default_preserved(): - root = qualify.ROOT - workflow = yaml.safe_load((root / ".github/workflows/ci-ready.yml").read_text()) - job = workflow["jobs"]["kv-task-qualification"] - assert job["runs-on"] == "ubuntu-22.04" - assert job["permissions"] == {"contents": "read"} - assert job["if"] == "github.event.pull_request.draft == false" - assert job["steps"][0]["with"]["persist-credentials"] is False - commands = "\n".join(step.get("run", "") for step in job["steps"]) - assert "env -i" in commands and "secrets." not in commands - assert "sysctl" not in commands and "--privileged" not in commands - assert "--plan evidence/kv-reclamation-20261030/plan/qualification-plan.json" in commands - assert "/qualification.json" in job["steps"][-1]["with"]["path"] - assert "/raw-evidence.zip" in job["steps"][-1]["with"]["path"] - parser = (root / ".github/workflows/skill-eval-qualify.yml").read_text() - assert "tools.skill_eval_qualify" in parser - assert "kv_reclamation" not in parser - - -def test_independently_frozen_verifier_suite(): - result = subprocess.run([sys.executable, "-m", "pytest", "-q", - str(qualify.INDEPENDENT / "test_verifier.py")], - capture_output=True, text=True, timeout=60, check=False) - assert result.returncode == 0, result.stdout + result.stderr - - -def test_independently_frozen_benchmark_suite(): - result = subprocess.run([sys.executable, "-m", "pytest", "-q", - str(qualify.EVIDENCE / "validator/test_benchmark.py")], - env={**os.environ, - "KV_FROZEN_ORACLE": str(qualify.INDEPENDENT / "oracle.py")}, - capture_output=True, text=True, timeout=60, check=False) - assert result.returncode == 0, result.stdout + result.stderr - - -def test_retained_actual_outputs_have_labelled_path_redactions(tmp_path): - private, output = tmp_path / "private", tmp_path / "output" - private.mkdir() - output.mkdir() - original = {"source": str(qualify.ROOT / "candidate.py"), "output": str(private / "trial"), - "cwd": "/tmp/kv-replay-example", "observations": [{"reclaimed": [1, 2]}]} - (private / "actual.json").write_text(json.dumps(original)) - (private / "not-evidence.txt").write_text("not retained") - receipt = qualify.retain_raw_evidence(private, output) - with zipfile.ZipFile(output / receipt["path"]) as archive: - value = json.loads(archive.read("actual.json")) - assert value["observations"] == original["observations"] - assert value["source"] == "/candidate.py" - assert value["output"] == "/trial" - assert value["cwd"] == "" - assert "not-evidence.txt" not in archive.namelist() - assert receipt["sha256"] == qualify.file_hash(output / receipt["path"]) diff --git a/tools/kv_edit_inputs.py b/tools/kv_edit_inputs.py deleted file mode 100644 index 98e4c548f..000000000 --- a/tools/kv_edit_inputs.py +++ /dev/null @@ -1,91 +0,0 @@ -"""Pure public input construction shared by scripted qualification and live freeze. - -No discovery, credentials, provider calls, ledger actions or hidden-trace input. -Expected development/benchmark data stays in the trusted workbench suite only. -""" - -import hashlib - -from inferrsi.harness.kv_edit_workbench import VisibleSuite -from inferrsi.model.anthropic_agent import canonical - -COMMON = {"candidate.py", "task.json", "CONTRACT.md", "workbench.md", "development.json"} -KNOWLEDGE = {"knowledge/SKILL.md", "knowledge/metadata.json", - "knowledge/references/source-notes.md"} -KNOWLEDGE_ROOT = "src/inferrsi/data/skill_library/repositories/vllm/audit-vllm-kv-lifetimes/" -PROMPT = ( - "Use only the exposed task, contract, starter and fixed visible workbench. " - "Only candidate.py may change. External links are not readable files. " - "Finish after current-revision checks and measurement. " - "Any supplied knowledge is fallible guidance." -) -WORKBENCH = """# Fixed synthetic CPU workbench - -Read task.json, CONTRACT.md, candidate.py and development.json. Only candidate.py -may change. The exposed development file contains public case IDs and events; -trusted expected outputs remain outside the read allowlist. - -Use write_candidate to retain a revision, run_visible_checks for bounded public -correctness feedback, and run_visible_benchmark for fixed whole-command AB/BA -timing against the starter. Measurement includes sandbox/interpreter startup and -requires correct complete output. Rewriting invalidates previous receipts. -Submit only after checks and measurement on the current revision. Hidden final -grading occurs after immutable submission and never feeds another edit. - -This synthetic feasibility task does not establish production performance or -knowledge efficacy. Any knowledge files explicitly exposed in the manifest are -fallible guidance; their outbound links do not expand the read allowlist. -""" - - -def common_actor_contract_sha256(config): - """Bind exact shared exposure/protocol/workload despite transport-specific IDs. - - Scripted callback identities include response fixture bytes and therefore - cannot equal live callback identities. No other edit-plan field is omitted. - Outer campaign IDs are separately frozen by each supervising controller. - """ - value = {k: config[k] for k in ( - "arm", "model", "client", "input_token_ceiling", "prompt", "suite_sha256", - "baseline_sha256", - )} - value["schema"] = "kv-common-actor-contract-v1" - value["edit_plan"] = {k: v for k, v in config["edit_plan"].items() - if k != "callbacks_sha256"} - return hashlib.sha256(canonical(value).encode()).hexdigest() - - -def build_inputs(baseline, contract, development, benchmark_events, benchmark_expected, - min_improvement_pct): - """Return (public byte map, common prompt, private visible-workbench suite). - - All workload/gate inputs must come from the prospectively frozen task. No - grader split is accepted, and no expected outputs enter the public byte map. - Both arms use this same result; callers add only the allowlisted K to A3. - """ - if (type(baseline) is not bytes or not 0 < len(baseline) <= 65536 - or type(contract) is not bytes or not contract): - raise ValueError("Explicit bounded starter and contract bytes required") - baseline.decode("utf-8") - contract.decode("utf-8") - suite = VisibleSuite(canonical({ - "schema": "kv-visible-suite-v1", "development": development, - "benchmark_events": benchmark_events, "benchmark_expected": benchmark_expected, - "min_improvement_pct": min_improvement_pct, - }).encode()) - public = { - "candidate.py": baseline, - "CONTRACT.md": contract, - "task.json": canonical({ - "request": "Optimize Reclaimer; preserve its complete contract.", - "workbench": "fixed public checks and whole-command AB/BA timing", - "scope": "synthetic CPU feasibility only; no production or knowledge-efficacy claim", - }).encode(), - "workbench.md": WORKBENCH.encode(), - "development.json": canonical({ - "schema": "kv-public-development-v1", - "traces": [{k: t[k] for k in ("id", "events")} - for t in suite.read()["development"]], - }).encode(), - } - return public, PROMPT, suite diff --git a/tools/kv_edit_pilot.py b/tools/kv_edit_pilot.py deleted file mode 100644 index c850cd00b..000000000 --- a/tools/kv_edit_pilot.py +++ /dev/null @@ -1,257 +0,0 @@ -"""Secret-free, SCRIPTED KV edit/check worker and existing-executor supervisor. - -This module cannot call a model provider. It exercises the real fixed-tool and -sandbox path with supplied response DATA. A future paid launcher is a separately -reviewed integration, not a switch or credential/config option offered here. -""" - -import argparse -import hashlib -import json -import math -import os -import sys -import time -from dataclasses import asdict, replace -from pathlib import Path - -# -I worker bootstrap: this exact trusted checkout, never cwd/PYTHONPATH/home. -ROOT = Path(__file__).resolve().parents[1] -if __package__ in {None, ""}: - sys.path.insert(0, str(ROOT / "src")) - -from inferrsi.harness import kv_edit_workbench as workbench # noqa: E402 -from inferrsi.harness import kv_reclamation as kv # noqa: E402 -from inferrsi.infra.artifacts import atomic_write_json # noqa: E402 -from inferrsi.infra.executors.local import execute_command # noqa: E402 -from inferrsi.model import anthropic_agent as provider # noqa: E402 -from inferrsi.model import anthropic_edit_agent as edit # noqa: E402 -from inferrsi.model.request_guard import ( # noqa: E402 - ENV_KEYS, - authorize_request, - gate_environment, -) - -SCHEMA = "kv-scripted-worker-v1" -FILES = ("tools/kv_edit_pilot.py", "tools/kv_worker_supervisor.py", - "src/inferrsi/harness/kv_edit_workbench.py", - "src/inferrsi/harness/kv_reclamation.py", - "src/inferrsi/harness/skill_eval_engineering.py", "src/inferrsi/harness/runner.py", - "src/inferrsi/harness/evaluation.py", "src/inferrsi/harness/reporting.py", - "src/inferrsi/infra/executors/local.py", "src/inferrsi/infra/processes.py", - "src/inferrsi/infra/artifacts.py", "src/inferrsi/runspec.py", - "src/inferrsi/claim.py", "src/inferrsi/summary_chart.py") -FIELDS = {"schema", "plan_sha256", "execution_id", "arm", "model", "client", "edit_plan", - "input_token_ceiling", "suite_sha256", "baseline_sha256", "responses_sha256", - "actor_sha256", "dependencies", "prompt"} - - -def _read_json(path, limit=16 * 1024 * 1024): - return json.loads(kv.read_regular(Path(path), limit), object_pairs_hook=provider.unique_object, - parse_constant=lambda _: (_ for _ in ()).throw(ValueError("nonfinite JSON"))) - - -def dependencies(): - return {name: hashlib.sha256((ROOT / name).read_bytes()).hexdigest() for name in FILES} - - -def _plan(value): - if type(value) is not dict or set(value) != { - "files", "limits", "check_ids", "metric_key", "metric_unit", "direction", - "callbacks_sha256"}: - raise ValueError("Wrong frozen edit-plan shape") - return edit.EditPlan(files=tuple(edit.ExposedFile(**f) for f in value["files"]), - limits=edit.EditLimits(**value["limits"]), check_ids=tuple(value["check_ids"]), - **{k: value[k] for k in ("metric_key", "metric_unit", "direction", "callbacks_sha256")}) - - -def prepare_scripted(inputs, workspace, plan, suite, baseline, responses, *, - plan_sha256, execution_id, arm, model, spec, prompt, input_token_ceiling): - """Freeze offline inputs. No discovery, credentials, provider calls or ledger writes.""" - inputs = Path(inputs) - if (not edit._sha(plan_sha256) or arm not in {"A2", "A3"} - or type(responses) is not list or not 0 < len(responses) <= spec["max_requests"] - or type(prompt) is not str): - raise ValueError("Explicit frozen scripted inputs required") - actual = edit.freeze_files(workspace, [f.path for f in plan.files]) - if actual != plan.files or kv.source_bytes(Path(workspace) / "candidate.py") != baseline: - raise ValueError("Staged inputs differ from frozen actor manifest/baseline") - scripts = provider.canonical(responses).encode() - if len(scripts) > spec["max_requests"] * provider.MAX_RESPONSE: - raise ValueError("Scripted response file exceeds fixed response bound") - deps = dependencies() - callback_binding = {"dependencies": deps, "suite_sha256": edit.digest(suite.serialized), - "baseline_sha256": edit.digest(baseline), - "responses_sha256": edit.digest(scripts), "prompt": prompt} - plan = replace(plan, callbacks_sha256=edit.fingerprint(callback_binding)) - identity = edit.frozen_identity(plan, model, spec, input_token_ceiling) - config = {"schema": SCHEMA, "plan_sha256": plan_sha256, "execution_id": execution_id, - "arm": arm, "model": model, "client": spec, "edit_plan": asdict(plan), - "input_token_ceiling": input_token_ceiling, - "suite_sha256": edit.digest(suite.serialized), - "baseline_sha256": edit.digest(baseline), "responses_sha256": edit.digest(scripts), - "actor_sha256": edit.fingerprint(identity), "dependencies": deps, "prompt": prompt} - inputs.mkdir(parents=True, exist_ok=False) - (inputs / "suite.json").write_bytes(suite.serialized) - (inputs / "baseline.py").write_bytes(baseline) - (inputs / "responses.json").write_bytes(scripts) - atomic_write_json(inputs / "worker.json", config) - return config - - -def checked_inputs(inputs): - inputs = Path(inputs) - config = _read_json(inputs / "worker.json", 2 * 1024 * 1024) - if (type(config) is not dict or set(config) != FIELDS or config["schema"] != SCHEMA - or config["arm"] not in {"A2", "A3"} or config["dependencies"] != dependencies()): - raise ValueError("Wrong scripted worker identity/dependencies") - plan = _plan(config["edit_plan"]) - binding = {k: config[k] for k in ("dependencies", "suite_sha256", "baseline_sha256", - "responses_sha256", "prompt")} - if plan.callbacks_sha256 != edit.fingerprint(binding): - raise ValueError("Visible callback/prompt/script binding changed") - identity = edit.frozen_identity(plan, config["model"], config["client"], - config["input_token_ceiling"]) - if edit.fingerprint(identity) != config["actor_sha256"]: - raise ValueError("Frozen actor identity changed") - raw = {} - for name, field, limit in (("suite.json", "suite_sha256", 16 * 1024 * 1024), - ("baseline.py", "baseline_sha256", plan.limits.max_source_bytes), - ("responses.json", "responses_sha256", - config["client"]["max_requests"] * provider.MAX_RESPONSE)): - raw[name] = kv.read_regular(inputs / name, limit) - if edit.digest(raw[name]) != config[field]: - raise ValueError("Frozen scripted input changed") - suite = workbench.VisibleSuite(raw["suite.json"]) - responses = _read_json(inputs / "responses.json", len(raw["responses.json"])) - if type(responses) is not list or not 0 < len(responses) <= config["client"]["max_requests"]: - raise ValueError("Invalid bounded scripted response collection") - return config, plan, suite, raw["baseline.py"], responses - - -def _known_usage(response, config, deadline): - """Only known compatible raw usage is relayed; never response text or errors.""" - try: - if (time.monotonic() >= deadline or type(response) is not dict - or response.get("type") != "message" or response.get("role") != "assistant" - or response.get("model") != config["model"] - or response.get("stop_reason") != "tool_use" - or type(response.get("content")) is not list - or any(not provider.valid_content_block(c) for c in response["content"]) - or len(provider.canonical(response).encode()) > provider.MAX_RESPONSE): - return None - usage = provider.token_usage(response.get("usage")) - if (usage["input_tokens"] > config["input_token_ceiling"] - or usage["output_tokens"] > config["client"]["max_output_tokens"]): - return None - return {k: response["usage"][k] for k in ("input_tokens", "output_tokens", - "cache_read_input_tokens", "cache_creation_input_tokens")} - except Exception: - return None - - -def scripted_worker(inputs, workspace, output, deadline, *, environment, emit): - """Fixed worker body. Test injection is via module mocks, never CLI commands.""" - output = Path(output) - output.mkdir(parents=True, exist_ok=False) - bridge = output.parent / "bridge" - result = {"schema": SCHEMA, "status": "failed", "model_calls": 0, - "provider_charges_usd": 0, "scope": "scripted adapter qualification only"} - try: - config, plan, suite, baseline, responses = checked_inputs(inputs) - gate = gate_environment(environment) - if (not math.isfinite(deadline) - or not 0 < deadline - time.monotonic() <= plan.limits.max_seconds): - raise ValueError("Invalid inherited absolute deadline") - # No actor/provider request occurs if the real sandbox cannot be established. - result["isolation"] = kv.probe_isolation(output / "isolation", deadline, - [str(Path(inputs).resolve()), str(bridge.resolve()), str(ROOT), - f"/proc/{os.getppid()}/environ"]) - bench = workbench.VisibleWorkbench(plan, suite, baseline, output / "workbench") - grant, sent = None, 0 - - def authorize(value, inherited): - nonlocal grant - if inherited != deadline: - return False - grant = dict(value) - atomic_write_json(bridge / "pending.json", grant) - authorize_request(provider.canonical(grant).encode(), environment=gate, - timeout=max(0.001, deadline - time.monotonic())) - return True - - def send(payload, spec, remaining): - nonlocal sent - if (grant is None or edit.digest(provider.canonical(payload).encode()) - != grant["request_sha256"] or sent >= len(responses)): - raise ValueError("No exact scripted request grant/response") - value = responses[sent] - sent += 1 - atomic_write_json(bridge / "complete.json", { - "grant": grant, "usage": _known_usage(value, config, deadline)}) - return value - - # These values describe simulated request slots, not a fresh paid claim. - requests = config["client"]["max_requests"] - slots = edit.RequestSlots(config["plan_sha256"], {config["arm"]: requests}, - 10170000, requests * 10170000, execution_id=config["execution_id"]) - answer = edit.run_edit_agent(workspace, plan, config["prompt"], config["model"], - config["client"], deadline, emit, arm=config["arm"], slots=slots, - authorize=authorize, send=send, visible_checks=bench.visible_checks, - visible_benchmark=bench.visible_benchmark, retain_revision=bench.retain_revision, - input_token_ceiling=config["input_token_ceiling"]) - source = answer.pop("artifact_source").encode("utf-8") - final = output / "final" - final.mkdir() - (final / "candidate.py").write_bytes(source) - atomic_write_json(final / "receipt.json", answer) - result.update(status="completed", artifact=answer, simulated_requests=sent) - except kv.SandboxUnavailable: - result.update(status="inconclusive", reason="sandbox_unavailable") - except provider.AgentError as error: - result["reason"] = str(error) - except Exception: - result["reason"] = "scripted_worker_error" - atomic_write_json(output / "result.json", result) - return 0 if result["status"] == "completed" else 1 - - -def run_supervised(inputs, workspace, output, deadline, *, admission=None, execute=execute_command): - """SCRIPTED ONLY; shared private supervisor retains all deadline/admission semantics.""" - from tools.kv_worker_supervisor import supervise - - config, plan, _, _, _ = checked_inputs(inputs) - - def command(inputs, workspace, worker_output, inherited): - return (sys.executable, "-I", str(Path(__file__).resolve()), "--offline-worker", - "--inputs", str(inputs.resolve()), "--workspace", str(workspace.resolve()), - "--output", str(worker_output), "--deadline", repr(inherited)) - - result = supervise(config, plan, inputs, workspace, output, deadline, checkout=ROOT, - command_factory=command, - environment_factory=lambda gate: {"PATH": os.environ.get("PATH", "/usr/bin:/bin"), - "LANG": "C", **gate}, - admission=admission, execute=execute) - result.update(schema=SCHEMA, simulated_slots_consumed=result.pop("slots_consumed"), - model_calls=0, provider_charges_usd=0) - atomic_write_json(Path(output) / "supervision.json", result) - return result - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--offline-worker", action="store_true", required=True) - parser.add_argument("--inputs", type=Path, required=True) - parser.add_argument("--workspace", type=Path, required=True) - parser.add_argument("--output", type=Path, required=True) - parser.add_argument("--deadline", type=float, required=True) - args = parser.parse_args() - # Only the inherited ephemeral request-gate capability is read. No provider - # credential discovery/configuration or live-transport path exists here. - environment = {key: os.environ.get(key, "") for key in ENV_KEYS} - return scripted_worker(args.inputs, args.workspace, args.output, args.deadline, - environment=environment, emit=lambda event: print(provider.canonical(event), flush=True)) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tools/kv_live_authorization.json b/tools/kv_live_authorization.json deleted file mode 100644 index dbab9c3af..000000000 --- a/tools/kv_live_authorization.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "author_login": "Dong1017", - "body_sha256": "085c799c5cf907254802d07edef5f58587a957b837fa36a70ceb1cc0a15c3549", - "comment_id": 6542223785, - "enabled": true, - "issue": 248, - "repository_id": 1409534281, - "schema": "kv-live-authorization-locator-v1" -} diff --git a/tools/kv_live_job.py b/tools/kv_live_job.py deleted file mode 100644 index 86e5f7496..000000000 --- a/tools/kv_live_job.py +++ /dev/null @@ -1,1058 +0,0 @@ -"""Fixed main-only KV launcher; disabled until a reviewed authorization is installed. - -No dispatch inputs, PR-code checkout, ledger initializer, refund or retry. Ordinary -CI tests use fake transports/stores only. Candidate source never executes on host. -""" - -import argparse -import json -import math -import os -import re -import shutil -import stat -import time -import urllib.request -from dataclasses import replace -from datetime import datetime, timedelta, timezone -from itertools import islice -from pathlib import Path - -from inferrsi.harness import kv_edit_workbench as workbench -from inferrsi.harness import kv_reclamation as kv -from inferrsi.infra.artifacts import atomic_write_json -from inferrsi.model import anthropic_agent as provider -from tools import kv_edit_inputs as inputs -from tools import kv_live_policy as policy -from tools import kv_live_worker as worker -from tools import kv_pilot_admission as admission -from tools import qualify_kv_actor_cleanup as cleanup -from tools.skill_eval_budget_git import GitHubDailyStore, GitHubGitStore - -ROOT = policy.ROOT -COMMON, KNOWLEDGE, KNOWLEDGE_ROOT = inputs.COMMON, inputs.KNOWLEDGE, inputs.KNOWLEDGE_ROOT - - -class DeadlineGitHub: - """Existing Git ledger API with bounded single-attempt IO, no new storage protocol.""" - - def __init__(self, token, deadline, *, opener=None): - self.token, self.deadline = token, deadline - self.opener = opener or urllib.request.build_opener( - urllib.request.ProxyHandler({}), policy.NoRedirect()) - - def request(self, method, path, data=None): - allowed = ((method == "GET" and re.fullmatch( - r"(?:|/git/ref/heads/[A-Za-z0-9/_-]+|/git/(?:commits|trees|blobs)/[0-9a-f]{40}" - r"|/pulls/[1-9][0-9]*|/actions/(?:runs|artifacts)/[1-9][0-9]*" - r"|/issues/comments/[1-9][0-9]*)", path)) - or (method == "POST" and path in {"/git/trees", "/git/commits"}) - or (method == "PATCH" and re.fullmatch( - r"/git/refs/heads/skill-eval-budget/1409534281/(?:[1-9][0-9]*|daily)", path))) - left = self.deadline - time.monotonic() - if method == "PATCH" and (type(data) is not dict - or set(data) != {"sha", "force"} or data["force"] is not False - or not policy._hex(data["sha"], 40)): - allowed = False - if not allowed or left <= 0: - raise ValueError("Controller API scope/deadline denied") - request = urllib.request.Request( - "https://api.github.com/repos/" + policy.REPOSITORY + path, method=method, - headers={"Authorization": "Bearer " + self.token, - "Accept": "application/vnd.github+json", "Content-Type": "application/json", - "X-GitHub-Api-Version": "2022-11-28"}, - data=None if data is None else provider.canonical(data).encode()) - try: - chunks, size = [], 0 - with self.opener.open(request, timeout=min(left, 10)) as response: - while True: - if time.monotonic() >= self.deadline: - raise ValueError("deadline") - chunk = response.read1(min(65536, 4 * 1024 * 1024 + 1 - size)) - if not chunk: - break - chunks.append(chunk) - size += len(chunk) - if size > 4 * 1024 * 1024: - raise ValueError("size") - if time.monotonic() >= self.deadline: - raise ValueError("deadline") - return json.loads(b"".join(chunks), object_pairs_hook=provider.unique_object) - except Exception: - # An uncertain write remains uncertain in the existing CAS adapter. - # No response body, URL, credential or urllib error text is retained. - raise ValueError("Controller API response unavailable") from None - - -def runway(expires_at, now=None): - now = datetime.now(timezone.utc) if now is None else now - local = now.astimezone(timezone(timedelta(hours=8))) - midnight = (local + timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0) - # Two 600s arms, plus bounded admission/launch margin. Not an extra model allowance. - if min(expires_at - now.timestamp(), (midnight - local).total_seconds()) < 1320: - raise ValueError("Insufficient expiry/Shanghai-day runway for the complete pair") - - -def _stage(plan, output): - suite = policy.artifact(plan["suite"], limit=16 * policy.MAX_JSON) - common_bytes, common_protocol = None, None - for arm in ("A2", "A3"): - value = plan["actors"][arm] - config = policy.read_json(ROOT / value["config"]["path"]) - folder, workspace = output / arm / "inputs", output / arm / "workspace" - folder.mkdir(parents=True) - workspace.mkdir() - names = [e["path"] for e in value["workspace"]] - expected = COMMON | (KNOWLEDGE if arm == "A3" else set()) - if set(names) != expected or len(names) != len(expected): - raise ValueError("Actor exposure differs from the fixed A2/A3 allowlist") - staged = {} - for entry in value["workspace"]: - if type(entry) is not dict or set(entry) != {"path", "source"}: - raise ValueError("Wrong exposure record") - name = policy.relative(entry["path"]) - source = entry["source"] - if name in KNOWLEDGE and source["path"] != KNOWLEDGE_ROOT + name[len("knowledge/"):]: - raise ValueError("Knowledge export is not the complete nominated owner") - if name == "candidate.py" and source["path"] != ( - "tests/fixtures/kv_reclamation/candidate.py"): - raise ValueError("Candidate starter differs from qualified task") - if name == "CONTRACT.md" and source["path"] != ( - "evidence/kv-reclamation-20261030/validator/independent/CONTRACT.md"): - raise ValueError("Neutral contract changed") - data = policy.artifact(source) - path = workspace / name - path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(data) - staged[name] = data - common = {name: staged[name] for name in COMMON} - if common_bytes is not None and common != common_bytes: - raise ValueError("A2/A3 common workbench bytes differ") - common_bytes = common - suite_data = workbench.VisibleSuite(suite).read() - expected_common, expected_prompt, _ = inputs.build_inputs( - staged["candidate.py"], staged["CONTRACT.md"], suite_data["development"], - suite_data["benchmark_events"], suite_data["benchmark_expected"], - suite_data["min_improvement_pct"]) - if common != expected_common or config["prompt"] != expected_prompt: - raise ValueError("Actor inputs differ from the shared qualified public workbench") - protocol = {k: config[k] for k in ("model", "client", "prompt", "input_token_ceiling")} - protocol["edit_plan"] = {k: v for k, v in config["edit_plan"].items() - if k not in {"files", "callbacks_sha256"}} - if common_protocol is not None and protocol != common_protocol: - raise ValueError("A2/A3 tool/metric/prompt limits differ") - common_protocol = protocol - edit_plan = worker.scripted._plan(config["edit_plan"]) - expected_config = worker.configuration(workspace, edit_plan, workbench.VisibleSuite(suite), - staged["candidate.py"], arm=arm, model=plan["provider"]["model"]["name"], - spec=plan["provider"]["client"], prompt=config["prompt"]) - if provider.canonical(config) != provider.canonical(expected_config): - raise ValueError("Frozen worker config differs from staged actor identity") - atomic_write_json(folder / "worker.json", config) - (folder / "suite.json").write_bytes(suite) - (folder / "baseline.py").write_bytes(staged["candidate.py"]) - atomic_write_json(folder / "binding.json", {"plan_sha256": policy.fingerprint(plan), - "execution_id": plan["execution_id"]}) - worker.checked_inputs(folder) - - -def replay_preflight(output, deadline): - """Same-host fixed tool replay, with response DATA and no provider transport. - - Rewrites the starter itself (not a hidden solution), checks/benchmarks that - exact revision and submits it. A failed speed gate is valid feedback, not a - reason to discard or rerun this mechanical qualification. - """ - results = {} - for arm in ("A2", "A3"): - config, plan, suite, baseline = worker.checked_inputs(output / arm / "inputs") - root = output / "preflight/replay" / arm - root.mkdir(parents=True) - workspace = root / "workspace" - shutil.copytree(output / arm / "workspace", workspace) - - def call(name, inputs, number): - return {"type": "tool_use", "id": f"replay-{number}", "name": name, "input": inputs} - - def response(content): - return {"type": "message", "role": "assistant", "model": config["model"], - "stop_reason": "tool_use", "content": content, - "usage": {"input_tokens": 1, "output_tokens": 1, - "cache_read_input_tokens": 0, "cache_creation_input_tokens": 0}} - - scripts = [response([call("read_file", {"path": "task.json"}, 1)]), - response([call("write_candidate", {"source": baseline.decode()}, 2), - call("run_visible_checks", {}, 3), call("run_visible_benchmark", {}, 4)]), - response([call("submit_candidate", {}, 5)])] - worker.scripted.prepare_scripted(root / "inputs", workspace, plan, suite, baseline, - scripts, plan_sha256=config["plan_sha256"], execution_id=config["execution_id"], - arm=arm, model=config["model"], spec=config["client"], prompt=config["prompt"], - input_token_ceiling=config["input_token_ceiling"]) - result = worker.scripted.run_supervised(root / "inputs", workspace, root / "run", deadline) - if (result["worker"].get("status") != "completed" - or result["execution"].get("status") != "completed" - or not result["request_completion_known"] - or result["simulated_slots_consumed"] != 3 - or kv.source_bytes(root / "run/worker/final/candidate.py") != baseline): - raise ValueError("Same-host scripted tool replay incomplete") - results[arm] = {"verdict": "PASS", "model_calls": 0, - "artifact_sha256": policy.sha(baseline), "scripted_requests": 3} - atomic_write_json(root / "receipt.json", results[arm]) - return results - - -def runtime_preflight(output, deadline, plan): - """Recheck actual mounted runtime/K/oracle isolation; hosted evidence is not a bypass.""" - from tools import kv_runtime_qualification - receipt = policy.validate_qualification(plan) - result = kv_runtime_qualification.qualify(output, deadline, root=ROOT, - independent=ROOT / "evidence/kv-reclamation-20261030/validator/independent", - fixtures=ROOT / "tests/fixtures/kv_reclamation", knowledge_root=ROOT / KNOWLEDGE_ROOT) - qualified = receipt["runtime_isolation"] - profile_source = receipt["dependencies"]["src/inferrsi/harness/skill_eval_engineering.py"] - result = {**result, "qualified_inventory_sha256": qualified["inventory_sha256"], - "profile_source_sha256": profile_source} - atomic_write_json(output / "receipt.json", result) - if (result.get("verdict") != "PASS" - or not policy._hex(result.get("inventory_sha256")) - or not policy._hex(qualified.get("inventory_sha256")) - or not policy._hex(profile_source) - or result.get("mount_count") != qualified.get("mount_count")): - raise ValueError("Same-host runtime/profile differs from hosted actor qualification") - # Inventory hashing is sensitive-set dependent, not a portable host fingerprint. - # The exact sandbox source/profile is bound by the canonical qualification - # closure; independently require this host's full inventory/probe PASS. - return result - - -def prepare(output): - if os.environ.get(provider.KEY_ENV): - raise ValueError("Preparation must precede provider-secret injection") - captured = {} - def capture(number): - comment = policy.fetch_comment(number) - captured.update({"id": comment["id"], "body": comment["body"], - "issue_url": comment["issue_url"], - "user": {k: comment["user"][k] for k in ("login", "type")}}) - return comment - authorization, locator = policy.frozen_authorization(fetch=capture) - plan = authorization["plan"] - revision, run_id = policy.runtime() - runway(plan["expires_at"]) - output = Path(output) - output.mkdir(parents=True, exist_ok=False) - atomic_write_json(output / "authorization.json", captured) - progress = {"schema": "kv-live-preflight-progress-v1", "implementation_sha": revision, - "run_id": run_id, "plan_sha256": authorization["plan_sha256"], - "stages": dict.fromkeys(("stage", "isolation", "cleanup", "runtime", "replay"), - "not_started")} - def step(name, callback): - progress["stages"][name] = "started_outcome_unknown" - atomic_write_json(output / "preflight.json", progress) - try: - result = callback() - except BaseException: - progress["stages"][name] = "failed_or_interrupted" - atomic_write_json(output / "preflight.json", progress) - raise - progress["stages"][name] = "completed" - atomic_write_json(output / "preflight.json", progress) - return result - step("stage", lambda: _stage(plan, output)) - deadline = time.monotonic() + 120 - isolation = step("isolation", lambda: kv.probe_isolation( - output / "preflight/isolation", deadline, - [str(ROOT), str(output / "A2/inputs"), str(output / "A3/workspace")])) - descendants = step("cleanup", lambda: cleanup.qualify( - output / "preflight/cleanup", deadline, 3)) - if descendants["verdict"] != "PASS": - raise ValueError("Same-host actor cleanup qualification incomplete") - runtime_profile = step("runtime", lambda: runtime_preflight( - output / "preflight/runtime", deadline, plan)) - replay = step("replay", lambda: replay_preflight(output, deadline)) - receipt = {"schema": "kv-live-prepared-v1", "implementation_sha": revision, - "run_id": run_id, "plan_sha256": authorization["plan_sha256"], - "locator_sha256": policy.fingerprint(locator), - "isolation": isolation, "cleanup": descendants, "replay": replay, - "runtime_profile": runtime_profile, - "inputs": kv.workspace_hashes(output)} - atomic_write_json(output / "prepared.json", receipt) - return receipt - - -def execute(output, *, api=None, stores=None, launch=worker.run_supervised): - output = Path(output) - authorization, locator = policy.frozen_authorization() - plan = authorization["plan"] - revision, run_id = policy.runtime() - prepared = policy.read_json(output / "prepared.json") - if (prepared["schema"] != "kv-live-prepared-v1" - or prepared["implementation_sha"] != revision or prepared["run_id"] != run_id - or prepared["plan_sha256"] != authorization["plan_sha256"] - or prepared["locator_sha256"] != policy.fingerprint(locator) - or prepared["cleanup"]["verdict"] != "PASS" - or prepared["runtime_profile"]["verdict"] != "PASS" - or set(prepared["replay"]) != {"A2", "A3"} - or any(v.get("verdict") != "PASS" or v.get("model_calls") != 0 - for v in prepared["replay"].values())): - raise ValueError("Prepared exact source/authorization/host identity changed") - current = kv.workspace_hashes(output) - current.pop("prepared.json", None) - if current != prepared["inputs"]: - raise ValueError("Prepared inputs changed before admission") - runway(plan["expires_at"]) - # Only the existing named injected key is used. Never inspect secret files or env dumps. - safe = provider.execution_environment({provider.KEY_ENV: os.environ.get(provider.KEY_ENV)}) - api = api or DeadlineGitHub(os.environ["GITHUB_TOKEN"], time.monotonic() + 120) - root_policy = policy.policy(plan, revision) - - def check_current(value): - if value != root_policy: - return False - fresh = policy.frozen_authorization(fetch=lambda n: api.request( - "GET", f"/issues/comments/{n}")) - if fresh != (authorization, locator): - return False - policy.verify_live(api, plan, revision, run_id) - return True - - if stores is None: - stores = (GitHubDailyStore(api, policy.DailyLedgerAnchor( - key=policy.budget.DailyKey(policy.REPOSITORY_ID), **plan["daily_ledger"])), - GitHubGitStore(api, policy.LedgerAnchor(key=root_policy.key, **plan["ledger"]), - trusted_limit=root_policy.trusted_limit)) - summary = {"schema": "kv-live-execution-v1", "implementation_sha": revision, - "run_id": run_id, "plan_sha256": authorization["plan_sha256"], - "execution_id": plan["execution_id"], "charge_pr": plan["charge_pr"], - "charge_head": plan["charge_head"], - "reservation_upper_bound_usd": "162.72", "reserved_usd": None, - "admission_outcome": "pending_durable_state_unknown_no_retry", - "actual_cost_usd": None, - "arms": {arm: {"status": "not_started", "request_completion_known": False, - "slots_consumed": 0} for arm in plan["arm_order"]}} - atomic_write_json(output / "execution.json", summary) - try: - session = admission.admit(root_policy, *stores, check_current=check_current) - except BaseException: - # Failed admission can have written a partial/uncertain durable claim. - # Preserve identity for no-secret grading, but never assert zero spend. - summary["admission_outcome"] = "not_granted_durable_state_unknown_no_retry" - atomic_write_json(output / "execution.json", summary) - raise - summary.update(admission_outcome="granted", reserved_usd="162.72") - atomic_write_json(output / "execution.json", summary) - try: - for arm in plan["arm_order"]: - summary["arms"][arm] = {"status": "launching_or_interrupted", - "request_completion_known": False, "slots_consumed": None} - atomic_write_json(output / "execution.json", summary) - deadline = time.monotonic() + 600 - api.deadline = deadline - result = launch(output / arm / "inputs", output / arm / "workspace", - output / arm / "run", deadline, admission=session, key=safe[provider.KEY_ENV]) - summary["arms"][arm] = {"status": result["worker"].get("status", "missing"), - "request_completion_known": result["request_completion_known"], - "slots_consumed": result["slots_consumed"]} - atomic_write_json(output / "execution.json", summary) - if not result["request_completion_known"]: - break - finally: - session.close() - return summary - - -def _validate_feedback(receipt, config, arm, execution): - fields = {"plan_sha256", "execution_id", "arm", "artifact_sha256", "revision", - "kind", "feedback", "receipt_sha256"} - if (type(receipt) is not dict or set(receipt) != fields or receipt["arm"] != arm - or receipt["plan_sha256"] != execution["plan_sha256"] - or receipt["execution_id"] != execution["execution_id"] - or receipt["kind"] not in {"checks", "benchmark"} - or type(receipt["revision"]) is not int or not 0 <= receipt["revision"] <= 8 - or not policy._hex(receipt["artifact_sha256"])): - raise ValueError("Wrong visible receipt identity") - value = {k: v for k, v in receipt.items() if k != "receipt_sha256"} - if receipt["receipt_sha256"] != policy.fingerprint({**value, - "actor_sha256": config["actor_sha256"]}): - raise ValueError("Visible feedback digest changed") - worker.edit._feedback(receipt["kind"], receipt["feedback"], - worker.scripted._plan(config["edit_plan"])) - - -def _validate_submission(root, receipt, source, config, arm, execution): - fields = {"plan_sha256", "execution_id", "arm", "artifact", "revision", "sha256", - "actor_sha256", "check_receipt", "benchmark_receipt"} - if (type(receipt) is not dict or set(receipt) != fields or receipt["arm"] != arm - or receipt["plan_sha256"] != execution["plan_sha256"] - or receipt["execution_id"] != execution["execution_id"] - or receipt["artifact"] != "candidate.py" or type(receipt["revision"]) is not int - or not 0 <= receipt["revision"] <= 8 - or receipt["actor_sha256"] != config["actor_sha256"] - or receipt["sha256"] != policy.sha(source)): - raise ValueError("Wrong final submission identity") - revision = root / "revisions" / f"revision-{receipt['revision']:04d}" - expected = {k: v for k, v in receipt.items() if k not in {"check_receipt", "benchmark_receipt"}} - if (policy.read_json(revision / "receipt.json", 16384) != expected - or kv.source_bytes(revision / "candidate.py") != source): - raise ValueError("Final source differs from retained revision") - found = set() - for number in range(1, 13): - path = root / "feedback" / f"{number:04d}.json" - if not path.exists(): - continue - feedback = policy.read_json(path, 8192) - _validate_feedback(feedback, config, arm, execution) - field = "check_receipt" if feedback["kind"] == "checks" else "benchmark_receipt" - if (feedback["revision"] == receipt["revision"] - and feedback["artifact_sha256"] == receipt["sha256"] - and feedback["receipt_sha256"] == receipt[field]): - found.add(field) - if found != {"check_receipt", "benchmark_receipt"}: - raise ValueError("Missing exact current-revision visible receipts") - - -def _public_evidence(root, destination, arm, execution): - """Bounded structured revision/feedback only; raw logs are never publication inputs.""" - result = {"revisions": [], "feedback": [], "status": "incomplete"} - try: - config = policy.read_json(root.parents[1] / "inputs/worker.json") - except (OSError, ValueError): - return result - for revision in range(9): - source = root / "workbench/revisions" / f"revision-{revision:04d}" - try: - receipt = policy.read_json(source / "receipt.json", 16384) - data = kv.source_bytes(source / "candidate.py") - fields = {"plan_sha256", "execution_id", "arm", "artifact", "revision", - "sha256", "actor_sha256"} - if (set(receipt) != fields or receipt["arm"] != arm - or receipt["plan_sha256"] != execution["plan_sha256"] - or receipt["execution_id"] != execution["execution_id"] - or receipt["actor_sha256"] != config["actor_sha256"] - or receipt["artifact"] != "candidate.py" or receipt["revision"] != revision - or receipt["sha256"] != policy.sha(data) - or not policy._hex(receipt["actor_sha256"])): - raise ValueError("Wrong revision identity") - target = destination / "revisions" / f"revision-{revision:04d}" - target.mkdir(parents=True) - (target / "candidate.py").write_bytes(data) - atomic_write_json(target / "receipt.json", receipt) - result["revisions"].append({"revision": revision, "sha256": policy.sha(data)}) - except (OSError, ValueError, KeyError, TypeError, provider.AgentError): - continue - for number in range(1, 13): - try: - receipt = policy.read_json(root / "feedback" / f"{number:04d}.json", 8192) - _validate_feedback(receipt, config, arm, execution) - (destination / "feedback").mkdir(parents=True, exist_ok=True) - atomic_write_json(destination / "feedback" / f"{number:04d}.json", receipt) - result["feedback"].append({"number": number, "sha256": policy.fingerprint(receipt)}) - except (OSError, ValueError, KeyError, TypeError, provider.AgentError): - continue - if result["revisions"] and result["feedback"]: - result["status"] = "retained" - return result - - -def _proof_projection(value): - """Exact public probe fields only: never scope/error/path/log/environment text.""" - if type(value) is not dict: - return {} - result = {} - for key in ("verdict", "executor_status", "worker_status"): - if value.get(key) in {"PASS", "FAIL", "INCONCLUSIVE", "completed", "failed", - "timeout", "cancelled", "error"}: - result[key] = value[key] - for key in ("readonly", "clean_env", "network_blocked", "fork_denied", - "normal_exit_process_group_gone", "detached_session_observed", - "visibility_complete", "explicit_cancel_sent", "candidate_profile_unchanged", - "outer_worker_supervised", "source_unchanged", "dirty", - "recovery_visibility_complete"): - if type(value.get(key)) is bool: - result[key] = value[key] - for key in ("model_calls", "observed_descendants", "surviving_descendants", "recovery_kills", - "recovery_survivors", "nodes", "mount_count", "escaped_symlinks", - "sensitive_files", "scripted_requests"): - if type(value.get(key)) is int and 0 <= value[key] <= 200000: - result[key] = value[key] - for key in ("probe_sha256", "qualifier_sha256", "snapshot_hash", "inventory_sha256", - "qualified_inventory_sha256", "profile_source_sha256", "artifact_sha256"): - if policy._hex(value.get(key)): - result[key] = value[key] - if policy._hex(value.get("revision"), 40): - result["revision"] = value["revision"] - denied = value.get("inaccessible") - if type(denied) is list and len(denied) <= 30000 and all(type(v) is bool for v in denied): - result["inaccessible"] = denied - limits = value.get("limits") - if (type(limits) is dict - and set(limits) == {"memory", "processes", "cpu", "output", "files", "core"}): - if all(type(v) is list and len(v) == 2 and all(type(n) is int and -1 <= n <= 2**40 - for n in v) for v in limits.values()): - result["limits"] = limits - for key in ("probe",): - if key in value: - result[key] = _proof_projection(value[key]) - if type(value.get("cases")) is dict: - result["cases"] = {mode: _proof_projection(value["cases"].get(mode)) - for mode in ("normal", "timeout", "cancel")} - chain = value.get("bound_chain") - if (type(chain) is dict and type(chain.get("chain_length")) is int - and 3 <= chain["chain_length"] <= 128): - result["bound_chain"] = {"chain_length": chain["chain_length"]} - for role in ("worker", "detached"): - identity = chain.get(role) - if (type(identity) is dict and set(identity) == {"pid", "start"} - and all(type(v) is int and 0 < v < 2**63 for v in identity.values())): - result["bound_chain"][role] = identity - return result - - -def _public_preflight(output, authorization, locator): - """Project fixed receipts, including partial preparation; no directory walk.""" - result = {"schema": "kv-live-public-preflight-v1", "status": "missing_or_invalid", - "plan_sha256": authorization["plan_sha256"], "stages": {}, "proofs": {}} - binding = None - try: - execution = policy.read_json(output / "execution.json") - binding = tuple(execution[k] for k in ("implementation_sha", "run_id", "plan_sha256")) - except (OSError, ValueError, KeyError, TypeError): - pass - for filename in ("preflight.json", "prepared.json"): - try: - value = policy.read_json(output / filename) - schema = ("kv-live-preflight-progress-v1" if filename == "preflight.json" - else "kv-live-prepared-v1") - if (value["schema"] != schema or value["plan_sha256"] != authorization["plan_sha256"] - or not policy._hex(value["implementation_sha"], 40) - or not re.fullmatch(r"[1-9][0-9]*", value["run_id"])): - raise ValueError("Unbound preparation") - identity = tuple(value[k] for k in ("implementation_sha", "run_id", "plan_sha256")) - if binding is not None and identity != binding: - raise ValueError("Preparation execution identity changed") - binding = identity - result.update(implementation_sha=value["implementation_sha"], run_id=value["run_id"]) - if filename == "preflight.json": - result["stages"] = {name: state if state in { - "not_started", "started_outcome_unknown", - "failed_or_interrupted", "completed"} else "missing_or_invalid" - for name in ("stage", "isolation", "cleanup", "runtime", "replay") - for state in [value.get("stages", {}).get(name)]} - result["status"] = "partial" - else: - if value.get("locator_sha256") != policy.fingerprint(locator): - raise ValueError("Unbound prepared locator") - result["locator_sha256"] = value["locator_sha256"] - for name in ("isolation", "cleanup", "runtime_profile"): - result["proofs"][name] = _proof_projection(value.get(name)) - result["proofs"]["replay"] = {a: _proof_projection(value.get("replay", {}).get(a)) - for a in ("A2", "A3")} - workspace = {arm: {entry["path"]: { - "expected_sha256": entry["source"]["sha256"], - "observed_sha256": value.get("inputs", {}).get( - f"{arm}/workspace/{entry['path']}")} - for entry in authorization["plan"]["actors"][arm]["workspace"] - if entry["path"] in COMMON | (KNOWLEDGE if arm == "A3" else set())} - for arm in ("A2", "A3")} - if any(not policy._hex(entry["observed_sha256"]) - or entry["observed_sha256"] != entry["expected_sha256"] - for paths in workspace.values() for entry in paths.values()): - raise ValueError("Missing staged workspace hashes") - result["workspace"] = workspace - result["status"] = "prepared" - except (OSError, ValueError, KeyError, TypeError, RecursionError): - continue - for name, path in (("isolation", "preflight/isolation/stdout.log"), - ("cleanup", "preflight/cleanup/qualification.json"), - ("runtime_profile", "preflight/runtime/receipt.json")): - if not result["proofs"].get(name): - try: - result["proofs"][name] = _proof_projection(policy.read_json(output / path)) - except (OSError, ValueError, TypeError, RecursionError): - result["proofs"][name] = {"verdict": "INCONCLUSIVE"} - for kind, names, pattern in (("replay", ("A2", "A3"), "preflight/replay/{}/receipt.json"), - ("cleanup", ("normal", "timeout", "cancel"), "preflight/cleanup/{}/result.json")): - target = result["proofs"].setdefault(kind, {}) - if kind == "cleanup": - target = target.setdefault("cases", {}) - for name in names: - if not target.get(name): - try: - target[name] = _proof_projection(policy.read_json( - output / pattern.format(name))) - except (OSError, ValueError, TypeError, RecursionError): - target[name] = {"verdict": "INCONCLUSIVE"} - return result - - -def _raw_kv_output(path, destination, events, *, verify, remaining): - """Retain exact raw JSON only after a closed KV-data schema/privacy check. - - Malformed candidate-only output is withheld, with bounded byte/hash diagnostics; - this is not a sanitizer for provider logs or private inputs. Frozen task inputs - identify allowed strings. Oversize/unsafe files are not opened or fully hashed. - """ - result = {"status": "missing_or_unreadable", "reason": "unavailable", - "bytes": None, "sha256": None, "truncated": None} - try: - if any(p.is_symlink() for p in (path, *path.parents)): - return {**result, "reason": "unsafe_path"} - info = path.lstat() - if not stat.S_ISREG(info.st_mode): - return {**result, "reason": "nonregular_file"} - if info.st_size > 4 * policy.MAX_JSON: - return {**result, "reason": "file_byte_limit", "bytes": info.st_size, - "truncated": True} - raw = kv.read_regular(path, 4 * policy.MAX_JSON) - result = {"status": "redacted_invalid_output", "reason": "json_parse_failed", - "bytes": len(raw), "sha256": policy.sha(raw), "truncated": False} - value = json.loads(raw, object_pairs_hook=provider.unique_object) - result["reason"] = "not_allowlisted_kv_data" - ids = {e["request_id"] for e in events if type(e) is dict - and type(e.get("request_id")) is str} - def integers(items): - return type(items) is list and len(items) <= 20000 and all( - type(n) is int for n in items) - def pending(items): - return type(items) is list and len(items) <= 10000 and all( - type(v) is list and len(v) == 3 and v[0] in ids - and type(v[1]) is int - and integers(v[2]) for v in items) - if (type(value) is not dict or set(value) != {"observations", "pending"} - or not pending(value["pending"]) or type(value["observations"]) is not list - or len(value["observations"]) > 10000): - raise ValueError("Not public KV output") - for observation in value["observations"]: - fields = {"pending"} if verify else set() - if (type(observation) is not dict - or set(observation) not in (fields | {"error"}, fields | {"reclaimed"}) - or (verify and not pending(observation["pending"])) - or ("error" in observation and observation["error"] != "ValueError") - or ("reclaimed" in observation and not integers(observation["reclaimed"]))): - raise ValueError("Not public observation") - if len(raw) > remaining[0]: - return {**result, "status": "total_evidence_limit", - "reason": "whole_evidence_budget_exhausted"} - result.update(status="projection_write_failed", reason="projection_write_failed") - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_bytes(raw) - remaining[0] -= len(raw) - result = {"status": "retained", "bytes": len(raw), "sha256": policy.sha(raw), - "data_sha256": policy.fingerprint(value), "truncated": False} - except OSError: - return result - except (ValueError, TypeError, RecursionError): - pass - return result - - -def _execution_projection(value): - result = {"status": "unavailable"} - if type(value) is dict: - if value.get("status") in {"completed", "failed", "timeout", "cancelled", "error"}: - result["status"] = value["status"] - for name in ("elapsed_seconds", "metric"): - number = value.get(name) - if type(number) in (int, float) and math.isfinite(number) and 0 <= number <= 3600: - result[name] = number - elif name in value and number is None: - result[name] = None - code = value.get("returncode") - if type(code) is int and -255 <= code <= 255: - result["returncode"] = code - elif "returncode" in value and code is None: - result["returncode"] = None - return result - - -def _grade_evidence(bundle, output, result, plan=None): - """Only known grader paths; all assigned cases/trials survive as explicit rows.""" - public = output / "public" - public.mkdir(exist_ok=True) - state = {"schema": "kv-live-grade-projection-v1", "status": "started_outcome_unknown"} - atomic_write_json(public / "projection.json", state) - try: - remaining = [32 * policy.MAX_JSON] # Whole two-arm raw evidence ceiling, not per file. - manifest = {"schema": "kv-live-grade-evidence-v1", "plan_sha256": result["plan_sha256"], - "arms": {a: {"status": "not_evaluated"} for a in ("A2", "A3")}} - if plan is not None: - captured = policy.read_json(bundle / "authorization.json") - atomic_write_json(public / "authorization.json", captured) - metadata = policy.read_json(bundle / "bundle.json") - manifest["execution"] = {k: metadata["execution"][k] for k in - ("implementation_sha", "run_id", "execution_id", "charge_pr", "charge_head")} - hidden = json.loads(policy.artifact(plan["hidden_traces"], limit=16 * policy.MAX_JSON)) - if type(hidden) is not list or not 0 < len(hidden) <= 256: - raise ValueError("Frozen grader trace-count evidence bound exceeded") - suite = workbench.VisibleSuite(policy.artifact( - plan["final_suite"], limit=16 * policy.MAX_JSON)).read() - for arm in ("A2", "A3"): - records = {"status": "recorded", "semantics": [], "performance": []} - manifest["arms"][arm] = records - arm_result = result["arms"].get(arm, {}) - source = arm_result.get("artifact_sha256", - arm_result.get("semantic", {}).get("artifact_sha256")) - records["artifact_sha256"] = source if policy._hex(source) else None - baseline_hash = next(e["source"]["sha256"] for e in plan["actors"][arm]["workspace"] - if e["path"] == "candidate.py") - checks = arm_result.get("semantic", {}).get("checks", {}) - for index, trace in enumerate(hidden): - name = f"{arm}/semantics/trace-{index:03d}.json" - trace_root = output / arm / "semantics" / f"trace-{index:03d}" - raw = _raw_kv_output( - trace_root / "stdout.log", - public / name, trace["events"], verify=True, remaining=remaining) - execution = {"status": "unavailable"} - try: - record = policy.read_json(trace_root / "execution.json", 16384) - if (type(record) is not dict or set(record) != { - "status", "returncode", "elapsed_seconds"}): - raise ValueError("Wrong trusted executor receipt") - execution = _execution_projection(record) - except (OSError, ValueError, TypeError): - pass - records["semantics"].append({"trace_id": trace["id"], - "trace_sha256": policy.fingerprint(trace), "output": {"path": name, **raw}, - "check": checks.get(trace["id"]), - "execution": execution}) - runs = output / arm / "benchmark/benchmark-001/runs" - run = None - try: - entries = list(islice(runs.iterdir(), 2)) - if (len(entries) == 1 - and re.fullmatch(r"\d{8}T\d{6}Z-[0-9a-f]{10}", entries[0].name) - and entries[0].is_dir() and not entries[0].is_symlink()): - run = entries[0] - except OSError: - pass - schedule = [(side, index + 1, f"{side}/trial-{index + 1:03d}") - for index in range(plan["final_repetitions"]) - for side in (("baseline", "candidate") if index % 2 == 0 - else ("candidate", "baseline"))] - schedule.append(("candidate", None, "validation")) - for side, trial, relative in schedule: - name = f"{arm}/performance/{relative}/output.json" - row = {"side": side, "trial": trial, - "phase": "quality" if trial is None else "measurement", - "source_sha256": baseline_hash if side == "baseline" else source, - "input_sha256": policy.fingerprint(suite["benchmark_events"]), - "expected_sha256": policy.fingerprint(suite["benchmark_expected"]), - "execution": {"status": "unavailable"}, - "source_verified": False, "output": {"status": "not_observed"}} - if run is not None: - row["output"] = {"path": name, **_raw_kv_output( - run / relative / "stdout.log", - public / name, suite["benchmark_events"], verify=False, - remaining=remaining)} - try: - row["execution"] = _execution_projection(policy.read_json( - run / relative / "execution.json")) - identity = policy.read_json(run / relative / "source.json") - row["source_snapshots"] = {k: identity.get(k, {}).get("snapshot_hash") - if policy._hex(identity.get(k, {}).get("snapshot_hash")) else None - for k in ("before", "after")} - row["source_verified"] = all( - identity.get(k, {}).get("available") is True - and identity[k].get("snapshot_hash") == row["source_sha256"] - for k in ("before", "after")) - except (OSError, ValueError, TypeError): - pass - records["performance"].append(row) - records["raw_outputs_complete"] = all(row["output"]["status"] == "retained" - for name in ("semantics", "performance") for row in records[name]) - atomic_write_json(public / "grade.json", result) - atomic_write_json(public / "evidence.json", manifest) # Completion marker written last. - except BaseException: - state["status"] = "failed_or_interrupted" - atomic_write_json(public / "projection.json", state) - raise - state.update(status="completed", evidence_sha256=policy.fingerprint(manifest), - grade_sha256=policy.fingerprint(result)) - atomic_write_json(public / "projection.json", state) - return manifest - - -def _worker_outcome(root): - """Recover only closed trusted outcome fields, even if controller writing stopped.""" - outcome = {"worker_status": "unavailable", "reason": None, "provider_attempts": None, - "transport_error_code": None, - "execution": {"status": "unavailable", "returncode": None, - "elapsed_seconds": None}} - try: - value = policy.read_json(root / "worker/result.json") - if type(value) is not dict or value.get("schema") != worker.SCHEMA: - raise ValueError("Wrong worker result") - if value.get("status") in {"completed", "failed", "inconclusive"}: - outcome["worker_status"] = value["status"] - if "reason" in value: - outcome["reason"] = worker.closed_reason(value["reason"]) - attempts = value.get("provider_attempts") - if type(attempts) is int and 0 <= attempts <= 8: - outcome["provider_attempts"] = attempts - if value.get("transport_error_code") is not None: - outcome["transport_error_code"] = worker.closed_reason(value["transport_error_code"]) - except (OSError, ValueError, KeyError, TypeError): - pass - try: - value = policy.read_json(root / "supervision.json") - if type(value) is not dict or value.get("schema") != "kv-worker-supervision-v1": - raise ValueError("Wrong supervision result") - projected = _execution_projection(value.get("execution")) - outcome["execution"].update({k: v for k, v in projected.items() - if k in outcome["execution"]}) - except (OSError, ValueError, KeyError, TypeError): - pass - return outcome - - -def package(output): - """Fixed bounded public projection. Never copy raw worker/provider logs or private paths.""" - output = Path(output) - public = output / "public" - public.mkdir(parents=True, exist_ok=False) - preflight_status = "missing_or_invalid_authorization" - try: - # Public bytes already verified before preparation; never credentials. - captured = policy.read_json(output / "authorization.json") - authorization, locator = policy.frozen_authorization(fetch=lambda _: captured, now=0) - atomic_write_json(public / "authorization.json", captured) - proof = _public_preflight(output, authorization, locator) - atomic_write_json(public / "preflight.json", proof) - preflight_status = proof["status"] - except (OSError, ValueError, KeyError, TypeError): - pass # Missing authorization is explicit INCONCLUSIVE in the no-spend grader. - execution = (policy.read_json(output / "execution.json") - if (output / "execution.json").is_file() else {}) - summary = {"schema": "kv-live-bundle-v1", "execution": execution, "artifacts": {}, - "requests": {}, "evidence": {}, "preflight_status": preflight_status} - summary["diagnostics"] = {arm: _worker_outcome(output / arm / "run") - for arm in ("A2", "A3")} - for arm in ("A2", "A3"): - try: - supervision = policy.read_json(output / arm / "run/supervision.json") - requests = supervision["completions"] - if type(requests) is not list or len(requests) > 8: - raise ValueError("Wrong request count") - safe = [] - for entry in requests: - raw = entry["usage"] - if raw is not None: - provider.token_usage(raw) - grant = entry["grant"] - if (grant["arm"] != arm or grant["plan_sha256"] != execution["plan_sha256"] - or not policy._hex(grant["request_sha256"])): - raise ValueError("Wrong completion identity") - safe.append({"slot": grant["slot"], "request_sha256": grant["request_sha256"], - "status": "known" if raw is not None else "unresolved", - "usage": None if raw is None else {k: raw[k] for k in ("input_tokens", - "output_tokens", "cache_read_input_tokens", - "cache_creation_input_tokens")}}) - summary["requests"][arm] = safe - except (OSError, ValueError, KeyError, TypeError, provider.AgentError): - summary["requests"][arm] = [] - for arm in ("A2", "A3"): - root = output / arm / "run/worker" - summary["evidence"][arm] = _public_evidence(root, public / arm, arm, execution) - try: - result = policy.read_json(root / "result.json") - if type(result) is not dict or result.get("status") != "completed": - raise ValueError("not submitted") - receipt = policy.read_json(root / "final/receipt.json", 16384) - data = kv.source_bytes(root / "final/candidate.py") - if (receipt["sha256"] != policy.sha(data) or receipt["arm"] != arm - or receipt["plan_sha256"] != execution["plan_sha256"]): - raise ValueError("wrong final artifact") - config = policy.read_json(output / arm / "inputs/worker.json") - _validate_submission(public / arm, receipt, data, config, arm, execution) - (public / arm).mkdir(exist_ok=True) - (public / arm / "candidate.py").write_bytes(data) - atomic_write_json(public / arm / "receipt.json", receipt) - summary["artifacts"][arm] = {"status": "submitted", "sha256": policy.sha(data), - "receipt_sha256": policy.fingerprint(receipt)} - except (OSError, ValueError, KeyError, TypeError, provider.AgentError): - summary["artifacts"][arm] = {"status": "missing_or_invalid"} - atomic_write_json(public / "bundle.json", summary) - return summary - - -def _grade_rejected(output, reason, plan_sha256=None): - output.mkdir(parents=True, exist_ok=True) - result = {"schema": "kv-live-final-grade-v1", "plan_sha256": plan_sha256, - "scope": "final input identity unavailable; no candidate evaluation or spending", - "arms": {arm: {"verdict": "INCONCLUSIVE", "reason": reason} - for arm in ("A2", "A3")}} - atomic_write_json(output / "grade.json", result) - _grade_evidence(None, output, result) - return result - - -def _grade_context(bundle): - """Shared no-network identity check for grading and interruption-only packaging.""" - try: - captured = policy.read_json(bundle / "authorization.json") - authorization, _ = policy.frozen_authorization(fetch=lambda _: captured, now=0) - except (OSError, ValueError, KeyError, TypeError): - raise ValueError("missing_or_invalid_authorization") from None - plan = authorization["plan"] - try: - metadata = policy.read_json(bundle / "bundle.json") - except (OSError, ValueError): - raise ValueError("missing_or_invalid_bundle") from None - if (type(metadata) is not dict or metadata.get("schema") != "kv-live-bundle-v1" - or type(metadata.get("execution")) is not dict - or metadata.get("execution", {}).get("plan_sha256") != authorization["plan_sha256"] - or metadata["execution"].get("run_id") != os.environ.get("GITHUB_RUN_ID") - or metadata["execution"].get("implementation_sha") != os.environ.get("GITHUB_SHA") - or metadata["execution"].get("execution_id") != plan["execution_id"] - or metadata["execution"].get("charge_pr") != plan["charge_pr"] - or metadata["execution"].get("charge_head") != plan["charge_head"]): - raise ValueError("invalid_execution_identity") - return authorization, metadata - - -def package_grade(bundle, output): - """Always-step recovery of already generated evidence; never invoke a candidate.""" - if os.environ.get(provider.KEY_ENV) or os.environ.get("GITHUB_TOKEN"): - raise ValueError("Final packaging must run without injected credentials") - bundle, output = Path(bundle), Path(output) - try: - authorization, _ = _grade_context(bundle) - except ValueError as exc: - return _grade_rejected(output, str(exc)) - plan = authorization["plan"] - try: - result = policy.read_json(output / "grade.json") - if (result.get("schema") != "kv-live-final-grade-v1" - or result.get("plan_sha256") != authorization["plan_sha256"] - or set(result.get("arms", {})) != {"A2", "A3"}): - raise ValueError("Unbound grade") - # Reuse the existing closed feedback contract; never publish arbitrary fields. - safe = {"schema": result["schema"], "plan_sha256": result["plan_sha256"], - "scope": "one synthetic pair; hidden semantics and nonheldout task timing", - "arms": {}} - hidden = json.loads(policy.artifact(plan["hidden_traces"], limit=16 * policy.MAX_JSON)) - ids = {t["id"] for t in hidden} - for arm, value in result["arms"].items(): - entry = {"verdict": "INCONCLUSIVE", "reason": "grading_interrupted"} - if value.get("reason") in {"not_evaluated", "grading_interrupted", - "missing_or_invalid_artifact"}: - entry["reason"] = value["reason"] - if policy._hex(value.get("artifact_sha256")): - entry["artifact_sha256"] = value["artifact_sha256"] - semantic = value.get("semantic", {}) - if (semantic.get("verdict") in {"PASS", "FAIL", "INCONCLUSIVE"} - and type(semantic.get("checks")) is dict and set(semantic["checks"]) <= ids - and all(type(v) is bool for v in semantic["checks"].values()) - and (semantic["verdict"] == "INCONCLUSIVE" or ( - set(semantic["checks"]) == ids - and (semantic["verdict"] == "PASS") == all(semantic["checks"].values()))) - and policy._hex(semantic.get("artifact_sha256"))): - entry["semantic"] = {k: semantic[k] for k in - ("verdict", "checks", "artifact_sha256")} - if semantic.get("reason") in {"deadline", "error", "timeout", "cancelled", - "sandbox_unavailable"}: - entry["semantic"]["reason"] = semantic["reason"] - feedback = value.get("performance") - if feedback is not None: - config = policy.read_json(ROOT / plan["actors"][arm]["config"]["path"]) - ep = worker.scripted._plan(config["edit_plan"]) - ep = replace(ep, limits=replace(ep.limits, - benchmark_repetitions=plan["final_repetitions"])) - try: - worker.edit._feedback("benchmark", feedback, ep) - entry["performance"] = feedback - except provider.AgentError: - pass - if "verdict" not in value and {"semantic", "performance"} <= set(entry): - entry.pop("verdict") - entry.pop("reason") - safe["arms"][arm] = entry - result = safe - except (OSError, ValueError, KeyError, TypeError, AttributeError): - result = {"schema": "kv-live-final-grade-v1", "plan_sha256": authorization["plan_sha256"], - "scope": "grade process incomplete; first outputs only, no re-execution", - "arms": {a: {"verdict": "INCONCLUSIVE", "reason": "grading_interrupted"} - for a in ("A2", "A3")}} - output.mkdir(parents=True, exist_ok=True) - return _grade_evidence(bundle, output, result, plan) - - -def grade(bundle, output): - """Separate no-secret hidden grade of immutable final bytes; never model feedback.""" - if os.environ.get(provider.KEY_ENV) or os.environ.get("GITHUB_TOKEN"): - raise ValueError("Final grading must run without injected provider/GitHub credentials") - # Revocation/expiry stops spending, not offline grading of already captured public bytes. - bundle, output = Path(bundle), Path(output) - try: - authorization, metadata = _grade_context(bundle) - except ValueError as exc: - return _grade_rejected(output, str(exc)) - plan = authorization["plan"] - output.mkdir(parents=True, exist_ok=False) - hidden = json.loads(policy.artifact(plan["hidden_traces"], limit=16 * policy.MAX_JSON)) - suite = workbench.VisibleSuite(policy.artifact(plan["final_suite"], limit=16 * policy.MAX_JSON)) - result = {"schema": "kv-live-final-grade-v1", "plan_sha256": authorization["plan_sha256"], - "scope": "one synthetic pair; hidden semantics and nonheldout task timing", - "arms": {arm: {"verdict": "INCONCLUSIVE", "reason": "not_evaluated"} - for arm in ("A2", "A3")}} - atomic_write_json(output / "grade.json", result) - try: - for arm in plan["arm_order"]: - try: - entry = metadata["artifacts"][arm] - source = kv.source_bytes(bundle / arm / "candidate.py") - receipt = policy.read_json(bundle / arm / "receipt.json", 16384) - config = policy.read_json(ROOT / plan["actors"][arm]["config"]["path"]) - if (entry["status"] != "submitted" or entry["sha256"] != policy.sha(source) - or entry["receipt_sha256"] != policy.fingerprint(receipt) - or receipt["sha256"] != policy.sha(source) or receipt["arm"] != arm - or receipt["actor_sha256"] != config["actor_sha256"] - or receipt["plan_sha256"] != authorization["plan_sha256"]): - raise ValueError("Final source/receipt identity changed") - _validate_submission(bundle / arm, receipt, source, config, arm, - metadata["execution"]) - deadline = time.monotonic() + plan["final_seconds"] - result["arms"][arm] = {"verdict": "INCONCLUSIVE", "reason": "grading_interrupted", - "artifact_sha256": policy.sha(source)} - atomic_write_json(output / "grade.json", result) - semantic = workbench.grade_final(source, policy.sha(source), hidden, - output / arm / "semantics", deadline) - ep = worker.scripted._plan(config["edit_plan"]) - ep = replace(ep, limits=replace(ep.limits, - benchmark_repetitions=plan["final_repetitions"])) - baseline = policy.artifact(next(e["source"] - for e in plan["actors"][arm]["workspace"] if e["path"] == "candidate.py")) - bench = workbench.VisibleWorkbench(ep, suite, baseline, output / arm / "benchmark") - result["arms"][arm] = {"semantic": semantic, "performance": { - "verdict": "INCONCLUSIVE", "reason": "grading_interrupted"}} - atomic_write_json(output / "grade.json", result) - result["arms"][arm]["performance"] = bench.visible_benchmark(source, deadline) - atomic_write_json(output / "grade.json", result) - except (OSError, ValueError, KeyError, TypeError, provider.AgentError): - result["arms"][arm].update(verdict="INCONCLUSIVE", - reason="missing_or_invalid_artifact") - finally: - atomic_write_json(output / "grade.json", result) - _grade_evidence(bundle, output, result, plan) - return result - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("action", choices=("prepare", "execute", "package", "grade", - "package-grade")) - parser.add_argument("--output", type=Path, required=True) - parser.add_argument("--bundle", type=Path) - args = parser.parse_args() - if args.action in {"grade", "package-grade"}: - if args.bundle is None: - parser.error("grade requires the fixed current-run bundle") - {"grade": grade, "package-grade": package_grade}[args.action](args.bundle, args.output) - elif args.bundle is not None: - parser.error("bundle input is only for final grading") - else: - {"prepare": prepare, "execute": execute, "package": package}[args.action](args.output) - - -if __name__ == "__main__": - main() diff --git a/tools/kv_live_policy.py b/tools/kv_live_policy.py deleted file mode 100644 index c00532b78..000000000 --- a/tools/kv_live_policy.py +++ /dev/null @@ -1,346 +0,0 @@ -"""Fixed public freeze/current-main validation. No secrets, writes or fallback policy.""" - -import hashlib -import io -import json -import os -import re -import subprocess -import time -import urllib.request -import zipfile -from pathlib import Path, PurePosixPath - -from inferrsi.harness import kv_reclamation as kv -from inferrsi.model import anthropic_agent as provider -from tools import kv_edit_inputs as inputs -from tools import kv_edit_pilot as scripted -from tools import kv_pilot_admission as admission -from tools import skill_eval_budget as budget -from tools.skill_eval_budget_git import DailyLedgerAnchor, LedgerAnchor - -ROOT = Path(__file__).resolve().parents[1] -REPOSITORY = "InferMatrixLab/InferRSI" -REPOSITORY_ID = 1409534281 -OPERATOR = "Dong1017" -WORKFLOW = ".github/workflows/kv-edit-pilot.yml" -LOCATOR = ROOT / "tools/kv_live_authorization.json" -PREFIX = "\n```json\n" -SUFFIX = "\n```" -MAX_JSON = 1024 * 1024 -PLAN_FIELDS = { - "schema", "repository_id", "issue", "campaign_id", "execution_id", "charge_pr", - "charge_head", "expires_at", "provider", "actors", "dependencies", "ledger", - "daily_ledger", "qualification", "suite", "final_suite", "hidden_traces", - "arm_order", "scope", "price_evidence", "prior_owner_group_statement", - "final_seconds", "final_repetitions", -} -DEPENDENCIES = ( - "tools/kv_edit_inputs.py", "tools/kv_live_policy.py", "tools/kv_live_job.py", - "tools/kv_live_worker.py", - "tools/kv_pilot_admission.py", "tools/kv_edit_pilot.py", - "src/inferrsi/model/anthropic_agent.py", "src/inferrsi/model/anthropic_edit_agent.py", - "src/inferrsi/model/request_guard.py", "tools/skill_eval_budget.py", - "tools/skill_eval_budget_git.py", "tools/skill_eval_pr.py", - "tools/qualify_kv_actor_cleanup.py", "src/inferrsi/infra/targets/__init__.py", - "src/inferrsi/infra/targets/git.py", - ".github/workflows/kv-edit-pilot.yml", "pyproject.toml", -) - -QUALIFICATION_DEPENDENCIES = { - "tools/kv_edit_inputs.py", - "tools/qualify_kv_edit_actor.py", "tools/qualify_kv_actor_cleanup.py", - "tools/kv_runtime_qualification.py", "tools/qualify_kv_reclamation.py", - "tools/kv_pilot_admission.py", "src/inferrsi/model/anthropic_edit_agent.py", - "src/inferrsi/model/anthropic_agent.py", "src/inferrsi/model/request_guard.py", -} - - -def read_json(path, limit=MAX_JSON): - return json.loads(kv.read_regular(Path(path), limit), object_pairs_hook=provider.unique_object, - parse_constant=lambda _: (_ for _ in ()).throw(ValueError("nonfinite JSON"))) - - -def sha(data): - return hashlib.sha256(data).hexdigest() - - -def fingerprint(value): - return sha(provider.canonical(value).encode()) - - -def relative(name): - if (type(name) is not str or not name or "\\" in name - or PurePosixPath(name).is_absolute() - or any(p in {"", ".", ".."} for p in name.split("/"))): - raise ValueError("Invalid frozen relative path") - return name - - -def _hex(value, size=64): - return type(value) is str and re.fullmatch(r"[0-9a-f]{" + str(size) + "}", value) is not None - - -def _positive(value): - return type(value) is int and value > 0 - - -def artifact(record, *, limit=MAX_JSON): - if (type(record) is not dict or set(record) != {"path", "bytes", "sha256"} - or not _positive(record["bytes"]) or record["bytes"] > limit - or not _hex(record["sha256"])): - raise ValueError("Invalid frozen artifact record") - path = ROOT / relative(record["path"]) - data = kv.read_regular(path, limit) - if len(data) != record["bytes"] or sha(data) != record["sha256"]: - raise ValueError("Frozen artifact changed") - return data - - -class NoRedirect(urllib.request.HTTPRedirectHandler): - def redirect_request(self, req, fp, code, msg, headers, newurl): - raise ValueError("Authorization redirect forbidden") - - -def fetch_comment(number): - request = urllib.request.Request( - f"https://api.github.com/repos/{REPOSITORY}/issues/comments/{number}", - headers={"Accept": "application/vnd.github+json", "User-Agent": "InferRSI-KV-controller"}) - with urllib.request.build_opener(urllib.request.ProxyHandler({}), NoRedirect()).open( - request, timeout=20) as response: - raw = response.read(MAX_JSON + 1) - if len(raw) > MAX_JSON: - raise ValueError("Authorization response too large") - return json.loads(raw, object_pairs_hook=provider.unique_object) - - -def frozen_authorization(*, fetch=fetch_comment, now=None): - locator = read_json(LOCATOR, 16384) - fields = {"schema", "enabled", "repository_id", "issue", "comment_id", "author_login", - "body_sha256"} - if (type(locator) is not dict or set(locator) != fields or locator.get("enabled") is not True - or locator["schema"] != "kv-live-authorization-locator-v1" - or locator["repository_id"] != REPOSITORY_ID or locator["author_login"] != OPERATOR - or not all(_positive(locator[k]) for k in ("issue", "comment_id")) - or not _hex(locator["body_sha256"])): - raise ValueError("Live KV authorization is disabled or incomplete") - comment = fetch(locator["comment_id"]) - body = comment.get("body") - if (comment.get("id") != locator["comment_id"] - or comment.get("issue_url") != - f"https://api.github.com/repos/{REPOSITORY}/issues/{locator['issue']}" - or comment.get("user", {}).get("login") != OPERATOR - or comment.get("user", {}).get("type") != "User" - or type(body) is not str or sha(body.encode()) != locator["body_sha256"] - or not body.startswith(PREFIX) or not body.endswith(SUFFIX)): - raise ValueError("Fixed authorization identity or exact bytes changed") - value = json.loads(body[len(PREFIX):-len(SUFFIX)], object_pairs_hook=provider.unique_object) - if (type(value) is not dict or set(value) != {"schema", "plan_sha256", "plan"} - or value["schema"] != "kv-live-authorization-v1" - or value["plan_sha256"] != fingerprint(value["plan"])): - raise ValueError("Authorization does not bind the exact plan") - validate_plan(value["plan"], now=time.time() if now is None else now) - if value["plan"]["issue"] != locator["issue"]: - raise ValueError("Authorization Issue changed") - return value, locator - - -def validate_plan(plan, *, now): - fields = PLAN_FIELDS | ({"budget_exception"} if type(plan) is dict - and plan.get("schema") == "kv-live-plan-v2" else set()) - if (type(plan) is not dict or set(plan) != fields - or plan["schema"] not in {"kv-live-plan-v1", "kv-live-plan-v2"} - or plan["repository_id"] != REPOSITORY_ID - or not all(_positive(plan[k]) for k in ("issue", "charge_pr", "expires_at")) - or not _hex(plan["charge_head"], 40) or plan["expires_at"] <= now - or plan["arm_order"] not in (["A2", "A3"], ["A3", "A2"]) - or plan["final_seconds"] != 180 or plan["final_repetitions"] != 8 - or type(plan["actors"]) is not dict or set(plan["actors"]) != {"A2", "A3"} - or any(type(plan[k]) is not str or not plan[k].strip() - for k in ("scope", "price_evidence", "prior_owner_group_statement"))): - raise ValueError("Wrong, expired or incomplete fixed KV scope") - for actor in plan["actors"].values(): - if (type(actor) is not dict or set(actor) != {"config", "workspace"} - or type(actor["workspace"]) is not list or not actor["workspace"]): - raise ValueError("Wrong frozen actor inputs") - artifact(actor["config"]) - key = budget.PRKey(REPOSITORY_ID, plan["charge_pr"]) - LedgerAnchor(key=key, **plan["ledger"]) - DailyLedgerAnchor(key=budget.DailyKey(REPOSITORY_ID), **plan["daily_ledger"]) - additional_attempt(plan) - sources = plan["dependencies"] - required = set(DEPENDENCIES) | set(scripted.dependencies()) - if (type(sources) is not dict or not required <= set(sources) - or any(not _hex(value) or sha(kv.read_regular(ROOT / relative(name), MAX_JSON)) != value - for name, value in sources.items())): - raise ValueError("Frozen executable dependencies changed") - for name in ("suite", "final_suite", "hidden_traces"): - artifact(plan[name], limit=16 * MAX_JSON) - validate_qualification(plan) - # Reuse strict cost identity validation; no new rates or token arithmetic here. - policy(plan, "0" * 40).reservation(now=now) - - -def qualification_dependencies(): - # Canonical closure belongs to the actor qualifier, including package - # initializers and transitive runtime/metric/scenario helpers. A checkout - # without that reviewed prerequisite cannot activate this launcher. - from tools import qualify_kv_edit_actor - return qualify_kv_edit_actor.dependencies() - - -def validate_qualification(plan): - q = plan["qualification"] - fields = {"run_id", "run_attempt", "head", "revision", "artifact_id", "archive", "receipt", - "dependencies_sha256"} - if (type(q) is not dict or set(q) != fields - or not all(_positive(q[k]) for k in ("run_id", "run_attempt", "artifact_id")) - or not _hex(q["head"], 40) or not _hex(q["revision"], 40)): - raise ValueError("Exact hosted actor qualification provenance required") - raw = artifact(q["receipt"]) - receipt = json.loads(raw, object_pairs_hook=provider.unique_object) - sources = receipt.get("dependencies", {}) - required = QUALIFICATION_DEPENDENCIES | set(scripted.dependencies()) - if (not required <= set(sources) or sources != qualification_dependencies() - or q["dependencies_sha256"] != fingerprint(sources) - or receipt.get("dependencies_sha256") != fingerprint(sources) - or any(not _hex(digest) - or sha(kv.read_regular(ROOT / relative(name), MAX_JSON)) != digest - for name, digest in sources.items())): - raise ValueError("Actor qualification dependency closure changed") - archive = artifact(q["archive"], limit=64 * MAX_JSON) - with zipfile.ZipFile(io.BytesIO(archive)) as bundle: - entries = bundle.infolist() - if (len(entries) != 2 or {e.filename for e in entries} - != {"qualification.json", "raw-evidence.zip"} - or any(e.flag_bits & 1 or e.file_size > 64 * MAX_JSON for e in entries) - or bundle.getinfo("qualification.json").file_size > MAX_JSON - or bundle.read("qualification.json") != raw): - raise ValueError("Hosted artifact does not contain the exact qualification receipt") - evidence = bundle.read("raw-evidence.zip") - if (receipt.get("schema") != "kv-scripted-actor-qualification-v1" - or receipt.get("revision") != q["revision"] or receipt.get("verdict") != "PASS" - or receipt.get("software_checks_passed") is not True - or receipt.get("source_unchanged") is not True - or receipt.get("model_calls") != 0 or receipt.get("provider_charges_usd") != 0 - or receipt.get("cleanup", {}).get("verdict") != "PASS" - or receipt.get("runtime_isolation", {}).get("verdict") != "PASS" - or not _hex(receipt.get("runtime_isolation", {}).get("inventory_sha256")) - or not _positive(receipt.get("runtime_isolation", {}).get("mount_count")) - or receipt.get("raw_evidence", {}).get("sha256") != sha(evidence) - or set(receipt.get("arms", {})) != {"A2", "A3"}): - raise ValueError("Exact model-free actor qualification required") - for arm in ("A2", "A3"): - result = receipt["arms"][arm] - config = read_json(ROOT / plan["actors"][arm]["config"]["path"]) - if (result.get("executor_status") != "completed" - or result.get("worker_status") != "completed" - or result.get("request_completion_known") is not True - or result.get("simulated_slots_consumed") != 4 - or result.get("visible_workbench_verified") is not True - or result.get("two_revision_receipts_retained") is not True - or result.get("hidden_correctness", {}).get("verdict") != "PASS" - or result.get("common_actor_contract_sha256") - != inputs.common_actor_contract_sha256(config) - or result.get("exposure") != config["edit_plan"]["files"] - or result.get("prompt_sha256") != sha(config["prompt"].encode())): - raise ValueError("Hosted actor exposure/tool evidence differs from frozen live arm") - return receipt - - -def additional_attempt(plan): - """Resolve only exact frozen v2 data; no ledger/worker/CLI cap authority.""" - if plan.get("schema", "kv-live-plan-v1") == "kv-live-plan-v1": - if "budget_exception" in plan: - raise ValueError("Legacy plan cannot carry a budget exception") - return None - value = plan.get("budget_exception") - fields = {"schema", "key", "pr_lifetime_cap_microusd", "pilot_lifetime_cap_microusd", - "additional_attempts", "prior_binding", "pr_checkpoint", "daily_checkpoint", - "campaign_id", "execution_id"} - if (plan.get("schema") != "kv-live-plan-v2" or type(value) is not dict - or set(value) != fields or value["schema"] != "kv-one-additional-attempt-v1" - or type(value["additional_attempts"]) is not int or value["additional_attempts"] != 1 - or type(value["pilot_lifetime_cap_microusd"]) is not int - or value["pilot_lifetime_cap_microusd"] != admission.PILOT_LIMIT_MICROUSD - or type(value["key"]) is not dict - or value["key"] != {"repository_id": REPOSITORY_ID, "pr_number": plan["charge_pr"]} - or type(value["prior_binding"]) is not dict - or set(value["prior_binding"]) != {"plan_digest", "run_id", "worst_case"} - or value["campaign_id"] != plan["campaign_id"] - or value["execution_id"] != plan["execution_id"] - or value["pr_checkpoint"] != plan["ledger"]["checkpoint_commit"] - or value["daily_checkpoint"] != plan["daily_ledger"]["checkpoint_commit"]): - raise ValueError("Invalid exact one-additional-attempt scope") - return admission.AdditionalAttempt( - budget.TrustedPRLimit(budget.PRKey(**value["key"]), value["pr_lifetime_cap_microusd"]), - budget.Binding(**value["prior_binding"]), value["campaign_id"], value["execution_id"], - value["pr_checkpoint"], value["daily_checkpoint"]) - - -def policy(plan, revision): - actors = [read_json(ROOT / plan["actors"][arm]["config"]["path"]) for arm in ("A2", "A3")] - return admission.PilotPolicy( - plan["campaign_id"], budget.PRKey(REPOSITORY_ID, plan["charge_pr"]), - plan["execution_id"], fingerprint(plan), plan["charge_head"], revision, - tuple(a["actor_sha256"] for a in actors), provider.canonical(plan["provider"]), - plan["expires_at"], additional_attempt(plan)) - - -def runtime(environment=None): - env = os.environ if environment is None else environment - expected = {"GITHUB_EVENT_NAME": "workflow_dispatch", "GITHUB_REF": "refs/heads/main", - "GITHUB_REPOSITORY": REPOSITORY, "GITHUB_RUN_ATTEMPT": "1", - "GITHUB_ACTOR": OPERATOR, "GITHUB_TRIGGERING_ACTOR": OPERATOR} - if any(env.get(k) != v for k, v in expected.items()): - raise ValueError("Only fixed first-attempt owner main dispatch is supported") - revision, run_id = env.get("GITHUB_SHA", ""), env.get("GITHUB_RUN_ID", "") - if not _hex(revision, 40) or not re.fullmatch(r"[1-9][0-9]*", run_id): - raise ValueError("Invalid runtime identity") - head = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip() - dirty = subprocess.check_output(["git", "status", "--porcelain"], cwd=ROOT, text=True) - if head != revision or dirty: - raise ValueError("Execution checkout differs or is dirty") - return revision, run_id - - -def verify_live(api, plan, revision, run_id): - if api.request("GET", "").get("id") != REPOSITORY_ID: - raise ValueError("Live repository changed") - main = api.request("GET", "/git/ref/heads/main") - if (main.get("ref") != "refs/heads/main" or main.get("object", {}).get("type") != "commit" - or main["object"].get("sha") != revision): - raise ValueError("Only exact current main may execute") - current = api.request("GET", f"/pulls/{plan['charge_pr']}") - if (current.get("number") != plan["charge_pr"] or current.get("head", {}).get("sha") - != plan["charge_head"] or current.get("draft") is not False - or current.get("base", {}).get("ref") != "main" - or current.get("base", {}).get("repo", {}).get("id") != REPOSITORY_ID - or current.get("merged") is not True): - raise ValueError("Final reviewed implementation charge PR changed or is not merged") - run = api.request("GET", f"/actions/runs/{run_id}") - if (run.get("event") != "workflow_dispatch" or run.get("head_branch") != "main" - or run.get("head_sha") != revision or run.get("run_attempt") != 1 - or run.get("status") != "in_progress" or run.get("path") != WORKFLOW - or run.get("repository", {}).get("id") != REPOSITORY_ID - or any(run.get(k, {}).get("login") != OPERATOR for k in ("actor", "triggering_actor"))): - raise ValueError("Live workflow identity changed") - qualified = api.request("GET", f"/actions/runs/{plan['qualification']['run_id']}") - if (qualified.get("head_sha") != plan["qualification"]["head"] - or qualified.get("path") != ".github/workflows/ci-ready.yml" - or qualified.get("status") != "completed" or qualified.get("conclusion") != "success" - or qualified.get("repository", {}).get("id") != REPOSITORY_ID - or qualified.get("run_attempt") != plan["qualification"]["run_attempt"]): - raise ValueError("Actor qualification run is not successful for the pinned source") - - q = plan["qualification"] - artifact_record = api.request("GET", f"/actions/artifacts/{q['artifact_id']}") - expected_name = f"kv-actor-qualification-{q['run_id']}-{q['run_attempt']}" - if (artifact_record.get("id") != q["artifact_id"] - or artifact_record.get("name") != expected_name - or artifact_record.get("expired") is not False - or artifact_record.get("digest") != "sha256:" + q["archive"]["sha256"] - or artifact_record.get("workflow_run", {}).get("id") != q["run_id"] - or artifact_record.get("workflow_run", {}).get("head_sha") != q["head"] - or artifact_record.get("workflow_run", {}).get("head_repository_id") != REPOSITORY_ID): - raise ValueError("Qualification archive is not the exact successful hosted run artifact") diff --git a/tools/kv_live_worker.py b/tools/kv_live_worker.py deleted file mode 100644 index 3f200fbf2..000000000 --- a/tools/kv_live_worker.py +++ /dev/null @@ -1,246 +0,0 @@ -"""Trusted fixed KV live worker. Not a candidate executor or scripted-CLI switch. - -Only a reviewed main controller may supply its frozen inputs and runtime key. -The worker has no GitHub authority; submitted source executes only via kv.invoke. -""" - -import argparse -import os -import sys -import time -from dataclasses import asdict, replace -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -if __package__ in {None, ""}: - sys.path.insert(0, str(ROOT)) - sys.path.insert(0, str(ROOT / "src")) - -from inferrsi.harness import kv_edit_workbench as workbench # noqa: E402 -from inferrsi.harness import kv_reclamation as kv # noqa: E402 -from inferrsi.infra.artifacts import atomic_write_json # noqa: E402 -from inferrsi.infra.executors.local import execute_command # noqa: E402 -from inferrsi.model import anthropic_agent as provider # noqa: E402 -from inferrsi.model import anthropic_edit_agent as edit # noqa: E402 -from inferrsi.model.request_guard import ENV_KEYS, authorize_request, gate_environment # noqa: E402 -from tools import kv_edit_pilot as scripted # noqa: E402 -from tools.kv_worker_supervisor import supervise # noqa: E402 - -SCHEMA = "kv-live-worker-v1" -REASONS = frozenset({ - "unknown", "sandbox_unavailable", "live_worker_error", "provider_transport_error", - "transport_error", "invalid_response_json", "response_bytes_limit", "credential_echo", - "invalid_credential_shape", "credential_in_payload", "request_authorization_denied", - "request_without_exact_grant", "request_bytes_limit", "timeout", "unknown_usage", - "incompatible_response", "input_token_limit", "output_token_limit", "request_limit", - "request_allocation_unavailable", "workspace_symlink", "invalid_workspace_file", - "tool_read_limit", "protected_workspace_changed", "unexposed_path", "invalid_candidate", - "candidate_bytes_limit", "invalid_visible_feedback", "feedback_bytes_limit", - "revision_retention_error", "deadline_exceeds_frozen_limit", "tool_call_limit", - "invalid_tool_call", "unsupported_tool", "task_not_read", "visible_callback_error", - "stale_or_missing_receipt", "edit_agent_error", "feedback_retention_limit", -}) -FIELDS = {"schema", "arm", "model", "client", "edit_plan", "input_token_ceiling", - "suite_sha256", "baseline_sha256", "actor_sha256", "dependencies", "prompt"} - - -def closed_reason(value): - """Fixed codes only; never exception text, provider bodies or credentials.""" - if type(value) is str and (value in REASONS or ( - len(value) == 8 and value.startswith("http_") and value[5:].isascii() - and value[5:].isdigit() and 100 <= int(value[5:]) <= 599)): - return value - return "unknown" - - -def dependencies(): - return {**scripted.dependencies(), - "tools/kv_live_worker.py": edit.digest(Path(__file__).read_bytes())} - - -def configuration(workspace, plan, suite, baseline, *, arm, model, spec, prompt): - """Offline freeze constructor; no root-plan digest cycle or credential access.""" - if (arm not in {"A2", "A3"} or type(prompt) is not str - or edit.freeze_files(workspace, [f.path for f in plan.files]) != plan.files - or kv.source_bytes(Path(workspace) / "candidate.py") != baseline): - raise ValueError("Wrong frozen live actor inputs") - deps = dependencies() - binding = {"dependencies": deps, "suite_sha256": edit.digest(suite.serialized), - "baseline_sha256": edit.digest(baseline), "prompt": prompt} - plan = replace(plan, callbacks_sha256=edit.fingerprint(binding)) - identity = edit.frozen_identity(plan, model, spec, 1000000) - return {"schema": SCHEMA, "arm": arm, "model": model, "client": spec, - "edit_plan": asdict(plan), "input_token_ceiling": 1000000, - "suite_sha256": binding["suite_sha256"], "baseline_sha256": binding["baseline_sha256"], - "actor_sha256": edit.fingerprint(identity), "dependencies": deps, "prompt": prompt} - - -def checked_inputs(inputs): - inputs = Path(inputs) - config = scripted._read_json(inputs / "worker.json", 2 * 1024 * 1024) - if (type(config) is not dict or set(config) != FIELDS or config["schema"] != SCHEMA - or config["arm"] not in {"A2", "A3"} or config["dependencies"] != dependencies() - or config["model"] != "claude-opus-5-5" or config["input_token_ceiling"] != 1000000 - or config["client"].get("max_requests") != 8 - or config["client"].get("max_output_tokens") != 4096): - raise ValueError("Live worker identity or limits changed") - plan = scripted._plan(config["edit_plan"]) - binding = {k: config[k] for k in ("dependencies", "suite_sha256", "baseline_sha256", "prompt")} - if (plan.callbacks_sha256 != edit.fingerprint(binding) - or edit.fingerprint(edit.frozen_identity(plan, config["model"], config["client"], - config["input_token_ceiling"])) != config["actor_sha256"]): - raise ValueError("Live actor callback/protocol identity changed") - suite = kv.read_regular(inputs / "suite.json", 16 * 1024 * 1024) - baseline = kv.source_bytes(inputs / "baseline.py") - if (edit.digest(suite) != config["suite_sha256"] - or edit.digest(baseline) != config["baseline_sha256"]): - raise ValueError("Live visible suite/baseline changed") - run = scripted._read_json(inputs / "binding.json", 8192) - if (type(run) is not dict or set(run) != {"plan_sha256", "execution_id"} - or not edit._sha(run["plan_sha256"]) or type(run["execution_id"]) is not str): - raise ValueError("Invalid outer binding") - return {**config, **run}, plan, workbench.VisibleSuite(suite), baseline - - -class GrantedSender: - """One IPC authorization and one exact HTTP attempt per consumed grant.""" - - def __init__(self, config, bridge, deadline, *, key, gate, - transport=provider.send_message, authorize_ipc=authorize_request): - self.config, self.bridge, self.deadline = config, Path(bridge), deadline - self.key, self.gate = key, gate_environment(gate) - self.transport, self.authorize_ipc = transport, authorize_ipc - self.grant, self.ready, self.sent, self.attempts = None, False, False, 0 - self.transport_error_code = None - - def authorize(self, grant, deadline): - if (deadline != self.deadline or time.monotonic() >= deadline - or (self.grant is not None and not self.sent)): - return False - self.grant, self.ready, self.sent = dict(grant), False, False - atomic_write_json(self.bridge / "pending.json", self.grant) - self.authorize_ipc(provider.canonical(grant).encode(), environment=self.gate, - timeout=max(0.001, deadline - time.monotonic())) - self.ready = True - return True - - def send(self, payload, spec, remaining): - left = self.deadline - time.monotonic() - if (not self.ready or self.sent or self.grant is None or left <= 0 - or spec != self.config["client"] - or edit.digest(provider.canonical(payload).encode()) != self.grant["request_sha256"] - or self.attempts >= self.config["client"]["max_requests"]): - raise provider.AgentError("request_without_exact_grant") - self.sent = True # Burn BEFORE transport; exceptions never permit resend. - self.attempts += 1 - try: - # authorize() already consumed the sole gate. Explicit empty runtime - # prevents send_message's implicit environment gate from double-counting. - value = self.transport(payload, spec, self.key, min(remaining, left), - environment={}, require_authorization=False) - except Exception as error: - # Keep a separate closed diagnostic before the edit agent generalizes - # transport failures. Counting this invocation does not prove HTTP delivery. - self.transport_error_code = closed_reason(error.args[0] if ( - isinstance(error, provider.AgentError) and len(error.args) == 1) else None) - atomic_write_json(self.bridge / "complete.json", {"grant": self.grant, "usage": None}) - raise provider.AgentError("provider_transport_error") from None - atomic_write_json(self.bridge / "complete.json", {"grant": self.grant, - "usage": scripted._known_usage(value, self.config, self.deadline)}) - return value - - -def live_worker(inputs, workspace, output, deadline, *, environment, emit, - transport=provider.send_message): - output = Path(output) - output.mkdir(parents=True, exist_ok=False) - result = {"schema": SCHEMA, "status": "failed", "provider_attempts": 0, - "scope": "one synthetic editing feasibility arm; no efficacy claim"} - sender = None - try: - config, plan, suite, baseline = checked_inputs(inputs) - safe = provider.execution_environment(environment) - bridge = output.parent / "bridge" - result["isolation"] = kv.probe_isolation(output / "isolation", deadline, - [str(Path(inputs).resolve()), str(bridge.resolve()), str(ROOT), - f"/proc/{os.getppid()}/environ"]) - bench = workbench.VisibleWorkbench(plan, suite, baseline, output / "workbench") - sender = GrantedSender(config, bridge, deadline, key=safe[provider.KEY_ENV], - gate=safe, transport=transport) - slots = edit.RequestSlots(config["plan_sha256"], {config["arm"]: 8}, - 10170000, 81360000, execution_id=config["execution_id"]) - (output / "feedback").mkdir() - feedback_count = 0 - - def retain_feedback(event): - nonlocal feedback_count - if event.get("name") in {"run_visible_checks", "run_visible_benchmark"}: - receipt = event["receipt"] - feedback_count += 1 - if (feedback_count > plan.limits.max_checks + plan.limits.max_benchmarks - or len(provider.canonical(receipt).encode()) - > plan.limits.max_feedback_bytes): - raise provider.AgentError("feedback_retention_limit") - atomic_write_json(output / "feedback" / f"{feedback_count:04d}.json", receipt) - emit(event) - - answer = edit.run_edit_agent(workspace, plan, config["prompt"], config["model"], - config["client"], deadline, retain_feedback, arm=config["arm"], slots=slots, - authorize=sender.authorize, send=sender.send, visible_checks=bench.visible_checks, - visible_benchmark=bench.visible_benchmark, retain_revision=bench.retain_revision, - input_token_ceiling=1000000) - source = answer.pop("artifact_source").encode("utf-8") - final = output / "final" - final.mkdir() - with (final / "candidate.py").open("xb") as stream: - stream.write(source) - stream.flush() - os.fsync(stream.fileno()) - atomic_write_json(final / "receipt.json", answer) - if time.monotonic() >= deadline: - raise provider.AgentError("timeout") - result.update(status="completed", artifact=answer) - except kv.SandboxUnavailable: - result.update(status="inconclusive", reason="sandbox_unavailable") - except provider.AgentError as error: - result["reason"] = closed_reason(error.args[0] if len(error.args) == 1 else None) - except Exception: - result["reason"] = "live_worker_error" - result["provider_attempts"] = sender.attempts if sender else 0 - result["transport_error_code"] = sender.transport_error_code if sender else None - atomic_write_json(output / "result.json", result) - return 0 if result["status"] == "completed" else 1 - - -def run_supervised(inputs, workspace, output, deadline, *, admission, key, - execute=execute_command): - if admission is None: - raise ValueError("Live worker requires parent-owned durable admission") - config, plan, _, _ = checked_inputs(inputs) - safe = provider.execution_environment({provider.KEY_ENV: key, "PATH": os.defpath, "LANG": "C"}) - - def command(inputs, workspace, worker_output, inherited): - return (sys.executable, "-I", str(Path(__file__).resolve()), - "--inputs", str(inputs.resolve()), "--workspace", str(workspace.resolve()), - "--output", str(worker_output), "--deadline", repr(inherited)) - - return supervise(config, plan, inputs, workspace, output, deadline, checkout=ROOT, - command_factory=command, environment_factory=lambda gate: { - k: v for k, v in {**safe, **gate}.items() if k != "PYTHONUTF8"}, - admission=admission, execute=execute) - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--inputs", type=Path, required=True) - parser.add_argument("--workspace", type=Path, required=True) - parser.add_argument("--output", type=Path, required=True) - parser.add_argument("--deadline", type=float, required=True) - args = parser.parse_args() - environment = {k: os.environ.get(k, "") for k in (*ENV_KEYS, provider.KEY_ENV)} - return live_worker(args.inputs, args.workspace, args.output, args.deadline, - environment=environment, emit=lambda e: print(provider.canonical(e), flush=True)) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tools/kv_pilot_admission.py b/tools/kv_pilot_admission.py deleted file mode 100644 index 6f01a7193..000000000 --- a/tools/kv_pilot_admission.py +++ /dev/null @@ -1,376 +0,0 @@ -"""Offline-capable admission seam for ONE paired KV editing feasibility pilot. - -No CLI, ledger bootstrap, credential discovery, transport, workflow dispatch or -new storage schema. Only trusted controller code may supply policy/stores/checks -or call these methods; candidate/author artifacts are never spending authority. -The existing ledgers retain the entire bound even after errors and completion. -""" - -import hashlib -import json -import math -import re -import threading -import time -from dataclasses import asdict, dataclass -from datetime import datetime, timezone -from decimal import Decimal - -from inferrsi.model import anthropic_agent as provider -from tools import skill_eval_budget as budget - -ARMS = ("A2", "A3") -CAMPAIGN_NAMESPACE = "kv-reclamation-one-pair-v1:" -MAX_ATTEMPTS_PER_ARM = 8 -PER_REQUEST_MICROUSD = 10_170_000 -FULL_RESERVATION_MICROUSD = 162_720_000 -MAX_ARM_SECONDS = 600 -PILOT_LIMIT_MICROUSD = 1_000_000_000 - - -def _identifier(value): - return type(value) is str and re.fullmatch(r"[A-Za-z0-9._-]{1,128}", value) is not None - - -def _sha(value, size=64): - return type(value) is str and re.fullmatch(r"[0-9a-f]{" + str(size) + "}", value) is not None - - -def _fingerprint(value): - return hashlib.sha256(provider.canonical(value).encode()).hexdigest() - - -@dataclass(frozen=True) -class AdditionalAttempt: - """One explicit issue-scoped continuation, supplied only by the trusted root. - - Old claims remain immutable. The fixed new identities are not caller-chosen - retry IDs, and checkpoints must be used by the independently anchored stores. - """ - - limit: budget.TrustedPRLimit - prior_binding: budget.Binding - campaign_id: str - execution_id: str - pr_checkpoint: str - daily_checkpoint: str - - def __post_init__(self): - if (type(self.limit) is not budget.TrustedPRLimit - or not budget.PR_LIMIT_MICROUSD < self.limit.limit_microusd <= PILOT_LIMIT_MICROUSD - or type(self.prior_binding) is not budget.Binding - or self.prior_binding.worst_case != FULL_RESERVATION_MICROUSD - or not self.prior_binding.run_id.startswith(CAMPAIGN_NAMESPACE) - or not _identifier(self.campaign_id) or not _identifier(self.execution_id) - or self.prior_binding.run_id == CAMPAIGN_NAMESPACE + self.campaign_id - or not _sha(self.pr_checkpoint, 40) or not _sha(self.daily_checkpoint, 40)): - raise budget.BudgetError("Invalid trusted one-additional-attempt exception") - - -@dataclass(frozen=True) -class PilotPolicy: - """Reviewed root configuration, NOT data accepted from a worker or PR. - - The plan digest binds visible task/K, verifier, schedule, artifact/isolation - qualification and all source revisions. Per-arm actor digests bind the exact - edit protocol/manifests. ``check_current`` independently revalidates this - policy, authorized PR/head, source revision and qualification before spending. - ``provider_json`` is canonical immutable JSON; mutable dicts are not retained. - """ - - campaign_id: str - key: budget.PRKey - execution_id: str - plan_sha256: str - pr_head: str - source_revision: str - actor_sha256: tuple[str, str] - provider_json: str - expires_at: int - additional_attempt: AdditionalAttempt | None = None - - def __post_init__(self): - if (not _identifier(self.campaign_id) or not _identifier(self.execution_id) - or type(self.key) is not budget.PRKey or not _sha(self.plan_sha256) - or not _sha(self.pr_head, 40) or not _sha(self.source_revision, 40) - or type(self.actor_sha256) is not tuple or len(self.actor_sha256) != 2 - or any(not _sha(value) for value in self.actor_sha256) - or type(self.expires_at) is not int or self.expires_at <= 0): - raise budget.BudgetError("Invalid trusted pilot binding") - extra = self.additional_attempt - if extra is not None and (type(extra) is not AdditionalAttempt - or extra.limit.key != self.key or extra.campaign_id != self.campaign_id - or extra.execution_id != self.execution_id): - raise budget.BudgetError("Additional attempt differs from exact reviewed scope") - self.reservation(now=0) - - @property - def trusted_limit(self): - return None if self.additional_attempt is None else self.additional_attempt.limit - - def reservation(self, *, now): - try: - config = json.loads(self.provider_json) - if (provider.canonical(config) != self.provider_json - or set(config) != {"model", "client", "pricing", "spend_limit"} - or config["model"] != {"name": "claude-opus-5-5", - "effort": "not_applicable"} - or config["client"]["kind"] != provider.EDIT_KIND - or config["client"]["max_requests"] != MAX_ATTEMPTS_PER_ARM - or config["client"]["max_output_tokens"] != 4096 - or config["pricing"]["currency"] != "USD" - or config["pricing"]["max_input_tokens"] != 1_000_000 - or self.expires_at > config["pricing"]["expires_at"] - or self.expires_at <= now): - raise ValueError("Unqualified pilot envelope") - result = provider.reservation(config["model"]["name"], config["client"], - config["pricing"], config["spend_limit"], 2, now=now) - if (result["requests"] != 16 - or Decimal(result["per_request"]) * 1_000_000 != PER_REQUEST_MICROUSD - or Decimal(result["amount"]) * 1_000_000 != FULL_RESERVATION_MICROUSD - or Decimal(result["limit"]) * 1_000_000 > budget.PR_LIMIT_MICROUSD): - raise ValueError("Changed pilot reservation") - return result - except (AttributeError, KeyError, TypeError, ValueError) as exc: - raise budget.BudgetError("Missing, changed or expired pilot pricing/bounds") from exc - - @property - def binding(self): - # Global run uniqueness plus the fixed namespace below prevents a new - # digest, PR, date, workflow run/attempt or campaign ID from resetting it. - value = asdict(self) - if self.additional_attempt is None: - value.pop("additional_attempt") # Preserve every historical v1 binding digest. - return budget.Binding(_fingerprint(value), - CAMPAIGN_NAMESPACE + self.campaign_id, - FULL_RESERVATION_MICROUSD) - - -def _now(clock): - now = clock() - budget.shanghai_day(now) # Reject naive or malformed clocks before timestamp(). - return now - - -def _read_pair(policy, daily_store, pr_store): - daily_key = budget.DailyKey(policy.key.repository_id) - daily = daily_store.read_verified(daily_key) - pr = pr_store.read_verified(policy.key) - if (type(daily) is not budget.DailySnapshot or daily.ledger.key != daily_key - or type(pr) is not budget.Snapshot or pr.ledger.key != policy.key - or pr.ledger.trusted_limit != policy.trusted_limit): - raise budget.BudgetError("Wrong verified ledger identity") - daily.ledger.require_active() - # An audited PR genesis is not enough if global historical obligations - # disagree. In particular, an old PR tip subtotal cannot erase daily imports. - retained_pr = sum(e.binding.worst_case for e in daily.ledger.entries if e.key == policy.key) - if retained_pr != pr.ledger.charged: - raise budget.BudgetError("Inconsistent historical daily/PR charges") - return daily, pr - - -def _campaign_history(policy, daily, pr, *, admitted=False): - """Namespace-wide one-shot guard; partial writes count, regardless of day/PR.""" - extra = policy.additional_attempt - expected = (() if extra is None else (extra.prior_binding,)) - if admitted: - expected += (policy.binding,) - entries = tuple(e for e in daily.entries if e.binding.run_id.startswith(CAMPAIGN_NAMESPACE)) - if (tuple(e.binding for e in entries) != expected - or any(e.key != policy.key or e.phase != "claimed" or e.actual_cost is not None - for e in entries)): - raise budget.BudgetError("Pilot campaign already bound or exact prior claim missing") - if extra is not None and (pr.genesis.legacy_charge != 0 - or tuple(e.binding for e in pr.entries) != expected - or any(e.phase != "claimed" or e.actual_cost is not None for e in pr.entries)): - raise budget.BudgetError("Additional attempt must preserve the exact prior PR claim") - total = sum(e.binding.worst_case for e in entries) - if total + (0 if admitted else FULL_RESERVATION_MICROUSD) > PILOT_LIMIT_MICROUSD: - raise budget.BudgetError("Pilot lifetime budget exhausted") - - -def _current(policy, check_current, wall_clock, admitted_day=None): - now = _now(wall_clock) - policy.reservation(now=now.timestamp()) - if check_current(policy) is not True: - raise budget.BudgetError("Trusted pilot authorization or qualification changed") - # The trusted live check can take time. Do not issue a stale boundary grant. - now = _now(wall_clock) - policy.reservation(now=now.timestamp()) - day = budget.shanghai_day(now) - if admitted_day is not None and day != admitted_day: - raise budget.BudgetError("Shanghai admission day ended") - return day - - -def admit(policy, daily_store, pr_store, *, check_current, - wall_clock=lambda: datetime.now(timezone.utc), monotonic_clock=time.monotonic): - """Reserve/claim once, daily first, then return a process-local controller. - - Stores implement the EXISTING verified-read/CAS contracts and must already - have audited genesis/checkpoints. No retries, missing-ledger initialization, - takeover or reconciliation-to-grant is performed. Persisted partial writes - burn capacity. A caller receiving an uncertain first write must stop, not - blindly retry: an absent observation does not prove a write cannot arrive. - """ - if (type(policy) is not PilotPolicy or not callable(check_current) - or not callable(wall_clock) or not callable(monotonic_clock)): - raise budget.BudgetError("Explicit trusted admission dependencies required") - admitted_day = _current(policy, check_current, wall_clock) - daily, pr = _read_pair(policy, daily_store, pr_store) - _campaign_history(policy, daily.ledger, pr.ledger) - # The explicit exception allows only its one additional immutable campaign. - # Unrelated obligations still count in daily_reserve; nothing is refunded. - binding = policy.binding - day_next = budget.daily_reserve(daily.ledger, policy.key, binding, admitted_day) - pr_next = budget.reserve(pr.ledger, binding) # Check BOTH caps before any write. - _current(policy, check_current, wall_clock, admitted_day) - for store, before, after in ((daily_store, daily, day_next), (pr_store, pr, pr_next)): - if (after == before.ledger - or store.compare_and_swap(before, after) is not budget.WriteResult.COMMITTED): - raise budget.BudgetError("Duplicate, conflicted or uncertain pilot reservation") - _current(policy, check_current, wall_clock, admitted_day) - budget.daily_claim_once(daily_store, policy.key, binding, - budget.shanghai_day(_now(wall_clock))) - budget.claim_once(pr_store, policy.key, binding) - session = _Admission(policy, daily_store, pr_store, admitted_day, - check_current, wall_clock, monotonic_clock) - session._check() # Both durable claims, price and day checked AFTER all writes. - return session - - -class _Admission: - """Trusted supervisor state; never serialize or pass this object to workers. - - Call start_arm BEFORE disclosure/launch. authorize matches the edit protocol's - exact grant and returns True, not a transferable token. complete requires the - SAME grant plus RAW four-class usage from trusted transport/worker observation, - never candidate-authored JSON or an unverified trace. Only one request may be - pending; missing/unknown usage in either arm stops both. No method refunds or - closes durable ledger charges. A process crash burns both durable claims. - """ - - def __init__(self, policy, daily_store, pr_store, day, check, wall, monotonic): - self.policy, self._daily_store, self._pr_store = policy, daily_store, pr_store - self._day, self._check_current = day, check - self._wall, self._monotonic = wall, monotonic - self._used = dict.fromkeys(ARMS, 0) - self._deadlines = {} - self._finished = set() - self._pending = None - self._stopped = False - self._lock = threading.Lock() - - def __reduce__(self): - raise TypeError("Pilot admission is process-local and nontransferable") - - def _check(self): - if self._stopped: - raise budget.BudgetError("Pilot admission stopped") - daily, pr = _read_pair(self.policy, self._daily_store, self._pr_store) - binding = self.policy.binding - if (not any(e.binding == binding and e.phase == "claimed" for e in pr.ledger.entries) - or not any(e.key == self.policy.key and e.binding == binding - and e.phase == "claimed" and e.admitted_day == self._day - for e in daily.ledger.entries)): - raise budget.BudgetError("Both exact durable pilot claims are required") - _campaign_history(self.policy, daily.ledger, pr.ledger, admitted=True) - _current(self.policy, self._check_current, self._wall, self._day) - - def context(self, arm): - if arm not in ARMS: - raise budget.BudgetError("Unknown pilot arm") - return f"{self.policy.binding.plan_digest}:{self.policy.binding.run_id}:{arm}" - - def start_arm(self, arm, absolute_deadline): - """Trusted supervisor only, immediately BEFORE task disclosure.""" - with self._lock: - try: - self._check() - remaining = absolute_deadline - self._monotonic() - if (arm not in ARMS or arm in self._deadlines or self._pending is not None - or set(self._deadlines) - self._finished - or type(absolute_deadline) not in (int, float) - or not math.isfinite(absolute_deadline) - or not 0 < remaining <= MAX_ARM_SECONDS): - raise budget.BudgetError("Invalid, repeated or overlapping arm launch") - self._deadlines[arm] = absolute_deadline - except BaseException: - self._stopped = True - raise - - def authorize(self, grant, absolute_deadline): - """Burn one per-arm/global attempt before live checks, then permit ONE send.""" - with self._lock: - try: - arm = grant.get("arm") if type(grant) is dict else None - if (self._stopped or self._pending is not None or arm not in self._deadlines - or arm in self._finished or self._used[arm] >= MAX_ATTEMPTS_PER_ARM): - raise budget.BudgetError("Pilot request allocation unavailable") - self._used[arm] += 1 # No failure after this point restores a slot. - expected = {"plan_sha256": self.policy.plan_sha256, - "execution_id": self.policy.execution_id, "arm": arm, - "slot": self._used[arm], - "request_sha256": grant.get("request_sha256"), - "actor_sha256": self.policy.actor_sha256[ARMS.index(arm)], - "reserved_microusd": PER_REQUEST_MICROUSD} - if (grant != expected or type(grant.get("slot")) is not int - or type(grant.get("reserved_microusd")) is not int - or not _sha(grant.get("request_sha256")) - or absolute_deadline != self._deadlines[arm]): - raise budget.BudgetError("Request differs from frozen pilot binding") - self._pending = dict(expected) # Lost/ambiguous grant stays unresolved. - self._check() - if self._monotonic() >= absolute_deadline: - raise budget.BudgetError("Frozen arm deadline ended") - return True - except BaseException: - self._stopped = True - raise - - def complete(self, grant, raw_usage): - """Record trusted observed usage only; never a refund or restart permit.""" - with self._lock: - try: - if (self._stopped or self._pending is None or type(grant) is not dict - or provider.canonical(grant) != provider.canonical(self._pending)): - raise budget.BudgetError("Missing or inconsistent request completion") - usage = provider.token_usage(raw_usage) - if usage["input_tokens"] > 1_000_000 or usage["output_tokens"] > 4096: - raise budget.BudgetError("Observed usage exceeds reserved request bound") - self._pending = None - except BaseException: - self._stopped = True - raise - - def finish_arm(self, arm): - """Keep failure/missing artifacts externally; unknown usage cannot finish.""" - with self._lock: - if (self._stopped or self._pending is not None or arm not in self._deadlines - or arm in self._finished): - self._stopped = True - raise budget.BudgetError("Unresolved, unknown or repeated arm completion") - self._finished.add(arm) - - def attempts(self): - with self._lock: - return dict(self._used) - - def close(self): - with self._lock: - self._stopped = True - - -def run_admitted(policy, daily_store, pr_store, *, launch, **dependencies): - """Invoke the injected TRUSTED launch callback once after both durable claims. - - Intended for fake/offline integration until independently reviewed real launch - wiring is supplied. This module itself performs no provider or process call. - """ - if not callable(launch): - raise budget.BudgetError("A trusted launch callback is required") - session = admit(policy, daily_store, pr_store, **dependencies) - try: - return launch(session) - finally: - session.close() diff --git a/tools/kv_runtime_qualification.py b/tools/kv_runtime_qualification.py deleted file mode 100644 index 397dcd3cb..000000000 --- a/tools/kv_runtime_qualification.py +++ /dev/null @@ -1,256 +0,0 @@ -"""Inventory only the production sandbox runtime; probe sensitive files fail-closed. - -This is a trusted software-control check, not a candidate execution alternative. -No environment values or raw absolute inventory paths are retained. -""" - -import csv -import hashlib -import importlib.metadata -import importlib.util -import io -import json -import os -import stat -import time -from pathlib import Path, PurePosixPath -from urllib.parse import urlsplit -from urllib.request import url2pathname - -from inferrsi.harness import kv_reclamation as kv -from inferrsi.harness.skill_eval_engineering import sandbox_command -from inferrsi.model import anthropic_edit_agent as edit - -FORBIDDEN_NAMES = {"inferrsi", "skill_library", "kv_reclamation", "oracle.py", "fixtures.json"} -MAX_NODES = 200000 -MAX_ESCAPES = 8192 -MAX_SENSITIVE = 20000 -MAX_DISTRIBUTIONS = 2000 -MAX_HASH_BYTES = 128 * 1024 * 1024 -MAX_SENSITIVE_BYTES = 32 * 1024 * 1024 - - -def _distribution_package(distribution): - """Resolve one declared package, never interpret SOURCES.txt as installed paths.""" - direct = distribution.read_text("direct_url.json") - direct = {} if direct is None else json.loads(direct) - if direct.get("dir_info", {}).get("editable") is True: - parsed = urlsplit(direct["url"]) - if (parsed.scheme != "file" or parsed.netloc not in {"", "localhost"} - or parsed.query or parsed.fragment): - raise ValueError("Unsupported editable source mapping") - source = Path(url2pathname(parsed.path)) - if not source.is_absolute(): - raise ValueError("Editable source must be absolute") - candidates = [source / "src/inferrsi", source / "inferrsi"] - candidates = [p for p in candidates if p.is_dir()] - if len(candidates) != 1: - raise ValueError("Editable package layout is missing or ambiguous") - return candidates[0] - return Path(distribution.locate_file("inferrsi")) - - -def runtime_mounts(work): - command = sandbox_command(work) - mounts = [] - for index, value in enumerate(command): - if value == "--ro-bind" and command[index + 2] != "/work": - mounts.append((Path(command[index + 1]), Path(command[index + 2]))) - if not mounts: - raise ValueError("No frozen runtime mounts") - return mounts - - -def sensitive_files(root, independent, fixtures, knowledge_root, deadline): - """Actual regular package, K, oracle and optimized-reference paths, not directories.""" - paths = [ - root / "src/inferrsi/__init__.py", - fixtures / "reference.py", - independent / "oracle.py", - independent / "fixtures.json", - knowledge_root / "SKILL.md", - knowledge_root / "metadata.json", - knowledge_root / "references/source-notes.md", - ] - if any(not p.is_file() or p.is_symlink() for p in paths): - raise ValueError("Required sensitive-file anchor missing") - - def add_tree(directory): - stack = [directory] - while stack: - if time.monotonic() >= deadline or len(paths) + len(stack) >= MAX_SENSITIVE: - raise ValueError("Sensitive-file inventory limit") - parent = stack.pop() - with os.scandir(parent) as children: - for item in children: - if item.is_symlink(): - raise ValueError("Sensitive inventory symlink") - if item.is_dir(follow_symlinks=False): - if item.name != "__pycache__": - stack.append(Path(item.path)) - elif item.is_file(follow_symlinks=False): - paths.append(Path(item.path)) - else: - raise ValueError("Sensitive inventory special node") - if len(paths) + len(stack) >= MAX_SENSITIVE: - raise ValueError("Sensitive-file inventory limit") - - def add_package(directory): - if (any(p.is_symlink() for p in (directory, *directory.parents)) - or (directory / "__init__.py").is_symlink() - or (directory / "data").is_symlink() - or not (directory / "__init__.py").is_file() - or not (directory / "data").is_dir()): - raise ValueError("Actual package source anchors incomplete") - add_tree(directory) - - add_tree(independent) - add_tree(knowledge_root) - spec = importlib.util.find_spec("inferrsi") - if spec is None or spec.origin is None: - raise ValueError("Cannot locate actual imported package") - paths.append(Path(spec.origin)) - for directory in spec.submodule_search_locations or (): - add_package(Path(directory)) - # Discover installed copies independently of the source checkout import path. - for index, distribution in enumerate(importlib.metadata.distributions()): - if index >= MAX_DISTRIBUTIONS or time.monotonic() >= deadline: - raise ValueError("Installed distribution inventory limit") - if distribution.metadata.get("Name", "").lower().replace("_", "-") == "inferrsi": - record = distribution.read_text("RECORD") - if record is not None: - # .files may silently filter missing nodes on newer Python. - # RECORD is the installed-file authority; parse its raw rows. - for row_number, row in enumerate(csv.reader(io.StringIO(record), strict=True)): - if (row_number >= MAX_SENSITIVE or len(paths) >= MAX_SENSITIVE - or time.monotonic() >= deadline): - raise ValueError("Installed package inventory limit") - if len(row) != 3 or not row[0]: - raise ValueError("Malformed installed package RECORD") - entry = PurePosixPath(row[0]) - if entry.parts[0] == "inferrsi": - if ".." in entry.parts or str(entry) != row[0]: - raise ValueError("Unsafe installed package RECORD path") - path = Path(distribution.locate_file(entry)) - if (any(p.is_symlink() for p in (path, *path.parents)) - or not path.is_file()): - raise ValueError("Installed package inventory incomplete") - paths.append(path) - elif "inferrsi" in entry.parts and entry.name != "inferrsi": - raise ValueError("Unsupported installed package RECORD mapping") - elif (distribution.read_text("PKG-INFO") is None - or distribution.read_text("installed-files.txt") is not None): - raise ValueError("Unsupported installed package inventory metadata") - # Editable/source egg-info SOURCES.txt lists source-archive paths, - # not installed nodes. Inventory the actual anchored package tree; - # retain independently installed wheel/editable copies as well. - add_package(_distribution_package(distribution)) - result = sorted({p.resolve(strict=True) for p in paths}) - if not result: - raise ValueError("No sensitive package/data inventory") - return result - - -def inventory(mounts, sensitive, deadline): - """Enumerate without following symlink directories; detect copies and escapes. - - Matching-size regular runtime files are compared to sensitive file hashes, - including the complete large hidden corpus, so renaming a complete file does - not evade the path-name check. Hash work is bounded and fails closed. - This attests this trusted host layout, not arbitrary hostile host integrity. - """ - roots = [source.resolve(strict=True) for source, _ in mounts] - sensitive = [p.resolve(strict=True) for p in sensitive] - sensitive_ancestors = {parent for p in sensitive for parent in (p, *p.parents)} - sensitive_hashes, sizes, hashed_bytes = set(), set(), 0 - - def bounded_hash(path, size): - nonlocal hashed_bytes - if size > MAX_SENSITIVE_BYTES or hashed_bytes + size > MAX_HASH_BYTES: - raise ValueError("Runtime hash byte limit") - hashed_bytes += size - data = kv.read_regular(path, MAX_SENSITIVE_BYTES) - if len(data) != size or time.monotonic() >= deadline: - raise ValueError("Runtime hash inventory incomplete") - return edit.digest(data) - - for path in sensitive: - if time.monotonic() >= deadline or len(sensitive) > MAX_SENSITIVE + 2: - raise ValueError("Sensitive hash inventory limit") - size = path.stat().st_size - if size: - sizes.add(size) - sensitive_hashes.add(bounded_hash(path, size)) - rows, escaped, rejected = [], [], False - for index, (source, destination) in enumerate(mounts): - # A bind source is resolved by the mount operation. Its target is the - # mounted tree, not an escaped descendant alias: enumerate its contents - # even when the source pathname itself is a symlink. - stack = [(source.resolve(strict=True), Path("."))] - while stack: - if time.monotonic() >= deadline or len(rows) >= MAX_NODES: - raise ValueError("Runtime inventory limit") - path, relative = stack.pop() - info = path.lstat() - row = { - "mount": index, - "relative": relative.as_posix(), - "mode": stat.S_IFMT(info.st_mode), - } - resolved = path.resolve(strict=False) - if ( - any(name.lower() in FORBIDDEN_NAMES for name in path.parts) - or resolved in sensitive_ancestors - ): - rejected = True - if stat.S_ISLNK(info.st_mode): - # Never traverse an escaped target. Test its in-sandbox alias for denial. - row["target_sha256"] = hashlib.sha256(os.readlink(path).encode()).hexdigest() - if not any(resolved == root or root in resolved.parents for root in roots): - escaped.append(str(destination / relative)) - if len(escaped) > MAX_ESCAPES: - raise ValueError("Runtime symlink limit") - row["escaped"] = True - elif stat.S_ISDIR(info.st_mode): - with os.scandir(path) as children: - stack.extend((Path(p.path), relative / p.name) for p in children) - elif stat.S_ISREG(info.st_mode): - row["bytes"] = info.st_size - if info.st_size in sizes: - row["sha256"] = bounded_hash(path, info.st_size) - if row["sha256"] in sensitive_hashes: - rejected = True - else: - raise ValueError("Special node in runtime mount") - rows.append(row) - rows.sort(key=lambda r: (r["mount"], r["relative"])) - return { - "verdict": "FAIL" if rejected else "PASS", - "nodes": len(rows), - "mount_count": len(mounts), - "inventory_sha256": edit.fingerprint({"nodes": rows}), - "escaped_symlinks": len(escaped), - "sensitive_files": len(sensitive), - }, escaped - - -def qualify(output, deadline, *, root, independent, fixtures, knowledge_root): - output = Path(output) - output.mkdir(parents=True, exist_ok=False) - result = {"verdict": "INCONCLUSIVE", "scope": "actual production candidate runtime mounts"} - try: - sensitive = sensitive_files(root, independent, fixtures, knowledge_root, deadline) - for name in ("knowledge-canary.txt", "oracle-canary.txt"): - path = output / name - path.write_text("synthetic sensitive-path isolation canary\n", encoding="utf-8") - sensitive.append(path) - result, escaped = inventory(runtime_mounts(output), sensitive, deadline) - if result["verdict"] == "PASS": - result["probe"] = kv.probe_isolation( - output / "probe", deadline, [str(p) for p in sensitive] + escaped - ) - except kv.SandboxUnavailable: - result.update(verdict="INCONCLUSIVE", reason="sandbox_unavailable") - except Exception: - result.update(verdict="INCONCLUSIVE", reason="runtime_inventory_incomplete") - return result diff --git a/tools/kv_worker_supervisor.py b/tools/kv_worker_supervisor.py deleted file mode 100644 index 4dd58e741..000000000 --- a/tools/kv_worker_supervisor.py +++ /dev/null @@ -1,139 +0,0 @@ -"""Private trusted worker supervision, shared by scripted and fixed live launchers. - -Factories are reviewed Python callbacks, never CLI/config/model command inputs. -No credential discovery, provider transport, ledger bootstrap or execution CLI. -""" - -import json -import math -import time -from pathlib import Path - -from inferrsi.harness import kv_reclamation as kv -from inferrsi.infra.artifacts import atomic_write_json -from inferrsi.infra.executors.local import execute_command -from inferrsi.model import anthropic_agent as provider -from inferrsi.model import anthropic_edit_agent as edit -from inferrsi.model.request_guard import ENV_KEYS, authorization_broker - - -def _read_json(path, limit): - return json.loads(kv.read_regular(Path(path), limit), object_pairs_hook=provider.unique_object, - parse_constant=lambda _: (_ for _ in ()).throw(ValueError("nonfinite JSON"))) - - -def supervise(config, plan, inputs, workspace, output, deadline, *, checkout, - command_factory, environment_factory, admission=None, execute=execute_command): - """One exact inherited deadline and parent-owned grant/completion latch.""" - if (not math.isfinite(deadline) - or not 0 < deadline - time.monotonic() <= plan.limits.max_seconds): - raise ValueError("Invalid inherited absolute deadline") - output = Path(output) - output.mkdir(parents=True, exist_ok=False) - bridge = output / "bridge" - bridge.mkdir(mode=0o700) - pending, consumed, failed = None, 0, False - completions = [] - - def complete(): - nonlocal pending, failed - if pending is None: - return - try: - value = _read_json(bridge / "complete.json", 8192) - if (type(value) is not dict or set(value) != {"grant", "usage"} - or value["grant"] != pending): - raise ValueError("Wrong usage receipt") - if admission is not None: - admission.complete(pending, value["usage"]) - elif value["usage"] is None: - raise ValueError("Unknown simulated usage") - else: - usage = provider.token_usage(value["usage"]) - if (usage["input_tokens"] > config["input_token_ceiling"] - or usage["output_tokens"] > config["client"]["max_output_tokens"]): - raise ValueError("Out-of-bound simulated usage") - completions[-1].update(status="known", - usage={k: value["usage"][k] for k in ("input_tokens", "output_tokens", - "cache_read_input_tokens", "cache_creation_input_tokens")}) - pending = None - (bridge / "complete.json").unlink() - except Exception: - failed = True - if admission is not None: - # Unknown, lost and malformed completion is never a refund. - try: - admission.complete(pending, None) - except Exception: - pass - raise ValueError("Unresolved request completion") from None - - def check(): - nonlocal pending, consumed, failed - try: - complete() - value = _read_json(bridge / "pending.json", 8192) - expected = {"plan_sha256", "execution_id", "arm", "slot", "request_sha256", - "actor_sha256", "reserved_microusd"} - if (failed or time.monotonic() >= deadline or type(value) is not dict - or set(value) != expected or value["plan_sha256"] != config["plan_sha256"] - or value["execution_id"] != config["execution_id"] - or value["arm"] != config["arm"] - or value["actor_sha256"] != config["actor_sha256"] - or value["slot"] != consumed + 1 or not edit._sha(value["request_sha256"]) - or value["reserved_microusd"] != 10170000 - or consumed >= config["client"]["max_requests"]): - raise ValueError("Wrong bounded grant") - consumed += 1 # Burn before any uncertain live admission check. - pending = value - completions.append({"grant": dict(value), "status": "unresolved", "usage": None}) - if admission is not None and admission.authorize(value, deadline) is not True: - raise ValueError("Admission denied") - return True - except Exception: - failed = True - raise ValueError("Scripted request authorization denied") from None - - try: - if admission is not None: - admission.start_arm(config["arm"], deadline) - context = f"{config['plan_sha256']}:{config['execution_id']}:{config['arm']}" - with authorization_broker(check, config["client"]["max_requests"], context=context) as gate: - environment = environment_factory(gate) - allowed = {"PATH", "LANG", provider.KEY_ENV, *ENV_KEYS} - if (type(environment) is not dict or set(environment) - allowed - or any(environment.get(k) != v for k, v in gate.items())): - raise ValueError("Worker environment exceeds fixed allowlist") - command = command_factory(Path(inputs), Path(workspace), output / "worker", deadline) - record = execute(command, checkout, output / "executor", deadline, - environment=environment) - try: - complete() - except ValueError: - pass - # Worker JSON cannot upgrade a failed or unknown outer execution outcome. - # Only explicit successful execution can authorize the following arm. - try: - worker = _read_json(output / "worker/result.json", 1024 * 1024) - except (ValueError, OSError): - worker = {"status": "missing", "reason": "worker_result_unavailable"} - returncode = record.get("returncode") - if (record.get("status") != "completed" - or type(returncode) is not int or returncode != 0 - or type(worker) is not dict or worker.get("status") not in { - "completed", "failed", "inconclusive"}): - failed = True - if admission is not None: - admission.close() - if admission is not None and not failed: - admission.finish_arm(config["arm"]) - except BaseException: - if admission is not None: - admission.close() - raise - result = {"schema": "kv-worker-supervision-v1", "execution": record, "worker": worker, - "request_completion_known": not failed and pending is None, - "slots_consumed": consumed, "completions": completions} - atomic_write_json(output / "supervision.json", result) - return result - diff --git a/tools/qualify_kv_actor_cleanup.py b/tools/qualify_kv_actor_cleanup.py deleted file mode 100644 index 1f9ac92bd..000000000 --- a/tools/qualify_kv_actor_cleanup.py +++ /dev/null @@ -1,329 +0,0 @@ -"""Model-free qualification of namespace cleanup with a trusted fork/setsid probe. - -No candidate input is accepted. Only this authored probe may fork; the real -candidate profile remains RLIMIT_NPROC=0. Every execution still uses the existing -bubblewrap boundary and execute_command. Missing isolation/visibility is not PASS. -""" - -import argparse -import hashlib -import json -import math -import os -import signal -import sys -import tempfile -import threading -import time -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -if __package__ in {None, ""}: - sys.path.insert(0, str(ROOT / "src")) - -from inferrsi.harness.skill_eval_engineering import ( # noqa: E402 - SandboxUnavailable, - read_regular, - sandbox_command, -) -from inferrsi.infra.artifacts import atomic_write_json # noqa: E402 -from inferrsi.infra.executors.local import execute_command # noqa: E402 -from inferrsi.infra.targets import repository_identity # noqa: E402 - -PROBE = '''import json, os, resource, sys, time -resource.setrlimit(resource.RLIMIT_CPU, (4, 4)) -resource.setrlimit(resource.RLIMIT_AS, (268435456, 268435456)) -resource.setrlimit(resource.RLIMIT_FSIZE, (65536, 65536)) -resource.setrlimit(resource.RLIMIT_NOFILE, (32, 32)) -resource.setrlimit(resource.RLIMIT_NPROC, (64, 64)) -resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) -try: - child = os.fork() -except OSError: - print(json.dumps({"role": "unavailable"}), flush=True) - raise SystemExit(1) -if child == 0: - try: - os.setsid() - except OSError: - print(json.dumps({"role": "unavailable"}), flush=True) - os._exit(1) - with open("/proc/self/stat") as stream: - start = int(stream.read().rsplit(")", 1)[1].split()[19]) - print(json.dumps({"role": "detached", "own_session": os.getsid(0) == os.getpid(), - "namespace_pid": os.getpid(), "start": start, - "pid_namespace": os.readlink("/proc/self/ns/pid")}), flush=True) - time.sleep(30) - os._exit(0) -print(json.dumps({"role": "parent"}), flush=True) -time.sleep(0.5 if sys.argv[1] == "normal" else 30) -''' - - -def _stat(path): - value = path.read_text().rsplit(")", 1)[1].split() - return {"state": value[0], "ppid": int(value[1]), "start": int(value[19])} - - -def _owned_snapshot(parent, known, *, proc=Path("/proc")): - """Read stat only; retain information only for this probe's descendants.""" - current, visible = {}, True - try: - entries = list(proc.iterdir()) - except OSError: - return {}, False - for entry in entries: - if not entry.name.isdigit(): - continue - try: - current[int(entry.name)] = _stat(entry / "stat") - except (FileNotFoundError, ProcessLookupError): - continue - except (OSError, ValueError, IndexError): - visible = False - selected = {pid: item for pid, item in current.items() - if known.get(pid) == item["start"]} - parents = {parent, *selected} - while True: - added = {pid: item for pid, item in current.items() - if item["ppid"] in parents and pid != parent and pid not in selected} - if not added: - break - selected.update(added) - parents.update(added) - return selected, visible - - -def _events(path): - try: - return [json.loads(line) for line in read_regular(path, 65536).decode().splitlines()] - except (OSError, ValueError, UnicodeError): - return [] - - -def _bind_detached(events, owned, *, proc=Path("/proc")): - """Match a trusted marker to an already-owned host PID, never a count heuristic.""" - markers = [e for e in events if e.get("role") == "detached"] - if len(markers) != 1: - return None - marker = markers[0] - if (set(marker) != {"role", "own_session", "namespace_pid", "start", "pid_namespace"} - or marker["own_session"] is not True - or type(marker["namespace_pid"]) is not int or marker["namespace_pid"] <= 0 - or type(marker["start"]) is not int or marker["start"] < 0 - or type(marker["pid_namespace"]) is not str): - return None - matches = [] - for pid, item in owned.items(): - if item["start"] != marker["start"]: - continue - try: - fields = (proc / str(pid) / "status").read_text().splitlines() - nspid = next(line.split()[1:] for line in fields if line.startswith("NSpid:")) - namespace = os.readlink(proc / str(pid) / "ns/pid") - if (int(nspid[-1]) == marker["namespace_pid"] - and namespace == marker["pid_namespace"]): - matches.append({"pid": pid, "start": item["start"]}) - except (OSError, ValueError, IndexError, StopIteration): - continue - return matches[0] if len(matches) == 1 else None - - -def _bind_chain(worker_events, detached, owned): - markers = [e for e in worker_events if e.get("role") == "worker"] - if detached is None or len(markers) != 1: - return None - worker = markers[0] - if (set(worker) != {"role", "pid", "start"} - or type(worker["pid"]) is not int or type(worker["start"]) is not int - or owned.get(worker["pid"], {}).get("start") != worker["start"]): - return None - chain, pid = [], detached["pid"] - while pid in owned and pid not in chain: - chain.append(pid) - if pid == worker["pid"]: - # At least one nested sandbox process must separate worker and probe. - return {"worker": {k: worker[k] for k in ("pid", "start")}, - "detached": detached, "chain_length": len(chain)} if len(chain) >= 3 else None - pid = owned[pid]["ppid"] - return None - - -def nested_worker(mode, work, output, deadline): - """Trusted stand-in for the actor's nested executor; no candidate input accepted.""" - work = Path(work) - if (mode not in {"normal", "timeout", "cancel"} or work.is_symlink() - or {p.name for p in work.iterdir()} != {"runner.py"} - or read_regular(work / "runner.py", 65536) != PROBE.encode()): - raise ValueError("Only the exact authored descendant probe is accepted") - own = _stat(Path("/proc/self/stat")) - print(json.dumps({"role": "worker", "pid": os.getpid(), "start": own["start"]}), flush=True) - record = execute_command((*sandbox_command(work), mode), work, output, deadline, - environment={"PATH": "/usr/bin:/bin", "LANG": "C"}) - return 0 if record["status"] == "completed" else 1 - - -def probe_worker(mode, output, deadline): - """Run only in the dedicated qualifier child; cancellation signals THIS child.""" - if mode not in {"normal", "timeout", "cancel"}: - raise ValueError("Unknown trusted cleanup probe") - output = Path(output) - output.mkdir(parents=True, exist_ok=False) - known, visibility, stop = {}, [True], threading.Event() - cancelled, bound_chain = [False], [None] - cancel_at = [None] - stdout = output / "sandbox/stdout.log" - worker_stdout = output / "worker/stdout.log" - - def observe(): - while not stop.is_set(): - found, visible = _owned_snapshot(os.getpid(), known) - visibility[0] = visibility[0] and visible - known.update({pid: item["start"] for pid, item in found.items()}) - observed = _bind_chain(_events(worker_stdout), - _bind_detached(_events(stdout), found), found) - if observed is not None: - bound_chain[0] = observed - if (mode == "cancel" and cancel_at[0] is None - and observed is not None): - cancel_at[0] = time.monotonic() + 0.05 - if (cancel_at[0] is not None and not cancelled[0] - and time.monotonic() >= cancel_at[0] and not stop.is_set()): - cancelled[0] = True - os.kill(os.getpid(), signal.SIGINT) - stop.wait(0.02) - - result = {"mode": mode, "verdict": "INCONCLUSIVE", "model_calls": 0, - "scope": "outer-worker nested sandbox cleanup; trusted probe, NOT candidate profile"} - observer = threading.Thread(target=observe, daemon=True) - try: - with tempfile.TemporaryDirectory(prefix="kv-tree-probe-") as temporary: - work = Path(temporary) - (work / "runner.py").write_text(PROBE) - sandbox_command(work) # Fail closed before spawning if unavailable. - command = (sys.executable, "-I", str(Path(__file__).resolve()), - "--nested-worker", mode, "--work", str(work), - "--output", str(output / "sandbox"), "--deadline", repr(deadline)) - observer.start() - try: - record = execute_command(command, work, output / "worker", deadline, - environment={"PATH": "/usr/bin:/bin", "LANG": "C"}) - finally: - stop.set() - observer.join(timeout=1) - found, visible = _owned_snapshot(os.getpid(), known) - visibility[0] = visibility[0] and visible and not observer.is_alive() - alive = [pid for pid, item in found.items() if item["state"] != "Z"] - expected_status = {"normal": "completed", "timeout": "timeout", "cancel": "cancelled"} - events = _events(stdout) - detached = bound_chain[0] is not None - ready = {"role": "parent"} in events and detached - result.update(executor_status=record["status"], observed_descendants=len(known), - detached_session_observed=detached, visibility_complete=visibility[0], - surviving_descendants=len(alive), explicit_cancel_sent=cancelled[0], - candidate_profile_unchanged=True, - outer_worker_supervised=True, bound_chain=bound_chain[0]) - if alive: - result["verdict"] = "FAIL" - # Recovery does not turn a failed cleanup test into PASS. Kill - # only PID+starttime identities observed as this probe's children. - killed = 0 - for pid in alive: - try: - if _stat(Path("/proc") / str(pid) / "stat")["start"] == known[pid]: - os.kill(pid, signal.SIGKILL) - killed += 1 - except (OSError, ValueError, IndexError): - pass - result["recovery_kills"] = killed - # SIGKILL delivery is asynchronous. Establish the recovery outcome - # without changing the failed verdict or granting more worker time. - recovery_deadline = time.monotonic() + 1 - while True: - remaining, visible = _owned_snapshot(os.getpid(), known) - survivors = [p for p, item in remaining.items() if item["state"] != "Z"] - if not survivors or time.monotonic() >= recovery_deadline: - result["recovery_survivors"] = len(survivors) - result["recovery_visibility_complete"] = visible - break - time.sleep(0.01) - elif (ready and visibility[0] and record["status"] == expected_status[mode] - and (mode == "normal" or record["returncode"] == -signal.SIGKILL)): - result["verdict"] = "PASS" - else: - result["reason"] = "isolation_or_observation_incomplete" - except (SandboxUnavailable, OSError): - result["reason"] = "sandbox_unavailable" - finally: - stop.set() - if observer.is_alive(): - observer.join(timeout=1) - atomic_write_json(output / "result.json", result) - return result - - -def qualify(output, deadline, probe_seconds): - """Own a fresh child for each normal/timeout/cancel case, no parent SIGINT.""" - if (type(probe_seconds) not in (float, int) or not 0 < probe_seconds <= 30 - or type(deadline) not in (float, int) or not math.isfinite(deadline) - or deadline <= time.monotonic()): - raise ValueError("Explicit bounded probe duration required") - identity = repository_identity(ROOT) - output = Path(output) - output.mkdir(parents=True, exist_ok=False) - result = {"schema": "kv-actor-descendant-qualification-v1", "model_calls": 0, - "probe_sha256": hashlib.sha256(PROBE.encode()).hexdigest(), "cases": {}, - "qualifier_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), - "revision": identity.get("head"), "snapshot_hash": identity.get("snapshot_hash"), - "dirty": identity.get("dirty"), - "scope": "outer-worker nested namespace cleanup; no actor efficacy or provider proof"} - for mode in ("normal", "timeout", "cancel"): - worker_deadline = min(deadline - 1, time.monotonic() + probe_seconds) - command = (sys.executable, "-I", str(Path(__file__).resolve()), "--probe-worker", mode, - "--output", str(output / mode), "--deadline", repr(worker_deadline)) - record = execute_command(command, ROOT, output / f"{mode}-supervisor", deadline, - environment={"PATH": "/usr/bin:/bin", "LANG": "C"}) - try: - case = json.loads(read_regular(output / mode / "result.json", 65536)) - except (OSError, ValueError): - case = {"verdict": "INCONCLUSIVE", "reason": "probe_worker_incomplete"} - if record["status"] != "completed": - case = {**case, "verdict": "INCONCLUSIVE", "worker_status": record["status"]} - result["cases"][mode] = case - verdicts = [v["verdict"] for v in result["cases"].values()] - result["verdict"] = ("FAIL" if "FAIL" in verdicts else "PASS" if set(verdicts) == {"PASS"} - else "INCONCLUSIVE") - after = repository_identity(ROOT) - result["source_unchanged"] = (identity.get("available") and after.get("available") - and identity.get("snapshot_hash") == after.get("snapshot_hash")) - if not result["source_unchanged"]: - result["verdict"] = "INCONCLUSIVE" - atomic_write_json(output / "qualification.json", result) - return result - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--probe-worker", choices=("normal", "timeout", "cancel")) - parser.add_argument("--nested-worker", choices=("normal", "timeout", "cancel")) - parser.add_argument("--work", type=Path) - parser.add_argument("--output", type=Path, required=True) - parser.add_argument("--deadline", type=float, required=True) - parser.add_argument("--probe-seconds", type=float) - args = parser.parse_args() - if args.nested_worker: - if args.probe_worker or args.work is None: - parser.error("Nested trusted worker requires its private probe workspace") - return nested_worker(args.nested_worker, args.work, args.output, args.deadline) - if args.probe_worker: - probe_worker(args.probe_worker, args.output, args.deadline) - else: - if args.probe_seconds is None: - parser.error("--probe-seconds must be explicit") - result = qualify(args.output, args.deadline, args.probe_seconds) - print(json.dumps({"verdict": result["verdict"], "model_calls": 0})) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tools/qualify_kv_edit_actor.py b/tools/qualify_kv_edit_actor.py deleted file mode 100644 index d388cd8c7..000000000 --- a/tools/qualify_kv_edit_actor.py +++ /dev/null @@ -1,446 +0,0 @@ -"""Fixed, secret-free actor-host qualification. Scripted data, never model calls.""" - -import argparse -import hashlib -import json -import re -import tempfile -import time -import zipfile -from pathlib import Path - -from inferrsi.harness import kv_edit_workbench as workbench -from inferrsi.harness import kv_reclamation as kv -from inferrsi.infra.artifacts import atomic_write_json -from inferrsi.infra.targets import repository_identity -from inferrsi.model import anthropic_edit_agent as edit -from tools import kv_edit_inputs as shared_inputs -from tools import kv_edit_pilot as pilot -from tools import kv_runtime_qualification as runtime -from tools import qualify_kv_actor_cleanup as cleanup -from tools import qualify_kv_reclamation as task - -ROOT = Path(__file__).resolve().parents[1] -TASK_PLAN = task.EVIDENCE / "plan/qualification-plan.json" - - -def checked_plan(path): - value = pilot._read_json(path) - if ( - value.get("schema") != "kv-actor-model-free-qualification-v1" - or task.file_hash(TASK_PLAN) != value["task_plan_sha256"] - or type(value["qualification_max_seconds"]) is not int - or not 0 < value["qualification_max_seconds"] <= 600 - or type(value["visible_workload_count"]) is not int - or not 0 < value["visible_workload_count"] <= 10000 - ): - raise ValueError("Unsupported actor qualification plan") - task.checked_plan(TASK_PLAN) # Includes independent original + supplemental verifier hashes. - expected = {"SKILL.md", "metadata.json", "references/source-notes.md"} - if ( - value["knowledge_root"] - != "src/inferrsi/data/skill_library/repositories/vllm/audit-vllm-kv-lifetimes" - or set(value["knowledge_files_sha256"]) != expected - ): - raise ValueError("Only the explicit complete pinned K owner is supported") - owner = ROOT / value["knowledge_root"] - actual_files = {p.relative_to(owner).as_posix() for p in owner.rglob("*") if p.is_file()} - if actual_files != expected or any(p.is_symlink() for p in owner.rglob("*")): - raise ValueError("Pinned K owner is not the complete declared regular-file set") - for name, digest in value["knowledge_files_sha256"].items(): - if task.file_hash(ROOT / value["knowledge_root"] / name) != digest: - raise ValueError("Pinned K bytes changed") - return value - - -def _message(calls): - return { - "type": "message", - "role": "assistant", - "model": "claude-opus-5-5", - "stop_reason": "tool_use", - "content": [ - {"type": "tool_use", "id": ident, "name": name, "input": inputs} - for ident, name, inputs in calls - ], - "usage": { - "input_tokens": 1, - "output_tokens": 1, - "cache_read_input_tokens": 0, - "cache_creation_input_tokens": 0, - }, - "fixture_origin": "scripted software control, not a provider response", - } - - -def _scripts(paths, baseline): - messages = [ - _message([(f"read-{i}", "read_file", {"path": path}) for i, path in enumerate(paths)]) - ] - for revision in (1, 2): - messages.append( - _message( - [ - ( - f"write-{revision}", - "write_candidate", - { - "source": baseline.decode() - + f"\n# Scripted revision {revision}; unchanged algorithm.\n" - }, - ), - (f"check-{revision}", "run_visible_checks", {}), - (f"benchmark-{revision}", "run_visible_benchmark", {}), - ] - ) - ) - messages.append(_message([("submit", "submit_candidate", {})])) - return messages - - -def _archive(private, output): - """Public generated controls/K only; never captures environment or gate tokens.""" - archive = output / "raw-evidence.zip" - names = [] - with zipfile.ZipFile(archive, "w", compression=zipfile.ZIP_DEFLATED) as bundle: - for path in sorted(private.rglob("*")): - if not path.is_file() or path.suffix not in { - ".json", - ".log", - ".py", - ".md", - ".html", - ".svg", - }: - continue - content = kv.read_regular(path, 32 * 1024 * 1024).decode("utf-8", errors="replace") - content = content.replace(str(ROOT), "") - content = content.replace(str(private), "") - content = content.replace(str(output), "") - content = re.sub(r"/tmp/kv-replay-[A-Za-z0-9_-]+", "", content) - content = re.sub(r"/tmp/kv-tree-probe-[A-Za-z0-9_-]+", "", content) - name = path.relative_to(private).as_posix() - names.append(name) - bundle.writestr(name, content) - bundle.writestr( - "REDACTIONS.txt", - "Paths replaced by , , " - ", , . No provider transport, " - "provider key, GitHub token or gate-token capture. All message/usage fixtures " - "are scripted and establish no real API usage or K efficacy.\n", - ) - return {"path": archive.name, "sha256": task.file_hash(archive), "files": names} - - -def visible_receipts(path, artifact, actor_sha256, repetitions, revisions): - """Independently read back exact current receipts and all expected control checks.""" - events = [ - json.loads(line) for line in kv.read_regular(path, 16 * 1024 * 1024).decode().splitlines() - ] - receipts = [ - event["receipt"] - for event in events - if event.get("type") == "agent.tool" and "receipt" in event - ] - if [(r["revision"], r["kind"]) for r in receipts] != [ - (1, "checks"), (1, "benchmark"), (2, "checks"), (2, "benchmark") - ] or artifact["revision"] != 2: - return False - retained = { - revision: edit.digest(kv.source_bytes(revisions / f"revision-{revision:04d}/candidate.py")) - for revision in (0, 1, 2) - } - artifact_events = [e for e in events if e.get("type") == "edit.artifact"] - if [e["revision"] for e in artifact_events] != [0, 1, 2]: - return False - for event in artifact_events: - directory = revisions / f"revision-{event['revision']:04d}" - metadata = {k: v for k, v in event.items() if k not in {"type", "bytes"}} - if ( - metadata != pilot._read_json(directory / "receipt.json") - or event["sha256"] != retained[event["revision"]] - or event["bytes"] != len(kv.source_bytes(directory / "candidate.py")) - or event["actor_sha256"] != actor_sha256 - or event["artifact"] != "candidate.py" - or any(event[k] != artifact[k] for k in ("plan_sha256", "execution_id", "arm")) - ): - return False - for receipt in receipts: - if ( - receipt["artifact_sha256"] != retained[receipt["revision"]] - or any(receipt[k] != artifact[k] for k in ("plan_sha256", "execution_id", "arm")) - ): - return False - body = {k: v for k, v in receipt.items() if k != "receipt_sha256"} - if receipt["receipt_sha256"] != edit.fingerprint({**body, "actor_sha256": actor_sha256}): - return False - feedback = receipt["feedback"] - if receipt["kind"] == "checks": - if feedback["verdict"] != "PASS": - return False - elif receipt["kind"] == "benchmark": - if ( - feedback["verdict"] == "INCONCLUSIVE" - or feedback["quality_verdict"] != "PASS" - or len(feedback["baseline_values"]) != repetitions - or len(feedback["candidate_values"]) != repetitions - ): - return False - else: - return False - final = {r["kind"]: r for r in receipts if r["revision"] == artifact["revision"]} - if set(final) != {"checks", "benchmark"}: - return False - return all( - r["artifact_sha256"] == artifact["sha256"] - and r["plan_sha256"] == artifact["plan_sha256"] - and r["execution_id"] == artifact["execution_id"] - and r["arm"] == artifact["arm"] - and r["receipt_sha256"] == artifact[field] - for kind, field in (("checks", "check_receipt"), ("benchmark", "benchmark_receipt")) - for r in [final[kind]] - ) - - -def dependencies(): - names = set(pilot.FILES) | { - "src/inferrsi/model/anthropic_edit_agent.py", - "src/inferrsi/model/anthropic_agent.py", - "src/inferrsi/model/request_guard.py", - "src/inferrsi/model/protocols/metrics.py", - "src/inferrsi/harness/scenarios/external.py", - "src/inferrsi/infra/targets/git.py", - "tools/qualify_kv_edit_actor.py", - "tools/qualify_kv_actor_cleanup.py", - "tools/kv_runtime_qualification.py", - "tools/kv_pilot_admission.py", - "tools/skill_eval_budget.py", - "tools/kv_edit_inputs.py", - "tools/qualify_kv_reclamation.py", - } - # Imported package initializers are executable dependencies too, including - # target identity and transitive runner metric/scenario imports. - for name in tuple(names): - for parent in Path(name).parents: - initializer = parent / "__init__.py" - if (ROOT / initializer).is_file(): - names.add(initializer.as_posix()) - return {name: task.file_hash(ROOT / name) for name in sorted(names)} - - -def qualify(plan_path, revision, output): - plan = checked_plan(plan_path) - identity = repository_identity(ROOT) - if ( - not re.fullmatch(r"[0-9a-f]{40}", revision) - or not identity.get("available") - or identity["head"] != revision - or identity["dirty"] - ): - raise ValueError("Actor qualification requires the exact clean Git revision") - output = Path(output) - output.mkdir(parents=True, exist_ok=False) - deadline = time.monotonic() + plan["qualification_max_seconds"] - result = { - "schema": "kv-scripted-actor-qualification-v1", - "revision": revision, - "snapshot_hash": identity["snapshot_hash"], - "plan_sha256": task.file_hash(plan_path), - "qualifier_sha256": task.file_hash(Path(__file__)), - "dependencies": dependencies(), - "dependencies_sha256": edit.fingerprint(dependencies()), - "model_calls": 0, - "provider_charges_usd": 0, - "verdict": "INCONCLUSIVE", - "software_checks_passed": False, - "scope": "scripted actual-host actor mechanics; NOT a paid A2/A3 or efficacy result", - "arms": {}, - } - with tempfile.TemporaryDirectory(prefix="kv-actor-qualification-") as temporary: - private = Path(temporary) - try: - result["cleanup"] = cleanup.qualify( - private / "cleanup", deadline, plan["descendant_probe_seconds"] - ) - result["runtime_isolation"] = runtime.qualify( - private / "runtime", - deadline, - root=ROOT, - independent=task.INDEPENDENT, - fixtures=task.FIXTURES, - knowledge_root=ROOT / plan["knowledge_root"], - ) - traces = task.load_module( - "fixed_verifier", task.INDEPENDENT / "verifier.py" - ).load_traces() - - def selected(split): - return [ - {k: t[k] for k in ("id", "split", "events", "expected")} - for t in traces - if t["split"] == split - ] - - development, hidden = selected("development"), selected("grader") - workload = task.load_module("fixed_workload", task.FIXTURES / "workload.py") - oracle = task.load_module("fixed_benchmark", task.EVIDENCE / "validator/benchmark.py") - baseline = kv.source_bytes(task.FIXTURES / "candidate.py") - common, prompt, suite = shared_inputs.build_inputs( - baseline, (task.INDEPENDENT / "CONTRACT.md").read_bytes(), development, - workload.workload(plan["visible_workload_count"]), - oracle.benchmark_expected(plan["visible_workload_count"]), - plan["visible_min_improvement_pct"], - ) - for arm in ("A2", "A3"): - root = private / arm - workspace = root / "workspace" - workspace.mkdir(parents=True) - for name, data in common.items(): - (workspace / name).write_bytes(data) - if arm == "A3": - for name in plan["knowledge_files_sha256"]: - target = workspace / "knowledge" / name - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes((ROOT / plan["knowledge_root"] / name).read_bytes()) - paths = [ - "task.json", # First read is identical in both scripted arms. - *sorted(shared_inputs.COMMON - {"task.json"}), - *( - sorted("knowledge/" + n for n in plan["knowledge_files_sha256"]) - if arm == "A3" - else [] - ), - ] - actor = edit.EditPlan( - edit.freeze_files(workspace, paths), - edit.EditLimits(**plan["actor_limits"]), - tuple(t["id"] for t in development), - "elapsed_seconds", - "seconds", - "minimize", - "0" * 64, - ) - config = pilot.prepare_scripted( - root / "inputs", - workspace, - actor, - suite, - baseline, - _scripts(paths, baseline), - plan_sha256=result["plan_sha256"], - execution_id="scripted-host-qualification", - arm=arm, - model=plan["model"], - spec=plan["client"], - prompt=prompt, - input_token_ceiling=plan["input_token_ceiling"], - ) - run = pilot.run_supervised( - root / "inputs", - workspace, - root / "run", - min(deadline, time.monotonic() + actor.limits.max_seconds), - ) - item = { - "executor_status": run["execution"]["status"], - "worker_status": run["worker"]["status"], - "request_completion_known": run["request_completion_known"], - "simulated_slots_consumed": run["simulated_slots_consumed"], - "actor_sha256": config["actor_sha256"], - "common_actor_contract_sha256": shared_inputs.common_actor_contract_sha256( - config - ), - "exposure": config["edit_plan"]["files"], - "prompt_sha256": hashlib.sha256(prompt.encode()).hexdigest(), - } - result["arms"][arm] = item - if run["worker"]["status"] != "completed": - item["reason"] = run["worker"].get("reason", "worker_incomplete") - continue - receipt = run["worker"]["artifact"] - item["visible_workbench_verified"] = visible_receipts( - root / "run/executor/stdout.log", - receipt, - config["actor_sha256"], - actor.limits.benchmark_repetitions, - root / "run/worker/workbench/revisions", - ) - source = kv.source_bytes(root / "run/worker/final/candidate.py") - item["final_artifact_sha256"] = receipt["sha256"] - item["hidden_correctness"] = workbench.grade_final( - source, receipt["sha256"], hidden, root / "post-submission-grading", deadline - ) - item["two_revision_receipts_retained"] = all( - ( - root / f"run/worker/workbench/revisions/revision-{i:04d}/candidate.py" - ).is_file() - for i in (0, 1, 2) - ) - complete = ( - all( - a["executor_status"] == "completed" - and a["worker_status"] == "completed" - and a["request_completion_known"] - and a["simulated_slots_consumed"] == 4 - and a.get("two_revision_receipts_retained") - and a.get("visible_workbench_verified") - and a.get("hidden_correctness", {}).get("verdict") == "PASS" - for a in result["arms"].values() - ) - and len(result["arms"]) == 2 - ) - result["software_checks_passed"] = ( - complete - and result["cleanup"]["verdict"] == "PASS" - and result["runtime_isolation"]["verdict"] == "PASS" - ) - if result["software_checks_passed"]: - result["verdict"] = "PASS" - elif ( - result["cleanup"]["verdict"] == "FAIL" - or result["runtime_isolation"]["verdict"] == "FAIL" - ) or any( - a.get("hidden_correctness", {}).get("verdict") == "FAIL" - for a in result["arms"].values() - ): - result["verdict"] = "FAIL" - finally: - try: - result["raw_evidence"] = _archive(private, output) - except Exception: - result.update( - verdict="INCONCLUSIVE", - software_checks_passed=False, - retention_error="raw_evidence_retention_failed", - ) - after = repository_identity(ROOT) - result["source_unchanged"] = after.get("available") and identity[ - "snapshot_hash" - ] == after.get("snapshot_hash") - if not result["source_unchanged"]: - result.update(verdict="INCONCLUSIVE", software_checks_passed=False) - atomic_write_json(output / "qualification.json", result) - return result - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--plan", type=Path, required=True) - parser.add_argument("--revision", required=True) - parser.add_argument("--output", type=Path, required=True) - args = parser.parse_args() - result = qualify(args.plan, args.revision, args.output) - print( - json.dumps( - { - "verdict": result["verdict"], - "model_calls": 0, - "software_checks_passed": result["software_checks_passed"], - } - ) - ) - return 0 if result["software_checks_passed"] else 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tools/qualify_kv_reclamation.py b/tools/qualify_kv_reclamation.py deleted file mode 100644 index d8f57b9a6..000000000 --- a/tools/qualify_kv_reclamation.py +++ /dev/null @@ -1,295 +0,0 @@ -"""Model-free synthetic task qualification through existing execution primitives. - -No provider, submitted artifact, arbitrary command or custom grader is accepted. -The numeric plan is explicit evidence, never a workflow/runtime default. -""" - -import argparse -import hashlib -import importlib.util -import json -import platform -import re -import tempfile -import time -import zipfile -from contextlib import contextmanager -from pathlib import Path - -from inferrsi.harness import kv_reclamation as kv -from inferrsi.harness.runner import run_experiment -from inferrsi.infra.artifacts import digest, write_json -from inferrsi.infra.targets import repository_identity -from inferrsi.runspec import RunSpec - -ROOT = Path(__file__).resolve().parents[1] -FIXTURES = ROOT / "tests/fixtures/kv_reclamation" -EVIDENCE = ROOT / "evidence/kv-reclamation-20261030" -INDEPENDENT = EVIDENCE / "validator/independent" - - -def load_module(name, path): - spec = importlib.util.spec_from_file_location(name, path) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -def file_hash(path): - return hashlib.sha256(path.read_bytes()).hexdigest() - - -def retain_raw_evidence(private, output): - """Retain actual outputs with labelled path redactions, never host environment.""" - archive = output / "raw-evidence.zip" - files = [] - with zipfile.ZipFile(archive, "w", compression=zipfile.ZIP_DEFLATED) as bundle: - for path in sorted(private.rglob("*")): - if path.suffix not in {".json", ".log", ".html", ".svg"}: - continue - data = kv.read_regular(path, 16 * 1024 * 1024).decode("utf-8", errors="replace") - data = data.replace(str(ROOT), "").replace(str(private), "") - data = data.replace(str(output), "") - data = re.sub(r"/tmp/kv-replay-[A-Za-z0-9_-]+", "", data) - name = path.relative_to(private).as_posix() - files.append(name) - bundle.writestr(name, data) - bundle.writestr("REDACTIONS.txt", "Labelled path redactions: , , " - ", . Original paths are omitted. " - "No environment dump or provider credentials were captured.\n") - return {"path": archive.name, "sha256": file_hash(archive), "files": files, - "scope": "Actual control outputs and runner evidence; labelled path redactions"} - - -@contextmanager -def retained_workspace(output, result): - with tempfile.TemporaryDirectory(prefix="kv-qualification-") as temporary: - private = Path(temporary) - try: - yield private - finally: - try: - result["raw_evidence"] = retain_raw_evidence(private, output) - except (OSError, ValueError): - result["verdict"] = "INCONCLUSIVE" - result["software_checks_passed"] = False - result["retention_error"] = "Raw evidence retention failed" - raise - - -def checked_plan(path): - plan = json.loads(path.read_text(encoding="utf-8")) - if plan.get("schema") != "kv-reclamation-model-free-plan-v1": - raise ValueError("Unsupported qualification plan") - for name, actual in (("verifier_freeze_sha256", file_hash(INDEPENDENT / "FREEZE.json")), - ("starter_sha256", file_hash(FIXTURES / "candidate.py")), - ("workload_sha256", file_hash(FIXTURES / "workload.py"))): - if actual != plan[name]: - raise ValueError("Frozen qualification input changed: " + name) - frozen = json.loads((INDEPENDENT / "FREEZE.json").read_text()) - for name, expected in frozen["files_sha256"].items(): - if file_hash(INDEPENDENT / name) != expected: - raise ValueError("Independent frozen verifier bytes changed") - supplement = json.loads((EVIDENCE / "validator/supplemental-freeze.json").read_text()) - if supplement["parent_freeze_sha256"] != plan["verifier_freeze_sha256"]: - raise ValueError("Benchmark supplement belongs to a different verifier freeze") - for name, expected in supplement["files_sha256"].items(): - if file_hash(EVIDENCE / "validator" / name) != expected: - raise ValueError("Independent benchmark supplement bytes changed") - return plan - - -def control_sources(): - starter = kv.source_bytes(FIXTURES / "candidate.py").decode("utf-8") - - def replace(old, new): - if starter.count(old) != 1: - raise ValueError("Fault injection no longer matches frozen starter") - return starter.replace(old, new).encode("utf-8") - - return { - "starter": starter.encode("utf-8"), - "reference": kv.source_bytes(FIXTURES / "reference.py"), - "ignore_save": replace('(not lease["save_required"] or lease["saved"])', "True"), - "ignore_fence": replace('and lease["write_step"] <= self._frontier', "and True"), - "ignore_generation": replace( - '(lease["request_id"], lease["generation"]) == (\n' - ' event["request_id"], event["generation"])', - 'lease["request_id"] == event["request_id"]'), - "empty_answers": replace("return reclaimed", "return []"), - "skip_events": replace(" def apply(self, event):\n", - " def apply(self, event):\n" - " if event.get('op') == 'save_complete':\n" - " return []\n"), - "missing_output": b"raise SystemExit(0)\n", - "spoof_output": (b"import json\nprint(json.dumps({'observations': [], 'pending': []}))\n" - b"raise SystemExit(0)\n"), - } - - -def protected_path_control(output): - output.mkdir() - (output / "candidate.py").write_bytes(b"# starter\n") - (output / "contract.md").write_bytes(b"frozen contract\n") - frozen = kv.workspace_hashes(output) - (output / "contract.md").write_bytes(b"tampered\n") - try: - kv.capture_artifact(output, frozen) - except ValueError: - return True - return False - - -def measure(name, before, after, events, expected, plan, output, deadline): - """Adapter only: the shared runner owns AB/BA, snapshots and adjudication.""" - sources = {"baseline": before, "candidate": after} - public_records = [] - - def execute(command, cwd, trial_dir, runner_deadline, metric_key): - label = command[1] - record, actual = kv.invoke(sources[label], {"mode": "benchmark", "events": events}, - trial_dir, min(deadline, runner_deadline)) - complete = digest(actual) == digest(expected) - record["metric"] = record["elapsed_seconds"] if complete else None - record["metric_error"] = None if complete else "complete trace output mismatch" - if record["status"] == "completed" and not complete: - record["status"] = "failed" - public_records.append({"side": label, "quality_check": metric_key is None, - "status": record["status"], "complete_trace": complete, - "elapsed_seconds": record["elapsed_seconds"], - "source_sha256": hashlib.sha256(sources[label]).hexdigest()}) - write_json(trial_dir / "execution.json", record) - return record - - timing = plan["timing"] - spec = RunSpec( - repo=str(ROOT), repo_type="synthetic-cpu-kv", backend="cpu", mode="code", - scenario="kv-reclamation-qualification", metric_key="elapsed_seconds", metric_unit="s", - comparison_scope="whole CPU task command including sandbox and interpreter startup", - baseline_command=("trusted-kv-control", "baseline"), - candidate_command=("trusted-kv-control", "candidate"), - validation_command=("trusted-kv-control", "candidate"), - repetitions=timing["paired_repetitions"], - min_improvement_pct=timing["min_improvement_pct"], - max_minutes=min(timing["comparison_max_seconds"], max(0.001, deadline - time.monotonic())) - / 60, - output_dir=str(output / name), agent="scripted-software-control", - ) - run = run_experiment(spec, {"platform": platform.system(), - "scope": "model-free trusted controls"}, execute=execute) - return {"evaluation": json.loads((run / "evaluation.json").read_text()), - "records": public_records} - - -def qualify(plan_path, revision, output): - plan = checked_plan(plan_path) - identity = repository_identity(ROOT) - if (not re.fullmatch(r"[0-9a-f]{40}", revision) or not identity.get("available") - or identity["head"] != revision): - raise ValueError("Qualification revision must equal the exact checked-out Git head") - started = time.monotonic() - deadline = started + plan["qualification_max_seconds"] - result = { - "schema": "kv-reclamation-qualification-v1", "revision": revision, - "snapshot_hash": identity["snapshot_hash"], "dirty": identity["dirty"], - "plan_sha256": file_hash(plan_path), "verifier_freeze_sha256": plan[ - "verifier_freeze_sha256"], "controller_sha256": file_hash(Path(kv.__file__)), - "qualifier_sha256": file_hash(Path(__file__)), - "benchmark_supplement_sha256": file_hash(EVIDENCE / "validator/supplemental-freeze.json"), - "python": platform.python_version(), "platform": platform.system(), - "verdict": "INCONCLUSIVE", "software_checks_passed": False, - "controls": {}, "measurements": {}, "model_calls": 0, "provider_charges_usd": 0, - "scope": "Synthetic model-free artifact qualification; actor isolation NOT qualified", - "chronology": "Independent and plan local freeze preceded reference; public plan posting " - "followed reference authorship and preceded all timing; independent " - "benchmark supplement frozen after reference authorship and before timing", - } - output.mkdir(parents=True, exist_ok=False) - try: - with retained_workspace(output, result) as private: - sentinel = private / "not-mounted-oracle.txt" - sentinel.write_text("synthetic isolation sentinel\n") - result["isolation"] = kv.probe_isolation(private / "probe", deadline, [ - str(sentinel), str(INDEPENDENT / "fixtures.json"), - str(ROOT / "src/inferrsi/data/skill_library/skill-index.json"), - ]) - sources = control_sources() - result["control_source_sha256"] = { - name: hashlib.sha256(source).hexdigest() for name, source in sources.items()} - verifier = load_module("kv_independent_verifier", INDEPENDENT / "verifier.py") - traces = verifier.load_traces() - result["trace_counts"] = {split: sum(t["split"] == split for t in traces) - for split in ("development", "grader")} - for name in plan["required_good"] + plan["required_bad"]: - result["controls"][name] = kv.verify_source( - sources[name], traces, private / name, deadline, verifier.compare) - result["protected_edit_rejected"] = protected_path_control(private / "protected") - record, _ = kv.invoke(b"while True: pass\n", {"mode": "verify", "events": []}, - private / "deadline", min(deadline, time.monotonic() - + plan["deadline_probe_seconds"])) - result["deadline"] = {"status": record["status"], - "elapsed_seconds": record["elapsed_seconds"], - "process_group_gone": kv.process_group_gone(record)} - controls = result["controls"] - good = all(controls[name]["verdict"] == "PASS" for name in plan["required_good"]) - bad = all(controls[name]["verdict"] == "FAIL" for name in plan["required_bad"]) - result["software_checks_passed"] = (good and bad and result["protected_edit_rejected"] - and record["status"] == "timeout" - and result["deadline"]["process_group_gone"]) - if not result["software_checks_passed"]: - result["verdict"] = ("INCONCLUSIVE" if any( - item["verdict"] == "INCONCLUSIVE" for item in controls.values()) else "FAIL") - return result - workload = load_module("kv_workload", FIXTURES / "workload.py") - benchmark = load_module("kv_benchmark_oracle", EVIDENCE / "validator/benchmark.py") - result["benchmark_oracle_sha256"] = file_hash(EVIDENCE / "validator/benchmark.py") - events = workload.workload(plan["timing"]["lease_count"]) - expected = benchmark.benchmark_expected(plan["timing"]["lease_count"]) - result["benchmark_input_sha256"] = digest(events) - result["benchmark_expected_sha256"] = digest(expected) - for name, after in (("reference", sources["reference"]), - ("unchanged", sources["starter"]), - ("no_op", sources["starter"])): - result["measurements"][name] = measure( - name, sources["starter"], after, events, expected, plan, private, deadline) - measured = result["measurements"] - reference = measured["reference"]["evaluation"] - negatives = [measured[name]["evaluation"] - for name in plan["required_negative_comparisons"]] - complete = all(e["improvement_pct"] is not None - and e["verdict"] != "INCONCLUSIVE" for e in [reference, *negatives]) - passed = (reference["verdict"] == "PASS" and all( - abs(e["improvement_pct"]) < plan["timing"]["negative_absolute_gain_limit_pct"] - for e in negatives)) if complete else False - result["verdict"] = "PASS" if passed else "FAIL" if complete else "INCONCLUSIVE" - except kv.SandboxUnavailable as error: - result["error"] = str(error) - result["software_checks_passed"] = False - finally: - after = repository_identity(ROOT) - result["source_unchanged"] = (after.get("available") and after.get("snapshot_hash") - == identity["snapshot_hash"]) - if not result["source_unchanged"]: - result["verdict"] = "INCONCLUSIVE" - result["software_checks_passed"] = False - result["elapsed_seconds"] = time.monotonic() - started - write_json(output / "qualification.json", result) - return result - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--plan", required=True, type=Path) - parser.add_argument("--revision", required=True) - parser.add_argument("--output", required=True, type=Path) - args = parser.parse_args() - result = qualify(args.plan, args.revision, args.output) - print(json.dumps({"qualification_verdict": result["verdict"], - "software_checks_passed": result["software_checks_passed"]})) - # CI checks software controls, not a campaign performance/efficacy threshold. - # The full qualification verdict must be read separately from a green check. - return 0 if result["software_checks_passed"] else 1 - - -if __name__ == "__main__": - raise SystemExit(main())