diff --git a/.github/workflows/bugbash-sandbox-image.yml b/.github/workflows/bugbash-sandbox-image.yml index 79443831888..8a11ab518c1 100644 --- a/.github/workflows/bugbash-sandbox-image.yml +++ b/.github/workflows/bugbash-sandbox-image.yml @@ -7,9 +7,9 @@ # key has that tag pushes nothing. Otherwise it builds fresh, pushes, attests that build's own # digest, and only then points the stable tag at the same digest. Nothing here deletes an image. # -# Runners still trust only the digest in image.json: a person copies it from the job summary into -# image.json in a pull request (no workflow writes to the repo). Until that PR merges, runners -# refuse a checkout whose image inputs changed (stale inputs key). +# Runners (tests/bugbash/sandbox/runner.ts) look up the stable tag of their checkout's key and +# accept it only with this workflow's attestation from main. Without one they build the image +# from their own checkout, so a pull request that changes an image input passes before the merge. name: Bug-bash sandbox image on: @@ -65,7 +65,7 @@ jobs: contents: read # Push ghcr.io/coder/xum-bugbash-sandbox. Only this job, only on main. packages: write - # Build provenance for audit (the runner trusts the digest in image.json). + # Build provenance: runners verify it before they use the stable tag's digest. id-token: write attestations: write steps: @@ -94,21 +94,23 @@ jobs: cat "$RUNNER_TEMP/inspect.err" >&2 exit 1 fi - make -s bugbash-sandbox-publish > image.json - # The subject of the attestation is this build's own output (image.json). Assignments, not - # $(...) inside echo: a failed jq must fail the step, or an empty digest would skip Attest. - image=$(jq -er .image image.json) - digest=$(jq -er .digest image.json) + record="$RUNNER_TEMP/publish.json" + make -s bugbash-sandbox-publish > "$record" + # The subject of the attestation is this build's own output (the publish record). + # Assignments, not $(...) inside echo: a failed jq must fail the step, or an empty + # digest would skip Attest. + image=$(jq -er .image "$record") + digest=$(jq -er .digest "$record") { echo "image=$image" echo "digest=$digest" echo "tag=$tag" } >> "$GITHUB_OUTPUT" { - echo '### tests/bugbash/sandbox/image.json' + echo '### Publish record' echo echo '```json' - cat image.json + cat "$record" echo '```' } >> "$GITHUB_STEP_SUMMARY" - name: Attest @@ -119,6 +121,9 @@ jobs: subject-name: ${{ steps.push.outputs.image }} subject-digest: ${{ steps.push.outputs.digest }} push-to-registry: true + # Runners verify the attestation bundle in GHCR and read no artifact storage record. + # Creating one needs artifact-metadata: write, and without it the action warns. + create-storage-record: false # The stable tag marks the key as published, so it comes only after the attestation: a failed # Attest leaves no tag, and a re-run builds and attests again instead of skipping the key. - name: Tag inputs- diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index fdf88e33de5..f4b8211d847 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -843,8 +843,9 @@ jobs: bugbash-sandbox: # The bug-bash sandbox (#5714, #5931): its self-check (isolation, proxy and export refusals, # against a fake upstream) and the repros of fixed bugs through its launcher, on the mock app - # AI. No secrets and no model. The launcher pulls the pinned image by digest without - # credentials, and exits nonzero on a refusal, a failed check or a container left `unknown`. + # AI. No secrets and no model. The launcher pulls main's attested image by digest without + # registry credentials (or builds one from this checkout), and exits nonzero on a refusal, a + # failed check or a container left `unknown`. # E2E shard 1 still runs the same repros on the host until this job proves stable. name: Test / Bug-bash sandbox needs: [changes] @@ -852,24 +853,25 @@ jobs: if: >- needs.changes.outputs.tests == 'true' && github.event.inputs.test_filter == '' && (github.event_name != 'pull_request' || needs.changes.outputs.e2e_tests == 'true') - # 45, not 30: a stale image key adds a local image build of up to 25 minutes (below). + # 45, not 30: a PR that changes the image inputs has no published image yet (main publishes + # it after the merge), so the launcher builds one locally, for up to 25 minutes. timeout-minutes: 45 runs-on: ${{ needs.changes.outputs.runner-4 }} - # Stale-key local build: only main publishes the image, so a PR that changes its inputs (Dockerfile, build.sh, - # playwright-core) has a stale image.json. With this opt-in the launcher then builds the image - # locally with build.sh, for these two jobs only, and prints a warning. Model-driven runs - # (make bug-bash, make mcp-apps-e2e) ignore it and keep the pinned digest. - env: - BUGBASH_SANDBOX_BUILD_STALE: "1" steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: persist-credentials: false - uses: ./.github/actions/setup-xum + # GH_TOKEN: the launcher's gh verifies the published image's attestation on the host. It + # never reaches Docker or a job container (the container env drops it). - name: Sandbox self-check run: make bug-bash-sandbox-check + env: + GH_TOKEN: ${{ github.token }} - name: Repros in the sandbox run: make test-bugbash-repros-sandbox + env: + GH_TOKEN: ${{ github.token }} test-windows: name: Test / Windows diff --git a/.github/workflows/stories.yml b/.github/workflows/stories.yml index 2a8382394a2..81515bffee8 100644 --- a/.github/workflows/stories.yml +++ b/.github/workflows/stories.yml @@ -107,9 +107,6 @@ jobs: # failed download apart from the jobs. - name: make electron-binary run: make electron-binary - # make test-stories refuses a stale sandbox image key (a PR that changes the Dockerfile): - # BUGBASH_SANDBOX_BUILD_STALE does not apply to model-driven jobs (launch.ts), so such a PR - # fails here until main publishes the image and image.json pins it. # One more run only when every failed job got a provider 5xx (summary.ts --upstream-fault): # the user's rule, so that an overloaded provider does not fail a queue entry. A test # failure, a 4xx or a refusal fails at once. Before that run, the first run's outputs move @@ -135,6 +132,9 @@ jobs: # The cap per job, so one run spends at most three times this, six with the retry. $3 # unless the caller passes budget_usd ($1 for the PR lane). BUGBASH_BUDGET_USD: ${{ inputs.budget_usd || '3' }} + # The launcher's gh verifies the published sandbox image's attestation on the host. It + # never reaches Docker or a job container (the container env drops it). + GH_TOKEN: ${{ github.token }} # The e2e page of each job, as in the PR comment (summary.ts). - name: Summary if: always() diff --git a/.xum/skills/bug-bash/SKILL.md b/.xum/skills/bug-bash/SKILL.md index a360e6e2a8f..a03710f6b30 100644 --- a/.xum/skills/bug-bash/SKILL.md +++ b/.xum/skills/bug-bash/SKILL.md @@ -26,20 +26,22 @@ Xum harness. Where the two differ, follow this skill. `tests/bugbash/sandbox/launch.ts`, one container per job. - The image `ghcr.io/coder/xum-bugbash-sandbox` (about 1.4 GB, linux/amd64) holds only tools: Node 22, Bun, git, the Chromium of `@e2e-dev/web`'s playwright-core, and Xvfb plus libgtk-3 - for Xum Electron (the Electron binary comes from the mounted node_modules). The runner pulls it only - by the digest in `tests/bugbash/sandbox/image.json`. It builds it only in the case below. + for Xum Electron (the Electron binary comes from the mounted node_modules). - The inputs key (`make bugbash-sandbox-key`) is a hash of the Dockerfile, `build.sh` and the - playwright-core version in `bun.lock`. The runner refuses an image whose key differs from the - checkout. One exception: with `BUGBASH_SANDBOX_BUILD_STALE=1` (exactly `1`), which only CI's - `Test / Bug-bash sandbox` job sets, the self-check and the exact-step repros build the image - locally with `build.sh` (local tag, no push, no registry login), check its inputs label, and - print a warning (a `::warning::` annotation in GitHub Actions). Explore charters, the MCP Apps - suite, the story tests, the agent repros and the preflight of `make bug-bash`, - `make mcp-apps-e2e` and `make test-stories` always refuse a stale key. -- An image change lands in three steps: the PR that changes the Dockerfile, `build.sh` or - playwright-core passes CI on a local build. Its merge push runs the workflow "Bug-bash sandbox + playwright-core version in `bun.lock`. Every job, in CI and on your machine, looks up the tag + `inputs-` anonymously and verifies its attestation on the host with + `gh attestation verify` (main's workflow "Bug-bash sandbox image", `refs/heads/main`). gh needs + your `gh auth login` or `GH_TOKEN`. The token stays on the host: no job container gets it. + A verified image is pulled by digest. +- Any other outcome (no tag, a lookup error, no gh, no token, a failed verification) logs the + reason, builds the image locally with `build.sh` (local tag, no push, no registry login) or + reuses that build, and prints a LOCAL IMAGE warning (a `::warning::` annotation in GitHub + Actions). Both images must carry the checkout's key as their inputs label. A local build adds + no power: each job already runs its checkout's files, `dist/` and `node_modules/`. +- An image change lands in one PR: the PR that changes the Dockerfile, `build.sh` or + playwright-core runs on a local build. Its merge push runs the workflow "Bug-bash sandbox image" on main, which publishes the image once for each inputs key (a manual run on main - retries a failed publish). Then a reviewed PR updates `image.json` to pin the new digest. An e2e update that keeps playwright-core keeps the key, so the image stays valid. + retries a failed publish). An e2e update that keeps playwright-core keeps the key, so the image stays valid. - The container gets `--network none`, `--read-only`, `--cap-drop ALL`, no-new-privileges, a non-root user, 4 GB memory and tmpfs scratch folders. It reads copies of the git-listed files, plus `dist/` and `node_modules/` read-only. An env allowlist passes no credential. @@ -184,7 +186,7 @@ artifact `stories-reports-` and, on a failure, the `results/` folders a `stories-evidence-` (both kept 7 days), and, except in the merge queue, saves the next seed: `bun tests/stories/summary.ts --next-seed ` (the caches of the newest web and Electron story jobs, less the seed entries either one deleted, at most 1000 entries and 128 MiB). GitHub scopes a saved cache to its ref, and a queue ref is temporary, so a -merge-queue run saves none. Only the nightly and on-demand runs on main change what main replays. A PR that changes the sandbox image inputs fails there until main publishes the image. +merge-queue run saves none. Only the nightly and on-demand runs on main change what main replays. A PR that changes the sandbox image inputs has no verified image yet, so each of its jobs builds the image locally (up to 25 minutes) and warns. After the PR lane, pr.yml's `Test / Stories (Haiku) comment` job posts the same page as one PR comment, which each push edits (`tests/stories/prComment.ts`; `--dry-run` prints the body). It runs the base commit's copy with no install scripts, writes only while the run's commit is still @@ -222,8 +224,8 @@ removes only those whose launcher is provably dead on this boot, and exits 3 whe for one. Check that container's `xum.bugbash.owner` and `xum.bugbash.checkout` labels with `docker inspect`, then `docker rm -f ` it. A SIGKILLed launcher also leaves `$TMPDIR/xum-bugbash-sandbox//` and a `xum-bugbash-docker-*` folder in `$TMPDIR`: -remove both by hand. A run with `BUGBASH_SANDBOX_BUILD_STALE=1` on a stale key leaves a local -image of about 1.4 GB: remove it with `docker image rm xum-bugbash-sandbox:inputs-`. +remove both by hand. A run without a verified image leaves a local image +of about 1.4 GB: remove it with `docker image rm xum-bugbash-sandbox:inputs-`. ## 5. Triage and verify diff --git a/Makefile b/Makefile index 0504937c39c..0c307d16bfe 100644 --- a/Makefile +++ b/Makefile @@ -236,14 +236,14 @@ bug-bash: ## Agent bug bash in the bug-bash sandbox: e2e explore charters x mode @bun tests/bugbash/run.ts $(BUGBASH_ARGS) # The bug-bash sandbox image (tests/bugbash/sandbox/build.sh, #5714). Each prints one value on -# stdout. Publishing runs only in the manual "Bug-bash sandbox image" workflow on main. +# stdout. Publishing runs only in the "Bug-bash sandbox image" workflow on main. bugbash-sandbox-key: ## Print the inputs key of the bug-bash sandbox image for this checkout @./tests/bugbash/sandbox/build.sh --key bugbash-sandbox-image: ## Build the bug-bash sandbox image locally (linux/amd64) and print its image ID @./tests/bugbash/sandbox/build.sh -bugbash-sandbox-publish: ## Workflow only: build and push the bug-bash sandbox image, print its image.json record +bugbash-sandbox-publish: ## Workflow only: build and push the bug-bash sandbox image, print its publish record @./tests/bugbash/sandbox/build.sh --push # The sandbox self-check (tests/bugbash/sandbox/selfCheckRun.ts, #5714): three jobs through the real diff --git a/docs/AGENTS.md b/docs/AGENTS.md index a6e42fc43d3..895c110e95d 100644 --- a/docs/AGENTS.md +++ b/docs/AGENTS.md @@ -259,7 +259,7 @@ Freely make breaking changes, and reorganize / cleanup IPC as needed. - To run a bug bash, the MCP Apps suite, the story tests or the sandbox checks, read the `bug-bash` skill (`.xum/skills/bug-bash/SKILL.md`). It covers the host needs, the env, costs, options, results, exit codes and leftovers. - Host pause (#5714): model-driven runs (`make bug-bash`, `make mcp-apps-e2e`, `make test-stories`, any `e2e explore` and every e2e command with `e2e.mcpapps.config.ts`, `tests/stories/e2e.config.ts` or `tests/stories/e2e.electron.config.ts`) run only inside the bug-bash sandbox, one Docker container per job. A model-driven e2e process on the host refuses with exit 2 (`tests/bugbash/hostPause.ts`), and outside the sandbox an `agent.*` step fails with `MODEL_UNAVAILABLE`. No override exists. -- The sandbox image is pinned by digest in `tests/bugbash/sandbox/image.json`. A change to `tests/bugbash/sandbox/Dockerfile`, `tests/bugbash/sandbox/build.sh` or the playwright-core version of `@e2e-dev/web` in `bun.lock` changes the inputs key (`make bugbash-sandbox-key`), and the runner then refuses the image until the "Bug-bash sandbox image" workflow publishes a new one (it runs on the push to main) and a PR updates `image.json`. One exception: with `BUGBASH_SANDBOX_BUILD_STALE=1`, which only CI's `Test / Bug-bash sandbox` job sets, the self-check and the exact-step repros build the image locally with `build.sh` and warn. So an image change lands in three steps: the Dockerfile PR passes CI on a local build, the merge push makes main publish the image (a manual run on main retries a failed publish), and an `image.json` PR pins the new digest. `make bug-bash`, `make mcp-apps-e2e`, `make test-stories` and their preflight always refuse a stale key. +- Each sandbox job, in CI and on a developer machine, looks up the image tag `ghcr.io/coder/xum-bugbash-sandbox:inputs-` of its inputs key (`make bugbash-sandbox-key`: a hash of `tests/bugbash/sandbox/Dockerfile`, `tests/bugbash/sandbox/build.sh` and the playwright-core version of `@e2e-dev/web` in `bun.lock`). It uses the tag's digest only when `gh attestation verify` proves that main's "Bug-bash sandbox image" workflow built it (gh needs your login or `GH_TOKEN`). Otherwise it builds the image locally with `build.sh` and warns. That workflow publishes each new key once, on the push to main (a manual run on main retries a failed publish), so a PR that changes an image input runs on a local build until it merges. - Every model-driven job and every real-AI app run with `XUM_DISABLE_AGENT_TOOLS=1`, `XUM_DISABLE_TERMINALS=1` and `XUM_DISABLE_PROJECT_AUTOMATION=1`, because AI replies are untrusted input for the explorer. Do not turn them off for a bug bash without a security review. Callers with their own `streamText` tools (refinement, memory harvest, continuous compaction) do not check `XUM_DISABLE_AGENT_TOOLS`. None of them runs shell commands in the seeded app, so a new caller of that kind needs the same check. Settings can still add stdio MCP servers and custom editor commands, so charters must keep explorers out of them. - One reviewed exception (#7027, #7029): `BUGBASH_SCENARIO=scripted-tools` runs the app with agent tools on, so repros can see the real tool block. Its only model is a fixed loopback script (`tests/bugbash/scriptedModel.ts`) that ignores typed text and calls only `todo_write` and `file_read` of the seeded README.md. `startApp.ts` refuses it outside the sandbox, in a model-driven job and with a mounted proxy socket, and keeps terminals and project automation off. Only the exact-step sandbox job (`make test-bugbash-repros-sandbox`: no network, no proxy, no explorer model) selects its `tools-on` repros. Do not add tool calls to that script, or run it with a model, without a new security review. - A finding is a bug only once a repro fails for the reported reason. Put it in `tests/bugbash/repros/.e2e.ts` (tracked). Use exact `screen`/`browser`/`expect` steps where they work. A repro may use `agent.*` steps only when each of its tests has the `agent` tag (`tests/bugbash/reproRules.test.ts` enforces this): the free runs pass `--exclude-tag agent`, so only a model-driven sandbox job runs it: `make test-stories` (needs `BUGBASH_BUDGET_USD`). CI's `Stories` workflow (`.github/workflows/stories.yml`) runs that target with Sonnet 5.5 in the merge queue (pr.yml's `Test / Stories` job, which `Required` counts only when the repo variable `STORIES_MODE` is `block`), nightly on main and on demand, and with Haiku 5.5 on each push to a PR that is not a draft (`Test / Stories (Haiku)`, where Sonnet 5.5 judges the story tests' asserts and Haiku the agent repros', and which `Required` counts only when `STORIES_PR_MODE` is `block`). That target also runs the story tests, user stories that an agent drives in `tests/stories/*.e2e.ts`, and the Electron stories in `tests/stories/electron/*.e2e.ts` on Xum Electron under Xvfb (it runs `make electron-binary` first). Prove it: restore the bug in a scratch edit, rebuild, see `ASSERTION_FAILED` on the bug's own assertion, then restore and see it pass. diff --git a/src/node/services/agentSkills/builtInSkillContent.generated.ts b/src/node/services/agentSkills/builtInSkillContent.generated.ts index 31803a649dc..90c0106fc1d 100644 --- a/src/node/services/agentSkills/builtInSkillContent.generated.ts +++ b/src/node/services/agentSkills/builtInSkillContent.generated.ts @@ -2316,7 +2316,7 @@ export const BUILTIN_SKILL_FILES: Record> = { "", "- To run a bug bash, the MCP Apps suite, the story tests or the sandbox checks, read the `bug-bash` skill (`.xum/skills/bug-bash/SKILL.md`). It covers the host needs, the env, costs, options, results, exit codes and leftovers.", "- Host pause (#5714): model-driven runs (`make bug-bash`, `make mcp-apps-e2e`, `make test-stories`, any `e2e explore` and every e2e command with `e2e.mcpapps.config.ts`, `tests/stories/e2e.config.ts` or `tests/stories/e2e.electron.config.ts`) run only inside the bug-bash sandbox, one Docker container per job. A model-driven e2e process on the host refuses with exit 2 (`tests/bugbash/hostPause.ts`), and outside the sandbox an `agent.*` step fails with `MODEL_UNAVAILABLE`. No override exists.", - '- The sandbox image is pinned by digest in `tests/bugbash/sandbox/image.json`. A change to `tests/bugbash/sandbox/Dockerfile`, `tests/bugbash/sandbox/build.sh` or the playwright-core version of `@e2e-dev/web` in `bun.lock` changes the inputs key (`make bugbash-sandbox-key`), and the runner then refuses the image until the "Bug-bash sandbox image" workflow publishes a new one (it runs on the push to main) and a PR updates `image.json`. One exception: with `BUGBASH_SANDBOX_BUILD_STALE=1`, which only CI\'s `Test / Bug-bash sandbox` job sets, the self-check and the exact-step repros build the image locally with `build.sh` and warn. So an image change lands in three steps: the Dockerfile PR passes CI on a local build, the merge push makes main publish the image (a manual run on main retries a failed publish), and an `image.json` PR pins the new digest. `make bug-bash`, `make mcp-apps-e2e`, `make test-stories` and their preflight always refuse a stale key.', + "- Each sandbox job, in CI and on a developer machine, looks up the image tag `ghcr.io/coder/xum-bugbash-sandbox:inputs-` of its inputs key (`make bugbash-sandbox-key`: a hash of `tests/bugbash/sandbox/Dockerfile`, `tests/bugbash/sandbox/build.sh` and the playwright-core version of `@e2e-dev/web` in `bun.lock`). It uses the tag's digest only when `gh attestation verify` proves that main's \"Bug-bash sandbox image\" workflow built it (gh needs your login or `GH_TOKEN`). Otherwise it builds the image locally with `build.sh` and warns. That workflow publishes each new key once, on the push to main (a manual run on main retries a failed publish), so a PR that changes an image input runs on a local build until it merges.", "- Every model-driven job and every real-AI app run with `XUM_DISABLE_AGENT_TOOLS=1`, `XUM_DISABLE_TERMINALS=1` and `XUM_DISABLE_PROJECT_AUTOMATION=1`, because AI replies are untrusted input for the explorer. Do not turn them off for a bug bash without a security review. Callers with their own `streamText` tools (refinement, memory harvest, continuous compaction) do not check `XUM_DISABLE_AGENT_TOOLS`. None of them runs shell commands in the seeded app, so a new caller of that kind needs the same check. Settings can still add stdio MCP servers and custom editor commands, so charters must keep explorers out of them.", "- One reviewed exception (#7027, #7029): `BUGBASH_SCENARIO=scripted-tools` runs the app with agent tools on, so repros can see the real tool block. Its only model is a fixed loopback script (`tests/bugbash/scriptedModel.ts`) that ignores typed text and calls only `todo_write` and `file_read` of the seeded README.md. `startApp.ts` refuses it outside the sandbox, in a model-driven job and with a mounted proxy socket, and keeps terminals and project automation off. Only the exact-step sandbox job (`make test-bugbash-repros-sandbox`: no network, no proxy, no explorer model) selects its `tools-on` repros. Do not add tool calls to that script, or run it with a model, without a new security review.", "- A finding is a bug only once a repro fails for the reported reason. Put it in `tests/bugbash/repros/.e2e.ts` (tracked). Use exact `screen`/`browser`/`expect` steps where they work. A repro may use `agent.*` steps only when each of its tests has the `agent` tag (`tests/bugbash/reproRules.test.ts` enforces this): the free runs pass `--exclude-tag agent`, so only a model-driven sandbox job runs it: `make test-stories` (needs `BUGBASH_BUDGET_USD`). CI's `Stories` workflow (`.github/workflows/stories.yml`) runs that target with Sonnet 5.5 in the merge queue (pr.yml's `Test / Stories` job, which `Required` counts only when the repo variable `STORIES_MODE` is `block`), nightly on main and on demand, and with Haiku 5.5 on each push to a PR that is not a draft (`Test / Stories (Haiku)`, where Sonnet 5.5 judges the story tests' asserts and Haiku the agent repros', and which `Required` counts only when `STORIES_PR_MODE` is `block`). That target also runs the story tests, user stories that an agent drives in `tests/stories/*.e2e.ts`, and the Electron stories in `tests/stories/electron/*.e2e.ts` on Xum Electron under Xvfb (it runs `make electron-binary` first). Prove it: restore the bug in a scratch edit, rebuild, see `ASSERTION_FAILED` on the bug's own assertion, then restore and see it pass.", diff --git a/tests/bugbash/run.test.ts b/tests/bugbash/run.test.ts index 18ab630e425..cadc5b4db55 100644 --- a/tests/bugbash/run.test.ts +++ b/tests/bugbash/run.test.ts @@ -294,7 +294,7 @@ test("U1: a refusal the launcher returns (after docker started) names its reason test("U1: a run whose every charter refuses, with no call made, keeps no run folder", async () => { const { deps, lines } = fakeLaunch(() => ({ - error: new Refusal("stale image"), + error: new Refusal("rootless Docker is not supported"), cleanup: "none", })); const stop = new AbortController().signal; @@ -302,7 +302,9 @@ test("U1: a run whose every charter refuses, with no call made, keeps no run fol expect(await runBugBash([...charters("a", "b")], env, stop, deps)).toBe(2); const runDir = lines.find((line) => line.includes(" -> "))!.split(" -> ")[1]; expect(fs.existsSync(runDir)).toBe(false); - expect(lines.filter((line) => line.endsWith("refused: stale image"))).toHaveLength(2); + expect( + lines.filter((line) => line.endsWith("refused: rootless Docker is not supported")) + ).toHaveLength(2); }); test("U1: Docker is checked once, before the probe and any folder; --preflight stops there", async () => { diff --git a/tests/bugbash/sandbox/Dockerfile b/tests/bugbash/sandbox/Dockerfile index 1cd1d77a59d..b4823cc9164 100644 --- a/tests/bugbash/sandbox/Dockerfile +++ b/tests/bugbash/sandbox/Dockerfile @@ -3,7 +3,7 @@ # an empty build context, and no cache. # # The image is published by the main branch only (.github/workflows/bugbash-sandbox-image.yml), -# and runners pull it by the digest in image.json. The base images are pinned by digest. The +# and runners pull it by the digest of its attested stable tag. The base images are pinned by digest. The # Debian apt mirror and the Playwright download CDN are not pinned: the build trusts them, and # the published digest fixes the bytes after the build. FROM oven/bun:1.4.3-slim@sha256:14a223fa35747ec728bce300ef85888bc5f7958164d15bbba6a54b22b52a3288 AS bun diff --git a/tests/bugbash/sandbox/build.sh b/tests/bugbash/sandbox/build.sh index 1fbcf6fca25..30ae790cdee 100755 --- a/tests/bugbash/sandbox/build.sh +++ b/tests/bugbash/sandbox/build.sh @@ -3,7 +3,7 @@ # # build.sh --key print the inputs key of this checkout # build.sh build for linux/amd64 into the local Docker, print the image ID -# build.sh --push the manual publish job only: build, push to GHCR, print the image.json record +# build.sh --push the publish workflow only: build, push to GHCR, print the publish record (JSON) # # The inputs key is the sha256 of three lines: the sha256 of the Dockerfile, the sha256 of this # script, and the playwright-core version of @e2e-dev/web in bun.lock. The image carries it as @@ -14,7 +14,8 @@ # The build gets an empty context, no cache and no build args from the env. --push always builds # fresh: it never checks the registry and never reuses an image. It pushes a new tag, so it # overwrites no tag, and it reports the name and digest from its own build output. Two publishes -# of one inputs key give two digests; the reviewed digest in image.json is what runners trust. +# of one inputs key give two digests; the publish workflow points the stable tag at one of them +# and attests it, and runners verify that attestation. # Nothing here deletes an image. set -euo pipefail diff --git a/tests/bugbash/sandbox/image.json b/tests/bugbash/sandbox/image.json deleted file mode 100644 index 8ce2c6f1f8c..00000000000 --- a/tests/bugbash/sandbox/image.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "image": "ghcr.io/coder/xum-bugbash-sandbox", - "digest": "sha256:e3804b5418a8e6783698d807c75cc256efae45b81b03400644ae7f13364c9a03", - "inputsKey": "2bf9e12d3ac7dcb10397698df2792bf1b244b3fcbe68225378302ef695ba302a", - "playwrightCore": "1.63.0", - "publishedFrom": "2e2a2b769f417aedcecc09f33abb379d6c8be7b1" -} diff --git a/tests/bugbash/sandbox/inputs.test.ts b/tests/bugbash/sandbox/inputs.test.ts index af5a69bfef6..f538ba61352 100644 --- a/tests/bugbash/sandbox/inputs.test.ts +++ b/tests/bugbash/sandbox/inputs.test.ts @@ -132,10 +132,8 @@ test("a mount source must be a plain folder of the checkout", () => { }); test("the container env passes only allowlisted names, and never a credential", () => { - // BUGBASH_SANDBOX_BUILD_STALE too: the host's opt-in to a local image is not the container's. - // prettier-ignore - const host = { BUGBASH_AI: "mock", HOME: "/home/u", ANTHROPIC_API_KEY: "sk", GH_TOKEN: "t", - BUGBASH_SANDBOX_BUILD_STALE: "1" }; + // GH_TOKEN: the host's gh verifies the image's attestation with it; the container never gets it. + const host = { BUGBASH_AI: "mock", HOME: "/home/u", ANTHROPIC_API_KEY: "sk", GH_TOKEN: "t" }; expect(containerEnv(host, jobEnv(".e2e/r"))).toEqual({ HOME: "/home/bugbash", TMPDIR: "/tmp", diff --git a/tests/bugbash/sandbox/launch.ts b/tests/bugbash/sandbox/launch.ts index 962dcd4cd65..58c916f0738 100644 --- a/tests/bugbash/sandbox/launch.ts +++ b/tests/bugbash/sandbox/launch.ts @@ -11,7 +11,7 @@ * (selfCheckRun.ts, `make bug-bash-sandbox-check`, against a fake upstream) * bun sandbox/launch.ts --recover (make bug-bash-sandbox-recover; see recover()) * - * The container runs the e2e CLI, Chromium and the seeded app with the pinned image (runner.ts). + * The container runs the e2e CLI, Chromium and the seeded app with the sandbox image (runner.ts). * It gets no network, no capabilities, a read-only root, copies of the git-listed inputs, and * read-only dist/ and node_modules/ (inputs.ts). Its output folder, the app log included, comes * back on its stdout as an export stream (exportStream.ts). @@ -49,7 +49,7 @@ import { Ledger, priced } from "./proxyPolicy"; // prettier-ignore import { checkMountSource, containerEnv, exactStepRefusal, exploreRefusal, jobEnv, outputDir, plainFolders, stage } from "./inputs"; import { namesAgentTag, onlyAgentTag } from "../hostPause"; -import { type CleanupState, IMAGE, Refusal, Session, Stopped } from "./runner"; +import { type CleanupState, Refusal, Session, Stopped } from "./runner"; import { DEFAULT_APP_MODEL } from "../aiMode"; const ROOT = fs.realpathSync(path.resolve(import.meta.dir, "../../..")); @@ -236,8 +236,9 @@ export async function probeApp( } /** - * The Docker part of a preflight: the local daemon answers, it is supported, and the pinned - * image is present (pulled when missing). The Makefile runs it before the build, so a missing + * The Docker part of a preflight: the local daemon answers, it is supported, and the image of + * this checkout's inputs key is present (pulled, or built when no verified image exists, which + * the jobs then reuse). The Makefile runs it before the build, so a missing * daemon refuses in about a second instead of after it. Throws a Refusal (or Stopped); leaves no * folder. Returns the cleanup state: an `unknown` one (a process group still runs) outranks a * refusal, as in launchJob, and the caller exits 3. @@ -497,15 +498,6 @@ function checkJob(args: string[], o: LaunchOptions) { // A model-driven `e2e run`: every test needs the explorer model, so the launcher probes it. const drivenRun = mcpApps || stories || electron || agentRepros; const driven = drivenRun || selfCheck || args[0] === "explore" ? modelJob(o.env) : undefined; - // The stale-key local build: CI's bugbash-sandbox job sets BUGBASH_SANDBOX_BUILD_STALE=1, so a pull request that - // changes the image inputs can pass before main publishes its image. Only the self-check and - // exact-step repros may use a local build: they reach no real model. By the job kind, not by - // `driven`: the self-check counts as model-driven. Explore charters, the MCP Apps suite, the - // story suite, the agent repros and checkDocker (the preflight of make bug-bash, make - // mcp-apps-e2e and make test-stories) keep the pinned digest. - // A positive check, not "none of the others": a job kind added later must not qualify by default. - const localBuild = - o.env.BUGBASH_SANDBOX_BUILD_STALE === "1" && (selfCheck || (exactStep && !agentRepros)); // The job's folder: its cwd in the container, and where its output comes back. const storyJob = stories || electron; const dir = storyJob ? storiesDir : bugbash; @@ -517,7 +509,7 @@ function checkJob(args: string[], o: LaunchOptions) { storyJob && judgeSpec !== "" ? pricedAnthropic(judgeSpec, `BUGBASH_JUDGE_MODEL ${JSON.stringify(judgeSpec)}`) : undefined; - return { dir, drivenRun, selfCheck, ai, driven, judgeModel, localBuild, storyJob, electron }; + return { dir, drivenRun, selfCheck, ai, driven, judgeModel, storyJob, electron }; } /** A job's checks and the Docker check, with no job: `--preflight` (Makefile, before the build). */ @@ -529,8 +521,10 @@ export async function preflight(args: string[], o: LaunchOptions): Promise { const log = o.log ?? logLine; - const { dir, drivenRun, selfCheck, ai, driven, judgeModel, localBuild, storyJob, electron } = - checkJob(args, o); + const { dir, drivenRun, selfCheck, ai, driven, judgeModel, storyJob, electron } = checkJob( + args, + o + ); const output = outputDir(args); const dest = path.join(dir, output); const records = `${dest}.proxy.jsonl`; @@ -598,13 +592,13 @@ export async function launchJob(args: string[], o: LaunchOptions): Promise "$bin/c.tmp"; mv "$bin/c.tmp" "$bin/containers"; } echo "$* [home=\${HOME-} cfg=\${DOCKER_CONFIG-}]" >> "$bin/calls.log" +env | grep -oE '^(GH|XDG|DBUS)_[A-Z_]*' | sed 's/^/docker got /' >> "$bin/calls.log" case "$1" in context) if [ "\${CONTEXT-}" = hang ]; then trap '' TERM; exec sleep 30; fi; echo "$HOST" ;; info) echo "$INFO" ;; images) [ "$IMAGES_RC" = 0 ] || { echo "daemon down" >&2; exit 1; } case "$*" in *xum-bugbash-sandbox:inputs-*) echo "$LOCAL" ;; *) echo "$IMAGES" ;; esac ;; - # build.sh's local build: it prints the ID of its tag with \`image inspect\` next. - buildx) [ "$BUILD" = fail ] && { echo "build failed" >&2; exit 1; }; exit 0 ;; + # The stable tag's lookup (TAG): its digest, quoted as by {{json}}, or the registry's error. + # Else build.sh's local build: it prints the ID of its tag with \`image inspect\` next. + buildx) if [ "$2" = imagetools ]; then for tag in "$@"; do :; done + case "$TAG" in + found) echo "\\"$DIGEST\\"" ;; + garbage) echo "$DIGEST" ;; + missing) echo "ERROR: $tag: not found" >&2; exit 1 ;; + *) echo "ERROR: $tag: dial tcp: lookup ghcr.io: no such host" >&2; exit 1 ;; + esac; exit 0; fi + [ "$BUILD" = fail ] && { echo "build failed" >&2; exit 1; }; exit 0 ;; pull) if [ "$PULL" = hang ]; then trap '' TERM; exec sleep 30; fi # A grandchild that holds no pipe: only a signal to the whole group reaches it. @@ -96,6 +105,14 @@ case "$1" in esac `; +// gh attestation verify: exits GH_RC with GH_ERR on stderr, and logs the env names it reads for +// the user's login. +const FAKE_GH = `#!/bin/sh +bin=$(dirname "$0"); . "$bin/fake.env" +echo "gh $* [token=\${GH_TOKEN-} config=\${GH_CONFIG_DIR-} xdg=\${XDG_CONFIG_HOME-} bus=\${DBUS_SESSION_BUS_ADDRESS-} run=\${XDG_RUNTIME_DIR-}]" >> "$bin/calls.log" +[ "$GH_RC" = 0 ] || { echo "$GH_ERR" >&2; exit "$GH_RC"; } +`; + let root = ""; let bin = ""; let key = ""; @@ -121,10 +138,10 @@ beforeEach(() => { encoding: "utf8", }).stdout.trim(); expect(key).toMatch(/^[0-9a-f]{64}$/); - lock(key); bin = path.join(root, "bin"); fs.mkdirSync(bin); fs.writeFileSync(path.join(bin, "docker"), FAKE, { mode: 0o755 }); + fs.writeFileSync(path.join(bin, "gh"), FAKE_GH, { mode: 0o755 }); fs.writeFileSync(path.join(bin, "containers"), ""); fake(); process.env.PATH = `${bin}:${savedPath ?? ""}`; @@ -143,20 +160,14 @@ afterEach(async () => { fs.rmSync(root, { recursive: true, force: true }); }); -function lock(inputsKey: string, digest = DIGEST) { - const record = { - image: "ghcr.io/coder/xum-bugbash-sandbox", - digest, - inputsKey, - playwrightCore: "1.63.0", - publishedFrom: "7".repeat(40), - }; - fs.writeFileSync(path.join(root, SANDBOX, "image.json"), JSON.stringify(record)); -} function fake(over: Record = {}) { const vars = { HOST: "unix:///var/run/docker.sock", INFO: '{"OSType":"linux","OperatingSystem":"Ubuntu 22.04","Architecture":"x86_64","SecurityOptions":["name=seccomp"]}', + TAG: "found", + DIGEST, + GH_RC: "0", + GH_ERR: "", IMAGES_RC: "0", IMAGES: "", LOCAL: "", @@ -199,26 +210,34 @@ const session = (stop = new AbortController(), graceMs?: number) => { return s; }; -test("the committed image.json is a valid lock, and a malformed one refuses", () => { - expect(readImageLock().digest).toMatch(/^sha256:[0-9a-f]{64}$/); - lock(key, "sha256:short"); - expect(() => readImageLock(root)).toThrow(Refusal); -}); - -test("a stale inputs key refuses before any docker command", async () => { - lock("0".repeat(64)); - expect(await failure(session().ensureImage())).toThrow(/stale image/); - await Bun.sleep(0); - expect(calls()).toBe(""); -}); - -test("a missing image is pulled by digest, with a private client config", async () => { - const s = session(); - expect(await s.ensureImage()).toBe(REF); +/** A session whose log lines (the fallback reason) the test reads. */ +const logged = () => { + const lines: string[] = []; + const s = new Session(new AbortController().signal, { root, log: (l) => lines.push(l) }); + sessions.push(s); + return { s, lines }; +}; +const WORKFLOW = "coder/xum/.github/workflows/bugbash-sandbox-image.yml"; + +test("a verified image of the stable tag is pulled by digest, with a private client config", async () => { + const { s, lines: log } = logged(); + expect(await s.ensureImage()).toEqual({ ref: REF, local: null }); + // One log line, and it names the image that runs. + expect(log).toHaveLength(1); + expect(log[0]).toContain(REF); const lines = calls().trim().split("\n"); - expect(lines.map((l) => l.split(" ")[0])).toEqual(["context", "info", "images", "pull", "image"]); - expect(lines[3]).toStartWith(`pull --quiet ${REF} `); - // After `context inspect`, docker gets no HOME and an empty config folder that cleanup removes. + const kinds = lines.map((l) => l.split(" ")[0]); + expect(kinds).toEqual(["context", "info", "buildx", "gh", "images", "pull", "image"]); + // The lookup is of this key's stable tag. gh accepts only main's publish workflow of coder/xum. + expect(lines[2]).toStartWith( + `buildx imagetools inspect --format {{json .Manifest.Digest}} ghcr.io/coder/xum-bugbash-sandbox:inputs-${key} ` + ); + expect(lines[3]).toStartWith( + `gh attestation verify oci://${REF} --repo coder/xum --bundle-from-oci --source-ref refs/heads/main --signer-workflow ${WORKFLOW} ` + ); + expect(lines[5]).toStartWith(`pull --quiet ${REF} `); + // After `context inspect`, docker (the lookup too) gets no HOME and an empty config folder + // that cleanup removes. const cfg = /cfg=(\S+)\]/.exec(lines[2])![1]; expect(lines[2]).toContain("[home= "); expect(fs.readdirSync(cfg)).toEqual([]); @@ -226,9 +245,9 @@ test("a missing image is pulled by digest, with a private client config", async expect(fs.existsSync(cfg)).toBe(false); }); -test("a present image is not pulled", async () => { +test("a present verified image is not pulled", async () => { fake({ IMAGES: "sha256:abc" }); - expect(await session().ensureImage()).toBe(REF); + expect(await session().ensureImage()).toEqual({ ref: REF, local: null }); expect(calls()).not.toContain("pull"); }); @@ -266,41 +285,71 @@ test("U6: an arm64 daemon refuses before the pull, naming the fix", async () => expect(error).toThrow(/linux\/amd64/); expect(error).toThrow(/aarch64/); expect(error).toThrow(/use an amd64 host/); - expect(calls()).not.toMatch(/^(images|pull)/m); + expect(calls()).not.toMatch(/^(images|pull|buildx)/m); }); test("U6: an amd64 daemon pulls the image", async () => { fake(); - expect(await session().ensureImage()).toBe(REF); + expect((await session().ensureImage()).ref).toBe(REF); expect(calls()).toContain("pull"); }); -// The stale-key local build: with `localBuild` and a stale key only, build.sh's local image replaces the pinned one. -// The launcher sets `localBuild` by the job kind (the launch tests below). -test("local build: a stale key with localBuild builds through build.sh, with the private client config", async () => { - lock("0".repeat(64)); - expect(await session().ensureImage({ localBuild: true })).toBe(LOCAL_ID); - const lines = calls().trim().split("\n"); - const build = lines.find((l) => l.startsWith("buildx build "))!; - expect(build).toContain(` --load --tag xum-bugbash-sandbox:inputs-${key} `); - const cfg = /cfg=(\S+)\]/.exec(lines[2])![1]; - expect(build).toEndWith(`[home= cfg=${cfg}]`); - // The label check is of the built image. Nothing is pulled, pushed or logged in to. - expect(lines.at(-1)).toStartWith(`image inspect --format {{json .Config.Labels}} ${LOCAL_ID} `); - expect(calls()).not.toMatch(/^(pull|push|login)|--push|ghcr\.io/m); -}); +// The local build: without a verified image of main's publish workflow for this key, build.sh +// builds one from this checkout. The reason goes to the log and to the caller (the warning). +test.each([ + ["a tag that is not published", { TAG: "missing" }, /:inputs-[0-9a-f]{64} is not published$/], + ["a failed tag lookup", { TAG: "down" }, /lookup of \S+ failed: .*no such host$/], + ["a lookup with no digest", { TAG: "garbage" }, /lookup of \S+ gave "sha256:6161/], + [ + "an attestation that does not verify", + { GH_RC: "1", GH_ERR: "Error: verifying with issuer: no matching attestations" }, + /@sha256:(61){32} is not verified \(gh exit 1\): Error: verifying .*no matching attestations$/, + ], + ["no gh login", { GH_RC: "4", GH_ERR: "try authenticating with: gh auth login" }, /no gh token/], +])( + "local build: %s builds through build.sh, with the private client config", + async (_name, over, why) => { + fake(over); + const { s, lines: log } = logged(); + const image = await s.ensureImage(); + expect(image.ref).toBe(LOCAL_ID); + expect(image.local).toMatch(why); + expect(log).toContain(`no verified image for inputs key ${key.slice(0, 16)}: ${image.local}`); + const lines = calls().trim().split("\n"); + const build = lines.find((l) => l.startsWith("buildx build "))!; + expect(build).toContain(` --load --tag xum-bugbash-sandbox:inputs-${key} `); + const cfg = /cfg=(\S+)\]/.exec(lines[2])![1]; + expect(build).toEndWith(`[home= cfg=${cfg}]`); + // gh runs only for a digest the lookup found. + expect(lines.some((l) => l.startsWith("gh "))).toBe("GH_RC" in over); + // The label check is of the built image. Nothing is pulled, pushed or logged in to. + expect(lines.at(-1)).toStartWith(`image inspect --format {{json .Config.Labels}} ${LOCAL_ID} `); + expect(calls()).not.toMatch(/^(pull|push|login)|--push/m); + } +); -test("local build: a present local image of the stale key is reused, not built again", async () => { - lock("0".repeat(64)); - fake({ LOCAL: LOCAL_ID }); - expect(await session().ensureImage({ localBuild: true })).toBe(LOCAL_ID); - expect(calls()).not.toMatch(/^(buildx|pull)/m); +test("local build: without gh on PATH, the image is built, not refused", async () => { + // A PATH of every system command but gh: the real gh must not answer for the fake. + const nogh = path.join(root, "nogh"); + fs.mkdirSync(nogh); + const linked = new Set(["gh"]); + for (const dir of (savedPath ?? "").split(":")) + for (const name of fs.existsSync(dir) ? fs.readdirSync(dir) : []) + if (!linked.has(name)) { + linked.add(name); + fs.symlinkSync(path.join(dir, name), path.join(nogh, name)); + } + fs.rmSync(path.join(bin, "gh")); + process.env.PATH = `${bin}:${nogh}`; + const image = await session().ensureImage(); + expect(image).toEqual({ ref: LOCAL_ID, local: "gh is not installed" }); + expect(calls()).toMatch(/^buildx build /m); }); -test("local build: a matching key takes the pinned digest and never builds, even with localBuild", async () => { - expect(await session().ensureImage({ localBuild: true })).toBe(REF); - expect(calls()).toMatch(/^pull --quiet /m); - expect(calls()).not.toMatch(/^buildx/m); +test("local build: a present local image of the key is reused, not built again", async () => { + fake({ TAG: "missing", LOCAL: LOCAL_ID }); + expect((await session().ensureImage()).ref).toBe(LOCAL_ID); + expect(calls()).not.toMatch(/^(buildx build|pull)/m); }); test.each([ @@ -311,9 +360,8 @@ test.each([ ], ["a failed build", { BUILD: "fail" }, /build\.sh: build failed/], ])("local build: a local image with %s refuses", async (_name, over, message) => { - lock("0".repeat(64)); - fake(over); - expect(await failure(session().ensureImage({ localBuild: true }))).toThrow(message); + fake({ TAG: "missing", ...over }); + expect(await failure(session().ensureImage())).toThrow(message); }); const JOB = { name: "xbb-1", owner: "boot:pid", checkout: "abc" }; @@ -1985,7 +2033,7 @@ test("U1: the Docker preflight reports an unknown cleanup instead of a refusal o expect(await checkDocker(o)).toBe("unknown: process groups 9 still run"); cleanup.mockResolvedValue("none"); expect(await checkDocker(o).catch((e: unknown) => e)).toBeInstanceOf(Refusal); - ensure.mockResolvedValue("ref"); + ensure.mockResolvedValue({ ref: "ref", local: null }); expect(await checkDocker(o)).toBe("none"); // A stop that lands during the cleanup still ends the preflight as a stop. const stop = new AbortController(); @@ -2002,30 +2050,48 @@ test("U1: the Docker preflight reports an unknown cleanup instead of a refusal o } }); -// The local build's gate in the launcher. Only the self-check and exact-step repros may use a local image, and -// only with BUGBASH_SANDBOX_BUILD_STALE=1 (CI's bugbash-sandbox job). -const OPT_IN = { BUGBASH_SANDBOX_BUILD_STALE: "1" }; +// Every job kind resolves its image the same way: main's verified image, else a local build of +// its checkout with a warning. Only the image differs, never a container flag. const SELF_CHECK_ARGS = ["self-check", "--output", ".e2e/s"]; +const KINDS = { + "an exact-step repro": "repro", + "a self-check": "self-check", + "an explore charter": "explore", + "the MCP Apps suite": MCP_APPS, + "the story suite": STORIES, + "the agent repros": AGENT_REPROS, +} as const; +type Kind = (typeof KINDS)[keyof typeof KINDS]; /** One job's `docker run` line and stderr, without its per-job values (name, folders, nonce). */ -async function jobRun(kind: "repro" | "self-check", env: Record) { - fs.rmSync(path.join(root, "tests/bugbash/.e2e"), { recursive: true, force: true }); +async function jobRun(kind: Kind, env: Record, over: Record = {}) { + for (const dir of ["tests/bugbash/.e2e", "tests/stories/.e2e"]) + fs.rmSync(path.join(root, dir), { recursive: true, force: true }); fs.rmSync(path.join(bin, "calls.log"), { force: true }); - const stderr: string[] = []; + let stderr: string[] = []; const error = spyOn(console, "error").mockImplementation( (line: string) => void stderr.push(line) ); try { - const outcome = - kind === "repro" - ? await launchJob(ARGS, { - root: fs.realpathSync(root), - cwd: prepare(), - env: { ...HOST_ENV, ...env }, - stop: new AbortController().signal, - log: () => undefined, - }) - : (await selfCheckLaunch({}, SELF_CHECK_ARGS, env)).outcome; - expect(outcome).toMatchObject({ code: 7, cleanup: "removed" }); + if (typeof kind !== "string") { + error.mockRestore(); // mcpLaunch reads stderr itself + const driven = await mcpLaunch(over, env, undefined, undefined, kind); + expect(driven.code).toBe(7); + stderr = driven.logged; + } else { + const outcome = + kind === "repro" + ? await launchJob(ARGS, { + root: fs.realpathSync(root), + cwd: prepare(over), + env: { ...HOST_ENV, ...env }, + stop: new AbortController().signal, + log: () => undefined, + }) + : kind === "self-check" + ? (await selfCheckLaunch(over, SELF_CHECK_ARGS, env)).outcome + : (await exploreLaunch(over, env)).outcome; + expect(outcome).toMatchObject({ code: 7, cleanup: "removed" }); + } } finally { error.mockRestore(); } @@ -2038,64 +2104,70 @@ async function jobRun(kind: "repro" | "self-check", env: Record) return { run, stderr, built: /^buildx build /m.test(calls()) }; } -test.each(["repro", "self-check"] as const)( - "local build: with the opt-in, a stale key gives the %s job a local image with the same container flags", - async (kind) => { - const pinned = await jobRun(kind, { ...OPT_IN, GITHUB_ACTIONS: "true" }); - expect(pinned.built).toBe(false); - expect(pinned.stderr.filter((l) => l.startsWith("::warning::"))).toEqual([]); - lock("0".repeat(64)); - const local = await jobRun(kind, { ...OPT_IN, GITHUB_ACTIONS: "true" }); +test.each(Object.entries(KINDS))( + "local build: without a verified image, %s gets a local image with the same container flags", + async (_name, kind) => { + const verified = await jobRun(kind, { GITHUB_ACTIONS: "true" }); + expect(verified.built).toBe(false); + expect(verified.run).toContain(` ${REF} `); + expect(verified.stderr.filter((l) => l.startsWith("::warning::"))).toEqual([]); + const local = await jobRun(kind, { GITHUB_ACTIONS: "true" }, { TAG: "missing" }); expect(local.built).toBe(true); // Only the image differs: every hardening flag stays. expect(local.run).toContain(` ${LOCAL_ID} `); - expect(local.run.replace(LOCAL_ID, REF)).toBe(pinned.run); - // One annotation in CI's log. - expect(local.stderr.filter((l) => l.startsWith("::warning::"))).toHaveLength(1); + expect(local.run.replace(LOCAL_ID, REF)).toBe(verified.run); + // One annotation in CI's log, with the reason. + const warnings = local.stderr.filter((l) => l.startsWith("::warning::")); + expect(warnings).toHaveLength(1); + expect(warnings[0]).toMatch(/LOCAL IMAGE sha256:b1b1.*:inputs-[0-9a-f]{64} is not published/); } ); -test.each([ - ["an explore charter", "1"], - ["the MCP Apps suite", "1"], - ["the story suite", "1"], - ["the agent repros", "1"], - ["a repro whose opt-in is not exactly 1", "true"], -])("local build: %s refuses a stale key, and nothing is built", async (job, value) => { - lock("0".repeat(64)); - const env = { BUGBASH_SANDBOX_BUILD_STALE: value }; - const driven: Record = { - "the MCP Apps suite": MCP_APPS, - "the story suite": STORIES, - "the agent repros": AGENT_REPROS, +test("the gh login env reaches gh on the host, and never docker or the job container", async () => { + // A token, a custom config folder, and the keyring's session bus. + const gh = { + GH_TOKEN: "ghs_hosttoken", + GH_CONFIG_DIR: "/home/u/.gh-custom", + XDG_CONFIG_HOME: "/home/u/.cfg", + DBUS_SESSION_BUS_ADDRESS: "unix:path=/run/user/1000/bus", + XDG_RUNTIME_DIR: "/run/user/1000", }; - const error = - job === "an explore charter" - ? ((await exploreLaunch({}, env)).outcome as { error: unknown }).error - : job in driven - ? (await mcpLaunch({}, env, undefined, undefined, driven[job])).code - : await launchIn({}, new AbortController(), { ...HOST_ENV, ...env }).catch( - (e: unknown) => e - ); - expect(error).toBeInstanceOf(Refusal); - expect((error as Error).message).toMatch(/stale image/); - expect(calls()).not.toMatch(/^buildx/m); -}); - -test("local build: the Docker preflight refuses a stale key, even with the opt-in", async () => { - lock("0".repeat(64)); - const saved = process.env.BUGBASH_SANDBOX_BUILD_STALE; - process.env.BUGBASH_SANDBOX_BUILD_STALE = "1"; + const saved = Object.fromEntries(Object.keys(gh).map((name) => [name, process.env[name]])); + Object.assign(process.env, gh); try { - const stop = new AbortController().signal; - const o = { root: fs.realpathSync(root), stop }; - // run.ts's preflight (make bug-bash), then launch.ts --preflight (make mcp-apps-e2e). - expect(await failure(checkDocker(o))).toThrow(/stale image/); - const env = { ...HOST_ENV, ...OPT_IN }; - expect(await failure(preflight(ARGS, { ...o, cwd: prepare(), env }))).toThrow(/stale image/); - expect(calls()).not.toMatch(/^buildx/m); + // The launcher's env is the host env (launch.ts's main passes process.env). + const { run } = await jobRun("repro", gh); + expect(run).toContain(` ${REF} `); + expect(calls()).toContain( + "[token=ghs_hosttoken config=/home/u/.gh-custom xdg=/home/u/.cfg " + + "bus=unix:path=/run/user/1000/bus run=/run/user/1000]" + ); + expect(calls()).not.toMatch(/docker got/); + for (const [name, value] of Object.entries(gh)) { + expect(run).not.toContain(name); + expect(run).not.toContain(value); + } + } finally { + for (const [name, value] of Object.entries(saved)) + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } +}); + +test("local build: the Docker preflight builds the local image instead of refusing", async () => { + const stop = new AbortController().signal; + const o = { root: fs.realpathSync(root), stop }; + // run.ts's preflight (make bug-bash), then launch.ts --preflight (make mcp-apps-e2e). + const error = spyOn(console, "error").mockImplementation(() => undefined); + try { + fake({ TAG: "missing" }); + expect(await checkDocker(o)).toBe("none"); + expect(calls()).toMatch(/^buildx build /m); + const cwd = prepare({ TAG: "missing" }); + fs.rmSync(path.join(bin, "calls.log"), { force: true }); + expect(await preflight(ARGS, { ...o, cwd, env: HOST_ENV })).toBe("none"); + expect(calls()).toMatch(/^buildx build /m); } finally { - if (saved === undefined) delete process.env.BUGBASH_SANDBOX_BUILD_STALE; - else process.env.BUGBASH_SANDBOX_BUILD_STALE = saved; + error.mockRestore(); } }); diff --git a/tests/bugbash/sandbox/runner.ts b/tests/bugbash/sandbox/runner.ts index 427311124dd..28c1dc08c0e 100644 --- a/tests/bugbash/sandbox/runner.ts +++ b/tests/bugbash/sandbox/runner.ts @@ -1,13 +1,12 @@ /** * The runner library of the bug-bash sandbox (#5714). It finds a local Docker daemon, gets the - * pinned image, runs commands as tracked async children, runs the job container, and cleans up - * after a job. launch.ts is its entry point. + * image, runs commands as tracked async children, runs the job container, and cleans up after a + * job. launch.ts is its entry point. * - * The trust anchor is the digest in image.json, which a reviewed pull request sets. The runner - * pulls only `name@digest`. It refuses when the inputs key of this checkout (`build.sh --key`) - * differs from image.json: that image was built for other inputs, and a person must publish a - * new one (workflow "Bug-bash sandbox image") and update image.json. The one exception is a - * local build for the jobs that the launcher allows (ensureImage's `localBuild`). + * The image: main's publish workflow ("Bug-bash sandbox image") pushes one stable tag per inputs + * key (`build.sh --key`) and attests it. ensureImage() looks up the tag of this checkout's key, + * verifies the attestation with gh, and pulls only `name@digest`. Without a verified image it + * builds one from this checkout with build.sh (or reuses that local build). */ import { spawn, type ChildProcess } from "node:child_process"; import type { Readable } from "node:stream"; @@ -17,6 +16,11 @@ import * as path from "node:path"; const ROOT = fs.realpathSync(path.resolve(import.meta.dir, "../../..")); export const IMAGE = "ghcr.io/coder/xum-bugbash-sandbox"; +/** The only workflow whose attestation ensureImage() accepts, and only from main. */ +const PUBLISH_WORKFLOW = "coder/xum/.github/workflows/bugbash-sandbox-image.yml"; +/** The host env names, besides PATH and HOME, that gh needs to find the user's login. */ +const GH_LOGIN_ENV = ["GH_TOKEN", "GH_CONFIG_DIR", "XDG_CONFIG_HOME", "DBUS_SESSION_BUS_ADDRESS", + "XDG_RUNTIME_DIR"]; // prettier-ignore /** How long build.sh gets for a local image: a fresh build with no cache. */ const BUILD_MS = 25 * 60_000; /** How long a child gets after SIGTERM before SIGKILL. */ @@ -41,27 +45,6 @@ export class Stopped extends Error { } } -export interface ImageLock { - image: string; - digest: string; - inputsKey: string; - playwrightCore: string; - publishedFrom: string; -} - -export function readImageLock(root = ROOT): ImageLock { - const file = path.join(root, "tests/bugbash/sandbox/image.json"); - const lock = JSON.parse(fs.readFileSync(file, "utf8")) as Partial; - const valid = - lock.image === IMAGE && - /^sha256:[0-9a-f]{64}$/.test(lock.digest ?? "") && - /^[0-9a-f]{64}$/.test(lock.inputsKey ?? "") && - /^\d+\.\d+\.\d+$/.test(lock.playwrightCore ?? "") && - /^[0-9a-f]{40}$/.test(lock.publishedFrom ?? ""); - if (!valid) throw new Refusal("tests/bugbash/sandbox/image.json is not a valid image lock"); - return lock as ImageLock; -} - interface Result { ok: boolean; /** The shell convention: the exit code, or 128 + the number of the signal that ended it. */ @@ -138,27 +121,31 @@ export class Session { } /** - * The pinned image, pulled when missing, as `name@digest`. With `localBuild` and a stale key - * only, a local image of this checkout instead, as its image ID. A matching key always - * takes the pinned image. + * The image of this checkout's inputs key. `local` is null for the image that main's publish + * workflow built and attested, pulled when missing, as `name@digest`. Otherwise `local` is the + * reason it is not that image, and the image is a local build of this checkout, as its ID. + * + * A local build adds no power over the published image: each job already stages the + * checkout's git-listed files and mounts its dist/ and node_modules/ (launch.ts). So a job runs + * either an image that main's publish workflow built and attested, or one built from its own + * checkout, never one that a registry, a tag or another branch chose. */ - async ensureImage(options: { localBuild?: boolean } = {}): Promise { - const lock = readImageLock(this.#root); + async ensureImage(): Promise<{ ref: string; local: string | null }> { const key = await this.#checkoutKey(); - const stale = key !== lock.inputsKey; - if (stale && options.localBuild !== true) - throw new Refusal( - `stale image: image.json has inputs key ${lock.inputsKey.slice(0, 16)}, this checkout ` + - `${key.slice(0, 16)}. Publish a new image, then update tests/bugbash/sandbox/image.json.` - ); await this.#connect(); this.#checkArchitecture(); - let ref = `${lock.image}@${lock.digest}`; - if (stale) ref = await this.#localImage(key); - else { + const published = await this.#publishedImage(key); + let ref: string; + if (published.ok) { + ref = published.ref; // An empty list with exit 0 means absent. Any failure refuses: it is not "absent". const listed = await this.#must(["images", "--no-trunc", "--quiet", ref], 15_000); if (listed.stdout === "") await this.#must(["pull", "--quiet", ref], 10 * 60_000); + // Names the image in the job log, as the fallback below names its reason. + this.#log(`verified image for inputs key ${key.slice(0, 16)}: ${ref}`); + } else { + this.#log(`no verified image for inputs key ${key.slice(0, 16)}: ${published.why}`); + ref = await this.#localImage(key); } // The same label check for both: the image must carry the key of this checkout. const inspect = ["image", "inspect", "--format", "{{json .Config.Labels}}", ref]; @@ -169,7 +156,56 @@ export class Session { const label = labels?.["org.xum.bugbash.inputs"]; if (label !== key) throw new Refusal(`${ref}: label org.xum.bugbash.inputs is ${label}, not ${key}`); - return ref; + return { ref, local: published.ok ? null : published.why }; + } + + /** + * The stable tag of `key`, looked up anonymously (the private client config), and its + * attestation, verified by gh on the host. Any outcome but a verified digest gives the reason. + * A stop still throws Stopped: it is no reason to build. + */ + async #publishedImage( + key: string + ): Promise<{ ok: true; ref: string } | { ok: false; why: string }> { + const tag = `${IMAGE}:inputs-${key}`; + const format = "{{json .Manifest.Digest}}"; + const args = ["buildx", "imagetools", "inspect", "--format", format, tag]; + const look = await this.#job("docker", args, this.#client!, 60_000); + // A missing tag fails with stderr `ERROR: : not found`. + if (!look.ok && look.error.endsWith(": not found")) + return { ok: false, why: `${tag} is not published` }; + if (!look.ok) return { ok: false, why: `the lookup of ${tag} failed: ${lastLine(look.error)}` }; + const digest = /^"(sha256:[0-9a-f]{64})"$/.exec(look.stdout)?.[1]; + if (digest == null) + return { ok: false, why: `the lookup of ${tag} gave ${JSON.stringify(look.stdout)}` }; + const ref = `${IMAGE}@${digest}`; + // gh gets the user's login and nothing else of this env: GH_TOKEN, or its config folder + // (GH_CONFIG_DIR, else XDG_CONFIG_HOME/gh, else HOME/.config/gh) with the token in it or in + // the desktop keyring, which gh reaches over the session bus (DBUS_SESSION_BUS_ADDRESS, else + // XDG_RUNTIME_DIR/bus). None of them reaches Docker or the container (containerEnv drops + // the token, and its allowlist has none of the others). + const env: Record = { + PATH: process.env.PATH ?? "", + HOME: process.env.HOME ?? "", + }; + for (const name of GH_LOGIN_ENV) if (process.env[name] != null) env[name] = process.env[name]; + // prettier-ignore + const verify = await this.#job("gh", ["attestation", "verify", `oci://${ref}`, "--repo", "coder/xum", + "--bundle-from-oci", "--source-ref", "refs/heads/main", "--signer-workflow", PUBLISH_WORKFLOW], env, 120_000); + if (verify.ok) return { ok: true, ref }; + // Node says `spawn gh ENOENT`, Bun `Executable not found in $PATH: "gh"`. + if (/ENOENT|Executable not found/.test(verify.error)) + return { ok: false, why: "gh is not installed" }; + // gh exits 4 when it has no login: CI sets GH_TOKEN for the launcher steps. + if (verify.code === 4) + return { + ok: false, + why: `${ref} is not verified: no gh token (set GH_TOKEN or run gh auth login)`, + }; + return { + ok: false, + why: `${ref} is not verified (gh exit ${verify.code}): ${lastLine(verify.error)}`, + }; } /** @@ -433,16 +469,16 @@ export class Session { } /** - * A pull request that changes the image inputs has no published image yet (only main - * publishes), so build.sh builds one from its committed source into the local Docker. It runs - * with the private client config: no registry login, and nothing is pushed. A present image + * A pull request that changes the image inputs has no published image yet (main publishes it + * after the merge), so build.sh builds one from its committed source into the local Docker. It + * runs with the private client config: no registry login, and nothing is pushed. A present image * under build.sh's local tag is reused: the self-check runs three jobs, then CI runs the repros. */ async #localImage(key: string): Promise { const tag = `xum-bugbash-sandbox:inputs-${key}`; // build.sh's local tag let id = (await this.#must(["images", "--no-trunc", "--quiet", tag], 15_000)).stdout; if (id === "") { - this.#log(`stale image key: building ${tag} with build.sh (up to 25 minutes)`); + this.#log(`building ${tag} with build.sh (up to 25 minutes)`); const script = path.join(this.#root, "tests/bugbash/sandbox/build.sh"); const built = await this.#job(script, [], this.#client!, BUILD_MS); // The tail only: the error holds the whole build log. @@ -590,6 +626,9 @@ export class Session { } } +/** The last line of a command's stderr: a reason line, not the whole log. */ +const lastLine = (error: string) => error.split("\n").at(-1) ?? ""; + // ESRCH: the group is empty. EPERM: its members belong to another user, so it is not a group // that this session started (every command runs as this user). Neither is an error here: these // run from timers, where a throw would end the launcher before cleanup. diff --git a/tests/bugbash/sandbox/selfCheckRun.ts b/tests/bugbash/sandbox/selfCheckRun.ts index c46daec9cbd..db30e47cbae 100644 --- a/tests/bugbash/sandbox/selfCheckRun.ts +++ b/tests/bugbash/sandbox/selfCheckRun.ts @@ -97,8 +97,7 @@ export async function runSelfCheck( BUGBASH_MODEL: "anthropic:claude-haiku-5-5", PATH: process.env.PATH, HOME: process.env.HOME, - // CI's opt-in to a local image on a stale key, and its warning annotation. - BUGBASH_SANDBOX_BUILD_STALE: process.env.BUGBASH_SANDBOX_BUILD_STALE, + // CI's warning annotation for a local image. GITHUB_ACTIONS: process.env.GITHUB_ACTIONS, }; const stamp = `${Date.now()}-${process.pid}`;