From 37b9627f0905d0f5f63d418a4a843484ef345e3f Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 27 Oct 2026 08:41:25 +0000 Subject: [PATCH 1/3] ci: publish the sandbox image on push to main, once per inputs key The publish job of the Bug-bash sandbox image workflow now also runs on pushes to main that change an image input. It skips a key whose stable tag inputs- exists, and otherwise builds fresh, pushes, points the stable tag at the build's digest and attests it. --- .github/workflows/bugbash-sandbox-image.yml | 52 +++++++++++++++---- .xum/skills/bug-bash/SKILL.md | 6 +-- docs/AGENTS.md | 2 +- .../builtInSkillContent.generated.ts | 2 +- 4 files changed, 47 insertions(+), 15 deletions(-) diff --git a/.github/workflows/bugbash-sandbox-image.yml b/.github/workflows/bugbash-sandbox-image.yml index ac695826b3c..b780931dd5c 100644 --- a/.github/workflows/bugbash-sandbox-image.yml +++ b/.github/workflows/bugbash-sandbox-image.yml @@ -1,16 +1,26 @@ # The bug-bash sandbox image (#5714): tests/bugbash/sandbox/Dockerfile, built by build.sh. # -# Pull requests build it with a read-only token and push nothing. Publishing is manual: only a -# workflow_dispatch run on main publishes, from that run's exact commit, with no cache and no -# artifact from any other run. Each publish builds fresh, and the attestation names that build's -# own output. A person then copies the digest from the job summary into image.json in a pull -# request: no workflow writes to the repo. Nothing here deletes an image. +# Pull requests build it with a read-only token and push nothing. Only main publishes: a push to +# main that changes an image input, or a manual run on main, from that run's exact commit, with no +# cache and no artifact from any other run. Publishing is create-if-missing (the user's pick, so +# an image change needs no person): each inputs key gets one stable tag, inputs-. A run whose +# key has that tag pushes nothing. Otherwise it builds fresh, pushes, points the stable tag at that +# build's own digest, and attests the digest. Nothing here deletes an image. # -# Until a new image is published and its digest PR merges, runners refuse a checkout whose image -# inputs changed (stale inputs key). +# Runners still trust only the digest in image.json: a person copies it from the job summary into +# image.json in a pull request (no workflow writes to the repo). Until that PR merges, runners +# refuse a checkout whose image inputs changed (stale inputs key). name: Bug-bash sandbox image on: + # The image inputs (build.sh --key) and this file: a change here publishes the current key once. + push: + branches: [main] + paths: + - tests/bugbash/sandbox/Dockerfile + - tests/bugbash/sandbox/build.sh + - bun.lock + - .github/workflows/bugbash-sandbox-image.yml pull_request: paths: - tests/bugbash/sandbox/Dockerfile @@ -18,6 +28,7 @@ on: - bun.lock - Makefile - .github/workflows/bugbash-sandbox-image.yml + # A re-run on main, for example after a failed publish. workflow_dispatch: {} permissions: {} @@ -46,11 +57,13 @@ jobs: publish: name: Publish - if: github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' + # A push to main or a manual run on main, never a pull request. The concurrency group above + # runs one publish at a time and never cancels it, so two pushes of one key cannot both push. + if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' runs-on: ubuntu-latest permissions: contents: read - # Push ghcr.io/coder/xum-bugbash-sandbox. Only this job, only a manual run on main. + # Push ghcr.io/coder/xum-bugbash-sandbox. Only this job, only on main. packages: write # Build provenance for audit (the runner trusts the digest in image.json). id-token: write @@ -69,11 +82,28 @@ jobs: - name: Build and push id: push run: | + # Create if missing: a key whose stable tag exists pushes nothing and sets no outputs. + # "not found" is the only lookup error that means missing: any other error fails the job. + key=$(make -s bugbash-sandbox-key) + tag="ghcr.io/coder/xum-bugbash-sandbox:inputs-$key" + if docker buildx imagetools inspect "$tag" > /dev/null 2> "$RUNNER_TEMP/inspect.err"; then + echo "\`$tag\` exists: nothing to publish." | tee -a "$GITHUB_STEP_SUMMARY" + exit 0 + fi + if ! grep -q ': not found$' "$RUNNER_TEMP/inspect.err"; then + cat "$RUNNER_TEMP/inspect.err" >&2 + exit 1 + fi make -s bugbash-sandbox-publish > image.json - # The subject of the attestation is this build's own output (image.json). + # The subject of the attestation is this build's own output (image.json). The stable tag + # names the same digest, so the attestation covers it too. + ref="$(jq -er .image image.json)@$(jq -er .digest image.json)" + docker buildx imagetools create --tag "$tag" "$ref" echo "image=$(jq -er .image image.json)" >> "$GITHUB_OUTPUT" echo "digest=$(jq -er .digest image.json)" >> "$GITHUB_OUTPUT" { + echo "Stable tag: \`$tag\`" + echo echo '### tests/bugbash/sandbox/image.json' echo echo '```json' @@ -81,6 +111,8 @@ jobs: echo '```' } >> "$GITHUB_STEP_SUMMARY" - name: Attest + # Skipped when the key already had an image: the push step set no subject. + if: steps.push.outputs.digest != '' uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 with: subject-name: ${{ steps.push.outputs.image }} diff --git a/.xum/skills/bug-bash/SKILL.md b/.xum/skills/bug-bash/SKILL.md index 75f2274d091..a360e6e2a8f 100644 --- a/.xum/skills/bug-bash/SKILL.md +++ b/.xum/skills/bug-bash/SKILL.md @@ -37,9 +37,9 @@ Xum harness. Where the two differ, follow this skill. suite, the story tests, the agent repros and the preflight of `make bug-bash`, `make mcp-apps-e2e` and `make test-stories` always refuse a stale key. - An image change lands in three steps: the PR that changes the Dockerfile, `build.sh` or - playwright-core passes CI on a local build. After it merges, a maintainer runs the manual - workflow "Bug-bash sandbox image" on main. Then a reviewed PR updates `image.json` to pin the - new digest. An e2e update that keeps playwright-core keeps the key, so the image stays valid. + playwright-core passes CI on a local build. Its merge push runs the workflow "Bug-bash sandbox + image" on main, which publishes the image once for each inputs key (a manual run on main + retries a failed publish). Then a reviewed PR updates `image.json` to pin the new digest. An e2e update that keeps playwright-core keeps the key, so the image stays valid. - The container gets `--network none`, `--read-only`, `--cap-drop ALL`, no-new-privileges, a non-root user, 4 GB memory and tmpfs scratch folders. It reads copies of the git-listed files, plus `dist/` and `node_modules/` read-only. An env allowlist passes no credential. diff --git a/docs/AGENTS.md b/docs/AGENTS.md index c90325856a3..a6e42fc43d3 100644 --- a/docs/AGENTS.md +++ b/docs/AGENTS.md @@ -259,7 +259,7 @@ Freely make breaking changes, and reorganize / cleanup IPC as needed. - To run a bug bash, the MCP Apps suite, the story tests or the sandbox checks, read the `bug-bash` skill (`.xum/skills/bug-bash/SKILL.md`). It covers the host needs, the env, costs, options, results, exit codes and leftovers. - Host pause (#5714): model-driven runs (`make bug-bash`, `make mcp-apps-e2e`, `make test-stories`, any `e2e explore` and every e2e command with `e2e.mcpapps.config.ts`, `tests/stories/e2e.config.ts` or `tests/stories/e2e.electron.config.ts`) run only inside the bug-bash sandbox, one Docker container per job. A model-driven e2e process on the host refuses with exit 2 (`tests/bugbash/hostPause.ts`), and outside the sandbox an `agent.*` step fails with `MODEL_UNAVAILABLE`. No override exists. -- The sandbox image is pinned by digest in `tests/bugbash/sandbox/image.json`. A change to `tests/bugbash/sandbox/Dockerfile`, `tests/bugbash/sandbox/build.sh` or the playwright-core version of `@e2e-dev/web` in `bun.lock` changes the inputs key (`make bugbash-sandbox-key`), and the runner then refuses the image until the manual "Bug-bash sandbox image" workflow publishes a new one and a PR updates `image.json`. One exception: with `BUGBASH_SANDBOX_BUILD_STALE=1`, which only CI's `Test / Bug-bash sandbox` job sets, the self-check and the exact-step repros build the image locally with `build.sh` and warn. So an image change lands in three steps: the Dockerfile PR passes CI on a local build, a maintainer runs the publish workflow on main, and an `image.json` PR pins the new digest. `make bug-bash`, `make mcp-apps-e2e`, `make test-stories` and their preflight always refuse a stale key. +- The sandbox image is pinned by digest in `tests/bugbash/sandbox/image.json`. A change to `tests/bugbash/sandbox/Dockerfile`, `tests/bugbash/sandbox/build.sh` or the playwright-core version of `@e2e-dev/web` in `bun.lock` changes the inputs key (`make bugbash-sandbox-key`), and the runner then refuses the image until the "Bug-bash sandbox image" workflow publishes a new one (it runs on the push to main) and a PR updates `image.json`. One exception: with `BUGBASH_SANDBOX_BUILD_STALE=1`, which only CI's `Test / Bug-bash sandbox` job sets, the self-check and the exact-step repros build the image locally with `build.sh` and warn. So an image change lands in three steps: the Dockerfile PR passes CI on a local build, the merge push makes main publish the image (a manual run on main retries a failed publish), and an `image.json` PR pins the new digest. `make bug-bash`, `make mcp-apps-e2e`, `make test-stories` and their preflight always refuse a stale key. - Every model-driven job and every real-AI app run with `XUM_DISABLE_AGENT_TOOLS=1`, `XUM_DISABLE_TERMINALS=1` and `XUM_DISABLE_PROJECT_AUTOMATION=1`, because AI replies are untrusted input for the explorer. Do not turn them off for a bug bash without a security review. Callers with their own `streamText` tools (refinement, memory harvest, continuous compaction) do not check `XUM_DISABLE_AGENT_TOOLS`. None of them runs shell commands in the seeded app, so a new caller of that kind needs the same check. Settings can still add stdio MCP servers and custom editor commands, so charters must keep explorers out of them. - One reviewed exception (#7027, #7029): `BUGBASH_SCENARIO=scripted-tools` runs the app with agent tools on, so repros can see the real tool block. Its only model is a fixed loopback script (`tests/bugbash/scriptedModel.ts`) that ignores typed text and calls only `todo_write` and `file_read` of the seeded README.md. `startApp.ts` refuses it outside the sandbox, in a model-driven job and with a mounted proxy socket, and keeps terminals and project automation off. Only the exact-step sandbox job (`make test-bugbash-repros-sandbox`: no network, no proxy, no explorer model) selects its `tools-on` repros. Do not add tool calls to that script, or run it with a model, without a new security review. - A finding is a bug only once a repro fails for the reported reason. Put it in `tests/bugbash/repros/.e2e.ts` (tracked). Use exact `screen`/`browser`/`expect` steps where they work. A repro may use `agent.*` steps only when each of its tests has the `agent` tag (`tests/bugbash/reproRules.test.ts` enforces this): the free runs pass `--exclude-tag agent`, so only a model-driven sandbox job runs it: `make test-stories` (needs `BUGBASH_BUDGET_USD`). CI's `Stories` workflow (`.github/workflows/stories.yml`) runs that target with Sonnet 5.5 in the merge queue (pr.yml's `Test / Stories` job, which `Required` counts only when the repo variable `STORIES_MODE` is `block`), nightly on main and on demand, and with Haiku 5.5 on each push to a PR that is not a draft (`Test / Stories (Haiku)`, where Sonnet 5.5 judges the story tests' asserts and Haiku the agent repros', and which `Required` counts only when `STORIES_PR_MODE` is `block`). That target also runs the story tests, user stories that an agent drives in `tests/stories/*.e2e.ts`, and the Electron stories in `tests/stories/electron/*.e2e.ts` on Xum Electron under Xvfb (it runs `make electron-binary` first). Prove it: restore the bug in a scratch edit, rebuild, see `ASSERTION_FAILED` on the bug's own assertion, then restore and see it pass. diff --git a/src/node/services/agentSkills/builtInSkillContent.generated.ts b/src/node/services/agentSkills/builtInSkillContent.generated.ts index 9190ebd351b..31803a649dc 100644 --- a/src/node/services/agentSkills/builtInSkillContent.generated.ts +++ b/src/node/services/agentSkills/builtInSkillContent.generated.ts @@ -2316,7 +2316,7 @@ export const BUILTIN_SKILL_FILES: Record> = { "", "- To run a bug bash, the MCP Apps suite, the story tests or the sandbox checks, read the `bug-bash` skill (`.xum/skills/bug-bash/SKILL.md`). It covers the host needs, the env, costs, options, results, exit codes and leftovers.", "- Host pause (#5714): model-driven runs (`make bug-bash`, `make mcp-apps-e2e`, `make test-stories`, any `e2e explore` and every e2e command with `e2e.mcpapps.config.ts`, `tests/stories/e2e.config.ts` or `tests/stories/e2e.electron.config.ts`) run only inside the bug-bash sandbox, one Docker container per job. A model-driven e2e process on the host refuses with exit 2 (`tests/bugbash/hostPause.ts`), and outside the sandbox an `agent.*` step fails with `MODEL_UNAVAILABLE`. No override exists.", - '- The sandbox image is pinned by digest in `tests/bugbash/sandbox/image.json`. A change to `tests/bugbash/sandbox/Dockerfile`, `tests/bugbash/sandbox/build.sh` or the playwright-core version of `@e2e-dev/web` in `bun.lock` changes the inputs key (`make bugbash-sandbox-key`), and the runner then refuses the image until the manual "Bug-bash sandbox image" workflow publishes a new one and a PR updates `image.json`. One exception: with `BUGBASH_SANDBOX_BUILD_STALE=1`, which only CI\'s `Test / Bug-bash sandbox` job sets, the self-check and the exact-step repros build the image locally with `build.sh` and warn. So an image change lands in three steps: the Dockerfile PR passes CI on a local build, a maintainer runs the publish workflow on main, and an `image.json` PR pins the new digest. `make bug-bash`, `make mcp-apps-e2e`, `make test-stories` and their preflight always refuse a stale key.', + '- The sandbox image is pinned by digest in `tests/bugbash/sandbox/image.json`. A change to `tests/bugbash/sandbox/Dockerfile`, `tests/bugbash/sandbox/build.sh` or the playwright-core version of `@e2e-dev/web` in `bun.lock` changes the inputs key (`make bugbash-sandbox-key`), and the runner then refuses the image until the "Bug-bash sandbox image" workflow publishes a new one (it runs on the push to main) and a PR updates `image.json`. One exception: with `BUGBASH_SANDBOX_BUILD_STALE=1`, which only CI\'s `Test / Bug-bash sandbox` job sets, the self-check and the exact-step repros build the image locally with `build.sh` and warn. So an image change lands in three steps: the Dockerfile PR passes CI on a local build, the merge push makes main publish the image (a manual run on main retries a failed publish), and an `image.json` PR pins the new digest. `make bug-bash`, `make mcp-apps-e2e`, `make test-stories` and their preflight always refuse a stale key.', "- Every model-driven job and every real-AI app run with `XUM_DISABLE_AGENT_TOOLS=1`, `XUM_DISABLE_TERMINALS=1` and `XUM_DISABLE_PROJECT_AUTOMATION=1`, because AI replies are untrusted input for the explorer. Do not turn them off for a bug bash without a security review. Callers with their own `streamText` tools (refinement, memory harvest, continuous compaction) do not check `XUM_DISABLE_AGENT_TOOLS`. None of them runs shell commands in the seeded app, so a new caller of that kind needs the same check. Settings can still add stdio MCP servers and custom editor commands, so charters must keep explorers out of them.", "- One reviewed exception (#7027, #7029): `BUGBASH_SCENARIO=scripted-tools` runs the app with agent tools on, so repros can see the real tool block. Its only model is a fixed loopback script (`tests/bugbash/scriptedModel.ts`) that ignores typed text and calls only `todo_write` and `file_read` of the seeded README.md. `startApp.ts` refuses it outside the sandbox, in a model-driven job and with a mounted proxy socket, and keeps terminals and project automation off. Only the exact-step sandbox job (`make test-bugbash-repros-sandbox`: no network, no proxy, no explorer model) selects its `tools-on` repros. Do not add tool calls to that script, or run it with a model, without a new security review.", "- A finding is a bug only once a repro fails for the reported reason. Put it in `tests/bugbash/repros/.e2e.ts` (tracked). Use exact `screen`/`browser`/`expect` steps where they work. A repro may use `agent.*` steps only when each of its tests has the `agent` tag (`tests/bugbash/reproRules.test.ts` enforces this): the free runs pass `--exclude-tag agent`, so only a model-driven sandbox job runs it: `make test-stories` (needs `BUGBASH_BUDGET_USD`). CI's `Stories` workflow (`.github/workflows/stories.yml`) runs that target with Sonnet 5.5 in the merge queue (pr.yml's `Test / Stories` job, which `Required` counts only when the repo variable `STORIES_MODE` is `block`), nightly on main and on demand, and with Haiku 5.5 on each push to a PR that is not a draft (`Test / Stories (Haiku)`, where Sonnet 5.5 judges the story tests' asserts and Haiku the agent repros', and which `Required` counts only when `STORIES_PR_MODE` is `block`). That target also runs the story tests, user stories that an agent drives in `tests/stories/*.e2e.ts`, and the Electron stories in `tests/stories/electron/*.e2e.ts` on Xum Electron under Xvfb (it runs `make electron-binary` first). Prove it: restore the bug in a scratch edit, rebuild, see `ASSERTION_FAILED` on the bug's own assertion, then restore and see it pass.", From ffafa560a9b93174bc7f6cf04aea40776d2d8882 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 27 Oct 2026 08:48:18 +0000 Subject: [PATCH 2/3] ci: tag inputs- only after the attestation, as a copy of the attested digest imagetools create wraps a single manifest in a new index by default, so the stable tag got an unattested digest. --prefer-index=false copies the manifest, and the step checks the tag's digest. The tag is the completion marker, so it now comes after Attest: a failed attestation leaves no tag and a re-run publishes again. --- .github/workflows/bugbash-sandbox-image.yml | 36 +++++++++++++++------ 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/.github/workflows/bugbash-sandbox-image.yml b/.github/workflows/bugbash-sandbox-image.yml index b780931dd5c..8bb3f475a36 100644 --- a/.github/workflows/bugbash-sandbox-image.yml +++ b/.github/workflows/bugbash-sandbox-image.yml @@ -4,8 +4,8 @@ # main that changes an image input, or a manual run on main, from that run's exact commit, with no # cache and no artifact from any other run. Publishing is create-if-missing (the user's pick, so # an image change needs no person): each inputs key gets one stable tag, inputs-. A run whose -# key has that tag pushes nothing. Otherwise it builds fresh, pushes, points the stable tag at that -# build's own digest, and attests the digest. Nothing here deletes an image. +# key has that tag pushes nothing. Otherwise it builds fresh, pushes, attests that build's own +# digest, and only then points the stable tag at the same digest. Nothing here deletes an image. # # Runners still trust only the digest in image.json: a person copies it from the job summary into # image.json in a pull request (no workflow writes to the repo). Until that PR merges, runners @@ -95,15 +95,13 @@ jobs: exit 1 fi make -s bugbash-sandbox-publish > image.json - # The subject of the attestation is this build's own output (image.json). The stable tag - # names the same digest, so the attestation covers it too. - ref="$(jq -er .image image.json)@$(jq -er .digest image.json)" - docker buildx imagetools create --tag "$tag" "$ref" - echo "image=$(jq -er .image image.json)" >> "$GITHUB_OUTPUT" - echo "digest=$(jq -er .digest image.json)" >> "$GITHUB_OUTPUT" + # The subject of the attestation is this build's own output (image.json). + { + echo "image=$(jq -er .image image.json)" + echo "digest=$(jq -er .digest image.json)" + echo "tag=$tag" + } >> "$GITHUB_OUTPUT" { - echo "Stable tag: \`$tag\`" - echo echo '### tests/bugbash/sandbox/image.json' echo echo '```json' @@ -118,3 +116,21 @@ jobs: subject-name: ${{ steps.push.outputs.image }} subject-digest: ${{ steps.push.outputs.digest }} push-to-registry: true + # The stable tag marks the key as published, so it comes only after the attestation: a failed + # Attest leaves no tag, and a re-run builds and attests again instead of skipping the key. + - name: Tag inputs- + if: steps.push.outputs.digest != '' + env: + TAG: ${{ steps.push.outputs.tag }} + REF: ${{ steps.push.outputs.image }}@${{ steps.push.outputs.digest }} + DIGEST: ${{ steps.push.outputs.digest }} + run: | + # --prefer-index=false copies the manifest as it is. The default wraps a single manifest + # in a new index, whose digest the attestation does not name. + docker buildx imagetools create --prefer-index=false --tag "$TAG" "$REF" + got=$(docker buildx imagetools inspect --format '{{json .Manifest.Digest}}' "$TAG" | jq -er .) + if [ "$got" != "$DIGEST" ]; then + echo "::error::$TAG has digest $got, not the attested $DIGEST" + exit 1 + fi + echo "Stable tag: \`$TAG\` names the attested \`$DIGEST\`." >> "$GITHUB_STEP_SUMMARY" From 70bca0723164d757182fbd796a84bd53c158bf2d Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 27 Oct 2026 08:57:32 +0000 Subject: [PATCH 3/3] ci: fail the publish step when the record has no digest jq inside echo arguments does not stop a bash -e script, so a bad record set an output and Attest no longer failed on it. Assign first. --- .github/workflows/bugbash-sandbox-image.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/bugbash-sandbox-image.yml b/.github/workflows/bugbash-sandbox-image.yml index 8bb3f475a36..79443831888 100644 --- a/.github/workflows/bugbash-sandbox-image.yml +++ b/.github/workflows/bugbash-sandbox-image.yml @@ -95,10 +95,13 @@ jobs: exit 1 fi make -s bugbash-sandbox-publish > image.json - # The subject of the attestation is this build's own output (image.json). + # The subject of the attestation is this build's own output (image.json). Assignments, not + # $(...) inside echo: a failed jq must fail the step, or an empty digest would skip Attest. + image=$(jq -er .image image.json) + digest=$(jq -er .digest image.json) { - echo "image=$(jq -er .image image.json)" - echo "digest=$(jq -er .digest image.json)" + echo "image=$image" + echo "digest=$digest" echo "tag=$tag" } >> "$GITHUB_OUTPUT" {