From 59c5d91e657f675ac762514b096b4f2a1bb41f6f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 31 Oct 2026 23:01:35 -0700 Subject: [PATCH 1/4] chore(quest): claim quest/m0/review-sandbox.md Quest-Claim: ee600bad93cc8ad4 From d2a8cd9d34b2e68521418ad3b05399e884fe83be Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 31 Oct 2026 23:27:33 -0700 Subject: [PATCH 2/4] test(review): live check that a reviewed head runs nothing `just review-sandbox` runs `quest review` with the installed Codex and Claude Code CLIs on a head that plants Codex's project MCP servers and hooks, an `.mcp.json` server, and Claude Code's settings hooks. It fails if any of them runs or the review trusts a new directory. Each CLI first runs once without quest's flags, so a plant that stops firing fails as a broken control instead of passing as a clean review. Each run uses a new config home that trusts the clone (and, for Codex, the planted hooks) with a copy of the user's login, handing a refreshed login back afterwards. The default vitest run excludes `test/live`. Co-Authored-By: Claude Opus 5.5 --- README.md | 7 +- justfile | 6 + quest/m0/README.md | 1 - quest/m0/codex-project-config.md | 2 +- quest/m0/review-harness.md | 8 +- quest/m0/review-sandbox.md | 28 --- test/live/review-sandbox.test.ts | 320 +++++++++++++++++++++++++++++++ vitest.config.ts | 13 +- 8 files changed, 345 insertions(+), 40 deletions(-) delete mode 100644 quest/m0/review-sandbox.md create mode 100644 test/live/review-sandbox.test.ts diff --git a/README.md b/README.md index ed9557bd..efce10f6 100644 --- a/README.md +++ b/README.md @@ -731,7 +731,12 @@ execpolicy rules, and Claude Code with only read tools and the Bash commands it judges read-only, in `dontAsk` mode. Claude Code loads no settings files, so no allow rule, even the user's own, widens that. Codex skips the user's config, whose project trust would load the checkout's `.codex/config.toml` and start its -MCP servers outside the sandbox. The base is fetched first, since +MCP servers outside the sandbox. Whether those flags isolate is up to each +CLI's release, so `just review-sandbox`, in this repository, plants project MCP +servers and hooks for both CLIs on a head, runs `quest review` with the +installed CLIs and copies of your logins, and fails if any of it runs or the +review trusts a new directory. It spends real usage, so CI never runs it; run it +after upgrading either CLI. The base is fetched first, since the sandbox has no network. The reviewer follows the base's `AGENTS.md` and `CLAUDE.md`, at the root and in each directory holding a changed path, never the checkout's, so a change cannot instruct its own reviewer. The base's `REVIEW.md` is optional, diff --git a/justfile b/justfile index dadcba80..ca2f2a67 100644 --- a/justfile +++ b/justfile @@ -40,6 +40,12 @@ _test: bash scripts/lifecycle-check.sh "$PWD/dist/quest" just worker-test +# Check that `quest review` runs nothing a reviewed head plants, with the real +# Codex and Claude Code CLIs. Uses each CLI's login and spends its usage; run it +# after upgrading either CLI. Arguments go to vitest, such as `-t codex`. +review-sandbox *args: + QUEST_LIVE=1 bun run vitest run {{args}} + # Apply formatters without changing program behavior. fix: bun run fix diff --git a/quest/m0/README.md b/quest/m0/README.md index 9a99ff17..4cbe8ea8 100644 --- a/quest/m0/README.md +++ b/quest/m0/README.md @@ -49,7 +49,6 @@ ownership-manifest design: - [Land checks at gate priority](/quest/m0/land-priority-sandbox.md) - `quest land serve` passes `QUEST_GATE_PRIORITY` to its sandbox - [Codex cloud off](/quest/m0/codex-cloud-off.md) - automatic Codex cloud reviews stop once no open PR relies on them - [Review handles expired and environment logins](/quest/m0/review-login.md) - `quest review` says to log in again instead of failing with a 401, and reviews with credentials from the environment -- [Live sandbox test](/quest/m0/review-sandbox.md) - the real reviewer CLIs run nothing a reviewed head plants - [Kix decision UI](/quest/m0/kix/README.md) - planning and implementation through prompts and summaries, run on the maintainer's own runners - [Agent test environment](/quest/m0/agent-test-env.md) - `just test` passes inside agent sessions - [Stale skill stubs](/quest/m0/init-prune.md) - `quest init` and `quest uninstall` remove stubs of retired skills such as `quest-ready` diff --git a/quest/m0/codex-project-config.md b/quest/m0/codex-project-config.md index 558e47d8..3cbe2a31 100644 --- a/quest/m0/codex-project-config.md +++ b/quest/m0/codex-project-config.md @@ -19,5 +19,5 @@ Decided 2026-10-28 (user, /quest-plan of the follow-ups from [moq-dev/quest#271] - Do not post upstream, on #49333 or a new issue; the maintainer declined. Only watch for the change. - This quest waits on Codex. Check #49333 and Codex release notes each time - ready work is triaged; [the live sandbox test](/quest/m0/review-sandbox.md) + ready work is triaged; the live sandbox test, `just review-sandbox`, catches a release that changes the behavior first. diff --git a/quest/m0/review-harness.md b/quest/m0/review-harness.md index 4445c0ec..7caeff92 100644 --- a/quest/m0/review-harness.md +++ b/quest/m0/review-harness.md @@ -13,14 +13,10 @@ Decided 2026-10-28 (user, /quest-plan of the follow-ups from [moq-dev/quest#271] the default tier is the newest general model at medium effort, and the deep tier the strongest one. Bump any that are stale, and check that both CLIs accept the new ids. -- Run [the live sandbox test](/quest/m0/review-sandbox.md) with the CLIs - installed at the time, and plan a quest for any failure. +- Run the live sandbox test, `just review-sandbox`, with the CLIs installed + at the time, and plan a quest for any failure. - This quest recurs. The PR that finishes this run creates `/quest/m1/review-harness.md` with this Goal, listed in m1's `Required`, and `Required: /quest/m0/README.md`, so the next run becomes ready when m0 completes. Each run creates the next milestone's run the same way, carrying forward the model ids it found and anything to check first. - -## Required - -- [Live sandbox test](/quest/m0/review-sandbox.md) - the test each run executes diff --git a/quest/m0/review-sandbox.md b/quest/m0/review-sandbox.md deleted file mode 100644 index 02bb0473..00000000 --- a/quest/m0/review-sandbox.md +++ /dev/null @@ -1,28 +0,0 @@ -# [S] Live test that a reviewed head runs nothing - -## Goal - -An opt-in recipe runs `quest review` with the real Codex and Claude Code CLIs -on a head that plants configuration, and fails if any of it runs: a -`.codex/config.toml` MCP server, `.codex/hooks.json` hooks, an `.mcp.json` -server, and `.claude/settings.json` hooks. It also fails if the review writes -a trust entry for the reviewed directory into the reviewer's user config. - -## Plan - -Decided 2026-10-28 (user, /quest-plan of the follow-ups from [moq-dev/quest#271](https://github.com/moq-dev/quest/pull/271)). - -- #271's fake-CLI tests check only that the isolating flags are passed. Its - P1 (a trusted clone's worktree loading the head's `.codex/config.toml` MCP - servers outside the sandbox) lived in CLI behavior that fakes cannot see. -- Each planted item records that it ran, such as a marker file or a request to - a local port, and the test asserts nothing recorded. Run each CLI against a - disposable config home that holds a login and trusts the reviewed clone, so - the trust path is exercised. -- [openai/codex#49333](https://github.com/openai/codex/issues/49333) reports - that `--ignore-user-config` skipping the project layer is a bug, and that - the flag still writes a trust entry. Quest's protection relies on the - former, so this test is what catches a Codex release that changes it. -- It needs real logins and spends API usage, so CI does not run it. - [The review harness check](/quest/m0/review-harness.md) runs it each - milestone, and anyone can run it after upgrading either CLI. diff --git a/test/live/review-sandbox.test.ts b/test/live/review-sandbox.test.ts new file mode 100644 index 00000000..c88fdc5e --- /dev/null +++ b/test/live/review-sandbox.test.ts @@ -0,0 +1,320 @@ +// Live: `quest review` with the real Codex and Claude Code CLIs runs nothing a +// reviewed head plants, and trusts nothing new. It spends the logins' usage, so +// only `just review-sandbox` runs it, never `just test` or CI. +// +// The fake-CLI tests in `test/review.test.ts` check only that quest passes the +// isolating flags; whether those flags isolate is the CLIs' behavior, which can +// change in any release (openai/codex#49333). Each CLI first runs once without +// quest's flags in the same checkout, so a plant that no longer fires fails as +// a broken control instead of passing as a clean review. +import { spawn, spawnSync } from "node:child_process"; +import { + appendFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + realpathSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { homedir, tmpdir } from "node:os"; +import { join } from "node:path"; +import { createInterface } from "node:readline"; +import { afterAll, beforeAll, describe, expect, test } from "vitest"; + +import { type HarnessName, strippedEnvironment } from "../../src/cli/agent"; +import { MODELS, review } from "../../src/cli/review"; + +/** The cheapest effort both CLIs accept; the isolation under test does not depend on it. */ +const EFFORT = "low"; + +interface Fixture { + /** + * The reviewed checkout: a clone the config homes trust, on the head. It is + * not a linked worktree, since Codex reads a worktree's hooks from its main + * checkout, which would not hold the head's. + */ + clone: string; + /** Where every plant records that it ran. */ + markers: string; +} + +let fixture: Fixture; +/** Holds the fixture and every config home, removed once all tests ran. */ +let scratch: string; + +const newHome = () => mkdtempSync(join(scratch, "home-")); + +beforeAll(() => { + scratch = realpathSync(mkdtempSync(join(tmpdir(), "quest-live-"))); + const markers = join(scratch, "markers"); + mkdirSync(markers); + const origin = join(scratch, "origin.git"); + git(scratch, "init", "-q", "--bare", "-b", "main", origin); + const clone = join(scratch, "clone"); + git(scratch, "init", "-q", "-b", "main", clone); + write(clone, "src/lib.ts", "export const average = (xs: number[]) => xs.reduce((a, b) => a + b, 0) / xs.length;\n"); + git(clone, "add", "."); + git(clone, "commit", "-qm", "base"); + git(clone, "remote", "add", "origin", origin); + git(clone, "push", "-q", "origin", "main"); + + git(clone, "checkout", "-qb", "feature"); + for (const [path, text] of Object.entries(plants(markers))) write(clone, path, text); + write( + clone, + "src/lib.ts", + "export const average = (xs: number[]) => xs.reduce((a, b) => a + b, 0) / (xs.length - 1);\n", + ); + git(clone, "add", "."); + git(clone, "commit", "-qm", "change"); + fixture = { clone, markers }; +}); + +afterAll(() => { + if (scratch) rmSync(scratch, { recursive: true, force: true }); +}); + +/** + * The configuration a head plants, each item touching a marker named for it + * when it runs: Codex's project MCP servers and hooks, and Claude Code's + * project MCP servers and hooks, its settings enabling every project MCP + * server. + */ +function plants(markers: string): Record { + const touch = (name: string) => `touch ${JSON.stringify(join(markers, name))}`; + const hooks = (prefix: string) => + JSON.stringify({ + hooks: Object.fromEntries( + ["SessionStart", "UserPromptSubmit"].map((event) => [ + event, + [{ hooks: [{ type: "command", command: touch(`${prefix}-${event}`) }] }], + ]), + ), + }); + const server = (name: string) => ({ command: "/bin/sh", args: ["-c", touch(name)] }); + return { + ".codex/config.toml": [ + "[mcp_servers.plant]", + `command = ${JSON.stringify(server("codex-config-mcp").command)}`, + `args = ${JSON.stringify(server("codex-config-mcp").args)}`, + "", + ].join("\n"), + ".codex/hooks.json": hooks("codex-hooks"), + ".mcp.json": JSON.stringify({ mcpServers: { plant: server("mcp-json") } }), + ".claude/settings.json": JSON.stringify({ + enableAllProjectMcpServers: true, + ...JSON.parse(hooks("claude-settings")), + }), + }; +} + +/** What each harness's own plants record when its CLI loads the head's configuration. */ +const CONTROL: Record = { + codex: ["codex-config-mcp", "codex-hooks-SessionStart", "codex-hooks-UserPromptSubmit"], + claude: ["claude-settings-SessionStart", "claude-settings-UserPromptSubmit", "mcp-json"], +}; + +/** + * A config home that holds a copy of the user's login and trusts the clone, + * and for Codex the planted hooks too, which it otherwise skips until a user + * trusts them. Each run gets a new one, so one run's writes cannot hide + * another's. + */ +interface Home { + harness: HarnessName; + /** The variables pointing the CLI at it. */ + env: Record; + /** The directories the home trusts. */ + trusted(): string[]; + /** Hand a login the CLI refreshed during the run back to the user's home. */ + release(): void; +} + +async function home(harness: HarnessName): Promise { + const { clone } = fixture; + const path = newHome(); + if (harness === "codex") { + const source = process.env.CODEX_HOME || join(homedir(), ".codex"); + const config = join(path, "config.toml"); + writeFileSync(config, `[projects.${JSON.stringify(clone)}]\ntrust_level = "trusted"\n`); + const listed = await codexHooks(path, clone); + const hooks = listed.hooks.filter((hook) => hook.source === "project"); + if (hooks.length === 0) throw new Error(`codex lists none of the planted hooks: ${listed.response}`); + const state = hooks.map( + (hook) => + `\n[hooks.state.${JSON.stringify(hook.key)}]\ntrusted_hash = ${JSON.stringify(hook.currentHash)}\n`, + ); + appendFileSync(config, state.join("")); + return { + harness, + env: { CODEX_HOME: path }, + trusted: () => + [...readFileSync(config, "utf8").matchAll(/^\[projects\.("(?:[^"\\]|\\.)*")\]/gm)].map((m) => + JSON.parse(m[1]), + ), + release: borrow(join(source, "auth.json"), join(path, "auth.json"), "CODEX_HOME"), + }; + } + const source = process.env.CLAUDE_CONFIG_DIR || join(homedir(), ".claude"); + const config = join(path, ".claude.json"); + writeFileSync(config, JSON.stringify({ projects: { [clone]: { hasTrustDialogAccepted: true } } })); + return { + harness, + env: { CLAUDE_CONFIG_DIR: path }, + trusted() { + const projects = ( + JSON.parse(readFileSync(config, "utf8")) as { + projects?: Record; + } + ).projects; + return Object.entries(projects ?? {}) + .filter(([, project]) => project.hasTrustDialogAccepted === true) + .map(([dir]) => dir); + }, + release: borrow(join(source, ".credentials.json"), join(path, ".credentials.json"), "CLAUDE_CONFIG_DIR"), + }; +} + +interface CodexHook { + key: string; + source: string; + currentHash: string; +} + +/** The hooks Codex finds for `cwd`, from its app server, which needs no login for this. */ +function codexHooks(codexHome: string, cwd: string): Promise<{ hooks: CodexHook[]; response: string }> { + return new Promise((resolve, reject) => { + const child = spawn("codex", ["app-server"], { + env: { ...strippedEnvironment(process.env), CODEX_HOME: codexHome }, + stdio: ["pipe", "pipe", "inherit"], + }); + const send = (message: object) => child.stdin.write(`${JSON.stringify({ jsonrpc: "2.0", ...message })}\n`); + const lines = createInterface({ input: child.stdout }); + lines.on("line", (line) => { + const message = JSON.parse(line) as { id?: number; result?: unknown; error?: unknown }; + if (message.id === 1) { + send({ method: "initialized" }); + send({ id: 2, method: "hooks/list", params: { cwds: [cwd] } }); + } else if (message.id === 2) { + child.kill(); + const result = message.result as { data?: { hooks?: CodexHook[] }[] } | undefined; + if (result?.data === undefined) reject(new Error(`codex hooks/list failed: ${line}`)); + else resolve({ hooks: result.data.flatMap((entry) => entry.hooks ?? []), response: line }); + } + }); + child.once("error", reject); + child.once("exit", (code) => reject(new Error(`codex app-server exited with ${code} before listing hooks`))); + send({ id: 1, method: "initialize", params: { clientInfo: { name: "quest-live-test", version: "0" } } }); + }); +} + +/** + * Copy the login file at `source` to `copy`, returning what writes a refreshed + * login back. A refresh may rotate the refresh token, which would sign the + * user's own home out; the write-back is skipped if that home has itself + * changed since. + */ +function borrow(source: string, copy: string, variable: string): () => void { + if (!existsSync(source)) { + throw new Error(`no login file at ${source}; log in, or set ${variable} to a config home that holds one`); + } + const original = readFileSync(source); + writeFileSync(copy, original, { mode: 0o600 }); + return () => { + const now = readFileSync(copy); + if (now.equals(original)) return; + if (!readFileSync(source).equals(original)) { + console.warn( + `${source} and its copy were both refreshed; log in again if the CLI reports an expired token`, + ); + return; + } + writeFileSync(source, now, { mode: 0o600 }); + }; +} + +/** The environment `quest review` gives a reviewer, pointed at `home`. */ +function environment(home: Home): NodeJS.ProcessEnv { + return { ...strippedEnvironment(process.env), ...home.env }; +} + +function recorded(): string[] { + return readdirSync(fixture.markers).sort(); +} + +function clearMarkers() { + rmSync(fixture.markers, { recursive: true }); + mkdirSync(fixture.markers); +} + +/** + * One plain turn with the CLI's defaults, which load the user's config and the + * checkout's. It blocks, so it carries its own bound for a hung CLI. + */ +function control(home: Home) { + const { model } = MODELS[home.harness].default; + const argv = + home.harness === "codex" + ? ["exec", "--json", "-m", model, "-c", `model_reasoning_effort="${EFFORT}"`, "-"] + : ["-p", "--output-format", "json", "--model", model, "--effort", EFFORT]; + return spawnSync(home.harness, argv, { + cwd: fixture.clone, + env: environment(home), + input: "Reply with the single word OK.", + encoding: "utf8", + timeout: 10 * 60_000, + }); +} + +const tail = (text: string | null) => (text ?? "").trim().split("\n").slice(-10).join("\n"); + +describe.each(["codex", "claude"] as const)("%s", (harness) => { + test("loads the planted configuration without quest's flags", async () => { + clearMarkers(); + const dir = await home(harness); + let run: ReturnType; + try { + run = control(dir); + } finally { + dir.release(); + } + const output = `exit ${run.status}\nstdout:\n${tail(run.stdout)}\nstderr:\n${tail(run.stderr)}`; + expect(recorded(), `a plant did not fire, so the review check would pass vacuously\n${output}`).toEqual( + expect.arrayContaining(CONTROL[harness]), + ); + }); + + test("quest review runs none of it and trusts nothing new", async () => { + clearMarkers(); + const dir = await home(harness); + const trusted = dir.trusted(); + let outcome: unknown; + try { + outcome = await review( + { root: fixture.clone, base: "origin/main", harness, effort: EFFORT, fresh: true }, + { env: environment(dir) }, + ).catch((error: unknown) => error); + } finally { + dir.release(); + } + expect(recorded(), "planted configuration ran during the review").toEqual([]); + expect(dir.trusted(), "the review trusted a new directory").toEqual(trusted); + if (outcome instanceof Error) throw outcome; + }); +}); + +function git(cwd: string, ...args: string[]) { + const run = spawnSync("git", ["-c", "user.name=Fixture", "-c", "user.email=fixture@localhost", ...args], { + cwd, + encoding: "utf8", + }); + if (run.status !== 0) throw new Error(`git ${args.join(" ")} failed: ${run.stderr}`); +} + +function write(dir: string, path: string, text: string) { + mkdirSync(join(dir, path, ".."), { recursive: true }); + writeFileSync(join(dir, path), text); +} diff --git a/vitest.config.ts b/vitest.config.ts index e735bb61..6aeb6521 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -1,8 +1,13 @@ -import { defineConfig } from "vitest/config"; +import { configDefaults, defineConfig } from "vitest/config"; + +// `test/live` runs the real agent CLIs on the user's logins, so it runs only +// with QUEST_LIVE=1, as `just review-sandbox` sets, and alone. A reviewer turn +// takes minutes; its timeout only bounds a hung CLI. +const live = process.env.QUEST_LIVE === "1"; -// Bun loads Markdown assets as text (bunfig.toml); load them the same way here. export default defineConfig({ plugins: [ + // Bun loads Markdown assets as text (bunfig.toml); load them the same way here. { name: "markdown-text", transform(code, id) { @@ -10,5 +15,7 @@ export default defineConfig({ }, }, ], - test: { include: ["test/**/*.test.ts"] }, + test: live + ? { include: ["test/live/**/*.test.ts"], testTimeout: 30 * 60_000, hookTimeout: 60_000 } + : { include: ["test/**/*.test.ts"], exclude: [...configDefaults.exclude, "test/live/**"] }, }); From 08ced209ea0a7bf555b2484e8d9cb477b46f1839 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 13:22:42 -0800 Subject: [PATCH 3/4] test(review): follow main's MODELS and hand a login back by rename Main flattened MODELS to one tier per harness, which broke the live test's control after the merge. Write a refreshed login back through a rename so an interrupted write never truncates the user's credential file. Co-Authored-By: Claude Opus 5.5 --- test/live/review-sandbox.test.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/test/live/review-sandbox.test.ts b/test/live/review-sandbox.test.ts index c88fdc5e..fabc0f11 100644 --- a/test/live/review-sandbox.test.ts +++ b/test/live/review-sandbox.test.ts @@ -16,6 +16,7 @@ import { readFileSync, readdirSync, realpathSync, + renameSync, rmSync, writeFileSync, } from "node:fs"; @@ -215,7 +216,8 @@ function codexHooks(codexHome: string, cwd: string): Promise<{ hooks: CodexHook[ * Copy the login file at `source` to `copy`, returning what writes a refreshed * login back. A refresh may rotate the refresh token, which would sign the * user's own home out; the write-back is skipped if that home has itself - * changed since. + * changed since. It replaces the file by rename, so an interrupted write + * never leaves the user's login truncated. */ function borrow(source: string, copy: string, variable: string): () => void { if (!existsSync(source)) { @@ -224,6 +226,7 @@ function borrow(source: string, copy: string, variable: string): () => void { const original = readFileSync(source); writeFileSync(copy, original, { mode: 0o600 }); return () => { + if (!existsSync(copy)) return; const now = readFileSync(copy); if (now.equals(original)) return; if (!readFileSync(source).equals(original)) { @@ -232,7 +235,9 @@ function borrow(source: string, copy: string, variable: string): () => void { ); return; } - writeFileSync(source, now, { mode: 0o600 }); + const staged = `${source}.quest-live-${process.pid}`; + writeFileSync(staged, now, { mode: 0o600 }); + renameSync(staged, source); }; } @@ -255,7 +260,7 @@ function clearMarkers() { * checkout's. It blocks, so it carries its own bound for a hung CLI. */ function control(home: Home) { - const { model } = MODELS[home.harness].default; + const { model } = MODELS[home.harness]; const argv = home.harness === "codex" ? ["exec", "--json", "-m", model, "-c", `model_reasoning_effort="${EFFORT}"`, "-"] From 2256f58f2243b25c62bfd40ae1ab7a0b0edf75bf Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 13:28:47 -0800 Subject: [PATCH 4/4] test(review): check an untrusted review writes no trust, and bound a hung reviewer A home that already trusts the clone cannot show a review trusting it, so each CLI also reviews from a home that trusts nothing and must leave it so. A review past 20 minutes has its reviewer CLI terminated and awaited, so a hung CLI never outlives its test or its config home. Co-Authored-By: Claude Opus 5.5 --- test/live/review-sandbox.test.ts | 90 ++++++++++++++++++++++---------- 1 file changed, 62 insertions(+), 28 deletions(-) diff --git a/test/live/review-sandbox.test.ts b/test/live/review-sandbox.test.ts index fabc0f11..a522665b 100644 --- a/test/live/review-sandbox.test.ts +++ b/test/live/review-sandbox.test.ts @@ -119,10 +119,10 @@ const CONTROL: Record = { }; /** - * A config home that holds a copy of the user's login and trusts the clone, - * and for Codex the planted hooks too, which it otherwise skips until a user - * trusts them. Each run gets a new one, so one run's writes cannot hide - * another's. + * A config home that holds a copy of the user's login and, with `trust`, trusts + * the clone, and for Codex the planted hooks too, which it otherwise skips + * until a user trusts them. Each run gets a new one, so one run's writes cannot + * hide another's. */ interface Home { harness: HarnessName; @@ -134,21 +134,23 @@ interface Home { release(): void; } -async function home(harness: HarnessName): Promise { +async function home(harness: HarnessName, trust = true): Promise { const { clone } = fixture; const path = newHome(); if (harness === "codex") { const source = process.env.CODEX_HOME || join(homedir(), ".codex"); const config = join(path, "config.toml"); - writeFileSync(config, `[projects.${JSON.stringify(clone)}]\ntrust_level = "trusted"\n`); - const listed = await codexHooks(path, clone); - const hooks = listed.hooks.filter((hook) => hook.source === "project"); - if (hooks.length === 0) throw new Error(`codex lists none of the planted hooks: ${listed.response}`); - const state = hooks.map( - (hook) => - `\n[hooks.state.${JSON.stringify(hook.key)}]\ntrusted_hash = ${JSON.stringify(hook.currentHash)}\n`, - ); - appendFileSync(config, state.join("")); + writeFileSync(config, trust ? `[projects.${JSON.stringify(clone)}]\ntrust_level = "trusted"\n` : ""); + if (trust) { + const listed = await codexHooks(path, clone); + const hooks = listed.hooks.filter((hook) => hook.source === "project"); + if (hooks.length === 0) throw new Error(`codex lists none of the planted hooks: ${listed.response}`); + const state = hooks.map( + (hook) => + `\n[hooks.state.${JSON.stringify(hook.key)}]\ntrusted_hash = ${JSON.stringify(hook.currentHash)}\n`, + ); + appendFileSync(config, state.join("")); + } return { harness, env: { CODEX_HOME: path }, @@ -161,7 +163,7 @@ async function home(harness: HarnessName): Promise { } const source = process.env.CLAUDE_CONFIG_DIR || join(homedir(), ".claude"); const config = join(path, ".claude.json"); - writeFileSync(config, JSON.stringify({ projects: { [clone]: { hasTrustDialogAccepted: true } } })); + writeFileSync(config, JSON.stringify({ projects: trust ? { [clone]: { hasTrustDialogAccepted: true } } : {} })); return { harness, env: { CLAUDE_CONFIG_DIR: path }, @@ -293,24 +295,56 @@ describe.each(["codex", "claude"] as const)("%s", (harness) => { }); test("quest review runs none of it and trusts nothing new", async () => { - clearMarkers(); const dir = await home(harness); const trusted = dir.trusted(); - let outcome: unknown; - try { - outcome = await review( - { root: fixture.clone, base: "origin/main", harness, effort: EFFORT, fresh: true }, - { env: environment(dir) }, - ).catch((error: unknown) => error); - } finally { - dir.release(); - } - expect(recorded(), "planted configuration ran during the review").toEqual([]); - expect(dir.trusted(), "the review trusted a new directory").toEqual(trusted); - if (outcome instanceof Error) throw outcome; + await reviewIn(dir, () => { + expect(recorded(), "planted configuration ran during the review").toEqual([]); + expect(dir.trusted(), "the review trusted a new directory").toEqual(trusted); + }); + }); + + // A home that already trusts the clone would not show the review trusting it. + test("quest review of an untrusted checkout trusts nothing", async () => { + const dir = await home(harness, false); + await reviewIn(dir, () => { + expect(recorded(), "planted configuration ran during the review").toEqual([]); + expect(dir.trusted(), "the review trusted a directory").toEqual([]); + }); }); }); +/** How long a review may run before its reviewer CLI is terminated, within the test timeout. */ +const REVIEW_BOUND = 20 * 60_000; + +/** + * Review the clone with `dir`, then run `verify` before rethrowing the + * review's error, so a failed review still checks what ran. A review that runs + * past `REVIEW_BOUND` has the reviewer CLI it spawned terminated and awaited, + * so a hung CLI never outlives its test or its config home. + */ +async function reviewIn(dir: Home, verify: () => void) { + clearMarkers(); + let timedOut = false; + const timer = setTimeout(() => { + timedOut = true; + spawnSync("pkill", ["-TERM", "-P", String(process.pid)]); + }, REVIEW_BOUND); + let outcome: unknown; + try { + outcome = await review( + { root: fixture.clone, base: "origin/main", harness: dir.harness, effort: EFFORT, fresh: true }, + { env: environment(dir) }, + ).catch((error: unknown) => error); + } finally { + clearTimeout(timer); + dir.release(); + } + verify(); + if (timedOut) + throw new Error(`the review ran past ${REVIEW_BOUND / 60_000} minutes, so its reviewer was terminated`); + if (outcome instanceof Error) throw outcome; +} + function git(cwd: string, ...args: string[]) { const run = spawnSync("git", ["-c", "user.name=Fixture", "-c", "user.email=fixture@localhost", ...args], { cwd,