From f1d246848330b9fe1f9859eb34421903725a469f Mon Sep 17 00:00:00 2001 From: T <60372079+mpaulosky@users.noreply.github.com> Date: Tue, 10 Nov 2026 10:48:54 -0800 Subject: [PATCH 1/3] chore: Apply repo-ci-baseline Template Co-Authored-By: Claude Sonnet 5.5 --- .claude/rules/sandcastle-hand-off.md | 2 +- .github/scripts/tests/pr-automerge.test.mjs | 152 ++++++++- .github/workflows/lint-actions.yml | 2 +- .github/workflows/pr-automerge.yml | 27 +- .github/workflows/sandcastle.yml | 52 +-- .sandcastle/lib/branches.mts | 2 +- .sandcastle/lib/branches.test.mts | 7 +- .sandcastle/lib/review-ledger.mts | 3 +- .sandcastle/lib/review-ledger.test.mts | 3 +- .sandcastle/lib/role-logs.mts | 163 +++++---- .sandcastle/lib/role-logs.test.mts | 361 +++++++++++++------- scripts/check-branch-name.sh | 14 +- scripts/tests/check-branch-name.test.sh | 12 +- 13 files changed, 548 insertions(+), 252 deletions(-) diff --git a/.claude/rules/sandcastle-hand-off.md b/.claude/rules/sandcastle-hand-off.md index 95a5e07..745f714 100644 --- a/.claude/rules/sandcastle-hand-off.md +++ b/.claude/rules/sandcastle-hand-off.md @@ -21,7 +21,7 @@ A pull request labelled before the repository began recording heads is not follo ## Which pull requests - Opened by the repository's owner (a session's counts), in the same repository (not a fork), into `main`, not a draft. -- A branch name `scripts/check-branch-name.sh` accepts. +- A branch name `scripts/check-branch-name.sh` accepts, but not `dependabot/*`. - Only a run on the default `Sandcastle` queue follows hand-offs: GitHub Actions always, a local run only with `SANDCASTLE_LABEL=Sandcastle`. - The repository runs Sandcastle in GitHub Actions with its `SANDCASTLE_GH_TOKEN` secret set to the account the runs use, since a workflow there records the head you label. Without that, nothing can be handed off. diff --git a/.github/scripts/tests/pr-automerge.test.mjs b/.github/scripts/tests/pr-automerge.test.mjs index 684c70d..9d6ac21 100644 --- a/.github/scripts/tests/pr-automerge.test.mjs +++ b/.github/scripts/tests/pr-automerge.test.mjs @@ -3,8 +3,11 @@ // client, so these tests cover exactly the code the workflow runs. // Usage: node --test .github/scripts/tests/pr-automerge.test.mjs import assert from "node:assert/strict"; -import { readFileSync } from "node:fs"; -import { test } from "node:test"; +import { spawnSync } from "node:child_process"; +import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { mock, test } from "node:test"; const WORKFLOW = new URL("../../workflows/pr-automerge.yml", import.meta.url); const OWNER = "octo"; @@ -346,6 +349,8 @@ async function evaluate(pr, events = [], options = {}) { registersAfterReads = 0, // An error thrown by the comment that records the ask. askCommentError, + // How long the comment that records the ask takes to persist, in ms. + askCommentDelayMs = 0, // The read-back window and its poll interval, in ms (the workflow's // COPILOT_ASK_WAIT_MS and COPILOT_ASK_POLL_MS). askWaitMs = 0, @@ -425,6 +430,8 @@ async function evaluate(pr, events = [], options = {}) { }; const removedLabels = []; const addedLabels = []; + // When each label was added (the PAT's call). + const addedLabelAt = []; const listRequests = []; let graphqlRequests = 0; const pullRequests = Array.isArray(pr) ? [...pr] : [pr]; @@ -526,6 +533,9 @@ async function evaluate(pr, events = [], options = {}) { return { data: { id: 9000 + exhaustionPosts.length - 1 } }; } order.push("record"); + if (askCommentDelayMs) { + await new Promise((resolve) => setTimeout(resolve, askCommentDelayMs)); + } if (askCommentError) { throw askCommentError; } @@ -771,6 +781,7 @@ async function evaluate(pr, events = [], options = {}) { // A label added with GITHUB_TOKEN starts no workflow. addLabels: viaPat("issues.addLabels", async (params) => { addedLabels.push(params); + addedLabelAt.push(Date.now()); if (addLabelError) { throw addLabelError; } @@ -814,7 +825,7 @@ async function evaluate(pr, events = [], options = {}) { return { merges, updates, posted, logs, eventRequests, queries, patTokens, calls, graphqlRequests, rulesRequests, removedLabels, addedLabels, listRequests, runRequests, headRunRequests, asked, requests, order, exhaustionPosts, deletedComments, - permissionRequests, commentRequests + permissionRequests, commentRequests, askedAt, addedLabelAt }; } @@ -1667,18 +1678,116 @@ test("reads a dropped request back for the whole window before calling in Claude assert.ok(order.filter((step) => step === "read-back").length >= 2, order.join(",")); }); -test("adds review:claude at once when the request fails, without waiting out the window (mpaulosky/dotfiles#360)", async () => { +test("adds review:claude only after the rest of the settle window when the request fails (mpaulosky/Application_Factory#23)", async () => { const started = Date.now(); const { addedLabels, requests, logs } = await evaluate(readyPr(unreviewed), [], { copilotRequest: Object.assign(new Error("Resource not accessible by personal access token"), { status: 403 }), - askWaitMs: 4000, - askPollMs: 1000 + askWaitMs: 300, + askPollMs: 100 }); assert.equal(requests.length, 1); assert.deepEqual(addedLabels, LABEL_FOR_CLAUDE); assert.ok(logs.some((line) => line.startsWith("warning: ") && line.includes("Resource not accessible")), logs.join("\n")); - assert.ok(Date.now() - started < 2000, `the run took ${Date.now() - started} ms`); + assert.ok(Date.now() - started >= 250, `the run took ${Date.now() - started} ms`); +}); + +// Claude Review's gate (the Python in claude-review.yml's "Skip while Copilot can +// review" step), run against a stub `gh` that serves one PR whose only comment is +// the ask record, written `ageSeconds` ago. Returns the step's can_review output. +function claudeReviewGate(ageSeconds) { + const lines = readFileSync(new URL("../../workflows/claude-review.yml", import.meta.url), "utf8").split("\n"); + const open = lines.findIndex((line) => line.includes("python3 -I - <<'PY'")); + assert.notEqual(open, -1, "claude-review.yml has no gate heredoc"); + const close = lines.findIndex((line, index) => index > open && line.trim() === "PY"); + const body = lines.slice(open + 1, close); + const indent = Math.min(...body.filter((line) => line.trim() !== "").map((line) => line.search(/\S/))); + const script = body.map((line) => line.slice(indent)).join("\n"); + + const dir = mkdtempSync(join(tmpdir(), "claude-gate-")); + try { + const created = new Date(Date.now() - ageSeconds * 1000).toISOString(); + const comment = JSON.stringify({ body: COPILOT_ASKED_MARKER, user: { login: "github-actions[bot]" }, created_at: created }); + writeFileSync(join(dir, "gh"), [ + "#!/bin/sh", + 'case "$2" in', + ' */pulls/7) echo \'{"head":{"sha":"abc123"}}\' ;;', + " */pulls/7/reviews) echo '[[]]' ;;", + ' */pulls/7/requested_reviewers) echo \'{"users":[]}\' ;;', + ` */issues/7/comments) echo '[[${comment}]]' ;;`, + " *) echo \"unexpected $2\" >&2; exit 1 ;;", + "esac" + ].join("\n")); + chmodSync(join(dir, "gh"), 0o755); + const output = join(dir, "output"); + writeFileSync(output, ""); + const done = spawnSync("python3", ["-I", "-"], { + input: script, + encoding: "utf8", + env: { PATH: `${dir}:${process.env.PATH}`, REPOSITORY: `${OWNER}/demo`, PR_NUMBER: "7", HEAD_SHA: HEAD, GITHUB_OUTPUT: output } + }); + assert.equal(done.status, 0, done.stderr); + return readFileSync(output, "utf8").trim(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +// Runs a promise under mock timers, ticking one second at a time once the run is waiting. +async function underClock(promise) { + let finished = false; + let failure; + let value; + promise.then((result) => { finished = true; value = result; }, (error) => { finished = true; failure = error; }); + for (let seconds = 0; !finished && seconds < 300; seconds++) { + await new Promise((resolve) => setImmediate(resolve)); + if (!finished) { + mock.timers.tick(1000); + } + } + assert.ok(finished, "the run never finished within 300 simulated seconds"); + if (failure) { + throw failure; + } + return value; +} + +test("a failed request after a slow record write holds review:claude until the persisted record has settled for every workflow (mpaulosky/Application_Factory#23)", async () => { + mock.timers.enable({ apis: ["Date", "setTimeout"], now: Date.parse("2026-01-02T00:00:00Z") }); + let result; + let recordAgeAtLabel; + let freshRun; + try { + // The real 30-second settle window, and a record that takes 5 seconds to persist. + result = await underClock(evaluate(readyPr(unreviewed), [], { + copilotRequest: Object.assign(new Error("Resource not accessible by personal access token"), { status: 403 }), + askCommentDelayMs: 5000, + askWaitMs: 30 * 1000, + askPollMs: 5000 + })); + assert.deepEqual(result.addedLabels, LABEL_FOR_CLAUDE); + assert.equal(result.askedAt.length, 1); + + // The label lands only once the record workflows read is 30 seconds old. + recordAgeAtLabel = (result.addedLabelAt[0] - result.askedAt[0]) / 1000; + assert.ok(recordAgeAtLabel >= 30, `the record was ${recordAgeAtLabel} seconds old when review:claude was added`); + + // A fresh PR Auto-Merge run (no copilotAskedHere flag) reads the persisted + // record at the moment the label was added, and keeps the label. + mock.timers.setTime(result.addedLabelAt[0]); + freshRun = await underClock(evaluate( + readyPr({ ...unreviewed, labels: labelled(CLAUDE_LABEL), comments: commentsOf(askedBy(ACTIONS, recordAgeAtLabel)) }) + )); + } finally { + mock.timers.reset(); + } + assert.deepEqual(freshRun.removedLabels, [], "PR Auto-Merge took review:claude off"); + assert.deepEqual(freshRun.asked, [], "a fresh run asked again"); + + // Claude Review's gate reads the same record at the same age and lets Claude review. + assert.equal(claudeReviewGate(recordAgeAtLabel), "can_review=false"); + // The same gate holds Claude back for a record 25 seconds old (what the old code allowed here). + assert.equal(claudeReviewGate(25), "can_review=true"); }); test("makes no request and adds no label when the record comment can't be posted (mpaulosky/dotfiles#360)", async () => { @@ -2154,6 +2263,25 @@ test("fails closed when Copilot's pending request can't be seen: too many reques } }); +test("only the Copilot bot's exact identity is Copilot's pending request, not a copilot-x user, bot or team (mpaulosky/Application_Factory#22)", async () => { + const pendingBy = (requestedReviewer) => ({ totalCount: 1, nodes: [{ requestedReviewer }] }); + const others = [ + { __typename: "User", login: "copilot-x" }, + { __typename: "User", login: "Copilot" }, + { __typename: "Bot", login: "copilot-x" }, + { __typename: "Bot", login: "my-copilot" }, + { __typename: "Team", slug: "copilot-team" } + ]; + for (const other of others) { + const called = await evaluate(readyPr({ ...unreviewed, reviewRequests: pendingBy(other), comments: commentsOf(askedBy(ACTIONS, 3600)) })); + assert.equal(called.addedLabels.length, 1, `${JSON.stringify(other)} isn't Copilot: Claude is called in`); + } + for (const login of ["copilot-pull-request-reviewer", "Copilot"]) { + const called = await evaluate(readyPr({ ...unreviewed, reviewRequests: pendingBy({ __typename: "Bot", login }), comments: commentsOf(askedBy(ACTIONS, 3600)) })); + assert.deepEqual(called.addedLabels, [], `${login} is Copilot: Claude isn't called in`); + } +}); + test("asks for Copilot's pending requests with their total", async () => { const { queries } = await evaluate(readyPr()); @@ -4978,6 +5106,16 @@ test("a writer's restart still counts while a Copilot request is pending for the const copilotRequestedAt = (at, reviewer = { login: "Copilot", type: "Bot" }) => ({ event: "review_requested", created_at: at, requested_reviewer: reviewer, actor: { login: "ruleset" } }); +test("a request for a copilot-x user or team does not save an add that PR Auto-Merge removed (mpaulosky/Application_Factory#22)", async () => { + for (const event of [ + { event: "review_requested", created_at: "2026-01-02T00:30:03Z", requested_reviewer: { login: "copilot-x", type: "User" }, actor: { login: "ruleset" } }, + { event: "review_requested", created_at: "2026-01-02T00:30:03Z", requested_team: { name: "copilot-team" }, actor: { login: "ruleset" } } + ]) { + const result = await evaluate(documentedLoop({ reviewRequests: COPILOT_PENDING }), [addedBy(OWNER), event, removedByActions("2026-01-02T00:30:05Z")]); + assert.equal(result.merges.length, 1, result.logs.join("\n")); + } +}); + test("a writer's restart survives PR Auto-Merge's removal made for a Copilot request that came after the add (mpaulosky/dotfiles#381, criterion 1)", async () => { // d d d, the owner's add while Copilot can't review, then a request goes pending (a ruleset on a push) and PR Auto-Merge takes the label off. const pr = documentedLoop({ reviewRequests: COPILOT_PENDING }); diff --git a/.github/workflows/lint-actions.yml b/.github/workflows/lint-actions.yml index 9115a8a..77d5603 100644 --- a/.github/workflows/lint-actions.yml +++ b/.github/workflows/lint-actions.yml @@ -42,7 +42,7 @@ jobs: # actions, dangerous triggers. advanced-security: false makes findings # fail the job instead of only uploading to code scanning. - name: Run zizmor - uses: zizmorcore/zizmor-action@53290badfa369cb0bf7ad16342cb62c2fa47ac7b # v0.8.0 + uses: zizmorcore/zizmor-action@45d7276c9ca790aabd705e31ca99c6185c3e76dd # v0.8.1 with: version: "1.30.1" min-severity: medium diff --git a/.github/workflows/pr-automerge.yml b/.github/workflows/pr-automerge.yml index cec6596..d8bc324 100644 --- a/.github/workflows/pr-automerge.yml +++ b/.github/workflows/pr-automerge.yml @@ -298,6 +298,15 @@ jobs: const CLAUDE_MARKER = ""; // Copilot's review comments are authored by this Bot. const COPILOT_REVIEWER = "copilot-pull-request-reviewer"; + // Whether a reviewer is the Copilot bot, by its exact login: the + // reviews API and GraphQL name it copilot-pull-request-reviewer, + // the requested-reviewer and timeline REST APIs name it Copilot. + // Only a Bot counts, so a user or team called copilot-x, or a + // user called Copilot, isn't taken for it. + const isCopilotBot = (login, type) => { + const name = (login ?? "").toLowerCase().replace(/\[bot\]$/, ""); + return type === "Bot" && (name === "copilot" || name === COPILOT_REVIEWER); + }; // At most this many unreplied threads are named in the notice. const UNREPLIED_NAMED = 3; // The second line of a Claude review with findings outside the @@ -1346,7 +1355,7 @@ jobs: // the add (mpaulosky/dotfiles#381). A review Copilot submitted // is a round, which bounds the removal already. const copilotRequests = events - .filter((event) => event.event === "review_requested" && /copilot/i.test(event.requested_reviewer?.login ?? event.requested_team?.name ?? "")) + .filter((event) => event.event === "review_requested" && isCopilotBot(event.requested_reviewer?.login, event.requested_reviewer?.type)) .map((event) => Date.parse(event.created_at)); const adds = claudeEvents .filter((event) => event.event === "labeled") @@ -1540,7 +1549,8 @@ jobs: // Fails closed: a request list that wasn't read, that holds more // than the page, or has a request with no reviewer counts as // pending, so Claude's review isn't taken for Copilot's. A Team's - // or Mannequin's request (no login comes back) isn't Copilot's. + // or Mannequin's request (no login comes back) isn't Copilot's, nor is a + // User's or Bot's with another login (copilot-x). function copilotPending(pr) { const requests = pr.reviewRequests; if (!requests || requests.totalCount > requests.nodes.length) { @@ -1548,7 +1558,7 @@ jobs: } return requests.nodes.some((request) => { const reviewer = request.requestedReviewer; - return !reviewer || /copilot/i.test(reviewer.login ?? ""); + return !reviewer || isCopilotBot(reviewer.login, reviewer.__typename); }); } @@ -1680,6 +1690,7 @@ jobs: } // Names this run's record among any others on the PR. const askId = `${Date.now()}-${Math.random().toString(36).slice(2)}`; + let askWrittenAt; try { await github.rest.issues.createComment({ owner, @@ -1697,6 +1708,9 @@ jobs: core.warning(`Couldn't record the ask for Copilot's review of PR #${pullNumber}, so it wasn't made: ${error.message}`); return false; } + // After the write returns, so a slow write can't shorten the wait: + // the record's own timestamp is no later than this. + askWrittenAt = Date.now(); // Two runs can both have read "no record" and both posted one. // Read the comments again: only the run whose record is // listed first makes the request (mpaulosky/dotfiles#369). GraphQL @@ -1764,6 +1778,13 @@ jobs: throw error; } core.warning(`Couldn't request Copilot's review of PR #${pullNumber}: ${error.message}`); + // Wait out what is left of COPILOT_ASK_SETTLE_MS (capped by + // COPILOT_ASK_WAIT_MS) before the caller adds review:claude: + // other workflows see only the record, which Claude Review + // skips and PR Auto-Merge strips while it is younger than that + // (mpaulosky/Application_Factory#23). + const remainingMs = askWrittenAt + COPILOT_ASK_SETTLE_MS - Date.now(); + await new Promise((resolve) => setTimeout(resolve, Math.max(0, Math.min(COPILOT_ASK_WAIT_MS, remainingMs)))); core.info(`Copilot's review of PR #${pullNumber} wasn't requested, so Claude is called in.`); return true; } diff --git a/.github/workflows/sandcastle.yml b/.github/workflows/sandcastle.yml index 2052f44..e12728a 100644 --- a/.github/workflows/sandcastle.yml +++ b/.github/workflows/sandcastle.yml @@ -126,12 +126,13 @@ jobs: # lib/skills.mts mounts the skills the roles name (SANDBOX_SKILLS) into # the sandbox from ~/.claude/skills, which a hosted runner doesn't have. # The repo lists them in .github/ci/sandcastle-skills.txt, a Seed data - # file: one dotfiles-sha line (the commit to take them from, 40 hex - # digits) and one skill line per skill name. This step is Owned and + # file: one dotfiles-sha line (the Application Factory commit to take them + # from, 40 hex digits; the key keeps its old name so existing files stay valid) and one skill line per skill name. This step is Owned and # reads that file as data only: it rejects any line that isn't blank, a # # comment in column 0 or one of those two shapes (LF line endings), fetches - # the pinned commit of the Template owner's public dotfiles repository, - # refuses a commit that isn't in that repository's main (GitHub serves a + # the pinned commit of the Application Factory (mpaulosky/Application_Factory, + # public), which keeps the skills in its .claude/skills, + # refuses a commit that isn't in the Factory's main (GitHub serves a # fork's commit by hash too), a skill that is missing and a symlink in a # listed skill or above it, and links only the listed skills into ~/.claude/skills. # This step runs no script of the repo's, so nothing its file says can @@ -144,7 +145,7 @@ jobs: - name: Install the skills if: steps.secrets.outputs.enabled == 'true' env: - DOTFILES_URL: https://github.com/mpaulosky/dotfiles.git + FACTORY_URL: https://github.com/mpaulosky/Application_Factory.git run: | set -euo pipefail list=.github/ci/sandcastle-skills.txt @@ -196,43 +197,43 @@ jobs: echo "::error file=$list::The file needs a dotfiles-sha line and at least one skill line, or neither." exit 1 fi - base="Support For New Projects/.claude/skills" - dotfiles="$(mktemp -d "${RUNNER_TEMP:-${TMPDIR:-/tmp}}/dotfiles.XXXXXX")" - git init --quiet "$dotfiles" + base=".claude/skills" + factory="$(mktemp -d "${RUNNER_TEMP:-${TMPDIR:-/tmp}}/factory.XXXXXX")" + git init --quiet "$factory" paths=() for skill in "${skills[@]}"; do paths+=("$base/$skill") done - git -C "$dotfiles" sparse-checkout set "${paths[@]}" - git -C "$dotfiles" fetch --quiet --depth 1 --filter=blob:none "$DOTFILES_URL" "$sha" - git -C "$dotfiles" checkout --quiet FETCH_HEAD - # GitHub serves a fork's commit by its hash, so the pin must be in dotfiles' own main. - git -C "$dotfiles" fetch --quiet --no-tags --filter=tree:0 "$DOTFILES_URL" refs/heads/main:refs/remotes/dotfiles/main - if ! git -C "$dotfiles" merge-base --is-ancestor "$sha" refs/remotes/dotfiles/main; then - echo "::error file=$list::The dotfiles-sha commit isn't in the main branch of dotfiles." + git -C "$factory" sparse-checkout set "${paths[@]}" + git -C "$factory" fetch --quiet --depth 1 --filter=blob:none "$FACTORY_URL" "$sha" + git -C "$factory" checkout --quiet FETCH_HEAD + # GitHub serves a fork's commit by its hash, so the pin must be in the Factory's own main. + git -C "$factory" fetch --quiet --no-tags --filter=tree:0 "$FACTORY_URL" refs/heads/main:refs/remotes/factory/main + if ! git -C "$factory" merge-base --is-ancestor "$sha" refs/remotes/factory/main; then + echo "::error file=$list::The dotfiles-sha commit isn't in the main branch of the Factory." exit 1 fi # The sandbox mount follows symlinks (lib/skills.mts mounts each skill's real path), so refuse one in # a listed skill, in the skill folder itself or in any folder above it. Others in the commit are not ours. - for folder in "$dotfiles/Support For New Projects" "$dotfiles/Support For New Projects/.claude" "$dotfiles/$base"; do + for folder in "$factory/.claude" "$factory/$base"; do if [ -L "$folder" ]; then - echo "::error::The dotfiles commit holds a symlink above the skills, which the sandbox mount would follow." + echo "::error::The Factory commit holds a symlink above the skills, which the sandbox mount would follow." exit 1 fi done for skill in "${skills[@]}"; do - if [ -n "$(find "$dotfiles/$base/$skill" -type l -print -quit)" ]; then + if [ -n "$(find "$factory/$base/$skill" -type l -print -quit)" ]; then echo "::error::The $skill skill holds a symlink, which the sandbox mount would follow." exit 1 fi - if [ ! -f "$dotfiles/$base/$skill/SKILL.md" ]; then - echo "::error::dotfiles $sha has no $skill skill." + if [ ! -f "$factory/$base/$skill/SKILL.md" ]; then + echo "::error::The Factory commit $sha has no $skill skill." exit 1 fi done mkdir -p "$HOME/.claude/skills" for skill in "${skills[@]}"; do - ln -sT "$dotfiles/$base/$skill" "$HOME/.claude/skills/$skill" + ln -sT "$factory/$base/$skill" "$HOME/.claude/skills/$skill" done # The host's git fetches and pushes authenticate through gh's credential @@ -304,12 +305,13 @@ jobs: # keep running) can change a file in it before this step uploads it or # the next step reads handbacks.json. The reviewers' logs reach its # reviewer-logs/ folder through lib/role-logs.mts, which writes an - # allowlisted view of each (the last host status lines, the last known - # tool names without arguments, and the last few error lines), named + # allowlisted view of each (the host status lines, the known tool names + # without arguments, and the error words that appeared, each only as a + # fixed line in a sorted set: no order, count, number or text), named # reviewer-.log and never for the source file, and leaves # out a view shaped like a secret. The view narrows what a steered - # reviewer can print into the artifact; it can't promise a few words on - # an error line stay out. The pre-push review's own log is never copied. + # reviewer can print into the artifact: no free text of a log line gets + # through. The pre-push review's own log is never copied. - name: Upload the run report if: always() && steps.secrets.outputs.enabled == 'true' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 diff --git a/.sandcastle/lib/branches.mts b/.sandcastle/lib/branches.mts index ffcd290..b4db9ab 100644 --- a/.sandcastle/lib/branches.mts +++ b/.sandcastle/lib/branches.mts @@ -42,7 +42,7 @@ export function isIssueBranch(branch: string, issueNumber: number): boolean { } // Whether a branch is one scripts/check-branch-name.sh accepts, without --pr: feature/, fix/ or hotfix/{n}-{slug}, or -// chore/{slug}. The branch of a pull request the owner handed to Sandcastle (lib/follow-up.mts#sweepSubject) has to pass +// chore/{slug}, but not the script's dependabot/* (the hook lets a person push a fix to Dependabot's branch; Sandcastle hands off none). The branch of a pull request the owner handed to Sandcastle (lib/follow-up.mts#sweepSubject) has to pass // it, because its name reaches a shell the way an issue branch's does ({{BRANCH}} in review-prompt.md, git in the sandbox). export function isHandOffBranch(branch: string): boolean { return /^(?:(?:feature|fix|hotfix)\/[0-9]+-[a-z0-9]+(?:-[a-z0-9]+)*|chore\/[a-z][a-z0-9]*(?:-[a-z0-9]+)*)$/.test(branch); diff --git a/.sandcastle/lib/branches.test.mts b/.sandcastle/lib/branches.test.mts index 2d3fdd4..88e081e 100644 --- a/.sandcastle/lib/branches.test.mts +++ b/.sandcastle/lib/branches.test.mts @@ -91,7 +91,7 @@ describe("isIssueBranch", () => { }); // The branches of a pull request handed to Sandcastle with the sandcastle:follow-up label: the same names -// scripts/check-branch-name.sh lets through, so a name that reaches a shell is as strict as an issue branch's. +// scripts/check-branch-name.sh lets through (but for its dependabot/*), so a name that reaches a shell is as strict as an issue branch's. describe("isHandOffBranch", () => { for (const branch of ["feature/5-add-search", "fix/12-stop-the-crash", "hotfix/1-patch", "chore/tidy-up", "chore/a", "chore/bump-v2-deps", "feature/314-x"]) { it(`accepts ${branch}`, () => { @@ -136,7 +136,10 @@ describe("isHandOffBranch", () => { "chore/a\t", "chore/a\r", "feature/5-x ", "chore/ ", ]; - for (const branch of corpus) assert.equal(isHandOffBranch(branch), passesBranchStandard(branch), JSON.stringify(branch)); + // The script also lets a person push to Dependabot's dependabot/* branches; a hand-off branch is never one. + for (const branch of corpus) { + assert.equal(isHandOffBranch(branch), !branch.startsWith("dependabot/") && passesBranchStandard(branch), JSON.stringify(branch)); + } }); // Not in the corpus above: bash's [a-z] and [0-9] take letters and digits of other scripts in some locales, so the diff --git a/.sandcastle/lib/review-ledger.mts b/.sandcastle/lib/review-ledger.mts index ebc228b..f46f082 100644 --- a/.sandcastle/lib/review-ledger.mts +++ b/.sandcastle/lib/review-ledger.mts @@ -135,7 +135,6 @@ export const OWNED_PATHS = [ ".claude/rules/sandcastle-hand-off.md", ".claude/skills/README.md", ".security/README.md", - "LICENSE", ".editorconfig", ".gitignore", ".markdownlint-cli2.jsonc", @@ -156,7 +155,7 @@ export const OWNED_PATHS = [ // local gate's .sandcastle/check.sh, whose watched paths and early exit decide // whether the Sandcastle check runs at all, global.json, which sets the SDK and the msbuild-sdks a build runs, // docs/PROCESS.md, which review-prompt.md tells a role to follow, and the gate chain (see OWNED_PATHS above). -// The other files that moved from Seed to Owned (LICENSE, CODE_OF_CONDUCT.md, +// The other files that moved from Seed to Owned (CODE_OF_CONDUCT.md, // the READMEs under .security/ and .claude/skills/) are audited text that no // role reads, and a wrong one only misleads a person, so they stay exempt. // The exemption judges a path, never the bytes, since a repository has no copy diff --git a/.sandcastle/lib/review-ledger.test.mts b/.sandcastle/lib/review-ledger.test.mts index 24db443..7a4e914 100644 --- a/.sandcastle/lib/review-ledger.test.mts +++ b/.sandcastle/lib/review-ledger.test.mts @@ -2669,7 +2669,7 @@ describe("repoSlugProblem and the repo command", () => { describe("OWNED_PATHS", () => { it("matches folders and globs as the rule means them", () => { - for (const path of [".github/workflows/ci.yml", ".github/hooks/tests/pre-push.test.sh", ".github/scripts/a/b/c.sh", "global.json", ".claude/skills/pre-push-review/SKILL.md", ".claude/skills/hardening-review/SKILL.md", ".claude/skills/hardening-review/scripts/mutate.py", ".claude/skills/hardening-review/references/brief.md", ".sandcastle/lib/build.mts", ".github/ci/coverage-threshold", ".claude/rules/sandcastle-hand-off.md", ".claude/skills/README.md", ".security/README.md", "LICENSE", "docs/CODE_OF_CONDUCT.md", ".github/ISSUE_TEMPLATE/bug_report.md", ".github/ISSUE_TEMPLATE/config.yml", ".github/ISSUE_TEMPLATE/feature_request.md"]) { + for (const path of [".github/workflows/ci.yml", ".github/hooks/tests/pre-push.test.sh", ".github/scripts/a/b/c.sh", "global.json", ".claude/skills/pre-push-review/SKILL.md", ".claude/skills/hardening-review/SKILL.md", ".claude/skills/hardening-review/scripts/mutate.py", ".claude/skills/hardening-review/references/brief.md", ".sandcastle/lib/build.mts", ".github/ci/coverage-threshold", ".claude/rules/sandcastle-hand-off.md", ".claude/skills/README.md", ".security/README.md", "docs/CODE_OF_CONDUCT.md", ".github/ISSUE_TEMPLATE/bug_report.md", ".github/ISSUE_TEMPLATE/config.yml", ".github/ISSUE_TEMPLATE/feature_request.md"]) { assert.equal(isOwnedPath(path), true, path); } for (const path of [ @@ -2687,6 +2687,7 @@ describe("OWNED_PATHS", () => { "xglobal.json", ".github/ISSUE_TEMPLATE/question.md", ".github/ISSUE_TEMPLATE/sub/bug_report.md", + "LICENSE", // a Seed file: the repository's own, whatever year it carries "docs/LICENSE", "LICENSE.bak", ".security/0001-an-entry.md", diff --git a/.sandcastle/lib/role-logs.mts b/.sandcastle/lib/role-logs.mts index 2128f8e..32bafb0 100644 --- a/.sandcastle/lib/role-logs.mts +++ b/.sandcastle/lib/role-logs.mts @@ -2,18 +2,20 @@ // 641-byte report and no log, so nothing named why. The artifact of a public repository can be // downloaded by any signed-in user and GitHub masks secrets in step logs, not artifacts, and a log // holds what an agent printed: model prose and command text anyone who can write an issue can steer. -// So a log is never copied whole. Only an allowlisted view is: the host's own status lines (the last few, the start line without its -// timestamp, since a forged line's digits are data), the name -// of each of the last few tool calls without its arguments (only a known tool's name, an MCP tool as `mcp__(...)`), and the few lines that name an -// error or an abort. Those error lines are the one -// place free text gets through, so they are cut short, limited in number, and left out when they hold a -// long run of token characters (where an encoded secret would sit). Whatever the view still holds is -// checked for the sandbox's secret values and for token shapes, also with its spacing taken out, and a -// view that fails the check is replaced by a note. This narrows what can leave the sandbox in a log; it -// can't promise that a steered reviewer cannot get a few words of its own onto an error line. +// So a log is never copied whole. Only an allowlisted view is, and it is three sorted sets of fixed lines, not a sequence: which of the host's own status +// lines appeared (every number in them masked as `#`, the start line without its timestamp, since a forged line's digits are data), which known tools +// were called (a known tool's name without its arguments, an MCP tool as `mcp__(...)`), and which error words appeared (as the fixed message +// `[error line: timed out]`, never the line itself, so no word of a log's free text reaches the view: a secret printed with invisible fillers between its +// characters, or as spaced-out hex, can't get through a check that only knows the shapes it looks for). A line holding a long run of token +// characters is not an error line here, it is left out. A view says nothing of how often, in what order or after what a line appeared, and has one +// `[... lines left out]` line, first, when anything else was in the log (or the log was cut). Whatever the view holds is still checked for the sandbox's +// secret values and for token shapes, also with its spacing taken out, and a view that fails the check is replaced by a note. Only the logs of builds this +// run started are copied (a sandbox that can write the logs folder could add files of its own), at most REPORT_LOG_LIMIT of them. What a steered reviewer can +// still choose, in the log of its own build, is which of the fixed lines appear: about 30 host kinds, 15 tools and 26 error words, one bit each, so about 72 bits +// a log; no text, no number, no order and no count. The index says only which issue each log is, whether it was copied, and two fixed lines about logs left out. // The output folder is emptied before it is written, in case an earlier run left files in it. // A reviewer run the host gave up on may still be writing its log, so a copy is a snapshot. A log is read only as far as its last REPORT_INPUT_LIMIT -// bytes, since the view keeps the last lines of each kind and the sandbox chooses how long a log is; a view of a longer log starts with the left-out marker. +// bytes, since the sandbox chooses how long a log is and the host's memory grows with what it reads; a view of a longer log starts with the left-out marker. import { closeSync, constants, fstatSync, lstatSync, mkdirSync, openSync, readdirSync, readSync, rmSync, writeFileSync } from "node:fs"; import { join } from "node:path"; @@ -27,30 +29,20 @@ export function reviewerLogsDir(reportDir: string): string { return join(reportDir, REVIEWER_LOGS_NAME); } -// The most of one kept error line, so a long line can't carry much. -export const REPORT_LINE_LIMIT = 200; - -// The most error lines of free text one view keeps (the last ones, where an abort shows), so a steered -// reviewer can't carry much out by printing the word `error` on many lines. -export const REPORT_ERROR_LINES = 20; - -// The most host status lines one view keeps (the last ones): their numbers are the only data a line shaped like one can carry, -// and a real run prints a few per iteration. -export const REPORT_HOST_LINES = 30; - -// The most tool-call lines one view keeps (the last ones): which tool a line names is data a forged line can choose. -export const REPORT_TOOL_LINES = 30; +// The most reviewer logs one run report copies (those of builds this run started, by sorted name), so the report's size doesn't depend on how many builds a +// run had; the rest are named in the index only by one fixed line saying more were not copied (no count). +export const REPORT_LOG_LIMIT = 20; // A run of this many token characters in an error line (base64, hex, a key) leaves the line out. export const REPORT_TOKEN_RUN = 24; -// The marker the view puts in place of a run of lines it left out, and in front of a log it cut to its last REPORT_INPUT_LIMIT bytes, so a reader -// can tell a cut log from a quiet one. No count: the number of lines an attacker printed would be data. +// The marker the view starts with when it left lines out, or the log was cut to its last REPORT_INPUT_LIMIT bytes, so a reader can tell a log with more in it +// from a quiet one. No count and no place: the number of lines an attacker printed, and where, would be data. const LEFT_OUT = "[... lines left out]"; // How much of a reviewer log the host reads: its last REPORT_INPUT_LIMIT bytes (characters, in allowlistedView), starting at a whole line. The sandbox writes -// the log, so its size is the sandbox's choice, and the view keeps only the last lines of each kind, so reading more only costs the host memory in -// proportion to the log (the string, then a line array and a same-length keep array) while generating its report. +// the log, so its size is the sandbox's choice, and reading more only costs the host memory in proportion to the log (the string, then a line array) +// while generating its report. export const REPORT_INPUT_LIMIT = 4 * 1024 * 1024; // `text` from its first whole line on: a text cut at its start begins in the middle of a line (or of a multi-byte character), so everything up to the @@ -83,6 +75,12 @@ const hostLine = new RegExp( ].join("|")})$`, ); +// A host status line with every number replaced by `#`: the numbers are free digits a forged line could use to carry a secret out, and the +// view says which kind of line ran, not how many. +function maskedHostLine(line: string): string { + return line.replace(/\d+/g, "#"); +} + // The tools a role is given, the only names the view keeps. An MCP tool (`mcp____`) is kept as `mcp__(...)`, // since the names there are free text. const knownTool = /^(?:Bash|Read|Grep|Glob|Edit|MultiEdit|Write|NotebookEdit|Task|Agent|TodoWrite|WebFetch|WebSearch|Skill)$/; @@ -100,7 +98,9 @@ function toolName(line: string): string | undefined { return /^mcp__[A-Za-z0-9-]{1,30}__[A-Za-z0-9_-]{1,40}$/.test(name) ? "mcp__(...)" : undefined; } -// A line that says a run went wrong, which the view keeps as free text (the last REPORT_ERROR_LINES of them) whoever wrote it. +// A line that says a run went wrong, whose word the view keeps as a fixed message, whoever wrote the line. +// It must keep no `g` or `y` flag (`test` and `exec` would then depend on lastIndex) and no `u` flag (a Unicode case-insensitive match +// lets U+017F or U+212A stand for `s` or `k`, and the message would copy them). const errorLine = /\b(?:abort(?:ed)?|error|errors|fail(?:ed|ure|ures)?|exception|fatal|panic|timed out|timeout|killed|terminated|exit(?:ed)? (?:code|with)|signal|crash(?:ed)?|unauthori[sz]ed|rate limit|overloaded)\b/i; // A run of token characters long enough to be an encoded secret. @@ -109,62 +109,44 @@ const tokenRun = new RegExp(`[A-Za-z0-9+/_=-]{${REPORT_TOKEN_RUN},}`); // A token shape without the word boundaries, for the view with its spacing taken out, where a boundary can't hold. const looseTokenShape = /sk-ant-[A-Za-z0-9_-]{16,}|gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}/; -// `text` cut to `limit` characters, never ending on half of a surrogate pair. -function cut(text: string, limit: number): string { - const head = text.slice(0, limit); - return /[\uD800-\uDBFF]$/.test(head) && text.length > limit ? head.slice(0, -1) : head; +// The fixed message that stands for an error line: the error word it matched, which only ASCII letters and single spaces can spell +// (a case-insensitive match without the `u` flag never lets a non-ASCII character stand for an ASCII one), never the line. Only a line `errorLine` matched comes here. +function errorMessage(line: string): string { + return `[error line: ${errorLine.exec(line)![0].toLowerCase()}]`; } -// The allowlisted view of a reviewer log, of its last REPORT_INPUT_LIMIT characters from a whole line (behind LEFT_OUT when it was longer): -// its last REPORT_HOST_LINES host status lines and the names of its last REPORT_TOOL_LINES known tool calls, with a -// `[... lines left out]` marker (no count) for each run of lines it left out. Of the lines that name an error, the last REPORT_ERROR_LINES -// are kept as free text cut to REPORT_LINE_LIMIT, unless one holds a run of REPORT_TOKEN_RUN token characters or -// `held` says it looks like a secret. Model prose, tool arguments and tool output are what it leaves out. +// The allowlisted view of a reviewer log, of its last REPORT_INPUT_LIMIT characters from a whole line: the host status lines that appeared (numbers masked), the +// known tools called (names only) and the fixed `[error line: ]` messages for the error words that appeared, each set sorted and each line once. +// A line that holds a run of REPORT_TOKEN_RUN token characters, or that `held` says looks like a secret, is no error line. Model prose, tool arguments and tool +// output are left out, and `[... lines left out]` comes first when any was, or the log was cut. export function allowlistedView(text: string, held: (line: string) => boolean = () => false): string { - // One character more than the limit, so a cut that lands on a line break leaves that line whole: the line break is the part dropped. A log - // liveRoleLogFs.read already cut starts with the marker line, which this cut then drops (it reaches no further back than its own line break). + // One character more than the limit, so a cut that lands on a line break leaves that line whole: the line break is the part dropped. const over = text.length > REPORT_INPUT_LIMIT; - const bounded = over ? fromWholeLine(text.slice(-(REPORT_INPUT_LIMIT + 1))) : text; - // The marker counts as a line left out, so the view starts with it whenever anything before the kept lines was dropped. - const lines = `${over ? `${LEFT_OUT}\n` : ""}${bounded}`.split(/\r?\n/); - const keep = new Array(lines.length).fill(undefined); - const errors: number[] = []; - const hosts: number[] = []; - const tools: number[] = []; - lines.forEach((line, index) => { - if (hostLine.test(line)) hosts.push(index); - else { - const tool = toolName(line); - if (tool !== undefined) tools.push(index); - else if (errorLine.test(line) && !tokenRun.test(line) && !held(line)) errors.push(index); + const lines = (over ? fromWholeLine(text.slice(-(REPORT_INPUT_LIMIT + 1))) : text).split(/\r?\n/); + let left = over; + const hosts = new Set(); + const tools = new Set(); + const errors = new Set(); + for (const line of lines) { + if (line.trim() === "") continue; + if (hostLine.test(line)) { + // The start line's timestamp is free digits, so it is kept without it. + hosts.add(line.startsWith("--- Run started") ? "--- Run started ---" : maskedHostLine(line)); + continue; } - }); - // The start line's timestamp is free digits, so it is kept without it. - for (const index of hosts.slice(-REPORT_HOST_LINES)) keep[index] = lines[index]!.startsWith("--- Run started") ? "--- Run started ---" : lines[index]; - for (const index of tools.slice(-REPORT_TOOL_LINES)) keep[index] = toolName(lines[index]!); - for (const index of errors.slice(-REPORT_ERROR_LINES)) { - keep[index] = cut(lines[index]!, REPORT_LINE_LIMIT); + const tool = toolName(line); + if (tool !== undefined) tools.add(tool); + else if (errorLine.test(line) && !tokenRun.test(line) && !held(line)) errors.add(errorMessage(line)); + else left = true; } - const out: string[] = []; - let left = 0; - const flush = () => { - if (left > 0) out.push(LEFT_OUT); - left = 0; - }; - keep.forEach((kept, index) => { - if (kept === undefined) { - if (lines[index]!.trim() !== "") left++; - } else { - flush(); - out.push(kept); - } - }); - flush(); + const out = [...(left ? [LEFT_OUT] : []), ...[...hosts].sort(), ...[...tools].sort(), ...[...errors].sort()]; return `${out.join("\n")}\n`; } -// Whether the view holds one of the secrets or a token shape, as written or with whitespace and control and format -// characters taken out (and ANSI colour codes first), so a token an error line printed spaced out or wrapped is whole again for the check. +// Whether the text holds one of the secrets or a token shape, as written or with whitespace and control and format +// characters taken out (and ANSI colour codes first), so a token printed spaced out or wrapped is whole again for the check. It runs on +// each error line (one that matched `errorLine` and holds no long run of token characters) before it is reduced to its fixed message (the `held` +// callback), and on the finished view as a second check. function looksLikeSecret(view: string, secrets: readonly string[]): boolean { const spaceless = view.replace(/\u001b\[[0-?]*[ -\/]*[@-~]/g, "").replace(/[\s\p{Cc}\p{Cf}\p{M}]/gu, ""); return [view, spaceless].some((form) => containsSecret(form, secrets) || looseTokenShape.test(form)); @@ -174,12 +156,12 @@ function looksLikeSecret(view: string, secrets: readonly string[]): boolean { export const REVIEWER_LOG_INDEX = "index.txt"; // The name Sandcastle gives a reviewer's log for a build's branch: the branch with the characters a file name can't hold replaced by a -// hyphen, then `-reviewer.log` (its buildLogFilename). A name that no longer matches this only leaves a log unlabelled in the index. +// hyphen, then `-reviewer.log` (its buildLogFilename). A name that no longer matches this means the log is not copied at all (copyReviewerLogs copies only the logs a BuildLogs record knows). export function reviewerLogFileName(branch: string): string { return `${branch.replace(/[/\\:*?"<>|]/g, "-")}-reviewer.log`; } -// The builds this run started, by the name of the reviewer log each one writes, so the index can say which build a numbered view is. +// The builds this run started, by the name of the reviewer log each one writes, so the index can say which build a numbered view is and the report copies only the logs of these builds. // Only the host's own issue numbers and branches go in, and only an issue number comes out: a log's name in the folder is whatever an // agent that can write there chose, so it is looked up here and never printed. That lookup is by name, so an agent could name a file // for another build's log and take its label: the label is a hint for a person reading a diagnostic, not proof of where a log came from. @@ -195,7 +177,7 @@ export class BuildLogs { } } -// The one record every build adds itself to. +// The one record every build adds itself to. It says which build wrote each log, and so which logs the report copies. export const buildLogs = new BuildLogs(); // Copy the allowlisted view of each `*-reviewer.log` in `logsDir` to `outDir`, as `reviewer-.log` (reviewerLogsDir of the run report's folder, which is outside the checkout on Actions), with a short @@ -203,8 +185,9 @@ export const buildLogs = new BuildLogs(); // are not a Sandcastle role's and are never copied (the lead's lists the subagents' answers). Never throws: it runs in the run's finally, and an unreadable file // or log directory is a warning (a directory that doesn't exist is no logs, and no warning). // `secrets` defaults to the values .sandcastle/.env gives the sandbox, read once per call. Returns the names it wrote (`reviewer-.log`), including a log it wrote the note for. Beside them it writes -// REVIEWER_LOG_INDEX, which says for each number the issue whose build wrote the log (from `builds`), or that the host started no build -// for it, and whether the view couldn't be copied. Each log is read only as far as its last REPORT_INPUT_LIMIT bytes (liveRoleLogFs.read). +// REVIEWER_LOG_INDEX, which says for each number the issue whose build wrote the log (from `builds`) and whether the view couldn't be copied. Only the logs of builds +// this run started are copied, at most REPORT_LOG_LIMIT of them by sorted name; the index ends with one fixed line (no count) for each of: more such logs than that, +// and other `*-reviewer.log` files in the folder, which a sandbox could have added. The returned names and the numbering cover only the copied logs. Each log is read only as far as its last REPORT_INPUT_LIMIT bytes (liveRoleLogFs.read). export function copyReviewerLogs( logsDir: string, outDir: string, @@ -223,22 +206,27 @@ export function copyReviewerLogs( throw new Error(`the output folder could not be emptied, so nothing was written to it: ${error instanceof Error ? error.message : String(error)}`); } // Sorted, so the numbers below don't depend on the order the filesystem lists in. - const names = fs + const all = fs .list(logsDir) .filter((name) => name.endsWith("-reviewer.log") && !name.endsWith("-pre-push-reviewer.log")) .sort(); - if (names.length === 0) return copied; + if (all.length === 0) return copied; + // Only the logs of builds this run started are copied: a sandbox that can write the logs folder could add files of its own, and each view is + // a few dozen bits it would choose, so copying them would multiply the channel by a number it picks. + const names = all.filter((name) => builds.issueOf(name) !== undefined); + const unknown = names.length < all.length; + const omitted = names.length > REPORT_LOG_LIMIT; + names.splice(REPORT_LOG_LIMIT); const known = secrets ?? [...sandboxSecrets().values()]; fs.mkdir(outDir); // A view is written under a name the host makes, never the source's: the source name is whatever an agent that can write the logs // folder chose, and the content checks don't see it, so a file named for the sandbox's credential would put the credential in the // artifact's file list. The same goes for the warnings below, which reach the runner's public log. The number is the file's place - // among the sorted names, and a log that can't be copied still takes its number. + // among the copied logs' sorted names, and a log that can't be copied still takes its number. const index: string[] = []; names.forEach((name, position) => { const outName = `reviewer-${position + 1}.log`; - const issue = builds.issueOf(name); - const label = issue === undefined ? "a build this run did not start" : `issue #${issue}`; + const label = `issue #${builds.issueOf(name)}`; try { const view = allowlistedView(fs.read(join(logsDir, name)), (line) => looksLikeSecret(line, known)); const held = looksLikeSecret(view, known); @@ -252,6 +240,8 @@ export function copyReviewerLogs( index.push(`${outName}: ${label} (not copied)`); } }); + if (omitted) index.push(`more than ${REPORT_LOG_LIMIT} reviewer logs: the rest were not copied`); + if (unknown) index.push("other reviewer logs in the folder were not copied: no build of this run wrote them"); try { fs.write(join(outDir, REVIEWER_LOG_INDEX), `${index.join("\n")}\n`); } catch (error) { @@ -270,7 +260,8 @@ export function copyReviewerLogs( // guards against the entry being swapped after that check, but only in part: O_NOFOLLOW refuses a symlink (ELOOP), O_NONBLOCK keeps the open of a // FIFO from waiting for a writer (it does nothing to a regular file), O_NOCTTY keeps a terminal from becoming the host's controlling one, and fstat // then refuses whatever was opened unless it is a regular file. A device node swapped in between the check and the open is still opened, and refused -// only after: that window needs write access to the logs folder and a won race, and is accepted. Windows has no O_NOFOLLOW, so there a swapped-in +// only after. The sandbox writes these logs, so it could win that race and have the device opened (O_NONBLOCK and O_NOCTTY limit what an open does). +// That is accepted: fstat refuses the node before any read. Windows has no O_NOFOLLOW, so there a swapped-in // symlink is followed. `isRegular` is a parameter so a test can stand in the swap. export function readLogTail(path: string, isRegular: (path: string) => boolean = (target) => lstatSync(target).isFile()): string { if (!isRegular(path)) throw new Error("not a regular file"); diff --git a/.sandcastle/lib/role-logs.test.mts b/.sandcastle/lib/role-logs.test.mts index 891c05d..58d3a2c 100644 --- a/.sandcastle/lib/role-logs.test.mts +++ b/.sandcastle/lib/role-logs.test.mts @@ -7,16 +7,14 @@ import { describe, it } from "node:test"; import { allowlistedView, BuildLogs, + buildLogs, copyReviewerLogs, REVIEWER_LOG_INDEX, reviewerLogFileName, reviewerLogsDir, REVIEWER_LOGS_NAME, - REPORT_ERROR_LINES, - REPORT_HOST_LINES, + REPORT_LOG_LIMIT, REPORT_INPUT_LIMIT, - REPORT_TOOL_LINES, - REPORT_LINE_LIMIT, REPORT_TOKEN_RUN, liveRoleLogFs, readLogTail, @@ -67,23 +65,79 @@ const hostLines = [ "Run complete: reached 3 iteration(s) without completion signal.", ]; -// copyReviewerLogs with the views going to /report, as most tests below read them from there. -const copyTo = (logsDir: string, fs?: RoleLogFs, warn?: (line: string) => void, secrets?: readonly string[]) => - copyReviewerLogs(logsDir, join(logsDir, "report"), fs, warn, secrets); +// A record of builds that wrote every `*-reviewer.log` in `names`, issue numbers in sorted order from 1: only a log of a build the run started is copied. +const knowing = (names: readonly string[]) => { + const builds = new BuildLogs(); + names.filter((name) => name.endsWith("-reviewer.log")).sort().forEach((name, i) => builds.add(i + 1, name.replace(/-reviewer\.log$/, ""))); + return builds; +}; + +// The files a folder lists, or none when it can't be listed (copyReviewerLogs says that itself). +const listed = (logsDir: string, fs?: RoleLogFs) => { + try { + return (fs ?? liveRoleLogFs).list(logsDir); + } catch { + return []; + } +}; + +// copyReviewerLogs with the views going to /report, as most tests below read them from there. Every log the folder lists is a build's. +const copyTo = (logsDir: string, fs?: RoleLogFs, warn?: (line: string) => void, secrets?: readonly string[], builds?: BuildLogs) => + copyReviewerLogs(logsDir, join(logsDir, "report"), fs, warn, secrets, builds ?? knowing(listed(logsDir, fs))); describe("allowlistedView", () => { - it("keeps every host status line as written, but for the start line's timestamp", () => { - const kept = hostLines.map((line) => (line.startsWith("--- Run started") ? "--- Run started ---" : line)); - assert.equal(allowlistedView(hostLines.join("\n")), `${kept.join("\n")}\n`); + it("keeps every host status line with its numbers masked, and the start line without its timestamp", () => { + const kept = hostLines.map((line) => (line.startsWith("--- Run started") ? "--- Run started ---" : line.replace(/\d+/g, "#"))); + assert.ok(kept.some((line) => line.includes("#")) && kept.every((line) => !/\d/.test(line))); + assert.equal(allowlistedView(hostLines.join("\n")), `${[...new Set(kept)].sort().join("\n")}\n`); + }); + + it("is a set, not a sequence: order, count and gaps change nothing, so they carry nothing", () => { + const a = allowlistedView("Agent started\nBash(1)\nfatal x\nAgent stopped\nRead(2)\npanic y"); + assert.equal(a, "Agent started\nAgent stopped\nBash(...)\nRead(...)\n[error line: fatal]\n[error line: panic]\n"); + // Reordered, repeated, interleaved with blank lines, with CRLF: the same view. + assert.equal(allowlistedView("panic y\r\nRead(2)\r\n\r\nAgent stopped\nfatal x\nBash(9)\nAgent started\nAgent started\nBash(1)\nBash(1)\npanic y"), a); + // A thousand lines of each kind are still one line of each. + const many = Array.from({ length: 1000 }, (_, i) => ["Agent started", `Bash(${i})`, `fatal ${i}`, "Agent stopped"][i % 4]!).join("\n"); + assert.equal(allowlistedView(many), "Agent started\nAgent stopped\nBash(...)\n[error line: fatal]\n"); + // A line of prose between two host lines does not show where it sat: only that something was left out. + assert.equal(allowlistedView("Agent started\nprose\nAgent stopped"), "[... lines left out]\nAgent started\nAgent stopped\n"); + assert.equal(allowlistedView("Agent started\n\n\nAgent stopped\n"), "Agent started\nAgent stopped\n"); + }); + + it("shows one line for each fixed host line, tool and error word, however often each appears: 30 host kinds, 15 tools, 26 error words", () => { + const masked = [...new Set(hostLines.map((line) => (line.startsWith("--- Run started") ? "--- Run started ---" : line.replace(/\d+/g, "#"))))].sort(); + assert.equal(allowlistedView(hostLines.concat(hostLines).join("\n")), `${masked.join("\n")}\n`); + const tools = ["Bash", "Read", "Grep", "Glob", "Edit", "MultiEdit", "Write", "NotebookEdit", "Task", "Agent", "TodoWrite", "WebFetch", "WebSearch", "Skill"]; + const toolLines = [...tools.map((name) => `${name}(x)`), "mcp__a__b(x)", "mcp__c__d(y)"]; + const toolView = allowlistedView(toolLines.concat(toolLines).join("\n")); + assert.equal(toolView, `${[...tools.map((name) => `${name}(...)`), "mcp__(...)"].sort().join("\n")}\n`); + assert.equal(toolView.split("\n").filter(Boolean).length, 15); + const words = ["abort", "aborted", "error", "errors", "fail", "failed", "failure", "failures", "exception", "fatal", "panic", "timed out", "timeout", "killed", + "terminated", "exit code", "exited code", "exit with", "exited with", "signal", "crash", "crashed", "unauthorized", "unauthorised", "rate limit", "overloaded"]; + const errorView = allowlistedView(words.concat(words).map((word) => `x ${word} y`).join("\n")); + assert.equal(errorView, `${words.map((word) => `[error line: ${word}]`).sort().join("\n")}\n`); + }); + + it("has 30 host kinds once numbers are masked, and shows each once", () => { + const phrases = ["Setting up sandbox", "Collecting commits", "Capturing session", "Expanding shell expressions", "No commits to sync out", "Syncing 2 commits to host", "Syncing 1 commit to host"]; + const forms = phrases.flatMap((phrase) => [phrase, `${phrase} done (1s)`, `${phrase} done (1.5s)`]); + const others = [ + "--- Run started: 2026-11-03T10:00:00.123Z ---", "Iteration 1/3", "Agent started", "Agent stopped", "Agent signaled completion after 2 iteration(s).", + "Context window: 42k", "Reached max iterations (3).", "Run complete: agent finished after 2 iteration(s).", "Run complete: reached 3 iteration(s) without completion signal.", + ]; + const lines = [...forms, ...others]; + const masked = [...new Set(lines.map((line) => (line.startsWith("--- Run started") ? "--- Run started ---" : line.replace(/\d+/g, "#"))))]; + assert.equal(masked.length, 30); + const view = allowlistedView(lines.join("\n")); + assert.equal(view, `${[...masked].sort().join("\n")}\n`); }); - it("keeps only the last REPORT_HOST_LINES host lines", () => { - assert.equal(REPORT_HOST_LINES, 30); - const lines = (count: number) => Array.from({ length: count }, (_, i) => `Iteration ${i}/99`); - assert.equal(allowlistedView(lines(REPORT_HOST_LINES).join("\n")), `${lines(REPORT_HOST_LINES).join("\n")}\n`); - const over = allowlistedView(lines(REPORT_HOST_LINES + 1).join("\n")); - assert.ok(over.startsWith("[... lines left out]\nIteration 1/99\n")); - assert.ok(over.endsWith(`Iteration ${REPORT_HOST_LINES}/99\n`)); + it("asks the caller's check only about an error line that holds no run of token characters, once for each", () => { + const asked: string[] = []; + const held = (line: string) => (asked.push(line), false); + allowlistedView(["prose", "Agent started", "Bash(x)", "fatal one", `fatal ${"A".repeat(REPORT_TOKEN_RUN)}`, "panic two", ""].join("\n"), held); + assert.deepEqual(asked, ["fatal one", "panic two"]); }); it("limits every number in a host line to three digits (two after a decimal point), and needs one", () => { @@ -97,7 +151,7 @@ describe("allowlistedView", () => { const line = (digits: number) => template.replace("{n}", "7".repeat(digits)); // `held` rejects every error line, so only a host line can be kept (one worded with an error word, as one is, would be kept as an error otherwise). const view = (text: string) => allowlistedView(text, () => true); - assert.equal(view(line(limit)), `${line(limit)}\n`, line(limit)); + assert.equal(view(line(limit)), `${line(limit).replace(/\d+/g, "#")}\n`, line(limit)); assert.equal(view(line(limit + 1)), "[... lines left out]\n", line(limit + 1)); assert.equal(view(line(0)), "[... lines left out]\n", line(0)); } @@ -118,20 +172,10 @@ describe("allowlistedView", () => { it("keeps a known tool's name and drops its arguments, also over several lines", () => { const view = allowlistedView('Bash(cat > f <<\'EOF\'\nthe hidden body\nEOF\n)\nRead(/etc/passwd)'); - assert.match(view, /^Bash\(\.\.\.\)\n/); - assert.ok(view.includes("Read(...)")); + assert.equal(view, "[... lines left out]\nBash(...)\nRead(...)\n"); assert.ok(!/hidden|passwd|cat > f/.test(view)); }); - it("keeps only the last REPORT_TOOL_LINES tool lines", () => { - assert.equal(REPORT_TOOL_LINES, 30); - const lines = (count: number) => Array.from({ length: count }, (_, i) => `${i % 2 === 0 ? "Bash" : "Read"}(${i})`); - const kept = (count: number) => lines(count).map((line) => `${line.split("(")[0]}(...)`); - assert.equal(allowlistedView(lines(REPORT_TOOL_LINES).join("\n")), `${kept(REPORT_TOOL_LINES).join("\n")}\n`); - const over = allowlistedView(lines(REPORT_TOOL_LINES + 1).join("\n")); - assert.equal(over, `[... lines left out]\n${kept(REPORT_TOOL_LINES + 1).slice(1).join("\n")}\n`); - }); - it("keeps each tool a role is given by name", () => { for (const name of ["Bash", "Read", "Grep", "Glob", "Edit", "MultiEdit", "Write", "NotebookEdit", "Task", "Agent", "TodoWrite", "WebFetch", "WebSearch", "Skill"]) { assert.equal(allowlistedView(`${name}(x)`), `${name}(...)\n`, name); @@ -164,9 +208,9 @@ describe("allowlistedView", () => { for (const line of lookalikes) assert.equal(allowlistedView(line), "[... lines left out]\n", line); }); - it("drops model prose and marks where lines were left out, once per run of them and without a count, blank lines being no run", () => { + it("drops model prose and marks that lines were left out, once, first and without a count, blank lines being none", () => { const view = allowlistedView("Agent started\nsome prose\nmore prose\n\nAgent stopped\nlast words"); - assert.equal(view, "Agent started\n[... lines left out]\nAgent stopped\n[... lines left out]\n"); + assert.equal(view, "[... lines left out]\nAgent started\nAgent stopped\n"); }); it("matches a host line whole: a prefix, a suffix, a malformed number or timestamp doesn't pass", () => { @@ -180,78 +224,93 @@ describe("allowlistedView", () => { it("keeps the singular sync line, whole-second durations and every other host form", () => { const forms = ["Syncing 1 commit to host", "Setting up sandbox done (3s)", "Reached max iterations (10).", "Capturing session", "Expanding shell expressions", "Iteration 123/123"]; - for (const line of forms) assert.equal(allowlistedView(line), `${line}\n`, line); + for (const line of forms) assert.equal(allowlistedView(line), `${line.replace(/\d+/g, "#")}\n`, line); }); - it("keeps a line for each word that names an error, and only as a word", () => { + it("masks the numbers of a host line, so a forged line carries no digits out", () => { + // 40 forged lines whose digits spell a token would, unmasked, carry it out; masked they are the same line. + const forged = Array.from({ length: 40 }, (_, i) => `Iteration ${100 + i}/${200 + i}`); + assert.equal(allowlistedView(forged.join("\n")), "Iteration #/#\n"); + assert.equal(allowlistedView("Iteration 1/3\nIteration 2/3"), allowlistedView("Iteration 9/9\nIteration 8/8")); + assert.equal(allowlistedView("Context window: 42k"), "Context window: #k\n"); + assert.equal(allowlistedView("Setting up sandbox done (1.5s)"), "Setting up sandbox done (#.#s)\n"); + }); + + it("keeps a fixed message for each word that names an error, and only as a word", () => { const words = [ "abort", "aborted", "error", "errors", "fail", "failed", "failure", "failures", "exception", "fatal", "panic", "timed out", "timeout", - "killed", "terminated", "exit code 1", "exited with 2", "signal", "crash", "crashed", "unauthorized", "unauthorised", "rate limit", "overloaded", + "killed", "terminated", "exit code", "exited code", "exit with", "exited with", "signal", "crash", "crashed", "unauthorized", "unauthorised", "rate limit", "overloaded", ]; - for (const word of words) assert.equal(allowlistedView(`it ${word} now`), `it ${word} now\n`, word); + assert.equal(words.length, 26); + for (const word of words) assert.equal(allowlistedView(`it ${word} 1 now`), `[error line: ${word}]\n`, word); for (const near of ["panicky", "fatally", "terrorism", "errorless", "exit codes", "unauthorizeds"]) { assert.equal(allowlistedView(`it ${near} now`), "[... lines left out]\n", near); } - assert.equal(allowlistedView("An ERROR happened"), "An ERROR happened\n"); + assert.equal(allowlistedView("An ERROR happened"), "[error line: error]\n"); }); - it("keeps an error line, not the prose around it, and the host lines and tool names near it", () => { + it("keeps an error line as a message, not the prose around it, and the host lines and tool names near it", () => { const view = allowlistedView("prose a\nBash(secret)\nthe claude-code process aborted\nAgent stopped\nprose b\n"); - assert.equal(view, "[... lines left out]\nBash(...)\nthe claude-code process aborted\nAgent stopped\n[... lines left out]\n"); + assert.equal(view, "[... lines left out]\nAgent stopped\nBash(...)\n[error line: aborted]\n"); }); - it("cuts an error line at the line limit exactly, and never in the middle of a surrogate pair", () => { - assert.equal(REPORT_LINE_LIMIT, 200); - const line = (length: number) => `error ${" x".repeat(length)}`.slice(0, length); - assert.equal(allowlistedView(line(REPORT_LINE_LIMIT - 1)), `${line(REPORT_LINE_LIMIT - 1)}\n`); - assert.equal(allowlistedView(line(REPORT_LINE_LIMIT)), `${line(REPORT_LINE_LIMIT)}\n`); - assert.equal(allowlistedView(line(REPORT_LINE_LIMIT + 1)), `${line(REPORT_LINE_LIMIT)}\n`); + it("publishes no text of an error line, however long, with any characters", () => { + const emoji = String.fromCodePoint(0x1f600); + for (const line of [`error ${" x".repeat(500)}`, `error ${emoji}${emoji} tail`, "error \uD83D lone", "fatal: the key is hunter2 ok"]) { + assert.match(allowlistedView(line), /^\[error line: [a-z ]+\]\n$/, line); + } }); - it("drops a surrogate pair the cut would split, and keeps one that ends at the limit", () => { - const emoji = String.fromCodePoint(0x1f600); - const withPairAt = (index: number, tail: string) => `${`error${" x".repeat(100)}`.slice(0, index)}${emoji}${tail}`; - const split = withPairAt(REPORT_LINE_LIMIT - 1, "yyyy"); - assert.equal(split.indexOf(emoji), REPORT_LINE_LIMIT - 1); - assert.equal(allowlistedView(split), `${split.slice(0, REPORT_LINE_LIMIT - 1)}\n`); - const whole = withPairAt(REPORT_LINE_LIMIT - 2, "yyyy"); - assert.equal(whole.indexOf(emoji), REPORT_LINE_LIMIT - 2); - assert.equal(allowlistedView(whole), `${whole.slice(0, REPORT_LINE_LIMIT)}\n`); - const loneHigh = `${`error${" x".repeat(100)}`.slice(0, REPORT_LINE_LIMIT - 1)}\uD83D`; - assert.equal(allowlistedView(loneHigh), `${loneHigh}\n`); - const exact = withPairAt(REPORT_LINE_LIMIT - 2, ""); - assert.equal(exact.length, REPORT_LINE_LIMIT); - assert.equal(allowlistedView(exact), `${exact}\n`); + it("does not publish a secret an error line spaced out with invisible fillers or as hex chunks", () => { + const secret = "s3cr3tValueForTesting12345678"; + const fillers = ["\u3164", "\u2800", "\u115F", "\u200B"]; + const hex = Buffer.from(secret).toString("hex").match(/.{1,4}/g)!.join(" "); + const spaced = [...fillers.map((filler) => `error ${secret.split("").join(filler)}`), `error ${hex}`]; + for (const line of spaced) { + assert.equal(allowlistedView(line, (candidate) => candidate.includes(secret)), "[error line: error]\n", line); + // copy() also holds a line whose characters, once format characters are taken out, spell the secret: only U+200B (a format character) is. + assert.equal(copy(line, [secret]), line.includes("\u200B") ? "[... lines left out]\n" : "[error line: error]\n", line); + } + // Plain, the secret is held; its run of token characters would leave the line out anyway. + assert.equal(allowlistedView(`error ${secret}`, (candidate) => candidate.includes(secret)), "[... lines left out]\n"); + }); + + it("keeps a message only for an ASCII error word: a look-alike character never spells one", () => { + // U+017F (long s) and U+212A (Kelvin sign) fold to s and k under a Unicode case-insensitive match, which errorLine must never use. + for (const line of ["\u017Fignal lost", "\u212Ailled", "err\u200Bor", "exit\u00A0code 1", "exit code 1"]) { + assert.equal(allowlistedView(line), "[... lines left out]\n", line); + } + assert.equal(allowlistedView("\u017Fignal \u212Ailled and a real ERROR"), "[error line: error]\n"); + assert.equal(allowlistedView("exit code 1"), "[error line: exit code]\n"); + assert.equal(allowlistedView("fatal\u0131"), "[error line: fatal]\n"); + assert.doesNotMatch(allowlistedView("\u017Fignal \u212Ailled and a real ERROR"), /[^\x00-\x7f]/); + }); + + it("replaces a view that holds a secret value with a note, even a value that is only a fixed message or host line", () => { + const note = "Left out: the view held something shaped like a secret.\n"; + assert.equal(copy("Agent stopped", ["Agent stopped"]), note); + assert.equal(copy("Bash(x)", ["Bash(...)"]), note); + assert.equal(copy("fatal", ["error line: fatal"]), note); + assert.equal(copy("Agent stopped", ["Agent stopped!"]), "Agent stopped\n"); }); it("leaves out an error line with a run of token characters, at n-1, n and n+1", () => { assert.equal(REPORT_TOKEN_RUN, 24); const withRun = (length: number) => `error ${"A".repeat(length)}`; - assert.equal(allowlistedView(withRun(REPORT_TOKEN_RUN - 1)), `${withRun(REPORT_TOKEN_RUN - 1)}\n`); + assert.equal(allowlistedView(withRun(REPORT_TOKEN_RUN - 1)), "[error line: error]\n"); assert.equal(allowlistedView(withRun(REPORT_TOKEN_RUN)), "[... lines left out]\n"); assert.equal(allowlistedView(withRun(REPORT_TOKEN_RUN + 1)), "[... lines left out]\n"); for (const ch of ["+", "/", "_", "=", "-", "9", "z"]) assert.equal(allowlistedView(`error ${ch.repeat(REPORT_TOKEN_RUN)}`), "[... lines left out]\n", ch); }); - it("leaves out an error line the caller's check holds, before the line is cut", () => { + it("leaves out an error line the caller's check holds", () => { const secret = `error ${"x ".repeat(98)}hunter2`; - assert.ok(secret.length > REPORT_LINE_LIMIT); assert.equal(allowlistedView(secret, (line) => line.includes("hunter2")), "[... lines left out]\n"); - assert.notEqual(allowlistedView(secret), "[... lines left out]\n"); - }); - - it("keeps only the last REPORT_ERROR_LINES error lines", () => { - assert.equal(REPORT_ERROR_LINES, 20); - const lines = (count: number) => Array.from({ length: count }, (_, i) => `error number ${i}`); - assert.equal(allowlistedView(lines(REPORT_ERROR_LINES).join("\n")), `${lines(REPORT_ERROR_LINES).join("\n")}\n`); - const over = allowlistedView(lines(REPORT_ERROR_LINES + 1).join("\n")); - assert.ok(!over.includes("error number 0\n")); - assert.ok(over.startsWith("[... lines left out]\nerror number 1\n")); - assert.ok(over.endsWith(`error number ${REPORT_ERROR_LINES}\n`)); + assert.equal(allowlistedView(secret), "[error line: error]\n"); }); it("keeps a first-line-only error log, and reads CRLF lines", () => { - assert.equal(allowlistedView("aborted"), "aborted\n"); + assert.equal(allowlistedView("aborted"), "[error line: aborted]\n"); assert.equal(allowlistedView("Agent started\r\nAgent stopped\r\n"), "Agent started\nAgent stopped\n"); }); }); @@ -260,7 +319,7 @@ describe("copyReviewerLogs", () => { it("copies only the reviewer logs, as their view, into the report folder", () => { const { fs, written } = fakeFs({ "a-reviewer.log": "aborted", "a-backend.log": "other" }); assert.deepEqual(copyTo("logs", fs, none, []), ["reviewer-1.log"]); - assert.deepEqual(written, { "logs/report/reviewer-1.log": "aborted\n", "logs/report/index.txt": "reviewer-1.log: a build this run did not start\n" }); + assert.deepEqual(written, { "logs/report/reviewer-1.log": "[error line: aborted]\n", "logs/report/index.txt": "reviewer-1.log: issue #1\n" }); }); it("never copies the review lead's log (issue 399), which lists the subagents' answers", () => { @@ -289,7 +348,7 @@ describe("copyReviewerLogs", () => { writeFileSync(join(out, "old-reviewer.log"), "left over\n"); mkdirSync(join(dir, "report")); writeFileSync(join(dir, "report", "planted-reviewer.log"), "planted\n"); - assert.deepEqual(copyReviewerLogs(dir, out, undefined, () => undefined, []), ["reviewer-1.log"]); + assert.deepEqual(copyReviewerLogs(dir, out, undefined, () => undefined, [], knowing(["ok-reviewer.log"])), ["reviewer-1.log"]); assert.deepEqual(readdirSync(out).sort(), ["index.txt", "reviewer-1.log"]); assert.deepEqual(readdirSync(join(dir, "report")), ["planted-reviewer.log"], "the checkout's report folder was touched"); assert.deepEqual(readdirSync(dir).sort(), ["ok-reviewer.log", "report"], "something was written under the logs folder"); @@ -326,7 +385,7 @@ describe("copyReviewerLogs", () => { writeFileSync(join(dir, "ok-reviewer.log"), "the reviewer aborted\n"); assert.deepEqual(copyTo(dir, undefined, () => undefined, []), ["reviewer-1.log"]); assert.ok(!existsSync(join(dir, "report", "planted-reviewer.log")), "a planted log survived"); - assert.equal(readFileSync(join(dir, "report", "reviewer-1.log"), "utf8"), "the reviewer aborted\n"); + assert.equal(readFileSync(join(dir, "report", "reviewer-1.log"), "utf8"), "[error line: aborted]\n"); // With no reviewer log at all, a planted file is still removed. rmSync(join(dir, "ok-reviewer.log")); writeFileSync(join(dir, "report", "planted-reviewer.log"), "planted\n"); @@ -342,7 +401,7 @@ describe("copyReviewerLogs", () => { copyTo(dir, undefined, () => undefined, []); assert.ok(existsSync(join(elsewhere, "keep-reviewer.log")), "the link's target was emptied"); assert.ok(!existsSync(join(elsewhere, "reviewer-1.log")), "a view was written through the link"); - assert.equal(readFileSync(join(dir, "report", "reviewer-1.log"), "utf8"), "aborted again\n"); + assert.equal(readFileSync(join(dir, "report", "reviewer-1.log"), "utf8"), "[error line: aborted]\n"); } finally { rmSync(elsewhere, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } @@ -391,18 +450,15 @@ describe("copyReviewerLogs", () => { } }); - it("replaces a view whose token is wrapped across error lines, each short enough to pass alone, with a note", () => { - const note = "Left out: the view held something shaped like a secret.\n"; - // The second line ends its run of token characters with `!`, so the word after it can't lengthen the body. + it("publishes no text of a token wrapped across error lines, each short enough to pass alone", () => { + // Each line is short enough to pass the token-run check alone; both come out as fixed messages, so no part of the token reaches the view. const wrapped = (head: string, filler: string, count: number) => copy(`error ${head}${filler.repeat(10)}\n${filler.repeat(count)}! failed`); - assert.equal(wrapped("ghp_", "D", 10), note); - assert.notEqual(wrapped("ghp_", "D", 9), note); - assert.equal(wrapped("sk-ant-", "E", 6), note); - assert.notEqual(wrapped("sk-ant-", "E", 5), note); - for (const prefix of ["gho_", "ghu_", "ghs_", "ghr_"]) assert.equal(wrapped(prefix, "F", 10), note, prefix); - assert.equal(wrapped("github_pat_", "G", 10), note); - assert.notEqual(wrapped("github_pat_", "G", 9), note); - assert.equal(copy("error hunter2\nhunter2 failed", ["hunter2hunter2"]), note); + for (const [head, filler] of [["ghp_", "D"], ["sk-ant-", "E"], ["gho_", "F"], ["ghu_", "F"], ["ghs_", "F"], ["ghr_", "F"], ["github_pat_", "G"]] as const) { + const out = wrapped(head, filler, 10); + assert.equal(out, "[error line: error]\n[error line: failed]\n", head); + assert.ok(!out.includes(filler) && !out.includes(head), head); + } + assert.equal(copy("error hunter2\nhunter2 failed", ["hunter2hunter2"]), "[error line: error]\n[error line: failed]\n"); }); it("publishes no secret that sits in prose, a tool's arguments or a tool-call lookalike, in any form, and none plain on an error line", () => { @@ -423,7 +479,7 @@ describe("copyReviewerLogs", () => { it("still copies an ordinary log whose words are spaced and coloured", () => { const out = copy("Agent started\n\u001b[32mgreen\u001b[0m prose about ghp and sk-ant\nthe run failed: exit code 1\n"); - assert.ok(out.includes("the run failed: exit code 1")); + assert.ok(out.includes("[error line: failed]") && !out.includes("exit code 1")); assert.ok(!out.startsWith("Left out")); }); @@ -478,7 +534,7 @@ describe("copyReviewerLogs", () => { const { fs, written } = fakeFs({ "dir-reviewer.log": "x" }); copyTo("logs", { ...fs, read: () => { throw new Error("EISDIR"); } }, (line) => warnings.push(line), []); assert.equal(warnings.length, 1); - assert.deepEqual(written, { "logs/report/index.txt": "reviewer-1.log: a build this run did not start (not copied)\n" }); + assert.deepEqual(written, { "logs/report/index.txt": "reviewer-1.log: issue #1 (not copied)\n" }); }); it("reads the sandbox's secrets from .sandcastle/.env by default", () => { @@ -493,7 +549,7 @@ describe("copyReviewerLogs", () => { process.chdir(dir); assert.deepEqual(copyTo("logs", undefined, () => undefined).sort(), ["reviewer-1.log", "reviewer-2.log"]); assert.equal(readFileSync(join(dir, "logs", "report", "reviewer-1.log"), "utf8"), "[... lines left out]\n"); - assert.equal(readFileSync(join(dir, "logs", "report", "reviewer-2.log"), "utf8"), "error plain\n"); + assert.equal(readFileSync(join(dir, "logs", "report", "reviewer-2.log"), "utf8"), "[error line: error]\n"); } finally { process.chdir(before); rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); @@ -510,7 +566,7 @@ describe("copyReviewerLogs", () => { mkdirSync(join(dir, "report"), { recursive: true }); assert.deepEqual(copyTo(dir, undefined, () => undefined, []), ["reviewer-2.log"]); assert.deepEqual(readdirSync(join(dir, "report")).sort(), ["index.txt", "reviewer-2.log"], "a symlinked log was copied"); - assert.equal(readFileSync(join(dir, "report", "reviewer-2.log"), "utf8"), "the reviewer aborted\n"); + assert.equal(readFileSync(join(dir, "report", "reviewer-2.log"), "utf8"), "[error line: aborted]\n"); assert.ok(!existsSync(join(dir, "report", "ok-backend.log"))); } finally { rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); @@ -518,6 +574,77 @@ describe("copyReviewerLogs", () => { }); }); +// Only the logs of builds this run started are copied, and at most REPORT_LOG_LIMIT of them: a sandbox that can write the logs folder +// could add files of its own, and each view is a few dozen bits it would choose. +describe("the log limit", () => { + const name = (i: number) => `log-${String(i).padStart(3, "0")}-reviewer.log`; + // Listed in reverse, each with a view of its own (a tool name that depends on its position), so which logs were copied and in what order shows. + const filesOf = (count: number) => Object.fromEntries(Array.from({ length: count }, (_, i) => count - 1 - i).map((i) => [name(i), `${["Bash", "Read", "Grep"][i % 3]}(${i})`])); + const more = `more than ${REPORT_LOG_LIMIT} reviewer logs: the rest were not copied`; + const others = "other reviewer logs in the folder were not copied: no build of this run wrote them"; + const buildsFor = (count: number) => { + const builds = new BuildLogs(); + for (let i = 0; i < count; i++) builds.add(100 + i, name(i).replace(/-reviewer\.log$/, "")); + return builds; + }; + + it("copies the first REPORT_LOG_LIMIT build logs by sorted name, whatever order they are listed in, and says in the index that the rest were not copied (n-1, n, n+1)", () => { + assert.equal(REPORT_LOG_LIMIT, 20); + for (const count of [REPORT_LOG_LIMIT - 1, REPORT_LOG_LIMIT, REPORT_LOG_LIMIT + 1, REPORT_LOG_LIMIT + 30]) { + const { fs, written } = fakeFs(filesOf(count)); + const copied = copyReviewerLogs("logs", "logs/report", fs, none, [], buildsFor(count)); + const limit = Math.min(count, REPORT_LOG_LIMIT); + assert.deepEqual(copied, Array.from({ length: limit }, (_, i) => `reviewer-${i + 1}.log`), `${count} logs`); + for (let i = 0; i < limit; i++) assert.equal(written[`logs/report/reviewer-${i + 1}.log`], `${["Bash", "Read", "Grep"][i % 3]}(...)\n`, `${count} logs, log ${i}`); + assert.ok(!(`logs/report/reviewer-${limit + 1}.log` in written), `${count} logs`); + const lines = Array.from({ length: limit }, (_, i) => `reviewer-${i + 1}.log: issue #${100 + i}`); + assert.equal(written["logs/report/index.txt"], `${[...lines, ...(count > REPORT_LOG_LIMIT ? [more] : [])].join("\n")}\n`, `${count} logs`); + } + }); + + it("copies by default only the logs of builds the global record knows, the record every build adds itself to", () => { + buildLogs.add(4242, "zz/default-record-test"); + const { fs, written } = fakeFs({ [reviewerLogFileName("zz/default-record-test")]: "Bash(1)", "x-reviewer.log": "Read(2)" }); + assert.deepEqual(copyReviewerLogs("logs", "logs/report", fs, none, []), ["reviewer-1.log"]); + assert.equal(written["logs/report/reviewer-1.log"], "Bash(...)\n"); + assert.equal(written["logs/report/index.txt"], `reviewer-1.log: issue #4242\n${others}\n`); + }); + + it("copies no log that no build of this run wrote, however many a sandbox adds, and says so once in the index", () => { + const builds = new BuildLogs(); + builds.add(9, "zzz/9-late"); + const files = { ...filesOf(REPORT_LOG_LIMIT + 5), "zzz-9-late-reviewer.log": "Write(x)" }; + const { fs, written } = fakeFs(files); + const copied = copyReviewerLogs("logs", "logs/report", fs, none, [], builds); + assert.deepEqual(copied, ["reviewer-1.log"]); + assert.equal(written["logs/report/reviewer-1.log"], "Write(...)\n"); + assert.equal(written["logs/report/index.txt"], `reviewer-1.log: issue #9\n${others}\n`); + assert.deepEqual(Object.keys(written).sort(), ["logs/report/index.txt", "logs/report/reviewer-1.log"]); + // With none known, nothing is copied and the index only says so. + const none3 = fakeFs(filesOf(3)); + assert.deepEqual(copyReviewerLogs("logs", "logs/report", none3.fs, none, [], new BuildLogs()), []); + assert.deepEqual(none3.written, { "logs/report/index.txt": `${others}\n` }); + }); + + it("says both fixed lines, the limit first, when there are more than the limit and files no build wrote", () => { + const { fs, written } = fakeFs({ ...filesOf(REPORT_LOG_LIMIT + 1), "extra-reviewer.log": "Bash(1)" }); + copyReviewerLogs("logs", "logs/report", fs, none, [], buildsFor(REPORT_LOG_LIMIT + 1)); + assert.ok(written["logs/report/index.txt"]!.endsWith(`${more}\n${others}\n`)); + }); + + it("gives a log that can't be read its number and its index line, and still omits the one after the limit", () => { + const { fs, written } = fakeFs(filesOf(REPORT_LOG_LIMIT + 1), name(REPORT_LOG_LIMIT - 1)); + const warnings: string[] = []; + const copied = copyReviewerLogs("logs", "logs/report", fs, (line) => warnings.push(line), [], buildsFor(REPORT_LOG_LIMIT + 1)); + assert.equal(copied.length, REPORT_LOG_LIMIT - 1); + assert.ok(!copied.includes(`reviewer-${REPORT_LOG_LIMIT}.log`)); + const index = written["logs/report/index.txt"]!.split("\n"); + assert.equal(index[REPORT_LOG_LIMIT - 1], `reviewer-${REPORT_LOG_LIMIT}.log: issue #${100 + REPORT_LOG_LIMIT - 1} (not copied)`); + assert.equal(index[REPORT_LOG_LIMIT], more); + assert.equal(warnings.length, 1); + }); +}); + // The numbered views carry no name, so the host says which build each one is (mpaulosky/dotfiles#428). describe("reviewer log index", () => { it("names a log the way Sandcastle does: the branch with the characters a file name can't hold as hyphens", () => { @@ -545,10 +672,10 @@ describe("reviewer log index", () => { assert.equal(REVIEWER_LOG_INDEX, "index.txt"); copyReviewerLogs("logs", "logs/report", fs, none, [], builds); - // Sorted names: feature-7..., fix-421..., unknown... + // Sorted names: feature-7..., fix-421...; unknown-reviewer.log is no build's and is not copied. assert.equal( written["logs/report/index.txt"], - "reviewer-1.log: issue #7\nreviewer-2.log: issue #421\nreviewer-3.log: a build this run did not start\n", + "reviewer-1.log: issue #7\nreviewer-2.log: issue #421\nother reviewer logs in the folder were not copied: no build of this run wrote them\n", ); }); @@ -560,13 +687,13 @@ describe("reviewer log index", () => { copyReviewerLogs("logs", "logs/report", fs, none, [], builds); - assert.equal(written["logs/report/index.txt"], `reviewer-1.log: issue #5 (not copied)\nreviewer-2.log: a build this run did not start\n`); + assert.equal(written["logs/report/index.txt"], `reviewer-1.log: issue #5 (not copied)\nother reviewer logs in the folder were not copied: no build of this run wrote them\n`); assert.ok(!JSON.stringify(written).includes(token)); }); it("writes no index when there is no reviewer log, and warns when the index can't be written", () => { const none2 = fakeFs({ "a-backend.log": "other" }); - copyReviewerLogs("logs", "logs/report", none2.fs, none, []); + copyReviewerLogs("logs", "logs/report", none2.fs, none, [], knowing(["a-reviewer.log"])); assert.deepEqual(none2.written, {}); const warnings: string[] = []; @@ -578,7 +705,7 @@ describe("reviewer log index", () => { fs.write(path, text); }, }; - assert.deepEqual(copyReviewerLogs("logs", "logs/report", noIndex, (line) => warnings.push(line), []), ["reviewer-1.log"]); + assert.deepEqual(copyReviewerLogs("logs", "logs/report", noIndex, (line) => warnings.push(line), [], knowing(["a-reviewer.log"])), ["reviewer-1.log"]); assert.deepEqual(warnings, ["Couldn't write the index of the reviewer logs: EACCES"]); assert.deepEqual(Object.keys(written), ["logs/report/reviewer-1.log"]); }); @@ -594,7 +721,7 @@ describe("reviewer log index", () => { fs.write(path, text); }, }; - copyReviewerLogs("logs", "logs/report", noIndex, (line) => warnings.push(line), []); + copyReviewerLogs("logs", "logs/report", noIndex, (line) => warnings.push(line), [], knowing(["a-reviewer.log"])); assert.deepEqual(warnings, ["Couldn't write the index of the reviewer logs: an error"]); } }); @@ -615,11 +742,11 @@ describe("reviewer log index", () => { // A reviewer log is written by the sandbox, so its size is the sandbox's choice: the host reads at most the end of it, which is all the view keeps. describe("the input limit", () => { - const early = "Error: early failure\n"; - const late = "Error: late failure\n"; + const early = "Fatal: early failure\n"; + const late = "Panic: late failure\n"; const LEFT_OUT = "[... lines left out]"; - // A log of exactly `length` characters: the early error line, one long line of filler, the late error line. - const logOf = (length: number) => `${early}${"a".repeat(length - early.length - late.length - 1)}\n${late}`; + // A log of exactly `length` characters: the early error line, one long line of blanks (which is no line left out), the late error line. + const logOf = (length: number) => `${early}${" ".repeat(length - early.length - late.length - 1)}\n${late}`; const inTemp = (body: (dir: string) => void) => { const root = mkdtempSync(join(tmpdir(), "role-logs-limit-")); try { @@ -633,8 +760,8 @@ describe("the input limit", () => { assert.equal(REPORT_INPUT_LIMIT, 4 * 1024 * 1024); for (const [length, cut] of [[REPORT_INPUT_LIMIT - 1, false], [REPORT_INPUT_LIMIT, false], [REPORT_INPUT_LIMIT + 1, true]] as const) { const view = allowlistedView(logOf(length)); - assert.equal(view.includes("Error: early failure"), !cut, `length ${length}`); - assert.ok(view.includes("Error: late failure"), `length ${length}`); + assert.equal(view.includes("[error line: fatal]"), !cut, `length ${length}`); + assert.ok(view.includes("[error line: panic]"), `length ${length}`); assert.equal(view.startsWith(`${LEFT_OUT}\n`), cut, `length ${length}`); } }); @@ -645,8 +772,8 @@ describe("the input limit", () => { for (const [bytes, cut] of [[REPORT_INPUT_LIMIT - 1, false], [REPORT_INPUT_LIMIT, false], [REPORT_INPUT_LIMIT + 1, true]] as const) { writeFileSync(file, logOf(bytes)); const text = liveRoleLogFs.read(file); - assert.equal(text.includes("Error: early failure"), !cut, `${bytes} bytes`); - assert.ok(text.includes("Error: late failure"), `${bytes} bytes`); + assert.equal(text.includes("Fatal: early failure"), !cut, `${bytes} bytes`); + assert.ok(text.includes("Panic: late failure"), `${bytes} bytes`); assert.equal(text.startsWith(`${LEFT_OUT}\n`), cut, `${bytes} bytes`); assert.ok(Buffer.byteLength(text) <= REPORT_INPUT_LIMIT + LEFT_OUT.length + 1, `${bytes} bytes`); } @@ -663,7 +790,7 @@ describe("the input limit", () => { inTemp((root) => { const file = join(root, "a-reviewer.log"); writeFileSync(file, logOf(REPORT_INPUT_LIMIT + 1)); - assert.equal(allowlistedView(liveRoleLogFs.read(file)), `${LEFT_OUT}\n${late}`); + assert.equal(allowlistedView(liveRoleLogFs.read(file)), `${LEFT_OUT}\n[error line: panic]\n`); }); }); @@ -671,20 +798,20 @@ describe("the input limit", () => { // The window (the last REPORT_INPUT_LIMIT + 1 characters) starts on a line break: the early line after it is whole, so it stays. const aligned = `x\n${early}${"a".repeat(REPORT_INPUT_LIMIT - early.length - late.length - 1)}\n${late}`; assert.equal(aligned.length, REPORT_INPUT_LIMIT + 2); - assert.ok(allowlistedView(aligned).includes("Error: early failure")); + assert.ok(allowlistedView(aligned).includes("[error line: fatal]")); // One character further in, the early line starts exactly where the window begins: nothing shows it began a line, so it goes, // which keeps the result within the limit. const exact = `x\n${early}${"a".repeat(REPORT_INPUT_LIMIT - early.length - late.length)}\n${late}`; assert.equal(exact.length, REPORT_INPUT_LIMIT + 3); - assert.ok(!allowlistedView(exact).includes("Error: early failure")); + assert.ok(!allowlistedView(exact).includes("[error line: fatal]")); inTemp((root) => { const file = join(root, "a-reviewer.log"); writeFileSync(file, aligned); - assert.ok(liveRoleLogFs.read(file).includes("Error: early failure")); + assert.ok(liveRoleLogFs.read(file).includes("Fatal: early failure")); // The marker read adds in front makes the text a little longer than the limit; the view's own cut still keeps the whole tail. - assert.ok(allowlistedView(liveRoleLogFs.read(file)).includes("Error: early failure")); + assert.ok(allowlistedView(liveRoleLogFs.read(file)).includes("[error line: fatal]")); writeFileSync(file, exact); - assert.ok(!liveRoleLogFs.read(file).includes("Error: early failure")); + assert.ok(!liveRoleLogFs.read(file).includes("Fatal: early failure")); }); }); @@ -699,16 +826,16 @@ describe("the input limit", () => { }); }); - it("cuts a log with CRLF line endings at a whole line and keeps its error lines free of the carriage return", () => { + it("cuts a log with CRLF line endings at a whole line", () => { const crlf = `${early.replace("\n", "\r\n")}${"a".repeat(REPORT_INPUT_LIMIT)}\r\n${late.replace("\n", "\r\n")}`; assert.ok(crlf.length > REPORT_INPUT_LIMIT); const view = allowlistedView(crlf); - assert.ok(view.includes("Error: late failure\n") && !view.includes("\r")); - assert.ok(!view.includes("Error: early failure")); + assert.ok(view.includes("[error line: panic]\n")); + assert.ok(!view.includes("[error line: fatal]")); inTemp((root) => { const file = join(root, "a-reviewer.log"); writeFileSync(file, crlf); - assert.ok(allowlistedView(liveRoleLogFs.read(file)).includes("Error: late failure\n")); + assert.ok(allowlistedView(liveRoleLogFs.read(file)).includes("[error line: panic]\n")); }); }); @@ -724,7 +851,7 @@ describe("the input limit", () => { const text = liveRoleLogFs.read(file); assert.ok(!text.includes("\uFFFD"), `offset ${offset}: a half character became U+FFFD`); assert.ok(!text.includes("é"), `offset ${offset}: the cut line was kept`); - assert.ok(text.includes("Error: late failure")); + assert.ok(text.includes("Panic: late failure")); } }); }); diff --git a/scripts/check-branch-name.sh b/scripts/check-branch-name.sh index a3ba6a1..7d0f096 100755 --- a/scripts/check-branch-name.sh +++ b/scripts/check-branch-name.sh @@ -6,7 +6,8 @@ # # The pre-push hook runs the first form on each pushed branch. ci.yml's # "Branch name" job runs the second on a PR's head branch, which also accepts -# the branches Actions and Dependabot create, since no hook ever sees them. +# the branches Actions create, since no hook ever sees them. Dependabot's +# dependabot/* is allowed in both forms. # CI is what catches a push that skipped the hook: a Claude cloud session's # claude/... branch, GitHub's web editor, or a clone without core.hooksPath. # Exits 0 when the name is allowed, 1 (with the reason) when it isn't. @@ -27,9 +28,14 @@ if [[ "$branch" =~ ^(feature|fix|hotfix)/[0-9]+-[a-z0-9]+(-[a-z0-9]+)*$ ]] \ exit 0 fi -# Made by release.yml, backfill-blog-posts.yml and Dependabot. -if [[ "$pr" == true ]] \ - && [[ "$branch" == docs/release-notes || "$branch" == docs/backfill-blog-posts || "$branch" == dependabot/* ]]; then +# Dependabot's branches can't be renamed, and a person pushes to one to fix its PR (mirroring a bump into the Template's file, say), +# so the hook allows them too, in shell-safe characters only. Sandcastle hands off no such branch (lib/branches.mts#isHandOffBranch). +if [[ "$branch" =~ ^dependabot/[A-Za-z0-9._-]+(/[A-Za-z0-9._-]+)*$ ]]; then + exit 0 +fi + +# Made by release.yml and backfill-blog-posts.yml. +if [[ "$pr" == true ]] && [[ "$branch" == docs/release-notes || "$branch" == docs/backfill-blog-posts ]]; then exit 0 fi diff --git a/scripts/tests/check-branch-name.test.sh b/scripts/tests/check-branch-name.test.sh index 55bd109..27d8d70 100644 --- a/scripts/tests/check-branch-name.test.sh +++ b/scripts/tests/check-branch-name.test.sh @@ -35,8 +35,16 @@ expect refused --pr claude/project-thread-ot9sjt expect refused --pr patch-1 expect refused "" -# Branches Actions and Dependabot create are allowed only as a PR's head. -for branch in docs/release-notes docs/backfill-blog-posts dependabot/nuget/all-nuget-1a2b3c; do +# Dependabot's branches are allowed to push to, so a person can fix its PR. +expect allowed dependabot/github_actions/main/minor-and-patch-actions-401d4b32a3 +expect allowed --pr dependabot/nuget/all-nuget-1a2b3c +expect refused dependabot +expect refused "dependabot/x;rm" +# shellcheck disable=SC2016 # the \$( is the point: a literal name, not an expansion +expect refused 'dependabot/$(x)' +expect refused "dependabot/a b" +# Branches Actions create are allowed only as a PR's head. +for branch in docs/release-notes docs/backfill-blog-posts; do expect allowed --pr "$branch" expect refused "$branch" done From 7433660bc93c898b83278cc70ca9c9e384f00f3b Mon Sep 17 00:00:00 2001 From: T <60372079+mpaulosky@users.noreply.github.com> Date: Tue, 10 Nov 2026 11:41:35 -0800 Subject: [PATCH 2/3] docs(ci): Point the skills list's comments at the Application Factory The re-Applied sandcastle.yml reads pinned skills from the Factory's main, not dotfiles. Comment lines only: the file still lists no pin and no skills. Co-Authored-By: Claude Sonnet 5.5 --- .github/ci/sandcastle-skills.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/ci/sandcastle-skills.txt b/.github/ci/sandcastle-skills.txt index ba7e861..4ec2b2b 100644 --- a/.github/ci/sandcastle-skills.txt +++ b/.github/ci/sandcastle-skills.txt @@ -6,10 +6,10 @@ # and runs no script of the repo's. Blank (or whitespace-only) lines and comments (a # in column 0) # are ignored, LF line endings only, and any other line is one of two kinds: # -# dotfiles-sha <40 lowercase hex digits> the commit of mpaulosky/dotfiles' main to take them from, given once -# skill a skill under "Support For New Projects/.claude/skills/" there, once each +# dotfiles-sha <40 lowercase hex digits> the commit of the Application Factory's main (mpaulosky/Application_Factory) to take them from, given once +# skill a skill under ".claude/skills/" there, once each # -# Pin the commit like an action, so a push to dotfiles can't change what the +# Pin the commit like an action, so a push to the Factory can't change what the # agents are told until the pin moves. The skills become the agents' # instructions, so have a person review every change to this file. # From 427f43ea00b2d8af37928cd7cadb8edcdd2368d0 Mon Sep 17 00:00:00 2001 From: T <60372079+mpaulosky@users.noreply.github.com> Date: Tue, 10 Nov 2026 12:43:52 -0800 Subject: [PATCH 3/3] chore: Apply repo-ci-baseline Template Co-Authored-By: Claude Sonnet 5.5 --- .github/scripts/tests/pr-automerge.test.mjs | 58 ++++++++++++++++++++- .github/workflows/pr-automerge.yml | 31 ++++++++--- scripts/check-branch-name.sh | 15 +++--- scripts/tests/check-branch-name.test.sh | 17 ++++++ 4 files changed, 105 insertions(+), 16 deletions(-) diff --git a/.github/scripts/tests/pr-automerge.test.mjs b/.github/scripts/tests/pr-automerge.test.mjs index 9d6ac21..203cd68 100644 --- a/.github/scripts/tests/pr-automerge.test.mjs +++ b/.github/scripts/tests/pr-automerge.test.mjs @@ -373,6 +373,10 @@ async function evaluate(pr, events = [], options = {}) { // The reads after the request that throw (the read-back), and what they throw. readBackErrors = 0, readBackError = new Error("Server error on the read-back"), + // When set, the reads after the request show the PR at this head (a push + // landed while the read-back ran), with its checks passing; the commit has + // to be in commits (graph()). + readBackHeadRefOid, // Whether the rival's record is listed before this run's own, whatever its // createdAt: GitHub lists comments oldest first, and the first record // listed is the one that decides. Unset, the rival is listed first when its @@ -623,6 +627,9 @@ async function evaluate(pr, events = [], options = {}) { }) } }; + const seenAfterRequest = (pr) => (requested && readBackHeadRefOid + ? { ...pr, headRefOid: readBackHeadRefOid, headRef: checksOn(readBackHeadRefOid, requiredCheck("Build Solution")) } + : pr); const github = { rest, graphql: async (query, variables = {}) => { @@ -655,9 +662,9 @@ async function evaluate(pr, events = [], options = {}) { if (snapshot instanceof Error) { throw snapshot; } - return { repository: { pullRequest: withRunsOwnWrites(snapshot) } }; + return { repository: { pullRequest: withRunsOwnWrites(seenAfterRequest(snapshot)) } }; } - return { repository: { pullRequest: withRunsOwnWrites(pullRequests[Math.min(graphqlRequests++, pullRequests.length - 1)]) } }; + return { repository: { pullRequest: withRunsOwnWrites(seenAfterRequest(pullRequests[Math.min(graphqlRequests++, pullRequests.length - 1)])) } }; }, paginate: async (method, params) => { clientCall("token", methodNames.get(method) ?? "unknown paginate method"); @@ -1690,6 +1697,53 @@ test("adds review:claude only after the rest of the settle window when the reque assert.deepEqual(addedLabels, LABEL_FOR_CLAUDE); assert.ok(logs.some((line) => line.startsWith("warning: ") && line.includes("Resource not accessible")), logs.join("\n")); assert.ok(Date.now() - started >= 250, `the run took ${Date.now() - started} ms`); + // The cap: the wait is COPILOT_ASK_WAIT_MS, not the 30-second window, so a run without the cap would take 30 s. + assert.ok(Date.now() - started < 5000, `the run took ${Date.now() - started} ms`); +}); + +// The other two ways askCopilotOnce returns true without a registered request: the read-back errors, or the head moved +// under it. Each leaves the record at least COPILOT_ASK_SETTLE_MS old before review:claude is added, as the failed +// request does, or Claude Review would skip a record younger than that and PR Auto-Merge would strip the label. +for (const [name, options] of [ + ["the read-back errors", { readBackErrors: 1 }], + // "moved" is an ordinary commit on top of the head, so the caller's merge checks accept it. + ["the head moved under the read-back", { readBackHeadRefOid: "moved", commits: graph({ moved: { parents: [HEAD], files: { app: "app3", lib: "lib0" } } }) }] +]) { + test(`adds review:claude only once the record has settled when ${name} (mpaulosky/Application_Factory#40)`, async () => { + mock.timers.enable({ apis: ["Date", "setTimeout"], now: Date.parse("2026-01-02T00:00:00Z") }); + let result; + let finishedAt; + try { + result = await underClock(evaluate(readyPr(unreviewed), [], { askWaitMs: 30 * 1000, askPollMs: 5000, ...options })); + finishedAt = Date.now(); + } finally { + mock.timers.reset(); + } + assert.equal(result.requests.length, 1); + assert.equal(result.askedAt.length, 1); + // Claude is called in (a moved head that failed the merge checks would add nothing, and the loop below would check nothing). + assert.deepEqual(result.addedLabels, LABEL_FOR_CLAUDE, result.logs.join("\n")); + assert.equal(result.addedLabelAt.length, 1); + // askCopilotOnce held the run until the record was 30 seconds old. + assert.ok(finishedAt - result.askedAt[0] >= 30 * 1000, `the run ended ${(finishedAt - result.askedAt[0]) / 1000} s after the record`); + for (const labelAt of result.addedLabelAt) { + assert.ok(labelAt - result.askedAt[0] >= 30 * 1000, `review:claude was added ${(labelAt - result.askedAt[0]) / 1000} s after the record`); + } + }); +} + +test("a registered request needs no settle wait: the run ends as soon as the read-back sees it (mpaulosky/Application_Factory#40)", async () => { + mock.timers.enable({ apis: ["Date", "setTimeout"], now: Date.parse("2026-01-02T00:00:00Z") }); + let result; + let finishedAt; + try { + result = await underClock(evaluate(readyPr(unreviewed), [], { copilotRequest: "registers", askWaitMs: 30 * 1000, askPollMs: 5000 })); + finishedAt = Date.now(); + } finally { + mock.timers.reset(); + } + assert.equal(result.requests.length, 1); + assert.ok(finishedAt - result.askedAt[0] < 30 * 1000, `the run ended ${(finishedAt - result.askedAt[0]) / 1000} s after the record`); }); // Claude Review's gate (the Python in claude-review.yml's "Skip while Copilot can diff --git a/.github/workflows/pr-automerge.yml b/.github/workflows/pr-automerge.yml index d8bc324..8950a96 100644 --- a/.github/workflows/pr-automerge.yml +++ b/.github/workflows/pr-automerge.yml @@ -84,7 +84,9 @@ name: PR Auto-Merge # mpaulosky/dotfiles#369; one whose record never shows requests anyway, since a # repeat request is idempotent, mpaulosky/dotfiles#374), requests the # review with the PAT, and reads the request back for up to COPILOT_ASK_WAIT_MS. -# A request that fails or doesn't register counts as dropped at once, and a +# A request that fails, or whose read-back errors or finds the head moved, counts as dropped once the +# record is COPILOT_ASK_SETTLE_MS old (a wait capped by COPILOT_ASK_WAIT_MS, mpaulosky/Application_Factory#23); +# one that doesn't register counts the same after the read-back window, and a # record COPILOT_ASK_SETTLE_MS old with no review of the head and no request # pending counts the same way, whoever wrote it (land.sh writes the same # record). Copilot is never asked again for the PR: a later head goes straight @@ -1711,6 +1713,17 @@ jobs: // After the write returns, so a slow write can't shorten the wait: // the record's own timestamp is no later than this. askWrittenAt = Date.now(); + // Wait out what is left of COPILOT_ASK_SETTLE_MS (capped by + // COPILOT_ASK_WAIT_MS) before a path that returns true without a + // registered request lets the caller add review:claude: other + // workflows see only the record, which Claude Review skips and PR + // Auto-Merge strips while it is younger than that + // (mpaulosky/Application_Factory#23). Every such path calls it: + // the failed request and the read-back's two early exits. + const waitOutSettle = () => new Promise((resolve) => setTimeout( + resolve, + Math.max(0, Math.min(COPILOT_ASK_WAIT_MS, askWrittenAt + COPILOT_ASK_SETTLE_MS - Date.now())) + )); // Two runs can both have read "no record" and both posted one. // Read the comments again: only the run whose record is // listed first makes the request (mpaulosky/dotfiles#369). GraphQL @@ -1778,13 +1791,7 @@ jobs: throw error; } core.warning(`Couldn't request Copilot's review of PR #${pullNumber}: ${error.message}`); - // Wait out what is left of COPILOT_ASK_SETTLE_MS (capped by - // COPILOT_ASK_WAIT_MS) before the caller adds review:claude: - // other workflows see only the record, which Claude Review - // skips and PR Auto-Merge strips while it is younger than that - // (mpaulosky/Application_Factory#23). - const remainingMs = askWrittenAt + COPILOT_ASK_SETTLE_MS - Date.now(); - await new Promise((resolve) => setTimeout(resolve, Math.max(0, Math.min(COPILOT_ASK_WAIT_MS, remainingMs)))); + await waitOutSettle(); core.info(`Copilot's review of PR #${pullNumber} wasn't requested, so Claude is called in.`); return true; } @@ -1795,11 +1802,13 @@ jobs: // again, and only a sweep ends on an error that ends the run. const deadline = Date.now() + COPILOT_ASK_WAIT_MS; let registered = false; + let headMoved = false; try { do { await new Promise((resolve) => setTimeout(resolve, Math.max(0, Math.min(COPILOT_ASK_POLL_MS, deadline - Date.now())))); const seen = await getPullRequestStatus(pullNumber); if (!seen || seen.headRefOid !== pr.headRefOid) { + headMoved = true; break; } registered = copilotPending(seen) || copilotAnsweredHead(seen); @@ -1809,8 +1818,14 @@ jobs: throw error; } core.warning(`Couldn't read PR #${pullNumber} back after requesting Copilot's review, so the request is taken as made: ${error.message}`); + await waitOutSettle(); return true; } + // The head moved or the PR couldn't be read: nothing was seen to + // register, so the record must settle before Claude is called in. + if (headMoved) { + await waitOutSettle(); + } core.info(registered ? `Copilot's review request for PR #${pullNumber} registered.` : `Copilot's review request for PR #${pullNumber} didn't register (GitHub drops it once the Copilot code review budget is used up).`); diff --git a/scripts/check-branch-name.sh b/scripts/check-branch-name.sh index 7d0f096..83142fd 100755 --- a/scripts/check-branch-name.sh +++ b/scripts/check-branch-name.sh @@ -6,8 +6,8 @@ # # The pre-push hook runs the first form on each pushed branch. ci.yml's # "Branch name" job runs the second on a PR's head branch, which also accepts -# the branches Actions create, since no hook ever sees them. Dependabot's -# dependabot/* is allowed in both forms. +# the branches Actions and Dependabot create, since no hook ever sees them. +# The pre-push hook also lets a person push to a Dependabot branch (see below). # CI is what catches a push that skipped the hook: a Claude cloud session's # claude/... branch, GitHub's web editor, or a clone without core.hooksPath. # Exits 0 when the name is allowed, 1 (with the reason) when it isn't. @@ -29,13 +29,16 @@ if [[ "$branch" =~ ^(feature|fix|hotfix)/[0-9]+-[a-z0-9]+(-[a-z0-9]+)*$ ]] \ fi # Dependabot's branches can't be renamed, and a person pushes to one to fix its PR (mirroring a bump into the Template's file, say), -# so the hook allows them too, in shell-safe characters only. Sandcastle hands off no such branch (lib/branches.mts#isHandOffBranch). -if [[ "$branch" =~ ^dependabot/[A-Za-z0-9._-]+(/[A-Za-z0-9._-]+)*$ ]]; then +# so the hook allows them too, in shell-safe characters only ('@' and '+' included, which a scoped package or a version can carry). +# Sandcastle hands off no such branch (lib/branches.mts#isHandOffBranch). The --pr form below takes any dependabot/* name, as it did +# before the hook did: CI quotes the name, and refusing a Dependabot branch for a character would block its PR. +if [[ "$branch" =~ ^dependabot/[A-Za-z0-9._@+-]+(/[A-Za-z0-9._@+-]+)*$ ]]; then exit 0 fi -# Made by release.yml and backfill-blog-posts.yml. -if [[ "$pr" == true ]] && [[ "$branch" == docs/release-notes || "$branch" == docs/backfill-blog-posts ]]; then +# Made by release.yml, backfill-blog-posts.yml and Dependabot. +if [[ "$pr" == true ]] \ + && [[ "$branch" == docs/release-notes || "$branch" == docs/backfill-blog-posts || "$branch" == dependabot/* ]]; then exit 0 fi diff --git a/scripts/tests/check-branch-name.test.sh b/scripts/tests/check-branch-name.test.sh index 27d8d70..f051fa9 100644 --- a/scripts/tests/check-branch-name.test.sh +++ b/scripts/tests/check-branch-name.test.sh @@ -39,6 +39,23 @@ expect refused "" expect allowed dependabot/github_actions/main/minor-and-patch-actions-401d4b32a3 expect allowed --pr dependabot/nuget/all-nuget-1a2b3c expect refused dependabot +# Scoped npm packages and build-metadata versions: '@' and '+' are shell-safe, so the hook takes them. +expect allowed dependabot/npm_and_yarn/@types/node-20.1.0 +expect allowed dependabot/nuget/Foo.Bar_1.2.3+build.5 +expect allowed dependabot/@scope/x +expect allowed dependabot/a+b +# The pattern is anchored at the start: dependabot/ inside another name is not a Dependabot branch. +expect refused claude/dependabot/x +expect refused xdependabot/a +# The --pr form takes any dependabot/* name, as before the hook did: CI quotes it, and a PR must not fail on a character. +expect allowed --pr dependabot/npm_and_yarn/types/node-20~1 +expect allowed --pr dependabot/ +# Shell-unsafe or malformed names stay refused for a push. +expect refused dependabot/ +expect refused dependabot/a/ +expect refused dependabot//a +expect refused Dependabot/x +expect refused $'dependabot/a\nb' expect refused "dependabot/x;rm" # shellcheck disable=SC2016 # the \$( is the point: a literal name, not an expansion expect refused 'dependabot/$(x)'