diff --git a/.github/ci/sandcastle-tests.sh b/.github/ci/sandcastle-tests.sh index 14727ef8..c2e84ec9 100755 --- a/.github/ci/sandcastle-tests.sh +++ b/.github/ci/sandcastle-tests.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Runs Sandcastle's tests (pnpm's test:sandcastle script) when anything they +# Runs Sandcastle's tests (pnpm's check:sandcastle script) when anything they # cover changed since , or always when is empty. The pre-push # gate (.github/ci/gate-checks.sh) and CI's Build Solution job # (.github/ci/prepare.sh build) both call this, so they watch the same paths. @@ -46,4 +46,4 @@ else fi echo "Sandcastle tests" "${pnpm[@]}" install --frozen-lockfile -"${pnpm[@]}" run test:sandcastle +"${pnpm[@]}" run check:sandcastle diff --git a/.github/scripts/tests/pr-automerge.test.mjs b/.github/scripts/tests/pr-automerge.test.mjs index 540627b4..cbb477c1 100644 --- a/.github/scripts/tests/pr-automerge.test.mjs +++ b/.github/scripts/tests/pr-automerge.test.mjs @@ -32,7 +32,18 @@ const AsyncFunction = Object.getPrototypeOf(async () => {}).constructor; const run = new AsyncFunction("github", "context", "core", "getOctokit", inlineScript()); // A same-repo PR into main that is ready to merge unless a test changes it. -function readyPr(overrides = {}) { +// Claude's reviews only count as backup for Copilot's quota note on the same +// commit, so each Claude review given here comes with that note, unless +// { quotaNotes: false } says Copilot never posted one, or the test's own +// copilotReviews already hold the notes it wants. +function readyPr({ quotaNotes = true, ...overrides } = {}) { + if (quotaNotes && overrides.claudeReviews) { + const reviewed = overrides.copilotReviews?.nodes ?? [{ commit: { oid: HEAD } }]; + const notes = overrides.claudeReviews.nodes + .filter((review) => !reviewed.some((other) => other.commit?.oid === review.commit.oid && COPILOT_NO_REVIEW_TEST.test(other.body ?? ""))) + .map((review) => ({ body: COPILOT_QUOTA_NOTE, commit: { oid: review.commit.oid, parents: review.commit.parents } })); + overrides = { ...overrides, copilotReviews: { nodes: [...reviewed, ...notes] } }; + } return { state: "OPEN", isDraft: false, @@ -72,6 +83,7 @@ function requiredCheck(name, overrides = {}) { // What Copilot posts, as a review of the head, when the account's Copilot // code review quota is used up: it reviewed nothing. +const COPILOT_NO_REVIEW_TEST = /^[ \t\r\n]*Copilot was unable to review/; const COPILOT_QUOTA_NOTE = "Copilot was unable to review this pull request because the user who requested the review has reached their quota limit."; @@ -1239,6 +1251,37 @@ test("merges on Claude's review of the head when Copilot only posted its quota n assert.equal(merges.length, 1); }); +test("doesn't count Claude's review of the head when Copilot posted no quota note", async () => { + const pr = readyPr({ + copilotReviews: copilotReviewsOf(), + claudeReviews: claudeReviewsOf(HEAD), + labels: labelled(CLAUDE_LABEL), + quotaNotes: false + }); + const { merges, logs } = await evaluate(pr); + + assert.deepEqual(merges, []); + assert.ok(logs.some((line) => line.includes("no Copilot or Claude review of")), logs.join("\n")); +}); + +test("doesn't count Claude's review of the head when the quota note is on an earlier head", async () => { + const pr = readyPr({ + copilotReviews: copilotReviewsOf({ oid: "older", note: true }), + claudeReviews: claudeReviewsOf(HEAD), + labels: labelled(CLAUDE_LABEL), + quotaNotes: false + }); + const { merges } = await evaluate(pr); + + assert.deepEqual(merges, []); +}); + +test("still counts Copilot's own review of the head without a quota note", async () => { + const { merges } = await evaluate(readyPr({ copilotReviews: copilotReviewsOf(HEAD) })); + + assert.equal(merges.length, 1); +}); + test("counts reviewed commits, not reviews, toward the cap", async () => { const { merges } = await evaluate(readyPr({ copilotReviews: copilotReviewsOf("one", "one", "two") })); @@ -1352,6 +1395,23 @@ test("takes review:claude off a PR at the review cap, even while a thread holds assert.ok(logs.some((line) => line.includes("removed review:claude")), logs.join("\n")); }); +test("takes review:claude off once Copilot has really reviewed the head, so it has budget (mpaulosky/dotfiles#270)", async () => { + const pr = readyPr({ copilotReviews: copilotReviewsOf(HEAD), labels: labelled(CLAUDE_LABEL) }); + const { removedLabels, logs } = await evaluate(pr); + + assert.deepEqual(removedLabels, [{ owner: OWNER, repo: "demo", issue_number: 7, name: CLAUDE_LABEL }]); + assert.ok(logs.some((line) => line.includes("Copilot has reviewed the head")), logs.join("\n")); +}); + +test("leaves review:claude on when Copilot's quota note or an earlier head's review is all it has", async () => { + for (const copilotReviews of [copilotReviewsOf({ oid: HEAD, note: true }), copilotReviewsOf("older")]) { + const pr = readyPr({ copilotReviews, labels: labelled(CLAUDE_LABEL), quotaNotes: false }); + const { removedLabels } = await evaluate(pr); + + assert.deepEqual(removedLabels, []); + } +}); + test("leaves review:claude on below the review cap and MAX_CLAUDE_ROUNDS", async () => { const pr = readyPr({ copilotReviews: copilotReviewsOf("one"), claudeReviews: claudeReviewsOf(HEAD), labels: labelled(CLAUDE_LABEL) }); const { removedLabels } = await evaluate(pr); @@ -1394,7 +1454,7 @@ test("takes review:claude off once Claude has reviewed MAX_CLAUDE_ROUNDS commits }); test("leaves review:claude on at a cap reached on Copilot's reviews alone", async () => { - const pr = readyPr({ copilotReviews: copilotReviewsOf("one", "two", HEAD), labels: labelled(CLAUDE_LABEL) }); + const pr = readyPr({ copilotReviews: copilotReviewsOf("one", "two", "three"), labels: labelled(CLAUDE_LABEL) }); const { removedLabels } = await evaluate(pr); assert.deepEqual(removedLabels, []); @@ -1464,7 +1524,7 @@ test("falls back to any Claude review at the cap when the label events can't be }); test("keeps review:claude at the cap when the label events can't be read and Claude never reviewed", async () => { - const pr = readyPr({ copilotReviews: copilotReviewsOf("one", "two", HEAD), labels: labelled(CLAUDE_LABEL) }); + const pr = readyPr({ copilotReviews: copilotReviewsOf("one", "two", "three"), labels: labelled(CLAUDE_LABEL) }); const { removedLabels } = await evaluate(pr, [], { eventsError: new Error("Server Error") }); assert.deepEqual(removedLabels, []); diff --git a/.github/workflows/claude-review.yml b/.github/workflows/claude-review.yml index e85e2d78..1c56cd86 100644 --- a/.github/workflows/claude-review.yml +++ b/.github/workflows/claude-review.yml @@ -1,22 +1,23 @@ --- name: Claude Review -# Claude reviews a PR as a backup to Copilot, whose review requests GitHub -# drops without a word once the account's Copilot code review budget is used -# up. It runs only on request: while the PR carries review:claude, which PR -# Auto-Merge adds (with RELEASE_PR_PAT) when Copilot's quota note answers a -# head, the repo-ci-baseline skill's reapply.sh and land.sh (in -# mpaulosky/dotfiles, not this repo) add when Copilot's request didn't -# register, and a person can add at any time. PR Auto-Merge also takes it off -# at the review cap and at MAX_CLAUDE_ROUNDS. It reviews every new head while -# the label is on. +# Claude reviews a PR as a backup to Copilot, whose review it stands in for +# only where Copilot answered the head with its quota note (the account's +# Copilot code review budget is used up). It runs only on request: while the PR +# carries review:claude, which PR Auto-Merge adds (with RELEASE_PR_PAT) when +# that note answers a head, as the repo-ci-baseline skill's land.sh (in +# mpaulosky/dotfiles, not this repo) does, and a person can add at any time; +# a review of a head without the note covers nothing. PR Auto-Merge also takes +# it off at the review cap and at MAX_CLAUDE_ROUNDS. It reviews every new head +# while the label is on. # # Claude never holds a write token. The review job checks the PR out with a # read-only token, gives Claude the diff against the base and read-only tools, # and takes its findings as structured JSON. The post job turns that JSON # into one review on the head commit (post_claude_review.py), with inline # threads like Copilot's, and a body that starts with : -# PR Auto-Merge accepts that review of the head as it does Copilot's. A +# PR Auto-Merge accepts that review of the head in place of Copilot's when +# Copilot's quota note is on the same commit. A # finding outside the diff can't be a thread, so it goes in the body under a # second marker, which PR Auto-Merge holds on until the next push gets a fresh # review (past the review cap too), and the post job warns. It doesn't @@ -176,13 +177,20 @@ jobs: || echo "::warning::Couldn't read the jobs of Claude Review run ${run}, so ${HEAD_SHA} may be reviewed twice." done < "${RUNNER_TEMP}/run-ids.txt" fi - # Commits Copilot or Claude Review (a marked github-actions[bot] review) reviewed. - # Copilot's note that it couldn't review (its quota is used up) covers + # Commits Copilot reviewed, or Claude Review (a marked github-actions[bot] + # review) did where Copilot answered the same commit with its note that + # it couldn't review (its quota is used up). The note itself covers # nothing, as PR Auto-Merge's COPILOT_NO_REVIEW decides. - gh api --paginate "repos/${REPOSITORY}/pulls/${PR_NUMBER}/reviews" --jq '.[] - | select((.user.login == "copilot-pull-request-reviewer[bot]" - and ((.body // "") | test("\\A[ \\t\\r\\n]*Copilot was unable to review") | not)) - or (.user.login == "github-actions[bot]" and ((.body // "") | startswith("")))) + # --slurp, so the note is seen whichever page of reviews it is on. + gh api --paginate --slurp "repos/${REPOSITORY}/pulls/${PR_NUMBER}/reviews" | jq -r ' + def declined: .user.login == "copilot-pull-request-reviewer[bot]" + and ((.body // "") | test("\\A[ \\t\\r\\n]*Copilot was unable to review")); + add as $reviews + | ([$reviews[] | select(declined) | .commit_id]) as $declined + | $reviews[] + | select((.user.login == "copilot-pull-request-reviewer[bot]" and (declined | not)) + or (.user.login == "github-actions[bot]" and ((.body // "") | startswith("")) + and (.commit_id as $commit | $declined | index($commit) != null))) | .commit_id' > "${RUNNER_TEMP}/reviewed.txt" python3 -I - <<'PY' import os, subprocess diff --git a/.github/workflows/pr-automerge.yml b/.github/workflows/pr-automerge.yml index 0a79821b..d356c389 100644 --- a/.github/workflows/pr-automerge.yml +++ b/.github/workflows/pr-automerge.yml @@ -5,7 +5,8 @@ name: PR Auto-Merge # pass, a reviewer has reviewed the PR's head commit, and every review thread # is resolved. The reviewer is Copilot, or Claude as its backup: Claude # Review (claude-review.yml) posts a review under github-actions[bot] whose -# body starts with CLAUDE_MARKER. The main ruleset asks Copilot to review +# body starts with CLAUDE_MARKER, and counts only for a head whose Copilot +# review is the quota note below. The main ruleset asks Copilot to review # each push but doesn't require its review, so merging on green alone would # usually land a PR before that review arrives. Each trigger below is a # moment one of those conditions can become true; the job re-checks all of @@ -41,7 +42,9 @@ name: PR Auto-Merge # reviewed MAX_CLAUDE_ROUNDS (the cap's 3) of the PR's commits, whoever added # it, since reviews of merge commits don't count toward the cap: that's the # owner's cost rule. -# At the cap the run takes review:claude off the PR (mpaulosky/dotfiles#170), once Claude has +# The run takes review:claude off a PR once Copilot has really reviewed its head +# (mpaulosky/dotfiles#270): Copilot has budget, and Claude's review would cover +# nothing. At the cap it takes it off too (mpaulosky/dotfiles#170), once Claude has # reviewed it since the label was last added: Claude Review reviews every # push while it's on, and each review can open threads that hold the merge, # so the PR would go on chasing them past the cap. @@ -885,7 +888,16 @@ jobs: return review.body?.startsWith(CLAUDE_MARKER); }); const copilotReviews = pr.copilotReviews.nodes.filter((review) => !COPILOT_NO_REVIEW.test(review.body ?? "")); - const reviewed = [...copilotReviews, ...claudeReviews] + // Claude is Copilot's backup only: its review of a commit + // qualifies when Copilot's quota note is on that same commit. + const declinedCommits = new Set( + pr.copilotReviews.nodes + .filter((review) => COPILOT_NO_REVIEW.test(review.body ?? "")) + .map((review) => review.commit?.oid) + .filter(Boolean) + ); + const backupReviews = claudeReviews.filter((review) => declinedCommits.has(review.commit?.oid)); + const reviewed = [...copilotReviews, ...backupReviews] .map((review) => review.commit) .filter(Boolean); const reviewedCommits = new Set(reviewed.map((commit) => commit.oid)); @@ -1188,7 +1200,14 @@ jobs: if (isOpenSameRepoPrIntoMain(pr) && pr.labels.nodes.some((label) => label.name === CLAUDE_LABEL)) { const { claudeReviews, capReached } = reviewState(pr); const roundsUsed = claudeRounds(claudeReviews) >= MAX_CLAUDE_ROUNDS; - if ((capReached || roundsUsed) && (await claudeReviewedSinceLabel(pullNumber, claudeReviews))) { + // Copilot has really reviewed the head, so it has budget and + // Claude's review would cover nothing: the label only costs money. + const copilotReviewedHead = pr.copilotReviews.nodes.some((review) => { + return review.commit?.oid === pr.headRefOid && !COPILOT_NO_REVIEW.test(review.body ?? ""); + }); + if (copilotReviewedHead) { + await removeClaudeLabel(pullNumber, "Copilot has reviewed the head"); + } else if ((capReached || roundsUsed) && (await claudeReviewedSinceLabel(pullNumber, claudeReviews))) { await removeClaudeLabel(pullNumber, capReached ? `Review cap (${COPILOT_REVIEW_CAP}) reached` : `Claude has reviewed ${MAX_CLAUDE_ROUNDS} of its commits (MAX_CLAUDE_ROUNDS)`); diff --git a/.sandcastle/.env.example b/.sandcastle/.env.example index 26685462..32d861e6 100644 --- a/.sandcastle/.env.example +++ b/.sandcastle/.env.example @@ -4,7 +4,19 @@ CLAUDE_CODE_OAUTH_TOKEN= # Or use an Anthropic API key instead — uncomment and fill in: # ANTHROPIC_API_KEY= # -# No GitHub token goes here. Sandcastle passes every key in this file into the -# sandbox, and agents that read issue text must not be able to reach GitHub. -# The host reads issues and opens pull requests with your own `gh auth login`, -# and main.mts refuses to start while GH_TOKEN or GITHUB_TOKEN is set here. +# Every key in this file is passed into the sandbox. Don't add a GitHub token: +# agents never reach GitHub, and the host refuses to start while GH_TOKEN or +# GITHUB_TOKEN is set here. The host uses its own `gh` login instead. +# +# A local run needs a queue scope, set on the command line rather than here +# (this file reaches the sandbox, and the host would refuse a push whose diff +# mentions a value from it): +# SANDCASTLE_ISSUE= pnpm run sandcastle builds only issue #n and its PR +# SANDCASTLE_LABEL=