Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/ci/sandcastle-tests.sh
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#!/usr/bin/env bash
# Runs Sandcastle's tests (pnpm's test:sandcastle script) when anything they
# Runs Sandcastle's tests (pnpm's check:sandcastle script) when anything they
# cover changed since <base>, or always when <base> is empty. The pre-push
# gate (.github/ci/gate-checks.sh) and CI's Build Solution job
# (.github/ci/prepare.sh build) both call this, so they watch the same paths.
Expand Down Expand Up @@ -46,4 +46,4 @@ else
fi
echo "Sandcastle tests"
"${pnpm[@]}" install --frozen-lockfile
"${pnpm[@]}" run test:sandcastle
"${pnpm[@]}" run check:sandcastle
66 changes: 63 additions & 3 deletions .github/scripts/tests/pr-automerge.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,18 @@ const AsyncFunction = Object.getPrototypeOf(async () => {}).constructor;
const run = new AsyncFunction("github", "context", "core", "getOctokit", inlineScript());

// A same-repo PR into main that is ready to merge unless a test changes it.
function readyPr(overrides = {}) {
// Claude's reviews only count as backup for Copilot's quota note on the same
// commit, so each Claude review given here comes with that note, unless
// { quotaNotes: false } says Copilot never posted one, or the test's own
// copilotReviews already hold the notes it wants.
function readyPr({ quotaNotes = true, ...overrides } = {}) {
if (quotaNotes && overrides.claudeReviews) {
const reviewed = overrides.copilotReviews?.nodes ?? [{ commit: { oid: HEAD } }];
const notes = overrides.claudeReviews.nodes
.filter((review) => !reviewed.some((other) => other.commit?.oid === review.commit.oid && COPILOT_NO_REVIEW_TEST.test(other.body ?? "")))
.map((review) => ({ body: COPILOT_QUOTA_NOTE, commit: { oid: review.commit.oid, parents: review.commit.parents } }));
overrides = { ...overrides, copilotReviews: { nodes: [...reviewed, ...notes] } };
}
return {
state: "OPEN",
isDraft: false,
Expand Down Expand Up @@ -72,6 +83,7 @@ function requiredCheck(name, overrides = {}) {

// What Copilot posts, as a review of the head, when the account's Copilot
// code review quota is used up: it reviewed nothing.
const COPILOT_NO_REVIEW_TEST = /^[ \t\r\n]*Copilot was unable to review/;
const COPILOT_QUOTA_NOTE =
"Copilot was unable to review this pull request because the user who requested the review has reached their quota limit.";

Expand Down Expand Up @@ -1239,6 +1251,37 @@ test("merges on Claude's review of the head when Copilot only posted its quota n
assert.equal(merges.length, 1);
});

test("doesn't count Claude's review of the head when Copilot posted no quota note", async () => {
const pr = readyPr({
copilotReviews: copilotReviewsOf(),
claudeReviews: claudeReviewsOf(HEAD),
labels: labelled(CLAUDE_LABEL),
quotaNotes: false
});
const { merges, logs } = await evaluate(pr);

assert.deepEqual(merges, []);
assert.ok(logs.some((line) => line.includes("no Copilot or Claude review of")), logs.join("\n"));
});

test("doesn't count Claude's review of the head when the quota note is on an earlier head", async () => {
const pr = readyPr({
copilotReviews: copilotReviewsOf({ oid: "older", note: true }),
claudeReviews: claudeReviewsOf(HEAD),
labels: labelled(CLAUDE_LABEL),
quotaNotes: false
});
const { merges } = await evaluate(pr);

assert.deepEqual(merges, []);
});

test("still counts Copilot's own review of the head without a quota note", async () => {
const { merges } = await evaluate(readyPr({ copilotReviews: copilotReviewsOf(HEAD) }));

assert.equal(merges.length, 1);
});

test("counts reviewed commits, not reviews, toward the cap", async () => {
const { merges } = await evaluate(readyPr({ copilotReviews: copilotReviewsOf("one", "one", "two") }));

Expand Down Expand Up @@ -1352,6 +1395,23 @@ test("takes review:claude off a PR at the review cap, even while a thread holds
assert.ok(logs.some((line) => line.includes("removed review:claude")), logs.join("\n"));
});

test("takes review:claude off once Copilot has really reviewed the head, so it has budget (mpaulosky/dotfiles#270)", async () => {
const pr = readyPr({ copilotReviews: copilotReviewsOf(HEAD), labels: labelled(CLAUDE_LABEL) });
const { removedLabels, logs } = await evaluate(pr);

assert.deepEqual(removedLabels, [{ owner: OWNER, repo: "demo", issue_number: 7, name: CLAUDE_LABEL }]);
assert.ok(logs.some((line) => line.includes("Copilot has reviewed the head")), logs.join("\n"));
});

test("leaves review:claude on when Copilot's quota note or an earlier head's review is all it has", async () => {
for (const copilotReviews of [copilotReviewsOf({ oid: HEAD, note: true }), copilotReviewsOf("older")]) {
const pr = readyPr({ copilotReviews, labels: labelled(CLAUDE_LABEL), quotaNotes: false });
const { removedLabels } = await evaluate(pr);

assert.deepEqual(removedLabels, []);
}
});

test("leaves review:claude on below the review cap and MAX_CLAUDE_ROUNDS", async () => {
const pr = readyPr({ copilotReviews: copilotReviewsOf("one"), claudeReviews: claudeReviewsOf(HEAD), labels: labelled(CLAUDE_LABEL) });
const { removedLabels } = await evaluate(pr);
Expand Down Expand Up @@ -1394,7 +1454,7 @@ test("takes review:claude off once Claude has reviewed MAX_CLAUDE_ROUNDS commits
});

test("leaves review:claude on at a cap reached on Copilot's reviews alone", async () => {
const pr = readyPr({ copilotReviews: copilotReviewsOf("one", "two", HEAD), labels: labelled(CLAUDE_LABEL) });
const pr = readyPr({ copilotReviews: copilotReviewsOf("one", "two", "three"), labels: labelled(CLAUDE_LABEL) });
const { removedLabels } = await evaluate(pr);

assert.deepEqual(removedLabels, []);
Expand Down Expand Up @@ -1464,7 +1524,7 @@ test("falls back to any Claude review at the cap when the label events can't be
});

test("keeps review:claude at the cap when the label events can't be read and Claude never reviewed", async () => {
const pr = readyPr({ copilotReviews: copilotReviewsOf("one", "two", HEAD), labels: labelled(CLAUDE_LABEL) });
const pr = readyPr({ copilotReviews: copilotReviewsOf("one", "two", "three"), labels: labelled(CLAUDE_LABEL) });
const { removedLabels } = await evaluate(pr, [], { eventsError: new Error("Server Error") });

assert.deepEqual(removedLabels, []);
Expand Down
40 changes: 24 additions & 16 deletions .github/workflows/claude-review.yml
Original file line number Diff line number Diff line change
@@ -1,22 +1,23 @@
---
name: Claude Review

# Claude reviews a PR as a backup to Copilot, whose review requests GitHub
# drops without a word once the account's Copilot code review budget is used
# up. It runs only on request: while the PR carries review:claude, which PR
# Auto-Merge adds (with RELEASE_PR_PAT) when Copilot's quota note answers a
# head, the repo-ci-baseline skill's reapply.sh and land.sh (in
# mpaulosky/dotfiles, not this repo) add when Copilot's request didn't
# register, and a person can add at any time. PR Auto-Merge also takes it off
# at the review cap and at MAX_CLAUDE_ROUNDS. It reviews every new head while
# the label is on.
# Claude reviews a PR as a backup to Copilot, whose review it stands in for
# only where Copilot answered the head with its quota note (the account's
# Copilot code review budget is used up). It runs only on request: while the PR
# carries review:claude, which PR Auto-Merge adds (with RELEASE_PR_PAT) when
# that note answers a head, as the repo-ci-baseline skill's land.sh (in
# mpaulosky/dotfiles, not this repo) does, and a person can add at any time;
# a review of a head without the note covers nothing. PR Auto-Merge also takes
# it off at the review cap and at MAX_CLAUDE_ROUNDS. It reviews every new head
# while the label is on.
#
# Claude never holds a write token. The review job checks the PR out with a
# read-only token, gives Claude the diff against the base and read-only tools,
# and takes its findings as structured JSON. The post job turns that JSON
# into one review on the head commit (post_claude_review.py), with inline
# threads like Copilot's, and a body that starts with <!-- claude-review -->:
# PR Auto-Merge accepts that review of the head as it does Copilot's. A
# PR Auto-Merge accepts that review of the head in place of Copilot's when
# Copilot's quota note is on the same commit. A
# finding outside the diff can't be a thread, so it goes in the body under a
# second marker, which PR Auto-Merge holds on until the next push gets a fresh
# review (past the review cap too), and the post job warns. It doesn't
Expand Down Expand Up @@ -176,13 +177,20 @@ jobs:
|| echo "::warning::Couldn't read the jobs of Claude Review run ${run}, so ${HEAD_SHA} may be reviewed twice."
done < "${RUNNER_TEMP}/run-ids.txt"
fi
# Commits Copilot or Claude Review (a marked github-actions[bot] review) reviewed.
# Copilot's note that it couldn't review (its quota is used up) covers
# Commits Copilot reviewed, or Claude Review (a marked github-actions[bot]
# review) did where Copilot answered the same commit with its note that
# it couldn't review (its quota is used up). The note itself covers
# nothing, as PR Auto-Merge's COPILOT_NO_REVIEW decides.
gh api --paginate "repos/${REPOSITORY}/pulls/${PR_NUMBER}/reviews" --jq '.[]
| select((.user.login == "copilot-pull-request-reviewer[bot]"
and ((.body // "") | test("\\A[ \\t\\r\\n]*Copilot was unable to review") | not))
or (.user.login == "github-actions[bot]" and ((.body // "") | startswith("<!-- claude-review -->"))))
# --slurp, so the note is seen whichever page of reviews it is on.
gh api --paginate --slurp "repos/${REPOSITORY}/pulls/${PR_NUMBER}/reviews" | jq -r '
def declined: .user.login == "copilot-pull-request-reviewer[bot]"
and ((.body // "") | test("\\A[ \\t\\r\\n]*Copilot was unable to review"));
add as $reviews
| ([$reviews[] | select(declined) | .commit_id]) as $declined
| $reviews[]
| select((.user.login == "copilot-pull-request-reviewer[bot]" and (declined | not))
or (.user.login == "github-actions[bot]" and ((.body // "") | startswith("<!-- claude-review -->"))
and (.commit_id as $commit | $declined | index($commit) != null)))
| .commit_id' > "${RUNNER_TEMP}/reviewed.txt"
python3 -I - <<'PY'
import os, subprocess
Expand Down
27 changes: 23 additions & 4 deletions .github/workflows/pr-automerge.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ name: PR Auto-Merge
# pass, a reviewer has reviewed the PR's head commit, and every review thread
# is resolved. The reviewer is Copilot, or Claude as its backup: Claude
# Review (claude-review.yml) posts a review under github-actions[bot] whose
# body starts with CLAUDE_MARKER. The main ruleset asks Copilot to review
# body starts with CLAUDE_MARKER, and counts only for a head whose Copilot
# review is the quota note below. The main ruleset asks Copilot to review
# each push but doesn't require its review, so merging on green alone would
# usually land a PR before that review arrives. Each trigger below is a
# moment one of those conditions can become true; the job re-checks all of
Expand Down Expand Up @@ -41,7 +42,9 @@ name: PR Auto-Merge
# reviewed MAX_CLAUDE_ROUNDS (the cap's 3) of the PR's commits, whoever added
# it, since reviews of merge commits don't count toward the cap: that's the
# owner's cost rule.
# At the cap the run takes review:claude off the PR (mpaulosky/dotfiles#170), once Claude has
# The run takes review:claude off a PR once Copilot has really reviewed its head
# (mpaulosky/dotfiles#270): Copilot has budget, and Claude's review would cover
# nothing. At the cap it takes it off too (mpaulosky/dotfiles#170), once Claude has
# reviewed it since the label was last added: Claude Review reviews every
# push while it's on, and each review can open threads that hold the merge,
# so the PR would go on chasing them past the cap.
Expand Down Expand Up @@ -885,7 +888,16 @@ jobs:
return review.body?.startsWith(CLAUDE_MARKER);
});
const copilotReviews = pr.copilotReviews.nodes.filter((review) => !COPILOT_NO_REVIEW.test(review.body ?? ""));
const reviewed = [...copilotReviews, ...claudeReviews]
// Claude is Copilot's backup only: its review of a commit
// qualifies when Copilot's quota note is on that same commit.
const declinedCommits = new Set(
pr.copilotReviews.nodes
.filter((review) => COPILOT_NO_REVIEW.test(review.body ?? ""))
.map((review) => review.commit?.oid)
.filter(Boolean)
);
const backupReviews = claudeReviews.filter((review) => declinedCommits.has(review.commit?.oid));
Comment thread
mpaulosky marked this conversation as resolved.
Resolved
const reviewed = [...copilotReviews, ...backupReviews]
.map((review) => review.commit)
.filter(Boolean);
const reviewedCommits = new Set(reviewed.map((commit) => commit.oid));
Expand Down Expand Up @@ -1188,7 +1200,14 @@ jobs:
if (isOpenSameRepoPrIntoMain(pr) && pr.labels.nodes.some((label) => label.name === CLAUDE_LABEL)) {
const { claudeReviews, capReached } = reviewState(pr);
const roundsUsed = claudeRounds(claudeReviews) >= MAX_CLAUDE_ROUNDS;
if ((capReached || roundsUsed) && (await claudeReviewedSinceLabel(pullNumber, claudeReviews))) {
// Copilot has really reviewed the head, so it has budget and
// Claude's review would cover nothing: the label only costs money.
const copilotReviewedHead = pr.copilotReviews.nodes.some((review) => {
return review.commit?.oid === pr.headRefOid && !COPILOT_NO_REVIEW.test(review.body ?? "");
});
if (copilotReviewedHead) {
await removeClaudeLabel(pullNumber, "Copilot has reviewed the head");
} else if ((capReached || roundsUsed) && (await claudeReviewedSinceLabel(pullNumber, claudeReviews))) {
await removeClaudeLabel(pullNumber, capReached
? `Review cap (${COPILOT_REVIEW_CAP}) reached`
: `Claude has reviewed ${MAX_CLAUDE_ROUNDS} of its commits (MAX_CLAUDE_ROUNDS)`);
Expand Down
20 changes: 16 additions & 4 deletions .sandcastle/.env.example
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,19 @@ CLAUDE_CODE_OAUTH_TOKEN=
# Or use an Anthropic API key instead — uncomment and fill in:
# ANTHROPIC_API_KEY=
#
# No GitHub token goes here. Sandcastle passes every key in this file into the
# sandbox, and agents that read issue text must not be able to reach GitHub.
# The host reads issues and opens pull requests with your own `gh auth login`,
# and main.mts refuses to start while GH_TOKEN or GITHUB_TOKEN is set here.
# Every key in this file is passed into the sandbox. Don't add a GitHub token:
# agents never reach GitHub, and the host refuses to start while GH_TOKEN or
# GITHUB_TOKEN is set here. The host uses its own `gh` login instead.
#
# A local run needs a queue scope, set on the command line rather than here
# (this file reaches the sandbox, and the host would refuse a push whose diff
# mentions a value from it):
# SANDCASTLE_ISSUE=<n> pnpm run sandcastle builds only issue #n and its PR
# SANDCASTLE_LABEL=<label> pnpm run sandcastle works the issues labelled <label>, e.g. Sandcastle:dev
# In GitHub Actions the queue is always the Sandcastle label, and setting
# either variable there stops the run.
#
# The run's time budget in minutes, also set on the command line or in the
# workflow's env rather than here (the host reads it from its own environment).
# No new round or role run starts once it has passed since the run started:
# SANDCASTLE_BUDGET_MINUTES=240 pnpm run sandcastle (the default is 240)
4 changes: 4 additions & 0 deletions .sandcastle/.gitignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
.env
logs/
worktrees/
# Notes passed between roles in a sandbox, such as the architect's design note.
work/
# The follow-up role's verdicts on a PR's review threads, read back by the host.
follow-up.json
Loading
Loading