From 0fbd1b733aa9f9f758c96c1f4566343999181129 Mon Sep 17 00:00:00 2001 From: T <60372079+mpaulosky@users.noreply.github.com> Date: Tue, 10 Nov 2026 12:23:48 -0800 Subject: [PATCH 1/2] fix(baseline): Hold review:claude until the ask record settles on every path askCopilotOnce returns true The settle wait of #23 covered only a failed request. A read-back that errors or finds the head moved returned true about 5 s after the request, so review:claude could be added while Claude Review still skipped a record that young and a later run stripped the label. Refs #40 Co-Authored-By: Claude Sonnet 5.5 --- .../scripts/tests/pr-automerge.test.mjs | 51 ++++++++++++++++++- .../owned/.github/workflows/pr-automerge.yml | 31 ++++++++--- .github/workflows/pr-automerge.yml | 31 ++++++++--- 3 files changed, 95 insertions(+), 18 deletions(-) diff --git a/.claude/skills/repo-ci-baseline/template/owned/.github/scripts/tests/pr-automerge.test.mjs b/.claude/skills/repo-ci-baseline/template/owned/.github/scripts/tests/pr-automerge.test.mjs index 9d6ac21..e161fc4 100644 --- a/.claude/skills/repo-ci-baseline/template/owned/.github/scripts/tests/pr-automerge.test.mjs +++ b/.claude/skills/repo-ci-baseline/template/owned/.github/scripts/tests/pr-automerge.test.mjs @@ -373,6 +373,9 @@ async function evaluate(pr, events = [], options = {}) { // The reads after the request that throw (the read-back), and what they throw. readBackErrors = 0, readBackError = new Error("Server error on the read-back"), + // When set, the reads after the request show the PR at this head (a push + // landed while the read-back ran). + readBackHeadRefOid, // Whether the rival's record is listed before this run's own, whatever its // createdAt: GitHub lists comments oldest first, and the first record // listed is the one that decides. Unset, the rival is listed first when its @@ -623,6 +626,7 @@ async function evaluate(pr, events = [], options = {}) { }) } }; + const seenAfterRequest = (pr) => (requested && readBackHeadRefOid ? { ...pr, headRefOid: readBackHeadRefOid } : pr); const github = { rest, graphql: async (query, variables = {}) => { @@ -655,9 +659,9 @@ async function evaluate(pr, events = [], options = {}) { if (snapshot instanceof Error) { throw snapshot; } - return { repository: { pullRequest: withRunsOwnWrites(snapshot) } }; + return { repository: { pullRequest: withRunsOwnWrites(seenAfterRequest(snapshot)) } }; } - return { repository: { pullRequest: withRunsOwnWrites(pullRequests[Math.min(graphqlRequests++, pullRequests.length - 1)]) } }; + return { repository: { pullRequest: withRunsOwnWrites(seenAfterRequest(pullRequests[Math.min(graphqlRequests++, pullRequests.length - 1)])) } }; }, paginate: async (method, params) => { clientCall("token", methodNames.get(method) ?? "unknown paginate method"); @@ -1690,6 +1694,49 @@ test("adds review:claude only after the rest of the settle window when the reque assert.deepEqual(addedLabels, LABEL_FOR_CLAUDE); assert.ok(logs.some((line) => line.startsWith("warning: ") && line.includes("Resource not accessible")), logs.join("\n")); assert.ok(Date.now() - started >= 250, `the run took ${Date.now() - started} ms`); + // The cap: the wait is COPILOT_ASK_WAIT_MS, not the 30-second window, so a run without the cap would take 30 s. + assert.ok(Date.now() - started < 5000, `the run took ${Date.now() - started} ms`); +}); + +// The other two ways askCopilotOnce returns true without a registered request: the read-back errors, or the head moved +// under it. Each leaves the record at least COPILOT_ASK_SETTLE_MS old before review:claude is added, as the failed +// request does, or Claude Review would skip a record younger than that and PR Auto-Merge would strip the label. +for (const [name, options] of [ + ["the read-back errors", { readBackErrors: 1 }], + ["the head moved under the read-back", { readBackHeadRefOid: "moved" }] +]) { + test(`adds review:claude only once the record has settled when ${name} (mpaulosky/Application_Factory#40)`, async () => { + mock.timers.enable({ apis: ["Date", "setTimeout"], now: Date.parse("2026-01-02T00:00:00Z") }); + let result; + let finishedAt; + try { + result = await underClock(evaluate(readyPr(unreviewed), [], { askWaitMs: 30 * 1000, askPollMs: 5000, ...options })); + finishedAt = Date.now(); + } finally { + mock.timers.reset(); + } + assert.equal(result.requests.length, 1); + assert.equal(result.askedAt.length, 1); + // askCopilotOnce held the run until the record was 30 seconds old. + assert.ok(finishedAt - result.askedAt[0] >= 30 * 1000, `the run ended ${(finishedAt - result.askedAt[0]) / 1000} s after the record`); + for (const labelAt of result.addedLabelAt) { + assert.ok(labelAt - result.askedAt[0] >= 30 * 1000, `review:claude was added ${(labelAt - result.askedAt[0]) / 1000} s after the record`); + } + }); +} + +test("a registered request needs no settle wait: the run ends as soon as the read-back sees it (mpaulosky/Application_Factory#40)", async () => { + mock.timers.enable({ apis: ["Date", "setTimeout"], now: Date.parse("2026-01-02T00:00:00Z") }); + let result; + let finishedAt; + try { + result = await underClock(evaluate(readyPr(unreviewed), [], { copilotRequest: "registers", askWaitMs: 30 * 1000, askPollMs: 5000 })); + finishedAt = Date.now(); + } finally { + mock.timers.reset(); + } + assert.equal(result.requests.length, 1); + assert.ok(finishedAt - result.askedAt[0] < 30 * 1000, `the run ended ${(finishedAt - result.askedAt[0]) / 1000} s after the record`); }); // Claude Review's gate (the Python in claude-review.yml's "Skip while Copilot can diff --git a/.claude/skills/repo-ci-baseline/template/owned/.github/workflows/pr-automerge.yml b/.claude/skills/repo-ci-baseline/template/owned/.github/workflows/pr-automerge.yml index d8bc324..8950a96 100644 --- a/.claude/skills/repo-ci-baseline/template/owned/.github/workflows/pr-automerge.yml +++ b/.claude/skills/repo-ci-baseline/template/owned/.github/workflows/pr-automerge.yml @@ -84,7 +84,9 @@ name: PR Auto-Merge # mpaulosky/dotfiles#369; one whose record never shows requests anyway, since a # repeat request is idempotent, mpaulosky/dotfiles#374), requests the # review with the PAT, and reads the request back for up to COPILOT_ASK_WAIT_MS. -# A request that fails or doesn't register counts as dropped at once, and a +# A request that fails, or whose read-back errors or finds the head moved, counts as dropped once the +# record is COPILOT_ASK_SETTLE_MS old (a wait capped by COPILOT_ASK_WAIT_MS, mpaulosky/Application_Factory#23); +# one that doesn't register counts the same after the read-back window, and a # record COPILOT_ASK_SETTLE_MS old with no review of the head and no request # pending counts the same way, whoever wrote it (land.sh writes the same # record). Copilot is never asked again for the PR: a later head goes straight @@ -1711,6 +1713,17 @@ jobs: // After the write returns, so a slow write can't shorten the wait: // the record's own timestamp is no later than this. askWrittenAt = Date.now(); + // Wait out what is left of COPILOT_ASK_SETTLE_MS (capped by + // COPILOT_ASK_WAIT_MS) before a path that returns true without a + // registered request lets the caller add review:claude: other + // workflows see only the record, which Claude Review skips and PR + // Auto-Merge strips while it is younger than that + // (mpaulosky/Application_Factory#23). Every such path calls it: + // the failed request and the read-back's two early exits. + const waitOutSettle = () => new Promise((resolve) => setTimeout( + resolve, + Math.max(0, Math.min(COPILOT_ASK_WAIT_MS, askWrittenAt + COPILOT_ASK_SETTLE_MS - Date.now())) + )); // Two runs can both have read "no record" and both posted one. // Read the comments again: only the run whose record is // listed first makes the request (mpaulosky/dotfiles#369). GraphQL @@ -1778,13 +1791,7 @@ jobs: throw error; } core.warning(`Couldn't request Copilot's review of PR #${pullNumber}: ${error.message}`); - // Wait out what is left of COPILOT_ASK_SETTLE_MS (capped by - // COPILOT_ASK_WAIT_MS) before the caller adds review:claude: - // other workflows see only the record, which Claude Review - // skips and PR Auto-Merge strips while it is younger than that - // (mpaulosky/Application_Factory#23). - const remainingMs = askWrittenAt + COPILOT_ASK_SETTLE_MS - Date.now(); - await new Promise((resolve) => setTimeout(resolve, Math.max(0, Math.min(COPILOT_ASK_WAIT_MS, remainingMs)))); + await waitOutSettle(); core.info(`Copilot's review of PR #${pullNumber} wasn't requested, so Claude is called in.`); return true; } @@ -1795,11 +1802,13 @@ jobs: // again, and only a sweep ends on an error that ends the run. const deadline = Date.now() + COPILOT_ASK_WAIT_MS; let registered = false; + let headMoved = false; try { do { await new Promise((resolve) => setTimeout(resolve, Math.max(0, Math.min(COPILOT_ASK_POLL_MS, deadline - Date.now())))); const seen = await getPullRequestStatus(pullNumber); if (!seen || seen.headRefOid !== pr.headRefOid) { + headMoved = true; break; } registered = copilotPending(seen) || copilotAnsweredHead(seen); @@ -1809,8 +1818,14 @@ jobs: throw error; } core.warning(`Couldn't read PR #${pullNumber} back after requesting Copilot's review, so the request is taken as made: ${error.message}`); + await waitOutSettle(); return true; } + // The head moved or the PR couldn't be read: nothing was seen to + // register, so the record must settle before Claude is called in. + if (headMoved) { + await waitOutSettle(); + } core.info(registered ? `Copilot's review request for PR #${pullNumber} registered.` : `Copilot's review request for PR #${pullNumber} didn't register (GitHub drops it once the Copilot code review budget is used up).`); diff --git a/.github/workflows/pr-automerge.yml b/.github/workflows/pr-automerge.yml index d8bc324..8950a96 100644 --- a/.github/workflows/pr-automerge.yml +++ b/.github/workflows/pr-automerge.yml @@ -84,7 +84,9 @@ name: PR Auto-Merge # mpaulosky/dotfiles#369; one whose record never shows requests anyway, since a # repeat request is idempotent, mpaulosky/dotfiles#374), requests the # review with the PAT, and reads the request back for up to COPILOT_ASK_WAIT_MS. -# A request that fails or doesn't register counts as dropped at once, and a +# A request that fails, or whose read-back errors or finds the head moved, counts as dropped once the +# record is COPILOT_ASK_SETTLE_MS old (a wait capped by COPILOT_ASK_WAIT_MS, mpaulosky/Application_Factory#23); +# one that doesn't register counts the same after the read-back window, and a # record COPILOT_ASK_SETTLE_MS old with no review of the head and no request # pending counts the same way, whoever wrote it (land.sh writes the same # record). Copilot is never asked again for the PR: a later head goes straight @@ -1711,6 +1713,17 @@ jobs: // After the write returns, so a slow write can't shorten the wait: // the record's own timestamp is no later than this. askWrittenAt = Date.now(); + // Wait out what is left of COPILOT_ASK_SETTLE_MS (capped by + // COPILOT_ASK_WAIT_MS) before a path that returns true without a + // registered request lets the caller add review:claude: other + // workflows see only the record, which Claude Review skips and PR + // Auto-Merge strips while it is younger than that + // (mpaulosky/Application_Factory#23). Every such path calls it: + // the failed request and the read-back's two early exits. + const waitOutSettle = () => new Promise((resolve) => setTimeout( + resolve, + Math.max(0, Math.min(COPILOT_ASK_WAIT_MS, askWrittenAt + COPILOT_ASK_SETTLE_MS - Date.now())) + )); // Two runs can both have read "no record" and both posted one. // Read the comments again: only the run whose record is // listed first makes the request (mpaulosky/dotfiles#369). GraphQL @@ -1778,13 +1791,7 @@ jobs: throw error; } core.warning(`Couldn't request Copilot's review of PR #${pullNumber}: ${error.message}`); - // Wait out what is left of COPILOT_ASK_SETTLE_MS (capped by - // COPILOT_ASK_WAIT_MS) before the caller adds review:claude: - // other workflows see only the record, which Claude Review - // skips and PR Auto-Merge strips while it is younger than that - // (mpaulosky/Application_Factory#23). - const remainingMs = askWrittenAt + COPILOT_ASK_SETTLE_MS - Date.now(); - await new Promise((resolve) => setTimeout(resolve, Math.max(0, Math.min(COPILOT_ASK_WAIT_MS, remainingMs)))); + await waitOutSettle(); core.info(`Copilot's review of PR #${pullNumber} wasn't requested, so Claude is called in.`); return true; } @@ -1795,11 +1802,13 @@ jobs: // again, and only a sweep ends on an error that ends the run. const deadline = Date.now() + COPILOT_ASK_WAIT_MS; let registered = false; + let headMoved = false; try { do { await new Promise((resolve) => setTimeout(resolve, Math.max(0, Math.min(COPILOT_ASK_POLL_MS, deadline - Date.now())))); const seen = await getPullRequestStatus(pullNumber); if (!seen || seen.headRefOid !== pr.headRefOid) { + headMoved = true; break; } registered = copilotPending(seen) || copilotAnsweredHead(seen); @@ -1809,8 +1818,14 @@ jobs: throw error; } core.warning(`Couldn't read PR #${pullNumber} back after requesting Copilot's review, so the request is taken as made: ${error.message}`); + await waitOutSettle(); return true; } + // The head moved or the PR couldn't be read: nothing was seen to + // register, so the record must settle before Claude is called in. + if (headMoved) { + await waitOutSettle(); + } core.info(registered ? `Copilot's review request for PR #${pullNumber} registered.` : `Copilot's review request for PR #${pullNumber} didn't register (GitHub drops it once the Copilot code review budget is used up).`); From a18d88b4abfb9fcc1bc88b24416fa4f864c2774d Mon Sep 17 00:00:00 2001 From: T <60372079+mpaulosky@users.noreply.github.com> Date: Tue, 10 Nov 2026 12:34:41 -0800 Subject: [PATCH 2/2] test(baseline): Give the moved-head case a valid commit so it asserts the label The moved head wasn't in the commit graph, so the caller stopped on the merge check and never added review:claude; the label-age assertions checked nothing. Refs #40 Co-Authored-By: Claude Sonnet 5.5 --- .../.github/scripts/tests/pr-automerge.test.mjs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.claude/skills/repo-ci-baseline/template/owned/.github/scripts/tests/pr-automerge.test.mjs b/.claude/skills/repo-ci-baseline/template/owned/.github/scripts/tests/pr-automerge.test.mjs index e161fc4..203cd68 100644 --- a/.claude/skills/repo-ci-baseline/template/owned/.github/scripts/tests/pr-automerge.test.mjs +++ b/.claude/skills/repo-ci-baseline/template/owned/.github/scripts/tests/pr-automerge.test.mjs @@ -374,7 +374,8 @@ async function evaluate(pr, events = [], options = {}) { readBackErrors = 0, readBackError = new Error("Server error on the read-back"), // When set, the reads after the request show the PR at this head (a push - // landed while the read-back ran). + // landed while the read-back ran), with its checks passing; the commit has + // to be in commits (graph()). readBackHeadRefOid, // Whether the rival's record is listed before this run's own, whatever its // createdAt: GitHub lists comments oldest first, and the first record @@ -626,7 +627,9 @@ async function evaluate(pr, events = [], options = {}) { }) } }; - const seenAfterRequest = (pr) => (requested && readBackHeadRefOid ? { ...pr, headRefOid: readBackHeadRefOid } : pr); + const seenAfterRequest = (pr) => (requested && readBackHeadRefOid + ? { ...pr, headRefOid: readBackHeadRefOid, headRef: checksOn(readBackHeadRefOid, requiredCheck("Build Solution")) } + : pr); const github = { rest, graphql: async (query, variables = {}) => { @@ -1703,7 +1706,8 @@ test("adds review:claude only after the rest of the settle window when the reque // request does, or Claude Review would skip a record younger than that and PR Auto-Merge would strip the label. for (const [name, options] of [ ["the read-back errors", { readBackErrors: 1 }], - ["the head moved under the read-back", { readBackHeadRefOid: "moved" }] + // "moved" is an ordinary commit on top of the head, so the caller's merge checks accept it. + ["the head moved under the read-back", { readBackHeadRefOid: "moved", commits: graph({ moved: { parents: [HEAD], files: { app: "app3", lib: "lib0" } } }) }] ]) { test(`adds review:claude only once the record has settled when ${name} (mpaulosky/Application_Factory#40)`, async () => { mock.timers.enable({ apis: ["Date", "setTimeout"], now: Date.parse("2026-01-02T00:00:00Z") }); @@ -1717,6 +1721,9 @@ for (const [name, options] of [ } assert.equal(result.requests.length, 1); assert.equal(result.askedAt.length, 1); + // Claude is called in (a moved head that failed the merge checks would add nothing, and the loop below would check nothing). + assert.deepEqual(result.addedLabels, LABEL_FOR_CLAUDE, result.logs.join("\n")); + assert.equal(result.addedLabelAt.length, 1); // askCopilotOnce held the run until the record was 30 seconds old. assert.ok(finishedAt - result.askedAt[0] >= 30 * 1000, `the run ended ${(finishedAt - result.askedAt[0]) / 1000} s after the record`); for (const labelAt of result.addedLabelAt) {