diff --git a/quest/m0/land-fixer-crash.md b/quest/m0/land-fixer-crash.md index bb7aab10..b70f76d0 100644 --- a/quest/m0/land-fixer-crash.md +++ b/quest/m0/land-fixer-crash.md @@ -1,4 +1,4 @@ -# [M] The land fixer stops crashing on kixel-ai/rind +# [XS] The land fixer stops crashing on kixel-ai/rind ## Goal @@ -9,7 +9,15 @@ On kixel-ai/rind, a PR the merge queue drops for `failed_checks` gets a land fix Planned 2026-11-15 with the maintainer in rind's `/quest-run` of 2026-11-14 and 2026-11-15: - Every fixer handback that run saw on rind said `Codex land fixer: fixer crashed (exit 1); run /quest-iterate.` (`src/cli/land-fixer.ts`), on kixel-ai/rind#742, #748 (twice), #758 (twice), #786 and #789. The maintainer's session ran `/quest-iterate` by hand each time. -- On kixel-ai/rind#789 the rind watcher logged `fixing 789` and `handback 789` 66 seconds apart, and the maintainer's host journal shows no fixer container starting in that minute, while containers for other runs started and finished cleanly. So the run exits 1 before Codex starts: in Quest's launch of the fixer, or in the host's restricted launcher it calls. -- Here: make the handback comment carry the failed launch's stderr tail and exit reason instead of `crashed (exit 1)` alone, reproduce the failure, and fix it with a regression test where the cause is in Quest's code. -- Where the cause is in the host's fixer service (its socket, container, or credentials), that fix is kixel-ai/dotfiles' work, per this repository's `AGENTS.md`: add it there as a follow-up, record it under this quest's `## Waiting` as `kixel-ai/dotfiles:` with the PR or quest that fixes it, since `## Required` links only quests in this tree, and finish this quest once rind's next dropped PR gets a real fixer run. - ✅ m0: the fixer is how [the queue watcher](/quest/m0/queue-watcher/README.md) absorbs the quest-file conflicts GitHub cannot resolve, so a crashing fixer leaves those for a human (rejected: m1). + +Found 2026-11-17 in this quest's first PR, which also makes a failed fixer's handback quote the tail of its own output after the gate's failure: + +- Each of those runs' job logs on the maintainer's host holds one line, `quest land check: upload exceeds 256 MiB`: the host's fixer client tars Quest's prepared checkout, `.git` and working tree, and refuses past its 256 MiB upload cap. Quest's checkout for kixel-ai/rind#789 tars to 382 MB, 97 MB of `.git` and 274 MB of working tree (235 MB of it `thirdparty/` test vectors and proofs). Quest's checkout is as small as the round allows (only the base, the PR, and the configured tags and branches), so the cause is in the host's fixer service. +- That fix is kixel-ai/dotfiles' work, per this repository's `AGENTS.md`. ✅ The host's fixer client uploads only `.git` and checks the base out inside the container, since the working tree is `HEAD`'s and the result already returns as `.git`; that puts rind at about 97 MB (rejected: raising the 256 MiB cap; compressing the upload, each of which only defers the next large repository). Settled by the maintainer on 2026-11-17 in moq-dev/quest#763. +- ✅ A failed fixer's handback quotes its output in every repository, public ones included, with tokens redacted before the tail is cut (rejected: private repositories only; never quoting output). Settled by the maintainer on 2026-11-17 in moq-dev/quest#763. +- Finish this quest once rind's next dropped PR gets a real fixer run. + +## Waiting + +- kixel-ai/dotfiles: its fixer client stops refusing rind's checkout (`quest land check: upload exceeds 256 MiB`), from a quest `/quest-plan` adds there from moq-dev/quest#763's follow-ups. Check that a rind fixer's job log on the maintainer's host (`watch-job-fix-*.json.log` in the rind watcher's state) shows a Codex run, or that rind's next fixer handback names a cause other than the upload. diff --git a/src/cli/land-fixer.ts b/src/cli/land-fixer.ts index 9cf009f6..a3457c34 100644 --- a/src/cli/land-fixer.ts +++ b/src/cli/land-fixer.ts @@ -45,8 +45,8 @@ export class ProtectedEdit extends Error {} /** * The failure log a handback quotes: the lines the fixer cited, else the tail of the gate's failing - * output, so a fixer that failed for its own reasons still shows why the gate dropped the PR. The - * fixer's own output stays in its job log on the host; the comment's reason names its error. + * output, so a fixer that failed for its own reasons still shows why the gate dropped the PR. Its + * own output's tail follows in `outputSection`; the whole stays in its job log on the host. */ export function handbackLog(log: string, lines?: [number, number]): string { if (!lines) return readLog(log, "tail").trimEnd().split("\n").slice(-100).join("\n"); @@ -59,7 +59,11 @@ export function handbackLog(log: string, lines?: [number, number]): string { .trimEnd(); } -/** Read a bounded window of a log, or "" when it is missing or not a regular file. */ +/** + * Read a bounded window of a log, or "" when it is missing or not a regular file. A tail window that + * starts mid-file drops its partial first line, or all of it without a line break, so a cut never splits a token from the prefix that + * `redactTokens` matches. + */ function readLog(path: string, window: "head" | "tail"): string { if (!path || !existsSync(path)) return ""; const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); @@ -67,8 +71,12 @@ function readLog(path: string, window: "head" | "tail"): string { const stat = fstatSync(fd); if (!stat.isFile()) return ""; const buffer = Buffer.alloc(Math.min(stat.size, 16_384)); - const length = readSync(fd, buffer, 0, buffer.length, window === "head" ? 0 : stat.size - buffer.length); - return buffer.subarray(0, length).toString("utf8"); + const start = window === "head" ? 0 : stat.size - buffer.length; + const length = readSync(fd, buffer, 0, buffer.length, start); + const text = buffer.subarray(0, length).toString("utf8"); + if (start === 0) return text; + const newline = text.indexOf("\n"); + return newline < 0 ? "" : text.slice(newline + 1); } finally { closeSync(fd); } @@ -113,6 +121,8 @@ export interface Process { /** Packs fetched from the daemon's repository at launch; their objects are already there. */ packs?: string[]; error?: string; + /** The tail of a failed fixer's own output, which its handback quotes so the failure names its cause. */ + output?: string; } /** @@ -412,14 +422,20 @@ export function settle( const restarted = job.daemon && daemon && job.daemon !== daemon ? "; the daemon restarted while it ran" : ""; if (existsSync(`${path}.exit`)) { const code = readFileSync(`${path}.exit`, "utf8"); - if (job.terminating !== undefined) job.error = "fixer timed out; run /quest-iterate"; - else if (code !== "0") job.error = `fixer ${exited(code)}${restarted}; run /quest-iterate`; - else + if (job.terminating === undefined && code === "0") try { read(); } catch (error) { + // The wrapper succeeded, so its output is the agent's transcript, not the cause. job.error = `invalid fixer result: ${String(error)}; run /quest-iterate`; } + else { + job.error = + job.terminating !== undefined + ? "fixer timed out; run /quest-iterate" + : `fixer ${exited(code)}${restarted}; run /quest-iterate`; + job.output = outputTail(path); + } } else if (deps.now() > deadline(path, job, deps.now(), timeout)) { if (job.terminating === undefined) { job.terminating = deps.now(); @@ -438,14 +454,40 @@ export function settle( } catch {} } job.error = "fixer timed out; run /quest-iterate to diagnose and retry"; + job.output = outputTail(path); } // Without a launch identity (an unreadable /proc), only its exit status or the timeout can end the job. else if (!job.identity || (job.pid && identity(job.pid) === job.identity)) return false; - else job.error = `fixer stopped without recording an exit status${restarted}; run /quest-iterate`; + else { + job.error = `fixer stopped without recording an exit status${restarted}; run /quest-iterate`; + job.output = outputTail(path); + } save(path, job); return true; } +/** Lines of a failed fixer's output its handback quotes: enough for a stack trace's end, few enough to skip its noise. */ +const OUTPUT_LINES = 20; +const OUTPUT_LIMIT = 2_000; + +/** + * The last lines the fixer wrapper at job `path` printed, tokens redacted, or "" when it printed + * nothing. Redacting before the cut keeps a cut through a token's prefix from posting the rest. + */ +export function outputTail(path: string): string { + const tail = redactTokens(readLog(`${path}.log`, "tail").trimEnd().split("\n").slice(-OUTPUT_LINES).join("\n")); + return tail.length > OUTPUT_LIMIT ? tail.slice(-OUTPUT_LIMIT) : tail; +} + +/** + * The handback section quoting a failed fixer's `output`, so a crash before the agent started, such as + * a launcher refusing the upload, names its cause on the PR. Empty when it does not fit in `room`. + */ +export function outputSection(output: string, room: number): string { + const section = `\n\nThe fixer's output ended with:\n\n
${excerpt(redactTokens(output))}`;
+ return section.length <= room ? section : "";
+}
+
/** New commits descend from the pin. Protected content/modes at the head must match the pin or its merge with `base`, the base the fixer was given. */
export function validateFix(root: string, entry: PickThe merge queue removed");
expect(history(state, 9)!.rounds).toMatchObject([{ outcome: "handback" }]);
// A crash ends the pick too, with a round of its own.
- // Its handback names the crash as the reason and quotes the gate's failure, not the fixer's output.
+ // Its handback names the crash as the reason, quotes the gate's failure, then the fixer's output, escaped.
const crash = github.comments[11].at(-1)!.body;
- expect(crash).toContain("fixer crashed (exit 3)");
+ expect(crash).toContain("fixer crashed (exit 1)");
expect(crash).toContain("running tests\nFAIL queue.test.ts");
- expect(crash).not.toContain("agent crashed");
+ expect(crash).toContain(
+ "The fixer's output ended with:\n\nstarting\nquest land check: upload exceeds 256 MiB <@maintainer>
",
+ );
expect(history(state, 11)!.rounds).toMatchObject([
{ outcome: "handback", before: crashed, failure: "Check / test: just test" },
]);
diff --git a/test/land.test.ts b/test/land.test.ts
index 5af0410d..143c13b9 100644
--- a/test/land.test.ts
+++ b/test/land.test.ts
@@ -32,6 +32,7 @@ import {
REQUEUE_LIMIT,
history,
historyPath,
+ outputTail,
redactTokens,
saveRound,
type Round,
@@ -1413,6 +1414,21 @@ printf '[{"number":1,"action":"handback","diagnosis":"saw %s","recommendation":"
expect(bodies).not.toContain(token);
expect(redactTokens(`a ${token} github_pat_${"x".repeat(30)} b`)).toBe("a [redacted token] [redacted token] b");
});
+ test("redacts a fixer's output before cutting its tail, so a cut never splits a token from its prefix", () => {
+ const token = `ghp_${"A1b2".repeat(9)}`;
+ const job = join(tempDir(), "job");
+ // The 2,000-character cut lands two characters into the token, splitting its `ghp_` prefix.
+ writeFileSync(`${job}.log`, `${"x".repeat(10)} ${token}${"y".repeat(2_000 - token.length + 2)}\n`);
+ const tail = outputTail(job);
+ expect(tail).not.toContain(token.slice(4));
+ expect(tail).toContain("[redacted token]");
+ // A tail window that starts mid-file drops its partial first line, which may hold a token's rest.
+ writeFileSync(`${job}.log`, `${token}${"z".repeat(16_384 - token.length + 10)}\nlast\n`);
+ expect(outputTail(job)).toBe("last");
+ // Without a line break, the whole window is a partial line.
+ writeFileSync(`${job}.log`, `${token}${" ".repeat(16_384 - token.length + 4)}`);
+ expect(outputTail(job)).toBe("");
+ });
test("bounds a fixer's import by what it added, not the history fetched at launch or its submodules", () => {
const root = tempDir();
const commit = (cwd: string, message: string) =>