From ecff8ff34d9c2b3824adffeaff6f398ed58ff411 Mon Sep 17 00:00:00 2001 From: "devintern-internal[bot]" <4622575+devintern-internal[bot]@users.noreply.github.com> Date: Sun, 23 Aug 2026 02:03:27 +0700 Subject: [PATCH] feat: implement DEV-76 - CI failure watcher acquirer: auto-fix failing checks on the agent's own PRs --- docs/code/worker.md | 16 +- packages/code/src/index.ts | 75 ++- packages/code/src/lib/address-review.ts | 330 ++++++---- .../src/lib/ci-failure-watcher-acquirer.ts | 580 ++++++++++++++++++ packages/code/src/lib/dashboard-api.ts | 2 +- packages/code/src/lib/github-reviews.ts | 210 +++++++ packages/code/src/lib/review-formatter.ts | 85 +++ packages/code/src/lib/run-recorder.ts | 4 +- packages/code/src/lib/worker-state.ts | 63 ++ .../code/src/lib/workspace/fleet-events.ts | 38 ++ .../src/lib/workspace/workspace-worker.ts | 63 ++ .../tests/ci-failure-watcher-acquirer.test.ts | 378 ++++++++++++ packages/code/tests/review-formatter.test.ts | 39 ++ packages/dashboard-ui/src/lib/api.ts | 2 +- packages/dashboard-ui/src/views/RunsView.tsx | 2 +- packages/dashboard-ui/src/views/StatsView.tsx | 2 +- 16 files changed, 1751 insertions(+), 138 deletions(-) create mode 100644 packages/code/src/lib/ci-failure-watcher-acquirer.ts create mode 100644 packages/code/tests/ci-failure-watcher-acquirer.test.ts diff --git a/docs/code/worker.md b/docs/code/worker.md index 21b12c9..3b87b16 100644 --- a/docs/code/worker.md +++ b/docs/code/worker.md @@ -3,7 +3,7 @@ title: "Worker Daemon" description: "Run devintern as a single long-running worker that reacts to PR reviews and tracker changes" section: "Server Automation" order: 0 -dateModified: 2026-08-17 +dateModified: 2026-08-23 --- # Worker Daemon @@ -95,6 +95,20 @@ When a watched PR falls behind its base branch and GitHub reports merge conflict This applies only to the agent's own PRs (the same watch list as review polling). Each conflict state is attempted once: a failed resolution is retried only after the branch or its base moves again. The same logic is available manually for any PR via `devintern resolve-conflicts `. +## CI failures on the agent's PRs + +The worker also watches GitHub Actions check runs and commit statuses on the agent's own PRs and fixes them without human intervention. When a check run completes with a `failure` conclusion (or a classic commit status reports failure) on the PR's current head SHA, the worker fetches the failing jobs' logs, reduces them to the error-relevant excerpt, and runs the agent through the same pipeline review feedback uses: worktree prep, sandboxed spawn, commit, push. The push restarts CI, closing the loop from "agent opened PR" to "PR green". + +Guardrails keep a red build from turning into churn: + +- Only terminal failures trigger a fix — pending and in-progress runs are ignored (the agent's own pushes constantly restart CI). +- A given failure is fixed at most once: attempts dedupe per head SHA plus check run id, and the mapping survives worker restarts. +- Consecutive failed autofix attempts are capped per PR (`CI_FIX_MAX_ATTEMPTS`, default 3). On exhaustion the worker posts a comment escalating to a human and stops retrying until someone pushes to the branch; CI going green resets the counter. +- Fork PRs are skipped quietly (Actions rarely runs on forks, and the agent cannot push there). +- When job logs cannot be downloaded (token scope, expired logs), the watcher falls back to the check run's annotations before giving up and letting the agent reproduce the failure locally. + +The same fix runs execute under your configured sandbox settings (`AGENT_SANDBOX`, `--sandbox`) exactly like review-feedback runs, and each attempt is recorded in the dashboard with the `ci_fix` origin. Downloading Actions job logs needs sufficient token scope (a PAT with `repo`, or a GitHub App with `actions:read`). + ## Mention the bot on any PR The worker also reacts to mentions on pull requests it did not create. When a teammate writes a comment like `@devintern address the review feedback` on any PR in the repository, the worker picks it up on the next poll and handles it through the same pipeline. Detection is a repository-wide sweep of new comments (two requests per interval, regardless of how many PRs are open), so mentions work without any webhook setup. diff --git a/packages/code/src/index.ts b/packages/code/src/index.ts index 0d05d4a..9358256 100755 --- a/packages/code/src/index.ts +++ b/packages/code/src/index.ts @@ -609,6 +609,8 @@ if (process.argv[2] === "init") { console.log(" WORKER_TASK_QUERY Task-selection query (same as --query)"); console.log(" WORKER_TASK_ARGS Extra CLI flags per task run (default: --create-pr)"); console.log(" WORKER_POLL_INTERVAL Polling interval in seconds (default: 60)"); + console.log(" CI_FIX_MAX_ATTEMPTS Consecutive CI autofix attempts per PR before"); + console.log(" escalating to a human (default: 3)"); console.log(" WORKER_RELAY_URL Relay base URL override (Mode 2; see worker connect)"); process.exit(0); } @@ -746,6 +748,70 @@ if (process.argv[2] === "init") { }), ); + // Tier 1 CI failure watcher: auto-fix failing checks on the agent's + // own PRs (same registry as review polling). Shares the interval; + // failures dedupe per head SHA + check run id. + const { CiFailureWatcherAcquirer, runCiFixViaCli } = + await import("./lib/ci-failure-watcher-acquirer"); + acquirers.push( + new CiFailureWatcherAcquirer({ + intervalSeconds, + workerState: new WorkerState(dbPath), + queue: new WebhookQueue({ dbPath, legacyDbPath: LEGACY_DB_PATH }), + github: { + fetchPr: (repo, n, etag) => + gh.conditionalGet(`/repos/${repo}/pulls/${n}`, ownerOf(repo), nameOf(repo), etag), + fetchCheckRuns: (repo, sha, etag) => + gh.getCheckRuns(ownerOf(repo), nameOf(repo), sha, etag), + fetchCommitStatus: (repo, sha, etag) => + gh.getCombinedStatus(ownerOf(repo), nameOf(repo), sha, etag), + fetchFailingJobLogs: async (repo, sha) => { + const owner = ownerOf(repo); + const name = nameOf(repo); + const runs = await gh + .getWorkflowRunsForSha(owner, name, sha) + .catch(() => [] as Awaited>); + const chunks: string[] = []; + for (const run of runs.slice(0, 3)) { + const jobs = await gh.getWorkflowRunJobs(owner, name, run.id).catch(() => []); + for (const job of jobs.filter((j) => j.conclusion === "failure").slice(0, 5)) { + if (!job.logs_url) { + continue; + } + const text = await gh.getJobLogs(owner, name, job.logs_url).catch(() => null); + if (text) { + chunks.push(`## Job: ${job.name}\n${text}`); + } + } + } + return chunks.length > 0 ? chunks.join("\n\n") : null; + }, + fetchCheckRunDetails: async (repo, checkRunId) => { + try { + const annotations = await gh.getCheckRunAnnotations( + ownerOf(repo), + nameOf(repo), + checkRunId, + ); + if (annotations.length === 0) { + return null; + } + return annotations + .map((a) => `${a.path ? `${a.path}: ` : ""}${a.message}`) + .join("\n"); + } catch { + return null; + } + }, + postComment: async (repo, n, body) => { + await gh.postPullRequestComment(ownerOf(repo), nameOf(repo), n, body); + }, + }, + fixPr: (repo, n, feedbackPath) => runCiFixViaCli(repo, n, feedbackPath), + verbose, + }), + ); + // Tier 2 mention sweep: react to @mentions on ANY PR in the repo (two // since-cursor requests per tick). Permission + mention gates apply in // the shared review pipeline; fork PRs without maintainer_can_modify @@ -1155,12 +1221,16 @@ if (process.argv[2] === "init") { let noPush = false; let noReply = false; let verbose = false; + let ciFeedbackPath: string | undefined; for (let i = 0; i < args.length; i++) { if (args[i] === "--no-push") { noPush = true; } else if (args[i] === "--no-reply") { noReply = true; + } else if (args[i] === "--ci-feedback") { + ciFeedbackPath = args[i + 1]; + i++; } else if (args[i] === "-v" || args[i] === "--verbose") { verbose = true; } else if (args[i] === "--help" || args[i] === "-h") { @@ -1176,6 +1246,8 @@ if (process.argv[2] === "init") { console.log("Options:"); console.log(" --no-push Don't push changes after fixing"); console.log(" --no-reply Don't post a reply comment on the PR"); + console.log(" --ci-feedback Fix CI failures from a feedback JSON file"); + console.log(" (written by the worker's CI failure watcher)"); console.log(" -v, --verbose Enable verbose logging"); console.log(" -h, --help Display this help message"); console.log(""); @@ -1199,7 +1271,7 @@ if (process.argv[2] === "init") { // Import and run address-review const { addressReview } = await import("./lib/address-review"); try { - await addressReview(prUrl, { noPush, noReply, verbose }); + await addressReview(prUrl, { noPush, noReply, verbose, ciFeedbackPath }); } catch (error) { // Close any run record addressReview opened before it failed (no-op // when none is active — addressReview also ends runs it completes). @@ -1469,6 +1541,7 @@ Subcommands: dashboard Serve the local observability dashboard (run history and stats) serve Deprecated alias for 'worker --listen' address-review Address review feedback on an existing pull request + (--ci-feedback fixes CI failures instead) resolve-conflicts Merge a PR's base branch into it, resolving conflicts login [method] Sign in (github | google | x | email; prompts if omitted) logout Clear local auth session diff --git a/packages/code/src/lib/address-review.ts b/packages/code/src/lib/address-review.ts index 3619fb2..ed476be 100644 --- a/packages/code/src/lib/address-review.ts +++ b/packages/code/src/lib/address-review.ts @@ -11,13 +11,15 @@ import { reapTree, resolveExecutablePathWithRetry, } from "@devintern/agent-harness"; +import { readFileSync } from "fs"; import { buildHeadlessAgentArgs, HEADLESS_AGENT_STDIO } from "./agent-spawn"; import { resolveAgentModel } from "./agent-model"; import { getSandbox } from "./sandbox"; import { GitHubReviewsClient } from "./github-reviews"; import { GitHubAppAuth } from "./github-app-auth"; import { beginRun, endRun, recordRunStage } from "./run-recorder"; -import { formatReviewPrompt } from "./review-formatter"; +import { formatCiFixPrompt, formatReviewPrompt } from "./review-formatter"; +import type { CiFailureFeedback } from "./review-formatter"; import { GIT_CLEAN_ARGS, Utils } from "./utils"; import { isCommitAlreadyComplete, runAgentHarnessToFixGitHook } from "./git-hook-fixer"; import type { @@ -30,6 +32,27 @@ export interface AddressReviewOptions { noPush?: boolean; noReply?: boolean; verbose?: boolean; + /** + * Path to a CI-failure feedback JSON file ({@link CiFailureFeedback}) + * written by the CI failure watcher: switches the command into fix-CI + * mode (no review/comment discovery, no replies). + */ + ciFeedbackPath?: string; +} + +/** + * Read and validate a CI-failure feedback file written by the watcher. + * + * @param filePath - Path passed via `--ci-feedback` + * @returns Parsed feedback + * @throws When the file is missing or not valid feedback JSON + */ +function readCiFeedbackFile(filePath: string): CiFailureFeedback { + const parsed = JSON.parse(readFileSync(filePath, "utf8")) as CiFailureFeedback; + if (!parsed || typeof parsed !== "object" || !Array.isArray(parsed.failures)) { + throw new Error(`Invalid CI feedback file: ${filePath}`); + } + return parsed; } interface ParsedPRUrl { @@ -342,147 +365,195 @@ export async function addressReview( throw new Error(`PR is ${pr.state}, not open. Cannot address review.`); } - // Get latest changes_requested review - console.log("\nšŸ”Ž Looking for changes_requested review..."); - const review = await getLatestChangesRequestedReview(githubClient, owner, repo, prNumber); + // CI-fix mode: a --ci-feedback JSON file (written by the CI failure + // watcher) carries failing-check metadata and a truncated log excerpt. + // The rest of the pipeline — worktree prep, sandboxed agent, commit/push + // with hook retries — is shared with the review flow below. + const ciFeedback = options.ciFeedbackPath + ? readCiFeedbackFile(options.ciFeedbackPath) + : undefined; - if (!review) { - console.log("āœ… No pending changes_requested reviews found."); - return; - } + let processedComments: ProcessedReviewComment[] = []; + let processedConversationComments: ProcessedConversationComment[] = []; + let prompt: string; + let commitSummary: string; - console.log(` Found review from @${review.reviewer}`); + if (ciFeedback) { + console.log( + `\nšŸ¤– CI failure mode: fixing ${ciFeedback.failures.length} failing check(s) ` + + `(${ciFeedback.failures.map((f) => f.name).join(", ")})`, + ); - // Fetch ALL review comments for the PR (not just from this review) - console.log("\nšŸ“„ Fetching review comments..."); - const rawComments = await githubClient.getPullRequestReviewComments(owner, repo, prNumber); + // Run record: begun only once there are actual failures to handle. + beginRun({ + origin: "ci_fix", + repo: `${owner}/${repo}`, + prNumber, + branch: pr.head.ref, + }); + recordRunStage("change_request", { + status: "succeeded", + summary: `fixing ${ciFeedback.failures.length} failing check(s) on ${pr.head.ref}`, + detail: { + failures: ciFeedback.failures, + hasLogs: Boolean(ciFeedback.logs), + }, + }); - // Check which comments already have a "hooray" reaction (marked as addressed) - const addressedCommentIds = new Set(); + prompt = formatCiFixPrompt({ + ...ciFeedback, + repository: ciFeedback.repository || `${owner}/${repo}`, + prTitle: pr.title, + branch: pr.head.ref, + }); + commitSummary = "Fix CI failures"; + } else { + // Get latest changes_requested review + console.log("\nšŸ”Ž Looking for changes_requested review..."); + const review = await getLatestChangesRequestedReview(githubClient, owner, repo, prNumber); + + if (!review) { + console.log("āœ… No pending changes_requested reviews found."); + return; + } - for (const comment of rawComments) { - try { - const reactions = await githubClient.getCommentReactions(owner, repo, comment.id); + console.log(` Found review from @${review.reviewer}`); - // Check if there's a "hooray" (šŸŽ‰) reaction - const hasHoorayReaction = reactions.some((r) => r.content === "hooray"); - if (hasHoorayReaction) { - addressedCommentIds.add(comment.id); - } - } catch (error) { - // Ignore errors fetching reactions, treat as not addressed - if (verbose) { - console.warn(` āš ļø Failed to fetch reactions for comment ${comment.id}`); + // Fetch ALL review comments for the PR (not just from this review) + console.log("\nšŸ“„ Fetching review comments..."); + const rawComments = await githubClient.getPullRequestReviewComments(owner, repo, prNumber); + + // Check which comments already have a "hooray" reaction (marked as addressed) + const addressedCommentIds = new Set(); + + for (const comment of rawComments) { + try { + const reactions = await githubClient.getCommentReactions(owner, repo, comment.id); + + // Check if there's a "hooray" (šŸŽ‰) reaction + const hasHoorayReaction = reactions.some((r) => r.content === "hooray"); + if (hasHoorayReaction) { + addressedCommentIds.add(comment.id); + } + } catch (error) { + // Ignore errors fetching reactions, treat as not addressed + if (verbose) { + console.warn(` āš ļø Failed to fetch reactions for comment ${comment.id}`); + } } } - } - const processedComments: ProcessedReviewComment[] = rawComments - .filter((c) => !addressedCommentIds.has(c.id)) // Filter out already addressed - .map((c) => ({ - id: c.id, - path: c.path, - line: c.line ?? c.original_line, - side: c.side, - diffHunk: c.diff_hunk, - body: c.body, - reviewer: c.user.login, - isReply: c.in_reply_to_id !== undefined, - })); - - const totalComments = rawComments.length; - const alreadyAddressed = totalComments - processedComments.length; - - console.log(` Found ${totalComments} comment(s)`); - if (alreadyAddressed > 0) { - console.log(` ${alreadyAddressed} already addressed (skipping)`); - } - console.log(` ${processedComments.length} remaining to address`); + processedComments = rawComments + .filter((c) => !addressedCommentIds.has(c.id)) // Filter out already addressed + .map((c) => ({ + id: c.id, + path: c.path, + line: c.line ?? c.original_line, + side: c.side, + diffHunk: c.diff_hunk, + body: c.body, + reviewer: c.user.login, + isReply: c.in_reply_to_id !== undefined, + })); + + const totalComments = rawComments.length; + const alreadyAddressed = totalComments - processedComments.length; + + console.log(` Found ${totalComments} comment(s)`); + if (alreadyAddressed > 0) { + console.log(` ${alreadyAddressed} already addressed (skipping)`); + } + console.log(` ${processedComments.length} remaining to address`); - // Fetch conversation comments (issue comments) - console.log("\nšŸ’¬ Fetching conversation comments..."); - const rawIssueComments = await githubClient.getIssueComments(owner, repo, prNumber); + // Fetch conversation comments (issue comments) + console.log("\nšŸ’¬ Fetching conversation comments..."); + const rawIssueComments = await githubClient.getIssueComments(owner, repo, prNumber); - // Filter to only include comments from the reviewer, created after the review - const reviewSubmittedAt = new Date(review.submittedAt); - const reviewerIssueComments = rawIssueComments.filter( - (c) => c.user.login === review.reviewer && new Date(c.created_at) >= reviewSubmittedAt, - ); + // Filter to only include comments from the reviewer, created after the review + const reviewSubmittedAt = new Date(review.submittedAt); + const reviewerIssueComments = rawIssueComments.filter( + (c) => c.user.login === review.reviewer && new Date(c.created_at) >= reviewSubmittedAt, + ); - // Check which issue comments already have a "hooray" reaction - const addressedIssueCommentIds = new Set(); + // Check which issue comments already have a "hooray" reaction + const addressedIssueCommentIds = new Set(); - for (const comment of reviewerIssueComments) { - try { - const reactions = await githubClient.getIssueCommentReactions(owner, repo, comment.id); + for (const comment of reviewerIssueComments) { + try { + const reactions = await githubClient.getIssueCommentReactions(owner, repo, comment.id); - const hasHoorayReaction = reactions.some((r) => r.content === "hooray"); - if (hasHoorayReaction) { - addressedIssueCommentIds.add(comment.id); - } - } catch (error) { - if (verbose) { - console.warn(` āš ļø Failed to fetch reactions for issue comment ${comment.id}`); + const hasHoorayReaction = reactions.some((r) => r.content === "hooray"); + if (hasHoorayReaction) { + addressedIssueCommentIds.add(comment.id); + } + } catch (error) { + if (verbose) { + console.warn(` āš ļø Failed to fetch reactions for issue comment ${comment.id}`); + } } } - } - const processedConversationComments: ProcessedConversationComment[] = reviewerIssueComments - .filter((c) => !addressedIssueCommentIds.has(c.id)) - .map((c) => ({ - id: c.id, - body: c.body, - author: c.user.login, - createdAt: c.created_at, - })); - - const totalIssueComments = reviewerIssueComments.length; - const alreadyAddressedIssue = totalIssueComments - processedConversationComments.length; - - console.log(` Found ${totalIssueComments} conversation comment(s) from reviewer`); - if (alreadyAddressedIssue > 0) { - console.log(` ${alreadyAddressedIssue} already addressed (skipping)`); - } - console.log(` ${processedConversationComments.length} remaining to address`); - - // If no comments remaining (neither review nor conversation), we're done - if (processedComments.length === 0 && processedConversationComments.length === 0) { - console.log("\nāœ… All review and conversation comments have been addressed already."); - console.log(` View PR: ${prUrl}`); - return; - } + processedConversationComments = reviewerIssueComments + .filter((c) => !addressedIssueCommentIds.has(c.id)) + .map((c) => ({ + id: c.id, + body: c.body, + author: c.user.login, + createdAt: c.created_at, + })); + + const totalIssueComments = reviewerIssueComments.length; + const alreadyAddressedIssue = totalIssueComments - processedConversationComments.length; + + console.log(` Found ${totalIssueComments} conversation comment(s) from reviewer`); + if (alreadyAddressedIssue > 0) { + console.log(` ${alreadyAddressedIssue} already addressed (skipping)`); + } + console.log(` ${processedConversationComments.length} remaining to address`); - // Build feedback object - const feedback: ProcessedReviewFeedback = { - prNumber, - prTitle: pr.title, - repository: `${owner}/${repo}`, - branch: pr.head.ref, - reviewer: review.reviewer, - reviewState: "changes_requested", - reviewBody: review.body, - comments: processedComments, - conversationComments: - processedConversationComments.length > 0 ? processedConversationComments : undefined, - }; + // If no comments remaining (neither review nor conversation), we're done + if (processedComments.length === 0 && processedConversationComments.length === 0) { + console.log("\nāœ… All review and conversation comments have been addressed already."); + console.log(` View PR: ${prUrl}`); + return; + } - // Run record: begun only once there is actual feedback to handle, so - // no-op invocations (nothing unaddressed) do not create run rows. - beginRun({ - origin: "pr_mention", - repo: `${owner}/${repo}`, - prNumber, - branch: pr.head.ref, - }); - recordRunStage("change_request", { - status: "succeeded", - summary: `addressing ${processedComments.length} review comment(s) and ${processedConversationComments.length} conversation comment(s) from @${review.reviewer}`, - detail: { + // Build feedback object + const feedback: ProcessedReviewFeedback = { + prNumber, + prTitle: pr.title, + repository: `${owner}/${repo}`, + branch: pr.head.ref, reviewer: review.reviewer, - reviewComments: processedComments.length, - conversationComments: processedConversationComments.length, - }, - }); + reviewState: "changes_requested", + reviewBody: review.body, + comments: processedComments, + conversationComments: + processedConversationComments.length > 0 ? processedConversationComments : undefined, + }; + + // Run record: begun only once there is actual feedback to handle, so + // no-op invocations (nothing unaddressed) do not create run rows. + beginRun({ + origin: "pr_mention", + repo: `${owner}/${repo}`, + prNumber, + branch: pr.head.ref, + }); + recordRunStage("change_request", { + status: "succeeded", + summary: `addressing ${processedComments.length} review comment(s) and ${processedConversationComments.length} conversation comment(s) from @${review.reviewer}`, + detail: { + reviewer: review.reviewer, + reviewComments: processedComments.length, + conversationComments: processedConversationComments.length, + }, + }); + + // Format prompt for Agent + prompt = formatReviewPrompt(feedback); + commitSummary = `Address review feedback from ${review.reviewer}`; + } // Prepare the review worktree console.log(`\n🌿 Preparing review worktree for branch: ${pr.head.ref}`); @@ -505,9 +576,6 @@ export async function addressReview( const workDir = worktreeResult.path!; console.log(`āœ… Worktree ready at: ${workDir}`); - // Format prompt for Agent - const prompt = formatReviewPrompt(feedback); - // Set git config for bot author if available (so Agent's commits are attributed to bot) let originalGitName: string | null = null; let originalGitEmail: string | null = null; @@ -577,6 +645,8 @@ export async function addressReview( if (!hasUncommitted && !hasUnpushed) { console.log("\nāš ļø No changes were made by @devintern/code"); console.log(` View PR: ${prUrl}`); + // Close the run record so it does not dangle as in_progress forever. + endRun("succeeded", "agent made no changes"); return; } @@ -633,11 +703,11 @@ export async function addressReview( while (commitAttempt <= hookRetries && !commitSuccess) { commitAttempt++; - const commitResult = await Utils.commitChanges( - `PR-${prNumber}`, - `Address review feedback from ${feedback.reviewer}`, - { verbose, author: gitAuthor, cwd: workDir }, - ); + const commitResult = await Utils.commitChanges(`PR-${prNumber}`, commitSummary, { + verbose, + author: gitAuthor, + cwd: workDir, + }); if (commitResult.success) { console.log("āœ… Changes committed successfully"); diff --git a/packages/code/src/lib/ci-failure-watcher-acquirer.ts b/packages/code/src/lib/ci-failure-watcher-acquirer.ts new file mode 100644 index 0000000..c45b101 --- /dev/null +++ b/packages/code/src/lib/ci-failure-watcher-acquirer.ts @@ -0,0 +1,580 @@ +/** + * CI failure watcher acquirer (worker Mode 1, Tier 1): watch GitHub Actions + * check runs and commit statuses on the agent's own PRs and auto-fix + * failures, closing the loop from "agent opened PR" to "PR green". + * + * Each tick, for every open PR in the `agent_prs` registry: + * 1. Conditional GET on the PR itself — closed/merged PRs leave the watch + * list; fork PRs are skipped gracefully (Actions rarely runs there). + * 2. ETag-cached conditional GETs on the head SHA's check runs and combined + * commit status. Only terminal `failure` conclusions are actionable — + * the agent's own pushes constantly re-run CI as `in_progress`. + * 3. When a new failure appears, fetch the failing jobs' logs (with a + * check-run annotation fallback), truncate them to the error-relevant + * tail, and run `devintern address-review --ci-feedback ` as a CLI + * subprocess — reusing the whole review pipeline (worktree prep, + * sandboxed agent spawn, commit/push with hook retries). + * + * Guardrails: + * - Failures dedupe per `headSha + checkRunId` via `processed_events`, so a + * failure triggers at most one fix attempt even across worker restarts. + * - Consecutive failed autofix attempts per PR are capped (`CI_FIX_MAX_ + * ATTEMPTS`, default 3). On exhaustion the watcher posts an escalation + * comment and stops retrying until the head moves again (a human push) — + * or CI passes, which resets the budget outright. + */ + +import { mkdtempSync, rmSync, writeFileSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import { spawn } from "child_process"; + +import type { WebhookQueue } from "./webhook-queue"; +import type { WorkerState } from "./worker-state"; +import type { Acquirer } from "../worker"; +import type { CiFailureFeedback } from "./review-formatter"; + +export interface PolledCiPr { + state: string; + head?: { + sha: string; + /** Head repository; differs from the base repo for fork PRs. */ + repo?: { full_name: string } | null; + }; +} + +export interface WatchedCheckRun { + id: number; + name: string; + /** `queued`, `in_progress`, or `completed`. */ + status: string; + /** Terminal outcome; null while still executing. */ + conclusion: string | null; + details_url?: string; +} + +export interface WatchedStatusState { + state: string; + statuses: Array<{ id: number; state: string; context?: string; target_url?: string | null }>; +} + +/** GitHub access used by the watcher (injected for tests). */ +export interface CiFailureWatcherGitHub { + fetchPr(repo: string, prNumber: number, etag?: string): Promise>; + fetchCheckRuns( + repo: string, + sha: string, + etag?: string, + ): Promise>; + fetchCommitStatus( + repo: string, + sha: string, + etag?: string, + ): Promise>; + /** + * Fetch raw log text of the failing Actions jobs for a SHA (workflow runs + * → jobs → `logs_url`). Returns null on 403/404/scope problems. + */ + fetchFailingJobLogs(repo: string, sha: string): Promise; + /** + * Fallback details for one check run when job logs are unavailable: + * its annotations rendered as text. Returns null when unsupported. + */ + fetchCheckRunDetails?(repo: string, checkRunId: number): Promise; + /** Best-effort escalation comment on the PR conversation. */ + postComment(repo: string, prNumber: number, body: string): Promise; +} + +export interface CiConditionalResult { + data: T | null; + etag?: string; + notModified: boolean; +} + +export interface CiFailureWatcherAcquirerOptions { + intervalSeconds: number; + workerState: WorkerState; + queue: WebhookQueue; + github: CiFailureWatcherGitHub; + /** + * Fix CI failures on one PR given a feedback JSON path (injected for + * tests). Resolves success when the fix was committed and pushed. + */ + fixPr: (repo: string, prNumber: number, feedbackPath: string) => Promise; + /** Max consecutive failed autofix attempts per PR (default 3). */ + maxAttempts?: number; + verbose?: boolean; +} + +/** Dedupe source for CI failures (keyed by head SHA + check run/status id). */ +const SOURCE = "github:ci"; + +/** Cursor source prefixes persisted per watched PR. */ +const PR_CURSOR_PREFIX = "github:cipr:"; +const CHECKS_CURSOR_PREFIX = "github:cichecks:"; +const STATUS_CURSOR_PREFIX = "github:cistatus:"; + +/** Default consecutive-attempt cap per PR (`CI_FIX_MAX_ATTEMPTS` override). */ +export const DEFAULT_CI_FIX_MAX_ATTEMPTS = 3; + +/** Log excerpt limits: raw CI logs can be megabytes. */ +const LOG_MAX_LINES = 200; +const LOG_CONTEXT_LINES = 5; +const LOG_MAX_CHARS = 16_000; + +/** ANSI escape sequences (colors/cursor control) polluting CI logs. */ +const ANSI_PATTERN = /\x1b\[[0-9;?]*[A-Za-z]|\x1b\][0-9;]*[A-Za-z]/g; + +/** + * Strip ANSI codes and reduce raw CI logs to the failure-relevant tail. + * + * Error lines are located across the WHOLE log (an error at the top of a + * multi-thousand-line build log is still the root cause) and kept with + * surrounding context, together with an always-included tail window. The + * result is capped at {@value LOG_MAX_CHARS} chars so the agent's context + * cannot be blown by a multi-megabyte build log. + * + * @param raw - Raw job-log text (may be empty/null) + * @param options - Line/char limits (overridable for tests) + * @returns Truncated excerpt, or `null` when there is nothing to show + */ +export function truncateCiLogs( + raw: string | null | undefined, + options: { + maxLines?: number; + contextLines?: number; + maxChars?: number; + } = {}, +): string | null { + if (!raw || !raw.trim()) { + return null; + } + + const maxLines = options.maxLines ?? LOG_MAX_LINES; + const contextLines = options.contextLines ?? LOG_CONTEXT_LINES; + const maxChars = options.maxChars ?? LOG_MAX_CHARS; + + const clean = raw.replace(ANSI_PATTERN, "").replace(/\r\n/g, "\n"); + const lines = clean.split("\n"); + + const errorPattern = + /(##\[error\])|(\berror\b)|(\bfail(ed|ure|ing)?\b)|(exception)|(assertion)|(āœ—)|(✘)|(exit code [1-9])/i; + const keep = new Set(); + const tailStart = Math.max(0, lines.length - Math.min(maxLines, 60)); + for (let i = tailStart; i < lines.length; i++) { + keep.add(i); + } + for (let i = 0; i < lines.length; i++) { + if (!errorPattern.test(lines[i])) { + continue; + } + for ( + let j = Math.max(0, i - contextLines); + j <= Math.min(lines.length - 1, i + contextLines); + j++ + ) { + keep.add(j); + } + } + + const keptIndices = [...keep].sort((a, b) => a - b); + const parts: string[] = []; + let previous = -2; + for (const index of keptIndices) { + if (index !== previous + 1 && parts.length > 0) { + parts.push("..."); + } + parts.push(lines[index]); + previous = index; + } + let excerpt = parts.join("\n"); + + if (excerpt.length > maxChars) { + excerpt = `...\n${excerpt.slice(-maxChars)}`; + } + return excerpt.trim() ? excerpt : null; +} + +/** + * Run `devintern address-review --ci-feedback ` as a CLI + * subprocess, reusing the manual flow (worktree, sandboxed agent, commit, + * push). The feedback JSON carries failing-check metadata and the truncated + * log excerpt. + * + * @param repo - `owner/repo` slug + * @param prNumber - Pull request number + * @param feedbackPath - Path to the CI feedback JSON file + * @param opts - Working directory and environment for the subprocess; + * workspace mode runs from the repo's base worktree with + * per-repo env, single-repo mode inherits both + */ +export function runCiFixViaCli( + repo: string, + prNumber: number, + feedbackPath: string, + opts: { cwd?: string; env?: Record } = {}, +): Promise { + const prUrl = `https://github.com/${repo}/pull/${prNumber}`; + return new Promise((resolve) => { + const child = spawn( + process.execPath, + [process.argv[1], "address-review", prUrl, "--ci-feedback", feedbackPath], + { + stdio: "inherit", + cwd: opts.cwd, + env: opts.env ?? process.env, + }, + ); + child.on("close", (code) => resolve(code === 0)); + child.on("error", (error) => { + console.error(`āŒ Failed to spawn ci-fix for ${prUrl}: ${error.message}`); + resolve(false); + }); + }); +} + +interface PendingFailure { + externalId: string; + name: string; + conclusion: string | null; + detailsUrl?: string; +} + +/** + * Watches CI on the agent's own PRs and triggers autofix runs. + */ +export class CiFailureWatcherAcquirer implements Acquirer { + readonly name = "poll:ci-failures"; + private options: CiFailureWatcherAcquirerOptions; + private timer: ReturnType | null = null; + private busy = false; + + constructor(options: CiFailureWatcherAcquirerOptions) { + this.options = { + ...options, + maxAttempts: options.maxAttempts ?? parseMaxAttemptsFromEnv(), + }; + } + + /** Start watching: immediate first tick, then on the configured interval. */ + async start(): Promise { + console.log( + `šŸ¤– Watching CI on agent PRs every ${this.options.intervalSeconds}s ` + + `(watching ${this.options.workerState.listOpenAgentPrs().length} open PR(s))`, + ); + await this.tick(); + this.timer = setInterval(() => void this.tick(), this.options.intervalSeconds * 1000); + } + + /** Stop watching (an in-flight tick finishes its current PR). */ + stop(): void { + if (this.timer) { + clearInterval(this.timer); + this.timer = null; + } + } + + /** One polling cycle over all watched PRs. Skipped while busy. */ + async tick(): Promise { + if (this.busy) { + return; + } + this.busy = true; + + try { + for (const pr of this.options.workerState.listOpenAgentPrs()) { + try { + await this.pollPr(pr.repo, pr.prNumber); + } catch (error) { + console.warn( + `āš ļø [${this.name}] polling ${pr.repo}#${pr.prNumber} failed: ${(error as Error).message}`, + ); + } + } + } finally { + this.busy = false; + } + } + + /** Poll a single PR; triggers at most one fix attempt. */ + private async pollPr(repo: string, prNumber: number): Promise { + const { workerState, github, verbose } = this.options; + + // 1. PR state (ETag-cached): unwatch closed/merged PRs, track head SHA. + const prSource = `${PR_CURSOR_PREFIX}${repo}#${prNumber}`; + const prCursor = workerState.getCursor(prSource); + const prResult = await github.fetchPr(repo, prNumber, prCursor?.etag); + if (!prResult.notModified) { + if (prResult.etag) { + workerState.setCursor(prSource, prResult.data?.head?.sha ?? "", prResult.etag); + } + if (prResult.data && prResult.data.state !== "open") { + console.log(`šŸ‘ļø [${this.name}] ${repo}#${prNumber} is ${prResult.data.state}; unwatching`); + workerState.markAgentPrClosed(repo, prNumber); + return; + } + } + + const headSha = prResult.notModified ? prCursor?.cursorValue : prResult.data?.head?.sha; + if (!headSha) { + return; + } + + // Fork PRs: check runs live on the head repo and Actions usually does not + // run there. Skip quietly instead of burning requests or commenting. + const headRepo = prResult.data?.head?.repo?.full_name; + if (headRepo && headRepo.toLowerCase() !== repo.toLowerCase()) { + if (verbose) { + console.log(` [${this.name}] ${repo}#${prNumber} is a fork PR (${headRepo}); skipping`); + } + return; + } + + const pending: PendingFailure[] = []; + let sawSuccess = false; + + // 2. Check runs (ETag-cached): terminal failures only. + const checksSource = `${CHECKS_CURSOR_PREFIX}${repo}#${prNumber}`; + const checksCursor = workerState.getCursor(checksSource); + const checksResult = await github.fetchCheckRuns(repo, headSha, checksCursor?.etag); + if (!checksResult.notModified && checksResult.data) { + if (checksResult.etag) { + workerState.setCursor(checksSource, headSha, checksResult.etag); + } + for (const check of checksResult.data) { + if (check.status !== "completed") { + continue; // agent pushes restart CI constantly; never act mid-run + } + if (check.conclusion === "success") { + sawSuccess = true; + continue; + } + if (check.conclusion !== "failure") { + continue; + } + pending.push({ + externalId: `check:${repo}#${prNumber}:${headSha}:${check.id}`, + name: check.name, + conclusion: check.conclusion, + detailsUrl: check.details_url, + }); + } + } + + // 3. Combined commit status (ETag-cached): non-Actions reporters. + const statusSource = `${STATUS_CURSOR_PREFIX}${repo}#${prNumber}`; + const statusCursor = workerState.getCursor(statusSource); + const statusResult = await github.fetchCommitStatus(repo, headSha, statusCursor?.etag); + if (!statusResult.notModified && statusResult.data) { + if (statusResult.etag) { + workerState.setCursor(statusSource, headSha, statusResult.etag); + } + if (statusResult.data.state === "success") { + sawSuccess = true; + } + for (const status of statusResult.data.statuses) { + if (status.state !== "failure" && status.state !== "error") { + continue; + } + pending.push({ + externalId: `check:${repo}#${prNumber}:${headSha}:status:${status.context ?? status.id}`, + name: status.context ?? `commit-status-${status.id}`, + conclusion: status.state, + detailsUrl: status.target_url ?? undefined, + }); + } + } + + // Fully green observation: zero the attempt counter. A mixed result + // (some checks pass, others fail) must NOT keep refunding the budget. + if (sawSuccess && pending.length === 0) { + this.resetRetryBudget(repo, prNumber, "CI passed"); + } + + // 4. Split failures into fresh vs already-handled, and mark fresh ones + // processed so a failure never triggers more than one fix attempt. + const fresh: PendingFailure[] = []; + for (const failure of pending) { + if (!this.options.queue.hasProcessed(SOURCE, failure.externalId)) { + fresh.push(failure); + this.options.queue.markProcessed(SOURCE, failure.externalId); + } + } + if (pending.length === 0) { + return; + } + + // 5. Retry cap & escalation bookkeeping — evaluated on every observation + // (fresh or not), so exhaustion escalates even when the failing run was + // already handled but the fix attempt produced no new CI run. + const state = workerState.getCiFixState(repo, prNumber); + if ( + state.escalatedSha && + state.escalatedSha !== headSha && + state.consecutiveFailures >= this.maxAttempts + ) { + // Head moved past the escalation point: someone (presumably a human) + // pushed. Grant a fresh budget. + console.log( + `ā™»ļø [${this.name}] ${repo}#${prNumber}: head moved past escalation point; retrying CI fixes`, + ); + state.consecutiveFailures = 0; + state.escalatedSha = undefined; + workerState.setCiFixState(repo, prNumber, state); + } + + if (state.consecutiveFailures >= this.maxAttempts) { + if (!state.escalatedSha) { + state.escalatedSha = headSha; + workerState.setCiFixState(repo, prNumber, state); + await this.escalateToHuman(repo, prNumber, pending); + } else if (verbose) { + console.log( + ` [${this.name}] ${repo}#${prNumber}: retry budget exhausted; waiting for human`, + ); + } + return; + } + + if (fresh.length === 0) { + return; + } + + console.log( + `\nšŸ¤– [${this.name}] CI failure(s) on ${repo}#${prNumber} @ ${headSha.slice(0, 7)}: ` + + fresh.map((f) => f.name).join(", "), + ); + + // 6. Gather failure-relevant logs and run one fix attempt. + const logs = await this.collectLogs(repo, headSha, fresh); + const feedback: CiFailureFeedback = { + repository: repo, + prNumber, + branch: undefined, + failures: fresh.map((f) => ({ + name: f.name, + conclusion: f.conclusion, + detailsUrl: f.detailsUrl, + })), + logs, + }; + + const feedbackDir = mkdtempSync(join(tmpdir(), "devintern-ci-fix-")); + const feedbackPath = join(feedbackDir, "ci-feedback.json"); + writeFileSync(feedbackPath, JSON.stringify(feedback)); + + try { + const ok = await this.options.fixPr(repo, prNumber, feedbackPath); + state.consecutiveFailures += 1; + workerState.setCiFixState(repo, prNumber, state); + console.log( + ok + ? `āœ… [${this.name}] ${repo}#${prNumber} CI fix pushed (attempt ` + + `${state.consecutiveFailures}/${this.maxAttempts})` + : `āš ļø [${this.name}] ${repo}#${prNumber} CI fix attempt did not complete`, + ); + } finally { + rmSync(feedbackDir, { recursive: true, force: true }); + } + } + + /** Job logs first; fall back to check-run annotations per failing check. */ + private async collectLogs( + repo: string, + headSha: string, + failures: PendingFailure[], + ): Promise { + const { github, verbose } = this.options; + + let rawLogs: string | null = null; + try { + rawLogs = await github.fetchFailingJobLogs(repo, headSha); + } catch (error) { + if (verbose) { + console.warn(` āš ļø [${this.name}] job log fetch failed: ${(error as Error).message}`); + } + } + + const excerpt = truncateCiLogs(rawLogs); + if (excerpt) { + return excerpt; + } + + // Annotations fallback (also covers non-Actions statuses via details URL). + const snippets: string[] = []; + for (const failure of failures) { + const match = + failure.detailsUrl?.match(/checks\/(\d+)/) ?? + failure.detailsUrl?.match(/check-runs\/(\d+)/); + const checkRunId = match ? parseInt(match[1], 10) : NaN; + if (!github.fetchCheckRunDetails || Number.isNaN(checkRunId)) { + continue; + } + try { + const detail = await github.fetchCheckRunDetails(repo, checkRunId); + if (detail) { + snippets.push(detail); + } + } catch { + // Details are best-effort only. + } + } + const annotationExcerpt = truncateCiLogs(snippets.join("\n") || null); + if (annotationExcerpt) { + return annotationExcerpt; + } + + console.warn( + `āš ļø [${this.name}] ${repo}: could not fetch logs (scope/fork?); proceeding without them`, + ); + return null; + } + + /** Zero the attempt counter because CI went green. */ + private resetRetryBudget(repo: string, prNumber: number, reason: string): void { + const state = this.options.workerState.getCiFixState(repo, prNumber); + if (state.consecutiveFailures > 0 || state.escalatedSha) { + console.log(`šŸ’š [${this.name}] ${repo}#${prNumber}: ${reason}; resetting CI fix counter`); + this.options.workerState.setCiFixState(repo, prNumber, { + consecutiveFailures: 0, + escalatedSha: undefined, + }); + } + } + + /** Post the give-up comment and freeze further attempts until head moves. */ + private async escalateToHuman( + repo: string, + prNumber: number, + failures: PendingFailure[], + ): Promise { + const names = failures.map((f) => `- ${f.name}`).join("\n"); + const body = + "āš ļø I could not fix the following CI failure(s) automatically after " + + `${this.maxAttempts} attempt(s):\n\n${names}\n\n` + + "I have stopped retrying to avoid churn. Push a new commit (or mention me) " + + "and I will take another look."; + try { + await this.options.github.postComment(repo, prNumber, body); + console.log( + `šŸ“£ [${this.name}] ${repo}#${prNumber}: posted escalation comment after ` + + `${this.maxAttempts} failed attempt(s)`, + ); + } catch (error) { + console.warn( + `āš ļø [${this.name}] could not post escalation comment on ${repo}#${prNumber}: ` + + `${(error as Error).message}`, + ); + } + } + + private get maxAttempts(): number { + return this.options.maxAttempts ?? DEFAULT_CI_FIX_MAX_ATTEMPTS; + } +} + +/** Read `CI_FIX_MAX_ATTEMPTS` (default {@link DEFAULT_CI_FIX_MAX_ATTEMPTS}). */ +function parseMaxAttemptsFromEnv(): number { + const parsed = parseInt(process.env.CI_FIX_MAX_ATTEMPTS || "", 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_CI_FIX_MAX_ATTEMPTS; +} diff --git a/packages/code/src/lib/dashboard-api.ts b/packages/code/src/lib/dashboard-api.ts index 627f937..8073c76 100644 --- a/packages/code/src/lib/dashboard-api.ts +++ b/packages/code/src/lib/dashboard-api.ts @@ -26,7 +26,7 @@ const RUN_STATUSES: RunStatus[] = [ "escalated", "abandoned", ]; -const RUN_ORIGINS: RunOrigin[] = ["task", "pr_mention"]; +const RUN_ORIGINS: RunOrigin[] = ["task", "pr_mention", "ci_fix"]; const STATS_WINDOWS: Record = { "7d": 7 * 24 * 60 * 60 * 1000, diff --git a/packages/code/src/lib/github-reviews.ts b/packages/code/src/lib/github-reviews.ts index 5bbd9d7..c471c93 100644 --- a/packages/code/src/lib/github-reviews.ts +++ b/packages/code/src/lib/github-reviews.ts @@ -45,6 +45,65 @@ export interface FileContent { sha: string; } +/** Actions check run on a commit (subset of the API shape the watcher needs). */ +export interface CheckRunSummary { + id: number; + name: string; + /** `queued`, `in_progress`, or `completed`. */ + status: string; + /** Terminal outcome; null while the run is still executing. */ + conclusion: string | null; + html_url?: string; + details_url?: string; + /** The workflow run this check belongs to (for job/log lookup). */ + check_suite?: { id: number }; +} + +/** Classic commit status (non-Actions CI reporter). */ +export interface CombinedStatus { + state: "error" | "failure" | "pending" | "success"; + total_count: number; + statuses: Array<{ + id: number; + state: string; + context?: string; + target_url?: string | null; + description?: string | null; + }>; +} + +/** One job of an Actions workflow run. */ +export interface ActionJob { + id: number; + run_id?: number; + name: string; + status: string; + conclusion: string | null; + /** Download URL for the job's log archive. */ + logs_url?: string; + steps?: Array<{ + name: string; + number: number; + conclusion: string | null; + }>; +} + +/** Actions workflow run associated with a commit SHA (subset). */ +export interface WorkflowRunSummary { + id: number; + name?: string; + status?: string; + conclusion: string | null; +} + +/** Annotation attached to a check run (log-download fallback). */ +export interface CheckAnnotation { + path?: string; + start_line?: number; + message: string; + annotation_level?: string; +} + /** * Client for interacting with GitHub's PR review APIs. */ @@ -685,6 +744,157 @@ export class GitHubReviewsClient { } } + /** + * Fetch the Actions check runs for a commit SHA (ETag-cached). + * + * A `304 Not Modified` response does not count against the rate limit, + * which makes per-tick polling of many watched PRs cheap. + * + * @param owner - Repository owner + * @param repo - Repository name + * @param sha - Commit SHA (usually the PR head) + * @param etag - ETag from the previous response, if any + */ + async getCheckRuns( + owner: string, + repo: string, + sha: string, + etag?: string, + ): Promise<{ + data: CheckRunSummary[] | null; + etag?: string; + notModified: boolean; + }> { + const result = await this.conditionalGet<{ + total_count: number; + check_runs: CheckRunSummary[]; + }>(`/repos/${owner}/${repo}/commits/${sha}/check-runs?per_page=100`, owner, repo, etag); + return { + data: result.data ? result.data.check_runs : null, + etag: result.etag, + notModified: result.notModified, + }; + } + + /** + * Fetch the combined commit status for a SHA (ETag-cached). + * + * Covers classic commit statuses (non-Actions CI reporters). + * + * @param owner - Repository owner + * @param repo - Repository name + * @param sha - Commit SHA (usually the PR head) + * @param etag - ETag from the previous response, if any + */ + async getCombinedStatus( + owner: string, + repo: string, + sha: string, + etag?: string, + ): Promise<{ data: CombinedStatus | null; etag?: string; notModified: boolean }> { + return this.conditionalGet( + `/repos/${owner}/${repo}/commits/${sha}/status`, + owner, + repo, + etag, + ); + } + + /** + * Fetch the Actions jobs of a workflow run (to locate failing jobs). + * + * @param owner - Repository owner + * @param repo - Repository name + * @param runId - Workflow run id + */ + async getWorkflowRunJobs(owner: string, repo: string, runId: number): Promise { + const data = await this.apiRequest<{ jobs: ActionJob[] }>( + "GET", + `/repos/${owner}/${repo}/actions/runs/${runId}/jobs?per_page=100`, + owner, + repo, + ); + return data.jobs; + } + + /** + * List Actions workflow runs for a commit SHA. + * + * @param owner - Repository owner + * @param repo - Repository name + * @param sha - Commit SHA (usually the PR head) + */ + async getWorkflowRunsForSha( + owner: string, + repo: string, + sha: string, + ): Promise { + const data = await this.apiRequest<{ workflow_runs: WorkflowRunSummary[] }>( + "GET", + `/repos/${owner}/${repo}/actions/runs?head_sha=${sha}&per_page=20`, + owner, + repo, + ); + return data.workflow_runs; + } + + /** + * Download the log archive text of one Actions job. + * + * The job's `logs_url` answers with a redirect to log storage; plain + * authenticated GET follows it and yields plain text. + * + * @param owner - Repository owner + * @param repo - Repository name + * @param logsUrl - Absolute `logs_url` of the job + * @returns Log text, or `null` on 403/404 (insufficient scope or expired) + */ + async getJobLogs(owner: string, repo: string, logsUrl: string): Promise { + let token: string | null; + try { + token = await this.getToken(owner, repo); + } catch { + return null; + } + const response = await Utils.fetchWithRetry(logsUrl, { + method: "GET", + headers: { + Authorization: `Bearer ${token}`, + Accept: "application/vnd.github.v3+json", + "User-Agent": "devintern", + }, + // Logs live behind a redirect to storage; follow it. + redirect: "follow", + }); + if (!response.ok) { + return null; + } + return response.text(); + } + + /** + * Fetch annotations of one check run. + * + * Used as a fallback when job-log download fails (403/404): annotations + * carry the lint/test failure messages GitHub surfaced on the check. + * + * @param owner - Repository owner + * @param repo - Repository name + * @param checkRunId - Check run id + */ + async getCheckRunAnnotations( + owner: string, + repo: string, + checkRunId: number, + ): Promise { + return this.apiRequest( + "GET", + `/repos/${owner}/${repo}/check-runs/${checkRunId}/annotations`, + owner, + repo, + ); + } + /** * Map raw GitHub review comments to the internal processed shape. * diff --git a/packages/code/src/lib/review-formatter.ts b/packages/code/src/lib/review-formatter.ts index 55b37b8..be4be1b 100644 --- a/packages/code/src/lib/review-formatter.ts +++ b/packages/code/src/lib/review-formatter.ts @@ -314,3 +314,88 @@ export function formatSingleCommentPrompt( return lines.join("\n"); } + +/** Structured CI-failure feedback handed to {@link formatCiFixPrompt}. */ +export interface CiFailureFeedback { + repository: string; + prNumber: number; + prTitle?: string; + branch?: string; + /** The failing checks/statuses observed on the PR's current head SHA. */ + failures: Array<{ + /** Check run name or commit-status context. */ + name: string; + conclusion: string | null; + detailsUrl?: string; + }>; + /** + * Failure-relevant log excerpt (already truncated by the watcher), or + * `null` when logs were unavailable (forks, scope, expiry). + */ + logs: string | null; +} + +/** + * Format CI failure feedback into an agent prompt. + * + * @param feedback - Failing check metadata and truncated log excerpt + * @returns Markdown prompt for the agent harness + */ +export function formatCiFixPrompt(feedback: CiFailureFeedback): string { + const lines: string[] = []; + + lines.push("# CI Failure - Fix Failing Checks"); + lines.push(""); + lines.push("## PR Information"); + lines.push(""); + lines.push(`- **Repository:** ${feedback.repository}`); + lines.push(`- **PR #${feedback.prNumber}:** ${feedback.prTitle ?? ""}`.trimEnd()); + if (feedback.branch) { + lines.push(`- **Branch:** \`${feedback.branch}\``); + } + lines.push(""); + lines.push("## Failing Checks"); + lines.push(""); + for (const failure of feedback.failures) { + const conclusion = failure.conclusion ?? "unknown"; + const detail = failure.detailsUrl ? ` (${failure.detailsUrl})` : ""; + lines.push(`- **${failure.name}** — \`${conclusion}\`${detail}`); + } + lines.push(""); + + if (feedback.logs && feedback.logs.trim()) { + lines.push("## Failure Logs (excerpt)"); + lines.push(""); + lines.push("```"); + lines.push(feedback.logs.trimEnd()); + lines.push("```"); + lines.push(""); + } else { + lines.push( + "## Failure Logs", + "", + "> Job logs were not available. Reproduce the failures locally by running", + "> the project's test/lint/build pipeline for the affected checks.", + "", + ); + } + + lines.push("## Instructions"); + lines.push(""); + lines.push("Please fix the code so every failing check above passes, without regressing"); + lines.push("the rest of the suite."); + lines.push(""); + lines.push("**Guidelines:**"); + lines.push("1. Identify the root cause from the log excerpt before editing"); + lines.push("2. Make minimal, focused changes that fix the failure"); + lines.push("3. If a failure looks like flaky infrastructure rather than a code issue,"); + lines.push(" say so explicitly in your final summary instead of masking it"); + lines.push("4. Run the relevant tests to verify your fix"); + lines.push(""); + lines.push("**IMPORTANT:**"); + lines.push("- After making your changes, commit them with a descriptive message"); + lines.push("- Your commit message should summarize how the failure was fixed"); + lines.push("- Do NOT push to the remote - that will be done automatically"); + + return lines.join("\n"); +} diff --git a/packages/code/src/lib/run-recorder.ts b/packages/code/src/lib/run-recorder.ts index fabb0e3..2aaaa48 100644 --- a/packages/code/src/lib/run-recorder.ts +++ b/packages/code/src/lib/run-recorder.ts @@ -15,7 +15,7 @@ import { Database } from "bun:sqlite"; import { prepareQueueDbDirectory, resolveQueueDbPath } from "./webhook-queue"; -export type RunOrigin = "task" | "pr_mention"; +export type RunOrigin = "task" | "pr_mention" | "ci_fix"; export type RunStatus = | "in_progress" @@ -408,7 +408,7 @@ export class RunStore { escalated: 0, abandoned: 0, }; - const byOrigin: Record = { task: 0, pr_mention: 0 }; + const byOrigin: Record = { task: 0, pr_mention: 0, ci_fix: 0 }; const weekCounts = new Map(); const harnesses = new Map< string, diff --git a/packages/code/src/lib/worker-state.ts b/packages/code/src/lib/worker-state.ts index 2795252..d6b59bc 100644 --- a/packages/code/src/lib/worker-state.ts +++ b/packages/code/src/lib/worker-state.ts @@ -33,6 +33,17 @@ export interface AgentPr { updatedAt: number; } +/** Consecutive-CI-autofix bookkeeping for one agent PR. */ +export interface CiFixState { + /** Fix attempts made since the last CI pass / escalation reset. */ + consecutiveFailures: number; + /** + * Head SHA at which retry budget was exhausted and a human was pinged. + * Any head move past this SHA clears the block. + */ + escalatedSha?: string; +} + /** * Parse an `owner/repo` slug and PR number from a GitHub PR URL. * @@ -104,6 +115,17 @@ export class WorkerState { CREATE INDEX IF NOT EXISTS idx_agent_prs_state ON agent_prs(state) `); + + this.db.run(` + CREATE TABLE IF NOT EXISTS ci_fix_state ( + repo TEXT NOT NULL, + pr_number INTEGER NOT NULL, + consecutive_failures INTEGER NOT NULL DEFAULT 0, + escalated_sha TEXT, + updated_at INTEGER NOT NULL, + PRIMARY KEY (repo, pr_number) + ) + `); } /** @@ -241,6 +263,47 @@ export class WorkerState { ); } + /** + * Read the CI-autofix retry bookkeeping for an agent PR. + * + * @param repo - Repo slug + * @param prNumber - PR number + */ + getCiFixState(repo: string, prNumber: number): CiFixState { + const row = this.db + .query( + `SELECT consecutive_failures, escalated_sha FROM ci_fix_state + WHERE repo = ? AND pr_number = ?`, + ) + .get(repo, prNumber) as Record | null; + if (!row) { + return { consecutiveFailures: 0 }; + } + return { + consecutiveFailures: row.consecutive_failures as number, + escalatedSha: (row.escalated_sha as string | null) ?? undefined, + }; + } + + /** + * Store the CI-autofix retry bookkeeping for an agent PR (upsert). + * + * @param repo - Repo slug + * @param prNumber - PR number + * @param state - New attempt count and/or escalation marker + */ + setCiFixState(repo: string, prNumber: number, state: CiFixState): void { + this.db.run( + `INSERT INTO ci_fix_state (repo, pr_number, consecutive_failures, escalated_sha, updated_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(repo, pr_number) DO UPDATE SET + consecutive_failures = excluded.consecutive_failures, + escalated_sha = excluded.escalated_sha, + updated_at = excluded.updated_at`, + [repo, prNumber, state.consecutiveFailures, state.escalatedSha ?? null, Date.now()], + ); + } + /** Close the underlying SQLite connection. */ close(): void { this.db.close(); diff --git a/packages/code/src/lib/workspace/fleet-events.ts b/packages/code/src/lib/workspace/fleet-events.ts index 669d883..966032d 100644 --- a/packages/code/src/lib/workspace/fleet-events.ts +++ b/packages/code/src/lib/workspace/fleet-events.ts @@ -12,6 +12,7 @@ */ import { runAddressReviewViaCli } from "../review-polling-acquirer"; +import { runCiFixViaCli } from "../ci-failure-watcher-acquirer"; import type { RepoConfig, WorkspaceConfig } from "./config"; import { buildRepoEnv, gitHubSlugFromRemote } from "./env"; import { toRoutableTask } from "./router"; @@ -29,6 +30,13 @@ export interface FleetEventDeps { prNumber: number, opts: { cwd: string; env: Record }, ) => Promise; + /** CI-fix runner (injected for tests; defaults to the CLI subprocess). */ + runCiFix?: ( + repo: string, + prNumber: number, + feedbackPath: string, + opts: { cwd: string; env: Record }, + ) => Promise; verbose?: boolean; } @@ -85,6 +93,36 @@ export function createFleetAddressPr( }; } +/** + * Build the fleet CI-fix runner: auto-fix failing checks on an agent PR from + * the repo's base worktree, mirroring {@link createFleetAddressPr}. + * + * @returns Handler resolving false when the slug maps to no workspace repo. + */ +export function createFleetCiFix( + deps: FleetEventDeps, +): (slug: string, prNumber: number, feedbackPath: string) => Promise { + const { config, workspaceDir, repoManager } = deps; + const runCiFix = deps.runCiFix ?? runCiFixViaCli; + + return async (slug, prNumber, feedbackPath) => { + const repo = repoBySlug(config, slug); + if (!repo) { + console.warn( + `āš ļø [fleet] CI failure for ${slug}#${prNumber} matches no workspace repo; skipping.`, + ); + return false; + } + await repoManager.ensureBareClone(repo); + await repoManager.fetch(repo.name); + const base = await repoManager.ensureBaseWorktree(repo); + return runCiFix(slug, prNumber, feedbackPath, { + cwd: base, + env: buildRepoEnv(repo, workspaceDir), + }); + }; +} + /** * Build the fleet mention handler: permission-gate the mentioning user, then * run the review pipeline for that PR. diff --git a/packages/code/src/lib/workspace/workspace-worker.ts b/packages/code/src/lib/workspace/workspace-worker.ts index 03d58cb..45b4cae 100644 --- a/packages/code/src/lib/workspace/workspace-worker.ts +++ b/packages/code/src/lib/workspace/workspace-worker.ts @@ -365,6 +365,7 @@ async function buildFleetEventAcquirers(options: { const { createFleetAddressPr, + createFleetCiFix, createFleetMentionHandler, createFleetTaskEvaluator, fleetGitHubSlugs, @@ -424,6 +425,68 @@ async function buildFleetEventAcquirers(options: { }), ); + // Tier 1: auto-fix CI failures on those same PRs. Fix runs execute as + // CLI subprocesses in each repo's base worktree with per-repo env. + const { CiFailureWatcherAcquirer } = await import("../ci-failure-watcher-acquirer"); + const fixPr = createFleetCiFix(eventDeps); + acquirers.push( + new CiFailureWatcherAcquirer({ + intervalSeconds, + workerState: state.workerState, + queue: state.queue, + github: { + fetchPr: (repo, n, etag) => + gh.conditionalGet(`/repos/${repo}/pulls/${n}`, ownerOf(repo), nameOf(repo), etag), + fetchCheckRuns: (repo, sha, etag) => + gh.getCheckRuns(ownerOf(repo), nameOf(repo), sha, etag), + fetchCommitStatus: (repo, sha, etag) => + gh.getCombinedStatus(ownerOf(repo), nameOf(repo), sha, etag), + fetchFailingJobLogs: async (repo, sha) => { + const [owner, name] = repo.split("/") as [string, string]; + const runs = await gh + .getWorkflowRunsForSha(owner, name, sha) + .catch(() => [] as Awaited>); + const chunks: string[] = []; + for (const run of runs.slice(0, 3)) { + const jobs = await gh.getWorkflowRunJobs(owner, name, run.id).catch(() => []); + for (const job of jobs.filter((j) => j.conclusion === "failure").slice(0, 5)) { + if (!job.logs_url) { + continue; + } + const text = await gh.getJobLogs(owner, name, job.logs_url).catch(() => null); + if (text) { + chunks.push(`## Job: ${job.name}\n${text}`); + } + } + } + return chunks.length > 0 ? chunks.join("\n\n") : null; + }, + fetchCheckRunDetails: async (repo, checkRunId) => { + try { + const annotations = await gh.getCheckRunAnnotations( + ownerOf(repo), + nameOf(repo), + checkRunId, + ); + if (annotations.length === 0) { + return null; + } + return annotations + .map((a) => `${a.path ? `${a.path}: ` : ""}${a.message}`) + .join("\n"); + } catch { + return null; + } + }, + postComment: async (repo, n, body) => { + await gh.postPullRequestComment(ownerOf(repo), nameOf(repo), n, body); + }, + }, + fixPr: (slug, prNumber, feedbackPath) => fixPr(slug, prNumber, feedbackPath), + verbose, + }), + ); + // Tier 2: one mention sweep per GitHub repo (cursor sources are already // namespaced by slug). The permission gate runs in the fleet handler. const { MentionSweepAcquirer } = await import("../mention-sweep-acquirer"); diff --git a/packages/code/tests/ci-failure-watcher-acquirer.test.ts b/packages/code/tests/ci-failure-watcher-acquirer.test.ts new file mode 100644 index 0000000..f3e10cd --- /dev/null +++ b/packages/code/tests/ci-failure-watcher-acquirer.test.ts @@ -0,0 +1,378 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { rmSync } from "fs"; +import { join } from "path"; +import { tmpdir } from "os"; + +import { CiFailureWatcherAcquirer, truncateCiLogs } from "../src/lib/ci-failure-watcher-acquirer"; +import type { + CiConditionalResult, + CiFailureWatcherGitHub, + PolledCiPr, + WatchedCheckRun, + WatchedStatusState, +} from "../src/lib/ci-failure-watcher-acquirer"; +import { WebhookQueue } from "../src/lib/webhook-queue"; +import { WorkerState } from "../src/lib/worker-state"; + +describe("truncateCiLogs", () => { + test("returns null for empty input", () => { + expect(truncateCiLogs(null)).toBeNull(); + expect(truncateCiLogs("")).toBeNull(); + expect(truncateCiLogs(" \n \n")).toBeNull(); + }); + + test("strips ANSI escape codes", () => { + const raw = "\x1b[31mError: build failed\x1b[0m\nplain line"; + const excerpt = truncateCiLogs(raw)!; + expect(excerpt).not.toContain(""); + expect(excerpt).toContain("Error: build failed"); + }); + + test("keeps lines around error markers with context", () => { + // 120 filler lines + one error line near the top + more filler. + const lines = Array.from({ length: 200 }, (_, i) => `pad-${String(i).padStart(4, "0")}`); + lines[10] = "ERROR: cannot find module"; + const raw = lines.join("\n"); + const excerpt = truncateCiLogs(raw, { contextLines: 2 })!; + expect(excerpt).toContain("ERROR: cannot find module"); + expect(excerpt).toContain("pad-0008"); // context before the error + expect(excerpt).toContain("pad-0012"); // context after the error + expect(excerpt).toContain("pad-0199"); // tail window always included + // Far-away noise outside error context and tail is dropped. + expect(excerpt).not.toContain("pad-0050"); + expect(excerpt).not.toContain("pad-0100"); + }); + + test("caps output at maxChars keeping the tail", () => { + const raw = Array.from({ length: 500 }, (_, i) => `filler ${i}`).join("\n"); + const excerpt = truncateCiLogs(raw, { maxChars: 200 })!; + expect(excerpt.length).toBeLessThanOrEqual(210); + expect(excerpt.startsWith("...")).toBe(true); + }); + + test("keeps only the last maxLines lines", () => { + const raw = Array.from({ length: 300 }, (_, i) => `line ${i}`).join("\n"); + const excerpt = truncateCiLogs(raw, { maxLines: 50 })!; + expect(excerpt).not.toContain("line 0\n"); + expect(excerpt).not.toContain("line 100"); + expect(excerpt).toContain("line 299"); + }); +}); + +describe("CiFailureWatcherAcquirer", () => { + let dbPath: string; + let workerState: WorkerState; + let queue: WebhookQueue; + + beforeEach(() => { + dbPath = join( + tmpdir(), + `ci-watch-test-${Date.now()}-${Math.random().toString(36).slice(2)}.db`, + ); + workerState = new WorkerState(dbPath); + queue = new WebhookQueue({ dbPath }); + }); + + afterEach(() => { + workerState.close(); + queue.close(); + for (const suffix of ["", "-wal", "-shm"]) { + rmSync(`${dbPath}${suffix}`, { force: true }); + } + }); + + interface FakeGitHubState { + prState: string; + headSha?: string; + headRepoFullName?: string; + prEtagHit?: boolean; + checkRuns: WatchedCheckRun[]; + checksEtagHit?: boolean; + status?: WatchedStatusState; + statusEtagHit?: boolean; + jobLogs?: string | null; + seenPrEtag?: string; + seenChecksEtag?: string; + seenStatusEtag?: string; + } + + function makeGithub(gh: FakeGitHubState): CiFailureWatcherGitHub { + return { + async fetchPr(_repo, _n, etag): Promise> { + gh.seenPrEtag = etag; + if (gh.prEtagHit) { + return { data: null, etag, notModified: true }; + } + return { + data: { + state: gh.prState, + head: gh.headSha + ? { + sha: gh.headSha, + repo: gh.headRepoFullName ? { full_name: gh.headRepoFullName } : null, + } + : undefined, + }, + etag: 'W/"pr-1"', + notModified: false, + }; + }, + async fetchCheckRuns(_repo, _sha, etag): Promise> { + gh.seenChecksEtag = etag; + if (gh.checksEtagHit) { + return { data: null, etag, notModified: true }; + } + return { data: gh.checkRuns, etag: 'W/"checks-1"', notModified: false }; + }, + async fetchCommitStatus(_repo, _sha, etag): Promise> { + gh.seenStatusEtag = etag; + if (!gh.status || gh.statusEtagHit) { + return { data: gh.status ?? null, etag, notModified: Boolean(gh.statusEtagHit) }; + } + return { data: gh.status, etag: 'W/"status-1"', notModified: false }; + }, + async fetchFailingJobLogs() { + return gh.jobLogs ?? null; + }, + async postComment() {}, + }; + } + + function makeAcquirer(gh: FakeGitHubState, overrides: { maxAttempts?: number } = {}) { + const fixed: string[] = []; + const comments: string[] = []; + const base = makeGithub(gh); + const github: CiFailureWatcherGitHub = { + ...base, + postComment: async (_repo, _n, body) => { + comments.push(body); + }, + }; + const options: ConstructorParameters[0] = { + intervalSeconds: 60, + workerState, + queue, + github, + fixPr: async (repo, n) => { + fixed.push(`${repo}#${n}`); + return true; + }, + verbose: false, + }; + if (overrides.maxAttempts !== undefined) { + options.maxAttempts = overrides.maxAttempts; + } + const acquirer = new CiFailureWatcherAcquirer(options); + return { acquirer, fixed, comments }; + } + + const sha1 = "aaaa1111aaaa1111aaaa1111aaaa1111aaaa1111"; + const sha2 = "bbbb2222bbbb2222bbbb2222bbbb2222bbbb2222"; + + function failingCheck(id: number): WatchedCheckRun { + return { id, name: `check-${id}`, status: "completed", conclusion: "failure" }; + } + + test("a failing completed check run triggers exactly one fix attempt", async () => { + workerState.recordAgentPr({ repo: "acme/widgets", prNumber: 42 }); + const gh: FakeGitHubState = { + prState: "open", + headSha: sha1, + checkRuns: [ + failingCheck(101), + { id: 102, name: "lint", status: "completed", conclusion: "success" }, + ], + }; + const { acquirer, fixed } = makeAcquirer(gh); + + await acquirer.tick(); + expect(fixed).toEqual(["acme/widgets#42"]); + + // Same failure next tick is deduped. + await acquirer.tick(); + expect(fixed).toEqual(["acme/widgets#42"]); + }); + + test("pending and non-failure conclusions never trigger", async () => { + workerState.recordAgentPr({ repo: "acme/widgets", prNumber: 42 }); + const gh: FakeGitHubState = { + prState: "open", + headSha: sha1, + checkRuns: [ + { id: 201, name: "build", status: "in_progress", conclusion: null }, + { id: 202, name: "build", status: "completed", conclusion: "success" }, + { id: 203, name: "skipped-job", status: "completed", conclusion: "skipped" }, + { id: 204, name: "cancelled-job", status: "completed", conclusion: "cancelled" }, + ], + }; + const { acquirer, fixed } = makeAcquirer(gh); + + await acquirer.tick(); + expect(fixed).toEqual([]); + }); + + test("dedupe survives worker restarts (new instance, shared queue)", async () => { + workerState.recordAgentPr({ repo: "acme/widgets", prNumber: 42 }); + const gh: FakeGitHubState = { + prState: "open", + headSha: sha1, + checkRuns: [failingCheck(301)], + }; + const first = makeAcquirer(gh); + await first.acquirer.tick(); + expect(first.fixed).toEqual(["acme/widgets#42"]); + + // A fresh acquirer over the same DB (restart) must not re-trigger. + const second = makeAcquirer(gh); + await second.acquirer.tick(); + expect(second.fixed).toEqual([]); + }); + + test("a failure on a NEW head SHA triggers again", async () => { + workerState.recordAgentPr({ repo: "acme/widgets", prNumber: 42 }); + const gh: FakeGitHubState = { + prState: "open", + headSha: sha1, + checkRuns: [failingCheck(401)], + }; + const { acquirer, fixed } = makeAcquirer(gh); + await acquirer.tick(); + + gh.headSha = sha2; + gh.checkRuns = [failingCheck(402)]; + await acquirer.tick(); + expect(fixed).toEqual(["acme/widgets#42", "acme/widgets#42"]); + }); + + test("commit status failures trigger and dedupe per context", async () => { + workerState.recordAgentPr({ repo: "acme/widgets", prNumber: 42 }); + const gh: FakeGitHubState = { + prState: "open", + headSha: sha1, + checkRuns: [], + status: { + state: "failure", + statuses: [ + { id: 1, state: "failure", context: "ci/travis" }, + { id: 2, state: "success", context: "ci/coverage" }, + ], + }, + }; + const { acquirer, fixed } = makeAcquirer(gh); + + await acquirer.tick(); + expect(fixed).toEqual(["acme/widgets#42"]); + + await acquirer.tick(); + expect(fixed).toEqual(["acme/widgets#42"]); + }); + + test("a closed PR is unwatched without triggering fixes", async () => { + workerState.recordAgentPr({ repo: "acme/widgets", prNumber: 42 }); + const gh: FakeGitHubState = { + prState: "closed", + headSha: sha1, + checkRuns: [failingCheck(501)], + }; + const { acquirer, fixed } = makeAcquirer(gh); + + await acquirer.tick(); + expect(fixed).toEqual([]); + expect(workerState.listOpenAgentPrs()).toHaveLength(0); + }); + + test("fork PRs are skipped gracefully", async () => { + workerState.recordAgentPr({ repo: "acme/widgets", prNumber: 42 }); + const gh: FakeGitHubState = { + prState: "open", + headSha: sha1, + headRepoFullName: "contributor/widgets-fork", + checkRuns: [failingCheck(601)], + }; + const { acquirer, fixed } = makeAcquirer(gh); + + await acquirer.tick(); + expect(fixed).toEqual([]); + }); + + test("ETags are stored and replayed on subsequent ticks", async () => { + workerState.recordAgentPr({ repo: "acme/widgets", prNumber: 42 }); + const gh: FakeGitHubState = { + prState: "open", + headSha: sha1, + checkRuns: [{ id: 701, name: "ok", status: "completed", conclusion: "success" }], + }; + const { acquirer } = makeAcquirer(gh); + + await acquirer.tick(); + expect(gh.seenPrEtag).toBeUndefined(); + expect(gh.seenChecksEtag).toBeUndefined(); + + gh.prEtagHit = true; + gh.checksEtagHit = true; + await acquirer.tick(); + expect(gh.seenPrEtag).toBe('W/"pr-1"'); + expect(gh.seenChecksEtag).toBe('W/"checks-1"'); + }); + + test("CI success resets the retry counter", async () => { + workerState.recordAgentPr({ repo: "acme/widgets", prNumber: 42 }); + const gh: FakeGitHubState = { + prState: "open", + headSha: sha1, + checkRuns: [failingCheck(801)], + }; + const { acquirer } = makeAcquirer(gh, { maxAttempts: 1 }); + + await acquirer.tick(); // attempt 1 → budget exhausted + expect(workerState.getCiFixState("acme/widgets", 42).consecutiveFailures).toBe(1); + + // Escalation was posted and no further attempt happens at the same head. + await acquirer.tick(); + expect(workerState.getCiFixState("acme/widgets", 42).escalatedSha).toBe(sha1); + + gh.checkRuns = [{ id: 802, name: "build", status: "completed", conclusion: "success" }]; + await acquirer.tick(); // CI passed → reset + const state = workerState.getCiFixState("acme/widgets", 42); + expect(state.consecutiveFailures).toBe(0); + expect(state.escalatedSha).toBeUndefined(); + }); + + test("exhausted budget escalates once and blocks until the head moves", async () => { + workerState.recordAgentPr({ repo: "acme/widgets", prNumber: 42 }); + const gh: FakeGitHubState = { + prState: "open", + headSha: sha1, + checkRuns: [failingCheck(901)], + }; + const { acquirer, fixed, comments } = makeAcquirer(gh, { maxAttempts: 2 }); + + await acquirer.tick(); // attempt 1 + expect(fixed).toHaveLength(1); + await acquirer.tick(); // new failure at new SHA? No — dedupe; nothing happens + + gh.headSha = sha2; + gh.checkRuns = [failingCheck(902)]; + await acquirer.tick(); // attempt 2 → exhausted + expect(fixed).toHaveLength(2); + + gh.headSha = "cccc3333cccc3333cccc3333cccc3333cccc3333"; + gh.checkRuns = [failingCheck(903)]; + await acquirer.tick(); // budget exhausted → escalate, no fix + expect(fixed).toHaveLength(2); + expect(comments).toHaveLength(1); + expect(comments[0]).toContain("stopped retrying"); + + // Still blocked at the escalation head even for brand-new failures. + gh.checkRuns = [failingCheck(904)]; + await acquirer.tick(); + expect(fixed).toHaveLength(2); + expect(comments).toHaveLength(1); // escalation posted exactly once + + // A human push moves the head past the escalation point → unblocked. + gh.headSha = "dddd4444dddd4444dddd4444dddd4444dddd4444"; + gh.checkRuns = [failingCheck(905)]; + await acquirer.tick(); + expect(fixed).toHaveLength(3); + }); +}); diff --git a/packages/code/tests/review-formatter.test.ts b/packages/code/tests/review-formatter.test.ts index 6e3bf8c..b2158f2 100644 --- a/packages/code/tests/review-formatter.test.ts +++ b/packages/code/tests/review-formatter.test.ts @@ -1,9 +1,11 @@ import { describe, test, expect } from "bun:test"; import { + formatCiFixPrompt, formatReviewPrompt, formatReplyMessage, formatSingleCommentPrompt, } from "../src/lib/review-formatter"; +import type { CiFailureFeedback } from "../src/lib/review-formatter"; import type { ProcessedReviewComment, ProcessedReviewFeedback } from "../src/types/github-webhooks"; describe("Review Formatter", () => { @@ -242,4 +244,41 @@ describe("Review Formatter", () => { expect(prompt).not.toContain("Line:"); }); }); + + describe("formatCiFixPrompt", () => { + const baseFeedback: CiFailureFeedback = { + repository: "acme/widgets", + prNumber: 42, + prTitle: "Add widget", + branch: "feature/dev-1", + failures: [ + { name: "build", conclusion: "failure" }, + { name: "ci/travis", conclusion: "error", detailsUrl: "https://ci.example.com/1" }, + ], + logs: "ERROR: cannot find module 'x'", + }; + + test("includes failing checks and log excerpt", () => { + const prompt = formatCiFixPrompt(baseFeedback); + + expect(prompt).toContain("CI Failure"); + expect(prompt).toContain("**build** — `failure`"); + expect(prompt).toContain("**ci/travis** — `error` (https://ci.example.com/1)"); + expect(prompt).toContain("ERROR: cannot find module 'x'"); + }); + + test("explains when logs are unavailable", () => { + const prompt = formatCiFixPrompt({ ...baseFeedback, logs: null }); + + expect(prompt).not.toContain("Failure Logs (excerpt)"); + expect(prompt).toContain("Job logs were not available"); + }); + + test("instructs the agent not to push and to flag flaky infra", () => { + const prompt = formatCiFixPrompt(baseFeedback); + + expect(prompt).toContain("Do NOT push"); + expect(prompt).toContain("flaky infrastructure"); + }); + }); }); diff --git a/packages/dashboard-ui/src/lib/api.ts b/packages/dashboard-ui/src/lib/api.ts index 2fe276c..3aa027d 100644 --- a/packages/dashboard-ui/src/lib/api.ts +++ b/packages/dashboard-ui/src/lib/api.ts @@ -5,7 +5,7 @@ import { useCallback, useEffect, useRef, useState } from "react"; -export type RunOrigin = "task" | "pr_mention"; +export type RunOrigin = "task" | "pr_mention" | "ci_fix"; export type RunStatus = | "in_progress" diff --git a/packages/dashboard-ui/src/views/RunsView.tsx b/packages/dashboard-ui/src/views/RunsView.tsx index d5f98f4..c8b084e 100644 --- a/packages/dashboard-ui/src/views/RunsView.tsx +++ b/packages/dashboard-ui/src/views/RunsView.tsx @@ -27,7 +27,7 @@ const STATUS_FILTERS: (RunStatus | "all")[] = [ "deferred", "abandoned", ]; -const ORIGIN_FILTERS: (RunOrigin | "all")[] = ["all", "task", "pr_mention"]; +const ORIGIN_FILTERS: (RunOrigin | "all")[] = ["all", "task", "pr_mention", "ci_fix"]; /** Paginated, filterable list of worker runs. */ export function RunsView({ onOpenRun }: { onOpenRun: (id: number) => void }) { diff --git a/packages/dashboard-ui/src/views/StatsView.tsx b/packages/dashboard-ui/src/views/StatsView.tsx index 5beca04..efc86f2 100644 --- a/packages/dashboard-ui/src/views/StatsView.tsx +++ b/packages/dashboard-ui/src/views/StatsView.tsx @@ -81,7 +81,7 @@ export function StatsView() {