diff --git a/docs/code/dashboard.md b/docs/code/dashboard.md index 633bece..4138695 100644 --- a/docs/code/dashboard.md +++ b/docs/code/dashboard.md @@ -1,6 +1,6 @@ --- title: "Observability Dashboard" -description: "A local web dashboard for worker run history: per-task timelines, stage-by-stage outcomes, aggregate stats, and worker logs" +description: "A local web dashboard for worker run history: per-task timelines, stage-by-stage outcomes, aggregate stats, run retries, and worker logs" section: "Server Automation" order: 2 dateModified: 2026-08-27 @@ -8,7 +8,7 @@ dateModified: 2026-08-27 # Observability Dashboard -`devintern dashboard` serves a local web dashboard over the worker's run history: every task, PR mention, and scheduled automation the worker handled, the stages each run went through (feasibility, implementation, self-review, change requests, outcome), aggregate stats like success rate and runs per week, and the worker's own log output. +`devintern dashboard` serves a local web dashboard over the worker's run history: every task, PR mention, and scheduled automation the worker handled, the stages each run went through (feasibility, implementation, self-review, change requests, outcome), aggregate stats like success rate and runs per week, a retry action for failed runs, and the worker's own log output. All data is read from the worker's local database (`.devintern-code/queue.db`). Nothing is uploaded anywhere: the dashboard runs on your machine and binds to localhost by default. @@ -38,6 +38,36 @@ Success and escalation rates are computed over finished runs only. Run duration The ticket link and description snapshot work for remote trackers whose web URLs can be derived from base configuration plus the task key (Jira, GitHub Issues, GitLab, Azure DevOps, Asana, Trello). Linear issue links need the organization slug, so those keys stay plain text there; markdown-file runs (including scheduled automations) have no tracker page at all. Descriptions are persisted when the run begins, so history keeps showing what was asked even if the ticket is later edited or deleted. +## Retrying a run + +Failed, escalated, and abandoned runs (and only those — succeeded runs have nothing to redo, deferred runs retry on their own schedule, and in-progress runs are still going) show a **Retry this run** action on the run detail page. Confirming it schedules the same flow a support engineer would run by hand: + +```bash +devintern PROJ-123 --force +``` + +How the retry executes depends on where the dashboard runs: + +- **Workspace worker (fleet mode)** — the default dashboard served by `devintern worker` inserts the retry into the shared workspace database, and the worker's retry-queue acquirer picks it up (default every 5 seconds, tunable with `WORKER_RETRY_INTERVAL_SECONDS`). The retry then runs through exactly the same pipeline as any fleet task: routing rules pick the repo, the task gets a disposable worktree from the bare clone, the per-repo environment applies, and the repo run lock serializes concurrent work. `--force` bypasses the incomplete-attempt retry gate, like the manual CLI flow. +- **Standalone `devintern dashboard`** — when the dashboard runs by itself inside a repo checkout, it spawns `devintern --force` as its own subprocess instead, mirroring the manual CLI flow; branch selection, worktree handling, and comments behave exactly like the CLI. + +In both modes the new attempt appears as a fresh run in the run list; the dashboard shows success/failure feedback for the trigger itself plus the new run's progress through its stages. + +Safeguards: + +- A confirmation prompt states exactly what will be re-run before anything starts. +- The actor must be signed in (`devintern login`); when `DASHBOARD_RETRY_EMAILS` is set, only those support-role email addresses may trigger retries. +- Retries are serialized per task: while a retry is already scheduled or running (including an attempt recorded by the worker), further triggers are refused. +- Every trigger is audited in `.devintern-code/queue.db` (`run_retry_audit`): who retried, when, and against which original run. The run detail page lists this history under "Retry history". + +### Environment variables + +| Variable | Description | +| ------------------------------- | ------------------------------------------------------------------------------------------------- | +| `DASHBOARD_PORT` | Port to listen on when `--port` is not given | +| `DASHBOARD_RETRY_EMAILS` | Comma-separated allowlist of emails authorized to trigger retries; unset means any signed-in user | +| `WORKER_RETRY_INTERVAL_SECONDS` | How often the workspace worker drains scheduled dashboard retries (default 5, minimum 1) | + ## Where the logs come from The worker daemon tees its own console output into `worker.stdout.log` and `worker.stderr.log` in the workspace home (`~/.devintern`), so the Logs tab works no matter how the daemon is launched — a systemd unit, launchd agent, cron wrapper, or a plain terminal — without the service definition having to redirect stdout/stderr. Each file rotates to `.1` past 8 MiB. The Logs tab tails those capture files — whether or not the worker is currently running — and only reads a bounded tail (the most recent ~256 KiB per file, at most 500 lines), so huge logs never slow down or bloat the dashboard. @@ -75,7 +105,8 @@ The dashboard is backed by a small read-only JSON API you can use directly, for | Endpoint | Returns | | --------------------------- | --------------------------------------------------------------------- | | `GET /api/runs` | Paginated run list (`limit`, `offset`, `status`, `origin`, `taskKey`); `origin=scheduled` is supported | -| `GET /api/runs/:id` | One run with its stage timeline | +| `GET /api/runs/:id` | One run with its stage timeline and retry metadata | +| `POST /api/runs/:id/retry` | Schedule a re-run of the task behind a failed/escalated/abandoned run (requires sign-in) | | `GET /api/stats?window=30d` | Aggregate stats (`7d`, `30d`, `90d`, or `all`) | | `GET /api/worker` | Worker liveness, queue counts, agent PRs, poll cursors | | `GET /api/logs` | Recent worker log entries (`limit` 1–1000, default 500; `level` all/info/warn/error) | diff --git a/packages/code/src/dashboard-server.ts b/packages/code/src/dashboard-server.ts index cb9b192..7625960 100644 --- a/packages/code/src/dashboard-server.ts +++ b/packages/code/src/dashboard-server.ts @@ -17,11 +17,13 @@ import { join, normalize, resolve } from "path"; import { DashboardData, handleLogs, + handleRetryRun, handleRuns, handleRunDetail, handleStats, handleWorkerStatus, } from "./lib/dashboard-api"; +import type { RetryHandlerDeps } from "./lib/dashboard-api"; export const DEFAULT_DASHBOARD_PORT = 4400; @@ -31,6 +33,14 @@ export interface DashboardServerOptions { dbPath?: string; /** Project root used to locate the worker lock file. */ workingDir?: string; + /** + * Retry execution mode (default `spawn`). The workspace worker passes + * `schedule` so dashboard retries are drained through the fleet pipeline; + * a standalone `devintern dashboard` keeps the detached-CLI spawn. + */ + retryMode?: "spawn" | "schedule"; + /** Collaborator overrides for the retry action (tests). */ + retryDeps?: RetryHandlerDeps; /** Directories to search for worker capture files (primary first). */ logDirs?: string[]; } @@ -93,6 +103,7 @@ export function startDashboardServer( const data = new DashboardData({ dbPath: options.dbPath, workingDir: options.workingDir, + retryMode: options.retryMode, logDirs: options.logDirs, }); const uiDir = resolveUiDir(); @@ -105,15 +116,20 @@ export function startDashboardServer( } const runDetailPattern = /^\/api\/runs\/([^/]+)$/; + const runRetryPattern = /^\/api\/runs\/([^/]+)\/retry$/; const server = Bun.serve({ port, hostname: host, - fetch(request: Request): Response { + async fetch(request: Request): Promise { const url = new URL(request.url); const { pathname } = url; if (pathname.startsWith("/api")) { + const retry = request.method === "POST" ? pathname.match(runRetryPattern) : null; + if (retry) { + return json(await handleRetryRun(data, retry[1], options.retryDeps)); + } if (request.method !== "GET") { return json({ status: 405, body: { error: "method not allowed" } }); } diff --git a/packages/code/src/lib/dashboard-api.ts b/packages/code/src/lib/dashboard-api.ts index 1c8fd11..6463ed5 100644 --- a/packages/code/src/lib/dashboard-api.ts +++ b/packages/code/src/lib/dashboard-api.ts @@ -14,6 +14,14 @@ import { LockManager } from "./lock-manager"; import { RunStore } from "./run-recorder"; import type { RunOrigin, RunRecord, RunStageRecord, RunStats, RunStatus } from "./run-recorder"; +import { + isRunRetriable, + resolveDashboardActor, + RunRetryAuditStore, + ScheduledRetryStore, + spawnCliForceRetry, +} from "./run-retry"; +import type { RetryActor, RunRetryAuditEntry, SpawnedRetryProcess } from "./run-retry"; import { readWorkerLogs } from "./worker-logs"; import type { LogEntry, WorkerLogLevel, WorkerLogsResult } from "./worker-logs"; import { resolveQueueDbPath, WebhookQueue } from "./webhook-queue"; @@ -57,6 +65,20 @@ export interface DashboardDataOptions { dbPath?: string; /** Project root used to locate the worker lock file. */ workingDir?: string; + /** + * How a retry is executed. `spawn` (default) starts a detached CLI + * subprocess — correct for a standalone `devintern dashboard` running + * inside a repo. `schedule` inserts a row into the workspace DB that the + * workspace worker drains through its normal fleet pipeline (routing, + * per-repo worktree, env, repo lock); used when the dashboard runs + * alongside the fleet worker. + */ + retryMode?: "spawn" | "schedule"; + /** + * How long a triggered retry blocks further retries of its task + * (tests); defaults to {@link INFLIGHT_RETRY_TTL_MS}. + */ + inflightRetryTtlMs?: number; /** Directories to search for worker capture files (primary first). */ logDirs?: string[]; /** Tail window per capture file; tests shrink this for truncation cases. */ @@ -69,6 +91,14 @@ interface Stores { queue: WebhookQueue; } +/** Retry metadata embedded in a run-detail response. */ +export interface RunRetryInfo { + eligible: boolean; + reason?: string; + /** Recent dashboard retries of this run, most recent first. */ + audit: RunRetryAuditEntry[]; +} + /** A log entry extended with the latest matching run, when one exists. */ export interface EnrichedLogEntry extends LogEntry { runId?: number; @@ -85,13 +115,27 @@ export interface EnrichedLogEntry extends LogEntry { export class DashboardData { readonly dbPath: string; readonly workingDir: string; + readonly retryMode: "spawn" | "schedule"; private readonly logDirs: string[]; private readonly maxLogBytesPerFile: number | undefined; private stores: Stores | null = null; + /** Lazy read-write connection for the retry audit trail. */ + private retryAuditStore: RunRetryAuditStore | null = null; + /** Lazy read-write connection for scheduled retries (schedule mode). */ + private scheduledRetryStore: ScheduledRetryStore | null = null; + /** + * Task keys with a dashboard-triggered retry in flight (task key → claimed + * at). The spawned CLI needs a moment to create its run row, so claims are + * held until the TTL lapses rather than released with the HTTP response. + */ + private inflightRetries = new Map(); + private inflightRetryTtlMs: number; constructor(options: DashboardDataOptions = {}) { this.dbPath = options.dbPath ?? resolveQueueDbPath(); this.workingDir = options.workingDir ?? process.cwd(); + this.retryMode = options.retryMode ?? "spawn"; + this.inflightRetryTtlMs = options.inflightRetryTtlMs ?? INFLIGHT_RETRY_TTL_MS; if (options.logDirs !== undefined) { // Explicit dirs keep tests hermetic; the workspace home is not probed. this.logDirs = options.logDirs; @@ -173,6 +217,59 @@ export class DashboardData { }); } + /** Eligibility plus audit history for a run's retry action. */ + getRetryInfo(runId: number): RunRetryInfo | null { + const detail = this.getRunDetail(runId); + if (!detail) { + return null; + } + const eligibility = isRunRetriable(detail.run); + return { + eligible: eligibility.eligible, + reason: eligibility.reason, + audit: this.getRetryAuditStore().listForRun(runId), + }; + } + + /** True while a dashboard-triggered retry for the task is in flight. */ + hasInflightRetry(taskKey: string): boolean { + this.pruneInflightRetries(); + return this.inflightRetries.has(taskKey); + } + + getRetryAuditStore(): RunRetryAuditStore { + if (!this.retryAuditStore) { + this.retryAuditStore = new RunRetryAuditStore(this.dbPath); + } + return this.retryAuditStore; + } + + /** Store backing the schedule mode of the retry action. */ + getScheduledRetryStore(): ScheduledRetryStore { + if (!this.scheduledRetryStore) { + this.scheduledRetryStore = new ScheduledRetryStore(this.dbPath); + } + return this.scheduledRetryStore; + } + + /** Claim the retry slot for a task; false when one is already held. */ + claimRetry(taskKey: string): boolean { + this.pruneInflightRetries(); + if (this.inflightRetries.has(taskKey)) { + return false; + } + this.inflightRetries.set(taskKey, Date.now()); + return true; + } + + private pruneInflightRetries(now = Date.now()): void { + for (const [taskKey, claimedAt] of this.inflightRetries) { + if (now - claimedAt > this.inflightRetryTtlMs) { + this.inflightRetries.delete(taskKey); + } + } + } + getStats(windowMs: number | null): RunStats | null { return this.read(null, (stores) => stores.runs.getStats(windowMs)); } @@ -244,6 +341,10 @@ export class DashboardData { this.stores.queue.close(); this.stores = null; } + this.retryAuditStore?.close(); + this.retryAuditStore = null; + this.scheduledRetryStore?.close(); + this.scheduledRetryStore = null; } } @@ -251,6 +352,189 @@ function badRequest(message: string): ApiResponse { return { status: 400, body: { error: message } }; } +function conflict(message: string): ApiResponse { + return { status: 409, body: { error: message } }; +} + +function forbidden(message: string): ApiResponse { + return { status: 403, body: { error: message } }; +} + +/** + * Injectable collaborators for {@link handleRetryRun}, so tests can fake the + * signed-in user and the spawned process. + */ +export interface RetryHandlerDeps { + /** Resolve the acting support engineer; null = not signed in. */ + resolveActor?: (workingDir: string) => Promise; + /** Start the CLI retry flow for a task. */ + spawn?: (taskKey: string, workingDir: string) => SpawnedRetryProcess; + /** + * Authorized support-role emails. When non-empty, only these may trigger a + * retry from the dashboard; when empty, any signed-in devintern user can. + */ + allowedEmails?: readonly string[]; +} + +/** + * Parse the comma-separated allowlist of authorized support roles + * (`DASHBOARD_RETRY_EMAILS`). + */ +export function resolveAllowedRetryEmails(env: NodeJS.ProcessEnv = process.env): string[] { + return (env.DASHBOARD_RETRY_EMAILS ?? "") + .split(",") + .map((entry) => entry.trim().toLowerCase()) + .filter((entry) => entry.length > 0); +} + +/** How long a triggered retry blocks further retries of the same task. */ +const INFLIGHT_RETRY_TTL_MS = 60_000; + +/** + * `POST /api/runs/:id/retry` — re-run this run's task. + * + * Safeguards, in order: + * 1. the run must exist, + * 2. it must be eligible (failed/escalated/abandoned with a task key), + * 3. the caller must be signed in (and on the support-role allowlist when + * `DASHBOARD_RETRY_EMAILS` is configured), + * 4. no other retry may be in flight for the task (dashboard or worker — an + * `in_progress` run for the same task key also blocks). + * + * Two execution modes, picked by the server options: + * - `schedule` (workspace worker): inserts a `pending` row into the + * `scheduled_retries` table; the worker drains it through the normal fleet + * pipeline (routing, per-repo worktree, env, repo lock) with `--force`. + * - `spawn` (standalone dashboard, default): spawns `devintern + * --force` as a detached subprocess of the dashboard server. + * + * Returns 202 once the retry is queued; the new attempt then shows up in the + * run list like any other run. + * + * @param data - Dashboard data source (also owns the in-flight guard) + * @param idParam - Raw id path segment + * @param deps - Injected collaborator overrides (tests) + */ +export async function handleRetryRun( + data: DashboardData, + idParam: string, + deps: RetryHandlerDeps = {}, +): Promise { + const id = parseInt(idParam, 10); + if (!Number.isFinite(id) || String(id) !== idParam) { + return badRequest("run id must be an integer"); + } + + const detail = data.getRunDetail(id); + if (!detail) { + return { status: 404, body: { error: `run ${id} not found` } }; + } + const run = detail.run; + + const eligibility = isRunRetriable(run); + if (!eligibility.eligible || !run.taskKey) { + return conflict(`not retriable: ${eligibility.reason ?? "unknown reason"}`); + } + const taskKey = run.taskKey; + + const actor = + (await deps.resolveActor?.(data.workingDir)) ?? (await resolveDashboardActor(data.workingDir)); + if (!actor) { + return forbidden("sign in first with `devintern login` to retry runs"); + } + + const allowedEmails = deps.allowedEmails ?? resolveAllowedRetryEmails(); + const actorEmail = actor.email?.toLowerCase(); + if ( + allowedEmails.length > 0 && + (!actorEmail || !allowedEmails.some((entry) => entry.toLowerCase() === actorEmail)) + ) { + return forbidden(`${actor.email ?? "this user"} is not authorized to retry runs`); + } + + // Concurrent retries: another dashboard retry for this task, or a live run + // for the task (the worker records one as soon as it picks the ticket up). + if (data.retryMode === "schedule") { + if (data.getScheduledRetryStore().hasActive(taskKey)) { + return conflict(`a retry of ${taskKey} is already scheduled or running`); + } + } else if (data.hasInflightRetry(taskKey)) { + return conflict(`a retry of ${taskKey} was just triggered and is starting`); + } + const activeRun = data + .listRuns({ taskKey, limit: MAX_LIMIT, offset: 0 }) + .runs.find((candidate) => candidate.status === "in_progress" && candidate.id !== run.id); + if (activeRun) { + return conflict( + `${taskKey} already has a run in progress (run ${activeRun.id}); wait for it to finish`, + ); + } + + if (data.retryMode === "schedule") { + const store = data.getScheduledRetryStore(); + const scheduled = store.schedule({ taskKey, runId: id, actor: actor.email ?? "unknown" }); + if (!scheduled.scheduled) { + return conflict(`a retry of ${taskKey} is already scheduled or running`); + } + data.getRetryAuditStore().record({ + runId: id, + taskKey, + actor: actor.email ?? "unknown", + action: "scheduled", + message: "queued for the worker", + }); + return { + status: 202, + body: { + status: "scheduled", + runId: id, + taskKey, + }, + }; + } + + if (!data.claimRetry(taskKey)) { + return conflict(`a retry of ${taskKey} was just triggered and is starting`); + } + + let spawned: SpawnedRetryProcess; + try { + spawned = deps.spawn + ? deps.spawn(taskKey, data.workingDir) + : spawnCliForceRetry({ taskKey, workingDir: data.workingDir }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + data.getRetryAuditStore().record({ + runId: id, + taskKey, + actor: actor.email ?? "unknown", + action: "failed", + message, + }); + return { status: 500, body: { error: `could not start the retry: ${message}` } }; + } + + data.getRetryAuditStore().record({ + runId: id, + taskKey, + actor: actor.email ?? "unknown", + action: "triggered", + command: spawned.command, + pid: spawned.pid, + }); + + return { + status: 202, + body: { + status: "triggered", + runId: id, + taskKey, + pid: spawned.pid, + command: spawned.command, + }, + }; +} + /** * `GET /api/runs` — paginated run list with optional filters. * @@ -289,7 +573,8 @@ export function handleRuns(data: DashboardData, params: URLSearchParams): ApiRes } /** - * `GET /api/runs/:id` — a run with its stages in insertion order. + * `GET /api/runs/:id` — a run with its stages in insertion order and its + * retry action metadata (eligibility + audit trail). * * @param data - Dashboard data source * @param idParam - Raw id path segment @@ -303,7 +588,13 @@ export function handleRunDetail(data: DashboardData, idParam: string): ApiRespon if (!detail) { return { status: 404, body: { error: `run ${id} not found` } }; } - return { status: 200, body: detail }; + const eligibility = isRunRetriable(detail.run); + const retry: RunRetryInfo = { + eligible: eligibility.eligible, + reason: eligibility.reason, + audit: data.getRetryAuditStore().listForRun(id), + }; + return { status: 200, body: { run: detail.run, stages: detail.stages, retry } }; } /** diff --git a/packages/code/src/lib/run-retry.ts b/packages/code/src/lib/run-retry.ts new file mode 100644 index 0000000..3852571 --- /dev/null +++ b/packages/code/src/lib/run-retry.ts @@ -0,0 +1,450 @@ +/** + * Run retry + * + * The Support Dashboard's "retry this run" action. A retry re-invokes the same + * underlying flow support would run by hand — `devintern --force` — as + * a detached subprocess of the dashboard server, so the CLI's own lock, + * entitlement, and worktree handling apply unchanged. + * + * Every trigger is audited in the queue database (`run_retry_audit`): who + * retried, when, and the original run id. Only retriable runs (failed, + * escalated, or abandoned terminal states with a task key) may be retried; + * succeeded/deferred runs and runs without a ticket are rejected up front. + */ + +import { createDefaultSupabaseAuthConfig, getAuthenticatedUser } from "@devintern/auth"; +import { Database } from "bun:sqlite"; +import { basename, join } from "path"; + +import { resolveConfigDir } from "@devintern/utils"; +import type { RunRecord, RunStatus } from "./run-recorder"; +import { prepareQueueDbDirectory } from "./webhook-queue"; + +/** Terminal, non-success statuses a support engineer can re-run. */ +export const RETRIABLE_RUN_STATUSES: readonly RunStatus[] = ["failed", "escalated", "abandoned"]; + +/** Why a run cannot currently be retried ("eligible" when it can). */ +export interface RetryEligibility { + eligible: boolean; + reason?: string; +} + +/** + * Whether a run is eligible for a dashboard retry. + * + * Eligible: a terminal failed/escalated/abandoned run that belongs to a task + * key (there must be something to re-invoke the CLI with). `deferred` is not + * eligible — the worker already scheduled its own retry; `in_progress` and + * `succeeded` are never eligible; PR mentions and automations without a task + * key have no CLI entry point to force. + */ +export function isRunRetriable(run: Pick): RetryEligibility { + if (!run.taskKey) { + return { eligible: false, reason: "this run has no task key to re-run" }; + } + if (RETRIABLE_RUN_STATUSES.includes(run.status)) { + return { eligible: true }; + } + switch (run.status) { + case "succeeded": + return { eligible: false, reason: "this run already succeeded" }; + case "in_progress": + return { eligible: false, reason: "this run is still in progress" }; + case "deferred": + return { + eligible: false, + reason: "this run is deferred and will be retried automatically", + }; + default: + return { eligible: false, reason: `runs in status "${run.status}" cannot be retried` }; + } +} + +/** A parsed devintern sign-in session, reduced to what auditing needs. */ +export interface RetryActor { + email: string | null; +} + +/** + * Resolve the acting user from the CLI login session + * (`.devintern-code/.auth-session.json`). Null when nobody is signed in or + * the session can no longer be validated. + */ +export async function resolveDashboardActor( + workingDir: string = process.cwd(), +): Promise { + try { + const configDir = resolveConfigDir({ + configDirName: ".devintern-code", + startDir: workingDir, + }); + const user = await getAuthenticatedUser( + createDefaultSupabaseAuthConfig(join(configDir, ".auth-session.json")), + ); + return user ? { email: user.email } : null; + } catch { + // Auth infrastructure unavailable (offline, corrupt session) — treat as + // signed out; the handler surfaces a sign-in hint. + return null; + } +} + +/** One audited retry attempt against an original run. */ +export interface RunRetryAuditEntry { + id: number; + runId: number; + taskKey?: string; + actor: string; + action: "triggered" | "scheduled" | "failed"; + command?: string; + pid?: number; + message?: string; + createdAt: number; +} + +interface AuditRow { + id: number; + run_id: number; + task_key: string | null; + actor: string; + action: string; + command: string | null; + pid: number | null; + message: string | null; + created_at: number; +} + +function rowToEntry(row: AuditRow): RunRetryAuditEntry { + return { + id: row.id, + runId: row.run_id, + taskKey: row.task_key ?? undefined, + actor: row.actor, + action: + row.action === "failed" ? "failed" : row.action === "scheduled" ? "scheduled" : "triggered", + command: row.command ?? undefined, + pid: row.pid ?? undefined, + message: row.message ?? undefined, + createdAt: row.created_at, + }; +} + +const MAX_AUDIT_ROWS = 200; + +/** + * SQLite-backed audit trail for dashboard retries, stored alongside the rest + * of the worker state in `queue.db`. Opened read-write and lazily: reads fall + * back to an empty trail while the table does not exist yet. + */ +export class RunRetryAuditStore { + private db: Database | null = null; + + constructor(private dbPath: string) {} + + private ensureDb(): Database { + if (this.db) { + return this.db; + } + prepareQueueDbDirectory(this.dbPath); + this.db = new Database(this.dbPath); + this.db.run("PRAGMA busy_timeout = 5000"); + this.db.run(` + CREATE TABLE IF NOT EXISTS run_retry_audit ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + run_id INTEGER NOT NULL, + task_key TEXT, + actor TEXT NOT NULL, + action TEXT NOT NULL, + command TEXT, + pid INTEGER, + message TEXT, + created_at INTEGER NOT NULL + ) + `); + this.db.run("CREATE INDEX IF NOT EXISTS idx_run_retry_audit_run_id ON run_retry_audit(run_id)"); + return this.db; + } + + /** Record one retry trigger (or spawn failure). */ + record(entry: Omit & { createdAt?: number }): void { + const db = this.ensureDb(); + db.run( + `INSERT INTO run_retry_audit (run_id, task_key, actor, action, command, pid, message, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + [ + entry.runId, + entry.taskKey ?? null, + entry.actor, + entry.action, + entry.command ?? null, + entry.pid ?? null, + entry.message ?? null, + entry.createdAt ?? Date.now(), + ], + ); + } + + /** List recent retries for a run, most recent first. */ + listForRun(runId: number): RunRetryAuditEntry[] { + // Read through a transient read-only connection when nothing has been + // written from this process yet; do not cache it, or a later write would + // fail against the read-only handle. + let tempDb: Database | null = null; + try { + if (this.db === null) { + tempDb = this.openReadonly(); + } + const db = this.db ?? tempDb; + if (!db) { + return []; + } + const rows = db + .query( + `SELECT * FROM run_retry_audit WHERE run_id = ? + ORDER BY created_at DESC, id DESC LIMIT ${MAX_AUDIT_ROWS}`, + ) + .all(runId) as AuditRow[]; + return rows.map(rowToEntry); + } catch { + // Missing DB file or table (DB created by an older version). + return []; + } finally { + tempDb?.close(); + } + } + + private openReadonly(): Database { + const conn = new Database(this.dbPath, { readonly: true }); + conn.run("PRAGMA busy_timeout = 5000"); + return conn; + } + + close(): void { + this.db?.close(); + this.db = null; + } +} + +export interface SpawnedRetryProcess { + pid?: number; + /** Human-readable command line used for the audit log. */ + command: string; +} + +/** A dashboard-scheduled retry waiting for (or being run by) the worker. */ +export interface ScheduledRetry { + id: number; + taskKey: string; + runId?: number; + actor: string; + status: "pending" | "running" | "done" | "failed"; + message?: string; + createdAt: number; + startedAt?: number; + finishedAt?: number; +} + +interface ScheduledRetryRow { + id: number; + task_key: string; + run_id: number | null; + actor: string; + status: string; + message: string | null; + created_at: number; + started_at: number | null; + finished_at: number | null; +} + +function rowToScheduledRetry(row: ScheduledRetryRow): ScheduledRetry { + return { + id: row.id, + taskKey: row.task_key, + runId: row.run_id ?? undefined, + actor: row.actor, + status: row.status as ScheduledRetry["status"], + message: row.message ?? undefined, + createdAt: row.created_at, + startedAt: row.started_at ?? undefined, + finishedAt: row.finished_at ?? undefined, + }; +} + +/** + * Durable queue of dashboard-scheduled retries, stored in the workspace DB + * (`scheduled_retries` table). The dashboard handler inserts a pending row; + * the worker's retry acquirer drains rows through the normal fleet pipeline + * (routing, per-repo worktree, env, repo lock) instead of the dashboard + * spawning a bare CLI subprocess from the workspace home. + */ +export class ScheduledRetryStore { + private db: Database | null = null; + + constructor(private dbPath: string) {} + + private ensureDb(): Database { + if (this.db) { + return this.db; + } + prepareQueueDbDirectory(this.dbPath); + this.db = new Database(this.dbPath); + this.db.run("PRAGMA busy_timeout = 5000"); + this.db.run(` + CREATE TABLE IF NOT EXISTS scheduled_retries ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + task_key TEXT NOT NULL, + run_id INTEGER, + actor TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + message TEXT, + created_at INTEGER NOT NULL, + started_at INTEGER, + finished_at INTEGER + ) + `); + this.db.run( + "CREATE INDEX IF NOT EXISTS idx_scheduled_retries_status ON scheduled_retries(status)", + ); + this.db.run( + "CREATE INDEX IF NOT EXISTS idx_scheduled_retries_task ON scheduled_retries(task_key)", + ); + return this.db; + } + + /** + * Schedule a retry; refuses while an active (pending/running) row already + * exists for the task, so double-clicks cannot double-run. + */ + schedule(entry: { taskKey: string; runId?: number; actor: string; createdAt?: number }): { + scheduled: boolean; + reason?: string; + } { + const db = this.ensureDb(); + if (this.hasActive(entry.taskKey)) { + return { scheduled: false, reason: "a retry is already scheduled or running" }; + } + db.run( + `INSERT INTO scheduled_retries (task_key, run_id, actor, status, created_at) + VALUES (?, ?, ?, 'pending', ?)`, + [entry.taskKey, entry.runId ?? null, entry.actor, entry.createdAt ?? Date.now()], + ); + return { scheduled: true }; + } + + /** Whether a pending or running retry exists for the task. */ + hasActive(taskKey: string): boolean { + try { + const row = this.ensureDb() + .query( + "SELECT 1 AS one FROM scheduled_retries WHERE task_key = ? AND status IN ('pending', 'running') LIMIT 1", + ) + .get(taskKey); + return row !== null; + } catch { + return false; + } + } + + /** + * Claim the oldest pending retry atomically (safe against concurrent + * claimers): the subquery only matches a still-pending row. + */ + claimNext(): ScheduledRetry | null { + const db = this.ensureDb(); + const row = db + .query( + `UPDATE scheduled_retries + SET status = 'running', started_at = ? + WHERE id = ( + SELECT id FROM scheduled_retries WHERE status = 'pending' + ORDER BY created_at ASC, id ASC LIMIT 1 + ) + RETURNING *`, + ) + .get(Date.now()) as ScheduledRetryRow | null; + return row ? rowToScheduledRetry(row) : null; + } + + /** Mark a claimed retry finished (`done` or `failed` with a message). */ + finish(id: number, outcome: "done" | "failed", message?: string): void { + this.ensureDb().run( + `UPDATE scheduled_retries + SET status = ?, message = ?, finished_at = ? + WHERE id = ? AND status = 'running'`, + [outcome, message ?? null, Date.now(), id], + ); + } + + /** Return a claimed retry to the queue (the repo was busy; try next tick). */ + requeue(id: number): void { + this.ensureDb().run( + `UPDATE scheduled_retries + SET status = 'pending', started_at = NULL + WHERE id = ? AND status = 'running'`, + [id], + ); + } + + /** + * Settle every row left `running` by a dead worker (startup orphan + * recovery). Rows are marked failed rather than requeued: the orphaned run + * itself already gets graceful-shutdown feedback, and silently re-running a + * forced retry on every worker restart could loop. Returns the settled rows + * for logging. + */ + failRunning(message: string): ScheduledRetry[] { + const db = this.ensureDb(); + const rows = db + .query( + `UPDATE scheduled_retries + SET status = 'failed', message = ?, finished_at = ? + WHERE status = 'running' + RETURNING *`, + ) + .all(message, Date.now()) as ScheduledRetryRow[]; + return rows.map(rowToScheduledRetry); + } + + /** Whether any pending retry exists (worker startup diagnostics). */ + hasPending(): boolean { + try { + const row = this.ensureDb() + .query("SELECT 1 AS one FROM scheduled_retries WHERE status = 'pending' LIMIT 1") + .get(); + return row !== null; + } catch { + return false; + } + } + + close(): void { + this.db?.close(); + this.db = null; + } +} + +/** + * Spawn `devintern --force` — the exact flow a support engineer + * runs from the CLI to bypass the retry gate — using the same interpreter and + * entry script that serve the dashboard. Detached so the HTTP response does + * not wait for a full agent run; the spawned CLI records its own new run. + */ +export function spawnCliForceRetry(options: { + taskKey: string; + workingDir: string; +}): SpawnedRetryProcess { + const script = process.argv[1]; + if (!script) { + throw new Error("could not determine the devintern entry point to invoke"); + } + const args = [script, options.taskKey, "--force"]; + const child = Bun.spawn([process.execPath, ...args], { + cwd: options.workingDir, + env: process.env, + // Outlive the dashboard request/process so long agent runs survive. + detached: true, + }); + child.unref(); + const command = `${basename(process.execPath)} devintern ${options.taskKey} --force`; + return { pid: child.pid, command }; +} diff --git a/packages/code/src/lib/workspace/retry-acquirer.ts b/packages/code/src/lib/workspace/retry-acquirer.ts new file mode 100644 index 0000000..fda8014 --- /dev/null +++ b/packages/code/src/lib/workspace/retry-acquirer.ts @@ -0,0 +1,112 @@ +/** + * Dashboard retry queue acquirer (fleet mode). + * + * The dashboard's "retry this run" action inserts a `pending` row into the + * shared `scheduled_retries` table (see `lib/run-retry.ts`); this acquirer + * drains those rows through the normal fleet pipeline — routing, bare-clone + * fetch, disposable per-task worktree, per-repo env, and the repo run lock — + * with `--force` prepended so the incomplete-attempt retry gate is bypassed. + * + * Rows are claimed atomically, so a dashboard retry is never lost on worker + * restart (a claimed-but-unfished row stays `running`; see the store) and a + * busy repo re-queues the row for the next tick. + */ + +import type { Acquirer } from "../../worker"; +import type { ScheduledRetry, ScheduledRetryStore } from "../run-retry"; +import type { TaskExecutionResult } from "../task-polling-acquirer"; +import { toRoutableTask } from "./router"; +import type { RoutableTask } from "./router"; + +export interface RetryQueueAcquirerOptions { + store: ScheduledRetryStore; + /** Fleet executor slice (same shape `createFleetTaskExecutor` returns). */ + execute: (taskKey: string, routable: RoutableTask) => Promise; + intervalSeconds: number; + verbose?: boolean; +} + +/** + * Poll the scheduled-retry table on a short interval so a dashboard-triggered + * retry is picked up quickly — ahead of the slower task/review pollers. + */ +export class RetryQueueAcquirer implements Acquirer { + readonly name = "retry-queue"; + private options: RetryQueueAcquirerOptions; + private timer: ReturnType | null = null; + private busy = false; + + constructor(options: RetryQueueAcquirerOptions) { + this.options = options; + } + + /** Start draining: immediate first tick, then on the configured interval. */ + async start(): Promise { + console.log(`🔁 Draining scheduled dashboard retries every ${this.options.intervalSeconds}s`); + await this.tick(); + this.timer = setInterval(() => void this.tick(), this.options.intervalSeconds * 1000); + } + + stop(): void { + if (this.timer) { + clearInterval(this.timer); + this.timer = null; + } + } + + /** + * Claim and run every pending retry, oldest first, sequentially. Stops at + * the first deferred row (it is back in the queue for the next tick), so a + * requeued row is never re-claimed by the same drain. + */ + async tick(): Promise { + if (this.busy) { + return; + } + this.busy = true; + try { + for (;;) { + const retry = this.options.store.claimNext(); + if (!retry) { + return; + } + if (!(await this.runOne(retry))) { + return; + } + } + } finally { + this.busy = false; + } + } + + /** Run one claimed retry; false when it was requeued for the next tick. */ + private async runOne(retry: ScheduledRetry): Promise { + const { store, execute, verbose } = this.options; + console.log(`🔁 [${this.name}] re-running ${retry.taskKey} (scheduled by ${retry.actor})`); + try { + const result = await execute( + retry.taskKey, + toRoutableTask({ key: retry.taskKey, labels: [], components: [] }), + ); + if (result === true) { + store.finish(retry.id, "done"); + } else if (result === "deferred") { + // Repo busy (or otherwise not started): back to the queue; stop this + // drain so the row is not re-claimed until the next tick. + store.requeue(retry.id); + console.log(`⏳ [${this.name}] ${retry.taskKey} deferred; will retry next tick`); + return false; + } else { + store.finish(retry.id, "failed", "task pipeline did not complete cleanly"); + } + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + store.finish(retry.id, "failed", message); + console.error(`❌ [${this.name}] ${retry.taskKey} failed: ${message}`); + } + if (verbose) { + console.log(` [${this.name}] ${retry.taskKey} settled`); + } + return true; + } +} diff --git a/packages/code/src/lib/workspace/workspace-worker.ts b/packages/code/src/lib/workspace/workspace-worker.ts index a5e3c8d..728755b 100644 --- a/packages/code/src/lib/workspace/workspace-worker.ts +++ b/packages/code/src/lib/workspace/workspace-worker.ts @@ -25,6 +25,7 @@ import { } from "../orphan-recovery"; import { RunStore } from "../run-recorder"; import { RetryStateStore } from "../retry-state"; +import { ScheduledRetryStore } from "../run-retry"; import type { TaskTrackerClient } from "../task-tracker-client"; import { findRepo, loadWorkspaceConfig } from "./config"; import type { RepoConfig, WorkspaceConfig } from "./config"; @@ -45,6 +46,7 @@ import { probePushAccess } from "../github-push-probe"; import { AutomationAcquirer } from "../automation-acquirer"; import type { AutomationConfig } from "../automation-config"; import { flushAnalytics, RUN_ORIGIN_ENV, trackWorkerStarted } from "../analytics"; +import { RetryQueueAcquirer } from "./retry-acquirer"; /** Orphaned-run feedback cutoff: `WORKER_ORPHAN_MAX_AGE_HOURS`, default 7 days. */ function orphanMaxAgeMs(): number { @@ -54,7 +56,8 @@ function orphanMaxAgeMs(): number { /** * Recover task runs left in progress by a previous (dead) worker before any * acquirer picks up new tickets: reap them and give their tickets the - * graceful-shutdown feedback (failure comment + move back to To Do). + * graceful-shutdown feedback (failure comment + move back to To Do). Also + * settles dashboard-scheduled retry rows left `running` by the crash. * * Replaces the old reap-only startup sweep, which also only ran when GitHub * credentials were configured; recovery here covers every workspace. @@ -110,6 +113,24 @@ export async function recoverOrphanedWorkspaceRuns(options: { retryStore?.recordIncompleteAttempt(taskKey, type, description), maxAgeMs: orphanMaxAgeMs(), }); + + // Dashboard-scheduled retries claimed by the previous worker: settle the + // rows so the dashboard's per-task guard unblocks (the operator can + // re-schedule) instead of reporting "already scheduled or running" + // forever. The orphaned run itself was reaped above. + const retryQueue = new ScheduledRetryStore(dbPath); + try { + const orphans = retryQueue.failRunning( + "worker restarted while this retry was running; schedule it again", + ); + for (const orphan of orphans) { + console.warn( + `⚠️ [fleet] scheduled retry of ${orphan.taskKey} was interrupted by a worker restart`, + ); + } + } finally { + retryQueue.close(); + } } finally { retryStore?.close(); runStore.close(); @@ -312,20 +333,26 @@ export type FleetExecutorDeps = Pick< /** * Build the fleet execute step: route a task to its repo and run it in a - * disposable worktree. Shared by the polling acquirer and the relay's task - * evaluation, which acquire tasks differently but execute identically. + * disposable worktree. Shared by the polling acquirer, the relay's task + * evaluation, and the dashboard retry queue, which acquire tasks differently + * but execute identically. * * Ambiguous/unrouted tasks are recorded as routing skips and count as * handled: dedupe keeps them out of the loop until the task changes again, * the same policy as failing tasks. + * + * @param deps - Routing, locking, and runner collaborators + * @param options - `extraArgs` overrides the per-task CLI args (the retry + * queue prepends `--force` to bypass the retry gate) */ export function createFleetTaskExecutor( deps: FleetExecutorDeps, + options: { extraArgs?: string[] } = {}, ): (taskKey: string, routable: RoutableTask) => Promise { const { config, workspaceDir, skips, repoManager } = deps; const runTask = deps.runTask ?? runTaskViaCli; const repoLock = deps.repoLock ?? ((name: string) => createRepoRunLock(name, workspaceDir)); - const extraArgs = fleetTaskArgs(config); + const extraArgs = options.extraArgs ?? fleetTaskArgs(config); return async (taskKey, routable) => { const decision = routeTask(routable, config); @@ -437,11 +464,22 @@ export async function runWorkspaceWorker(options: RunWorkspaceWorkerOptions): Pr const query = config.defaults.taskQuery; const intervalSeconds = config.defaults.pollIntervalSeconds; + + // Dashboard retries ride the shared workspace DB: the dashboard inserts a + // pending row, this worker drains it through the fleet executor below. + const retryQueue = new ScheduledRetryStore(workspaceDbPath(workspaceDir)); + if (!query && config.automations.length === 0) { - console.error( - "❌ Workspace mode needs a task query: set [defaults].task_query in workspace.toml.", - ); - process.exit(1); + if (retryQueue.hasPending()) { + console.warn( + "⚠️ No task query or automations configured; the worker will only drain scheduled dashboard retries.", + ); + } else { + console.error( + "❌ Workspace mode needs a task query: set [defaults].task_query in workspace.toml.", + ); + process.exit(1); + } } const state = openWorkspaceState(workspaceDir); @@ -468,6 +506,22 @@ export async function runWorkspaceWorker(options: RunWorkspaceWorkerOptions): Pr const acquirers: import("../../worker").Acquirer[] = []; + // First in the list and on a short interval: a dashboard-scheduled retry + // gets picked up ahead of the slower pollers. + acquirers.push( + new RetryQueueAcquirer({ + store: retryQueue, + execute: createFleetTaskExecutor( + { config, workspaceDir, skips: state.skips, repoManager }, + // `--force` bypasses the incomplete-attempt retry gate, exactly like + // the manual `devintern --force` the dashboard action mirrors. + { extraArgs: ["--force", ...fleetTaskArgs(config)] }, + ), + intervalSeconds: parseEnvInteger("WORKER_RETRY_INTERVAL_SECONDS", 5, { min: 1 }), + verbose: options.verbose, + }), + ); + if (config.automations.length > 0) { const semanticErrors: string[] = []; for (const automation of config.automations) { @@ -536,7 +590,9 @@ export async function runWorkspaceWorker(options: RunWorkspaceWorkerOptions): Pr if (config.workspace.dashboard) { try { const { startDashboardServer } = await import("../../dashboard-server"); - startDashboardServer({ port: config.workspace.dashboardPort }); + // `schedule`: retries are drained by this worker's retry-queue acquirer + // through the normal pipeline (never spawned from the workspace home). + startDashboardServer({ port: config.workspace.dashboardPort, retryMode: "schedule" }); } catch (error) { console.warn( `⚠️ Dashboard could not start (${(error as Error).message}); the worker will continue.`, diff --git a/packages/code/tests/dashboard-api.test.ts b/packages/code/tests/dashboard-api.test.ts index 3a830d6..b5490b8 100644 --- a/packages/code/tests/dashboard-api.test.ts +++ b/packages/code/tests/dashboard-api.test.ts @@ -409,4 +409,90 @@ describe("dashboard server", () => { server.stop(true); } }); + + test("POST /api/runs/:id/retry triggers the CLI flow end-to-end", async () => { + const store = new RunStore(dbPath); + const id = store.createRun({ origin: "task", taskKey: "PROJ-9" }); + store.finishRun(id, "failed"); + store.close(); + + const spawned: string[] = []; + const server = startDashboardServer({ + port: 0, + dbPath, + workingDir: dir, + retryDeps: { + resolveActor: async () => ({ email: "sup@example.com" }), + spawn: (taskKey: string) => { + spawned.push(taskKey); + return { pid: 1234, command: `bun devintern ${taskKey} --force` }; + }, + }, + }); + try { + const base = `http://127.0.0.1:${server.port}`; + + // GET exposes retry metadata on the detail payload. + const detail = (await (await fetch(`${base}/api/runs/${id}`)).json()) as { + retry: { eligible: boolean; audit: unknown[] }; + }; + expect(detail.retry.eligible).toBe(true); + expect(detail.retry.audit).toEqual([]); + + const response = await fetch(`${base}/api/runs/${id}/retry`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: "{}", + }); + expect(response.status).toBe(202); + const body = (await response.json()) as { status: string; taskKey?: string; pid?: number }; + expect(body.status).toBe("triggered"); + expect(body.taskKey).toBe("PROJ-9"); + expect(body.pid).toBe(1234); + expect(spawned).toEqual(["PROJ-9"]); + + // The audit entry is now visible on the detail payload. + const after = (await (await fetch(`${base}/api/runs/${id}`)).json()) as { + retry: { audit: { action: string }[] }; + }; + expect(after.retry.audit.map((entry) => entry.action)).toEqual(["triggered"]); + + // A second POST immediately afterwards hits the in-flight guard. + const repeat = await fetch(`${base}/api/runs/${id}/retry`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: "{}", + }); + expect(repeat.status).toBe(409); + + // Unauthorized requests are refused before anything spawns. + const deniedId = (() => { + const s = new RunStore(dbPath); + const failed = s.createRun({ origin: "task", taskKey: "PROJ-10" }); + s.finishRun(failed, "failed"); + s.close(); + return failed; + })(); + const noAuth = startDashboardServer({ + port: 0, + dbPath, + workingDir: dir, + retryDeps: { resolveActor: async () => null, spawn: () => ({ command: "" }) }, + }); + try { + const denied = await fetch(`http://127.0.0.1:${noAuth.port}/api/runs/${deniedId}/retry`, { + method: "POST", + }); + expect(denied.status).toBe(403); + } finally { + noAuth.stop(true); + } + + // GET is not allowed on the retry route. + const wrongMethod = await fetch(`${base}/api/runs/${id}/retry`); + expect(wrongMethod.status).toBe(404); + } finally { + server.stop(true); + } + }); }); diff --git a/packages/code/tests/dashboard-retry.test.ts b/packages/code/tests/dashboard-retry.test.ts new file mode 100644 index 0000000..7c8cd5e --- /dev/null +++ b/packages/code/tests/dashboard-retry.test.ts @@ -0,0 +1,406 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdirSync, rmSync } from "fs"; +import { join } from "path"; +import { tmpdir } from "os"; + +import { + DashboardData, + handleRetryRun, + handleRunDetail, + resolveAllowedRetryEmails, +} from "../src/lib/dashboard-api"; +import type { RetryHandlerDeps } from "../src/lib/dashboard-api"; +import { isRunRetriable, ScheduledRetryStore } from "../src/lib/run-retry"; +import type { SpawnedRetryProcess } from "../src/lib/run-retry"; +import { RunStore } from "../src/lib/run-recorder"; +import type { RunStatus } from "../src/lib/run-recorder"; + +const ACTOR: NonNullable = async () => ({ + email: "sup@example.com", +}); + +/** Deps with a fake spawn that records the task key it was invoked for. */ +function stubDeps(overrides: Partial = {}): { + deps: RetryHandlerDeps; + spawned: string[]; + pids: number[]; +} { + const spawned: string[] = []; + const pids: number[] = []; + let nextPid = 4000; + const deps: RetryHandlerDeps = { + resolveActor: ACTOR, + spawn: (taskKey: string) => { + spawned.push(taskKey); + pids.push(nextPid++); + return { pid: pids[pids.length - 1], command: `bun devintern ${taskKey} --force` }; + }, + ...overrides, + }; + return { deps, spawned, pids }; +} + +describe("isRunRetriable", () => { + test("failed, escalated, and abandoned runs with a task key are eligible", () => { + for (const status of ["failed", "escalated", "abandoned"] as RunStatus[]) { + expect(isRunRetriable({ status, taskKey: "PROJ-1" })).toEqual({ eligible: true }); + } + }); + + test("succeeded, in_progress, and deferred runs are not eligible", () => { + for (const status of ["succeeded", "in_progress", "deferred"] as RunStatus[]) { + const result = isRunRetriable({ status, taskKey: "PROJ-1" }); + expect(result.eligible).toBe(false); + expect(result.reason).toBeString(); + } + }); + + test("runs without a task key can never be retried", () => { + expect(isRunRetriable({ status: "failed" }).eligible).toBe(false); + expect(isRunRetriable({ status: "succeeded" }).reason).toContain("no task key"); + }); +}); + +describe("handleRetryRun", () => { + let dir: string; + let dbPath: string; + let data: DashboardData; + + beforeEach(() => { + dir = join(tmpdir(), `retry-test-${Date.now()}-${Math.random().toString(36).slice(2)}`); + mkdirSync(dir, { recursive: true }); + dbPath = join(dir, "queue.db"); + data = new DashboardData({ dbPath, workingDir: dir }); + }); + + afterEach(() => { + data.close(); + rmSync(dir, { recursive: true, force: true }); + }); + + function seedFailedRun(taskKey = "PROJ-1"): number { + const store = new RunStore(dbPath); + const id = store.createRun({ origin: "task", taskKey, harness: "claude-code" }); + store.finishRun(id, "failed", "agent exited non-zero"); + store.close(); + return id; + } + + test("validates the run id and rejects unknown runs", async () => { + const { deps } = stubDeps(); + seedFailedRun(); + + expect((await handleRetryRun(data, "abc", deps)).status).toBe(400); + expect((await handleRetryRun(data, "9999", deps)).status).toBe(404); + }); + + test("spawns devintern TASK --force and returns 202 with audit details", async () => { + const id = seedFailedRun("PROJ-7"); + const { deps, spawned, pids } = stubDeps(); + + const response = await handleRetryRun(data, String(id), deps); + expect(response.status).toBe(202); + const body = response.body as { status: string; taskKey?: string; pid?: number }; + expect(body.status).toBe("triggered"); + expect(body.taskKey).toBe("PROJ-7"); + expect(body.pid).toBe(pids[0]); + expect(spawned).toEqual(["PROJ-7"]); + + // The retry shows up in the run's detail payload. + const detailBody = handleRunDetail(data, String(id)).body as { + retry: { eligible: boolean; audit: { action: string; actor: string; pid?: number }[] }; + }; + expect(detailBody.retry.eligible).toBe(true); + expect(detailBody.retry.audit.length).toBe(1); + expect(detailBody.retry.audit[0].action).toBe("triggered"); + expect(detailBody.retry.audit[0].actor).toBe("sup@example.com"); + expect(detailBody.retry.audit[0].pid).toBe(pids[0]); + }); + + test("rejects non-terminal or ineligible statuses with 409", async () => { + const store = new RunStore(dbPath); + const succeeded = store.createRun({ origin: "task", taskKey: "PROJ-1" }); + store.finishRun(succeeded, "succeeded"); + const deferred = store.createRun({ origin: "task", taskKey: "PROJ-1" }); + store.finishRun(deferred, "deferred"); + const mention = store.createRun({ origin: "pr_mention" }); + store.finishRun(mention, "failed"); // no task key + store.close(); + const { deps } = stubDeps(); + + for (const ineligible of [succeeded, deferred, mention]) { + const response = await handleRetryRun(data, String(ineligible), deps); + expect(response.status).toBe(409); + expect((response.body as { error: string }).error).toContain("not retriable"); + } + }); + + test("requires a signed-in actor", async () => { + const id = seedFailedRun(); + const { deps } = stubDeps({ resolveActor: async () => null }); + + const response = await handleRetryRun(data, String(id), deps); + expect(response.status).toBe(403); + expect((response.body as { error: string }).error).toContain("devintern login"); + }); + + test("honors the support-role allowlist when configured", async () => { + const id = seedFailedRun(); + const stranger = stubDeps({ allowedEmails: ["support-team@example.com"] }); + + const denied = await handleRetryRun(data, String(id), stranger.deps); + expect(denied.status).toBe(403); + + const granted = await handleRetryRun( + data, + String(seedFailedRun()), + stubDeps({ + allowedEmails: ["SUP@Example.com"], + resolveActor: async () => ({ email: "sup@example.com" }), + }).deps, + ); + expect(granted.status).toBe(202); + }); + + test("blocks concurrent retries of the same task", async () => { + const id = seedFailedRun("PROJ-1"); + const first = stubDeps(); + expect((await handleRetryRun(data, String(id), first.deps)).status).toBe(202); + expect(first.spawned).toEqual(["PROJ-1"]); + + // While the previous retry's claim is held (the spawned CLI has not + // recorded its run row yet), another trigger for the same task is refused. + const second = stubDeps(); + const again = await handleRetryRun(data, String(id), second.deps); + expect(again.status).toBe(409); + expect((again.body as { error: string }).error).toContain("just triggered"); + expect(second.spawned).toEqual([]); + }); + + test("claims expire after the TTL so a later retry can proceed", async () => { + // Fresh instance: the claim lives on the data source, not the handler. + const claiming = new DashboardData({ dbPath, workingDir: dir, inflightRetryTtlMs: -1 }); + expect(claiming.claimRetry("PROJ-1")).toBe(true); + // TTL of -1 → the claim is already stale on the next access. + expect(claiming.hasInflightRetry("PROJ-1")).toBe(false); + + const id = seedFailedRun("PROJ-1"); + const { deps, spawned } = stubDeps(); + expect((await handleRetryRun(claiming, String(id), deps)).status).toBe(202); + expect(spawned).toEqual(["PROJ-1"]); + claiming.close(); + }); + + test("a live in-progress run for the task blocks a retry", async () => { + const store = new RunStore(dbPath); + const failedId = store.createRun({ origin: "task", taskKey: "PROJ-LIVE" }); + store.finishRun(failedId, "failed"); + store.createRun({ origin: "task", taskKey: "PROJ-LIVE" }); // stays in_progress + store.close(); + + const { deps } = stubDeps(); + const response = await handleRetryRun(data, String(failedId), deps); + expect(response.status).toBe(409); + expect((response.body as { error: string }).error).toContain("in progress"); + }); + + test("records a failed audit entry when the CLI could not be spawned", async () => { + const id = seedFailedRun("PROJ-1"); + const failing: RetryHandlerDeps = { + resolveActor: ACTOR, + spawn: (): SpawnedRetryProcess => { + throw new Error("entry point missing"); + }, + }; + + const response = await handleRetryRun(data, String(id), failing); + expect(response.status).toBe(500); + expect((response.body as { error: string }).error).toContain("entry point missing"); + + const detailBody = handleRunDetail(data, String(id)).body as { + retry: { audit: { action: string; message?: string }[] }; + }; + expect(detailBody.retry.audit[0].action).toBe("failed"); + expect(detailBody.retry.audit[0].message).toContain("entry point missing"); + }); +}); + +describe("handleRetryRun (schedule mode)", () => { + let dir: string; + let dbPath: string; + let data: DashboardData; + + beforeEach(() => { + dir = join(tmpdir(), `retry-sched-test-${Date.now()}-${Math.random().toString(36).slice(2)}`); + mkdirSync(dir, { recursive: true }); + dbPath = join(dir, "queue.db"); + data = new DashboardData({ dbPath, workingDir: dir, retryMode: "schedule" }); + }); + + afterEach(() => { + data.close(); + rmSync(dir, { recursive: true, force: true }); + }); + + function seedFailedRun(taskKey = "PROJ-1"): number { + const store = new RunStore(dbPath); + const id = store.createRun({ origin: "task", taskKey, harness: "claude-code" }); + store.finishRun(id, "failed", "agent exited non-zero"); + store.close(); + return id; + } + + test("inserts a pending row and returns 202 without spawning", async () => { + const id = seedFailedRun("PROJ-9"); + const { deps, spawned } = stubDeps(); + + const response = await handleRetryRun(data, String(id), deps); + expect(response.status).toBe(202); + const body = response.body as { status: string; taskKey?: string; pid?: number }; + expect(body.status).toBe("scheduled"); + expect(body.taskKey).toBe("PROJ-9"); + expect(body.pid).toBeUndefined(); + expect(spawned).toEqual([]); + + const store = data.getScheduledRetryStore(); + expect(store.hasActive("PROJ-9")).toBe(true); + + const detailBody = handleRunDetail(data, String(id)).body as { + retry: { audit: { action: string; actor: string }[] }; + }; + expect(detailBody.retry.audit[0].action).toBe("scheduled"); + expect(detailBody.retry.audit[0].actor).toBe("sup@example.com"); + }); + + test("refuses a second retry while one is scheduled or running", async () => { + const id = seedFailedRun("PROJ-1"); + const { deps } = stubDeps(); + + expect((await handleRetryRun(data, String(id), deps)).status).toBe(202); + const again = await handleRetryRun(data, String(id), stubDeps().deps); + expect(again.status).toBe(409); + expect((again.body as { error: string }).error).toContain("already scheduled or running"); + }); + + test("unblocks once the worker settled the scheduled retry", async () => { + const id = seedFailedRun("PROJ-1"); + const { deps } = stubDeps(); + expect((await handleRetryRun(data, String(id), deps)).status).toBe(202); + + const store = data.getScheduledRetryStore(); + const row = store.claimNext(); + expect(row?.taskKey).toBe("PROJ-1"); + store.finish(row!.id, "done"); + expect(store.hasActive("PROJ-1")).toBe(false); + + expect((await handleRetryRun(data, String(id), stubDeps().deps)).status).toBe(202); + }); + + test("a live in-progress run for the task blocks a scheduled retry", async () => { + const store = new RunStore(dbPath); + const failedId = store.createRun({ origin: "task", taskKey: "PROJ-LIVE" }); + store.finishRun(failedId, "failed"); + store.createRun({ origin: "task", taskKey: "PROJ-LIVE" }); // stays in_progress + store.close(); + + const response = await handleRetryRun(data, String(failedId), stubDeps().deps); + expect(response.status).toBe(409); + expect((response.body as { error: string }).error).toContain("in progress"); + }); +}); + +describe("ScheduledRetryStore", () => { + let dir: string; + let dbPath: string; + + beforeEach(() => { + dir = join(tmpdir(), `retry-store-test-${Date.now()}-${Math.random().toString(36).slice(2)}`); + mkdirSync(dir, { recursive: true }); + dbPath = join(dir, "queue.db"); + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + test("claims the oldest pending row atomically", () => { + const store = new ScheduledRetryStore(dbPath); + store.schedule({ taskKey: "PROJ-2", actor: "a@x.com", createdAt: 2000 }); + store.schedule({ taskKey: "PROJ-1", actor: "b@x.com", createdAt: 1000 }); + + const first = store.claimNext(); + expect(first?.taskKey).toBe("PROJ-1"); + expect(first?.status).toBe("running"); + // The other row stays pending; a second claimer gets it, not null races. + expect(store.claimNext()?.taskKey).toBe("PROJ-2"); + expect(store.claimNext()).toBeNull(); + store.close(); + }); + + test("requeue returns a claimed row to the pending queue", () => { + const store = new ScheduledRetryStore(dbPath); + store.schedule({ taskKey: "PROJ-1", actor: "a@x.com" }); + + const row = store.claimNext(); + store.requeue(row!.id); + expect(store.hasActive("PROJ-1")).toBe(true); + expect(store.hasPending()).toBe(true); + expect(store.claimNext()?.taskKey).toBe("PROJ-1"); + store.close(); + }); + + test("finish only settles rows still claimed as running", () => { + const store = new ScheduledRetryStore(dbPath); + store.schedule({ taskKey: "PROJ-1", actor: "a@x.com" }); + const row = store.claimNext(); + store.finish(row!.id, "failed", "pipeline failed"); + // A late finish (or double finish) must not resurrect the row. + store.finish(row!.id, "done"); + + expect(store.hasActive("PROJ-1")).toBe(false); + expect(store.hasPending()).toBe(false); + store.close(); + }); + + test("failRunning settles every running row and leaves others alone", () => { + const store = new ScheduledRetryStore(dbPath); + store.schedule({ taskKey: "PROJ-1", actor: "a@x.com" }); + store.schedule({ taskKey: "PROJ-2", actor: "b@x.com" }); + const running = store.claimNext(); // PROJ-1 + store.claimNext(); // PROJ-2 + + store.finish(running!.id, "done"); + store.schedule({ taskKey: "PROJ-3", actor: "c@x.com" }); // stays pending + + const settled = store.failRunning("worker restarted"); + expect(settled.map((row) => row.taskKey)).toEqual(["PROJ-2"]); + expect(store.hasActive("PROJ-2")).toBe(false); + expect(store.hasActive("PROJ-1")).toBe(false); + expect(store.hasPending()).toBe(true); // PROJ-3 untouched + expect(store.claimNext()?.taskKey).toBe("PROJ-3"); + store.close(); + }); + + test("refuses a duplicate while active but allows re-schedule after finish", () => { + const store = new ScheduledRetryStore(dbPath); + expect(store.schedule({ taskKey: "PROJ-1", actor: "a@x.com" }).scheduled).toBe(true); + expect(store.schedule({ taskKey: "PROJ-1", actor: "b@x.com" })).toEqual({ + scheduled: false, + reason: "a retry is already scheduled or running", + }); + + store.finish(store.claimNext()!.id, "done"); + expect(store.schedule({ taskKey: "PROJ-1", actor: "b@x.com" }).scheduled).toBe(true); + store.close(); + }); +}); + +describe("resolveAllowedRetryEmails", () => { + test("parses a comma-separated allowlist case-insensitively", () => { + expect(resolveAllowedRetryEmails({ DASHBOARD_RETRY_EMAILS: " A@x.com ,b@y.io," })).toEqual([ + "a@x.com", + "b@y.io", + ]); + expect(resolveAllowedRetryEmails({})).toEqual([]); + }); +}); diff --git a/packages/code/tests/retry-queue-acquirer.test.ts b/packages/code/tests/retry-queue-acquirer.test.ts new file mode 100644 index 0000000..bc0d1b3 --- /dev/null +++ b/packages/code/tests/retry-queue-acquirer.test.ts @@ -0,0 +1,122 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdirSync, rmSync } from "fs"; +import { join } from "path"; +import { tmpdir } from "os"; + +import { ScheduledRetryStore } from "../src/lib/run-retry"; +import { RetryQueueAcquirer } from "../src/lib/workspace/retry-acquirer"; +import type { TaskExecutionResult } from "../src/lib/task-polling-acquirer"; + +describe("RetryQueueAcquirer", () => { + let dir: string; + let dbPath: string; + let store: ScheduledRetryStore; + + beforeEach(() => { + dir = join( + tmpdir(), + `retry-acquirer-test-${Date.now()}-${Math.random().toString(36).slice(2)}`, + ); + mkdirSync(dir, { recursive: true }); + dbPath = join(dir, "queue.db"); + store = new ScheduledRetryStore(dbPath); + }); + + afterEach(() => { + store.close(); + rmSync(dir, { recursive: true, force: true }); + }); + + function makeAcquirer( + behavior: (taskKey: string) => Promise | TaskExecutionResult, + calls: string[], + ): RetryQueueAcquirer { + return new RetryQueueAcquirer({ + store, + execute: async (taskKey) => { + calls.push(taskKey); + return await behavior(taskKey); + }, + intervalSeconds: 3600, // ticks are driven manually in tests + }); + } + + test("drains pending retries through the executor and marks them done", async () => { + store.schedule({ taskKey: "PROJ-1", actor: "a@x.com" }); + store.schedule({ taskKey: "PROJ-2", actor: "b@x.com" }); + + const calls: string[] = []; + const acquirer = makeAcquirer(() => true, calls); + await acquirer.tick(); + + expect(calls).toEqual(["PROJ-1", "PROJ-2"]); + expect(store.hasActive("PROJ-1")).toBe(false); + expect(store.hasActive("PROJ-2")).toBe(false); + expect(store.hasPending()).toBe(false); + }); + + test("marks failed rows with a message when the pipeline fails", async () => { + store.schedule({ taskKey: "PROJ-1", actor: "a@x.com" }); + + const calls: string[] = []; + const acquirer = makeAcquirer(() => false, calls); + await acquirer.tick(); + + expect(calls).toEqual(["PROJ-1"]); + expect(store.hasActive("PROJ-1")).toBe(false); + }); + + test("requeues deferred rows for the next tick", async () => { + store.schedule({ taskKey: "PROJ-1", actor: "a@x.com" }); + + const calls: string[] = []; + let busy = true; + const acquirer = makeAcquirer(() => (busy ? "deferred" : true), calls); + + await acquirer.tick(); + expect(calls).toEqual(["PROJ-1"]); + expect(store.hasPending()).toBe(true); + + busy = false; + await acquirer.tick(); + expect(calls).toEqual(["PROJ-1", "PROJ-1"]); + expect(store.hasPending()).toBe(false); + }); + + test("records the error message when the executor throws", async () => { + store.schedule({ taskKey: "PROJ-1", actor: "a@x.com" }); + + const acquirer = makeAcquirer(() => { + throw new Error("boom"); + }, []); + await acquirer.tick(); + + expect(store.hasActive("PROJ-1")).toBe(false); + }); + + test("a tick while draining is a no-op (busy guard)", async () => { + store.schedule({ taskKey: "PROJ-1", actor: "a@x.com" }); + + const calls: string[] = []; + let release: () => void = () => {}; + const gate = new Promise((resolve) => { + release = resolve; + }); + const acquirer = new RetryQueueAcquirer({ + store, + execute: async (taskKey) => { + calls.push(taskKey); + await gate; + return true; + }, + intervalSeconds: 3600, + }); + + const first = acquirer.tick(); + await acquirer.tick(); // blocked by the busy guard, must not claim + release(); + await first; + + expect(calls).toEqual(["PROJ-1"]); + }); +}); diff --git a/packages/code/tests/workspace-orphan-recovery.test.ts b/packages/code/tests/workspace-orphan-recovery.test.ts index 6590b2f..5d030e6 100644 --- a/packages/code/tests/workspace-orphan-recovery.test.ts +++ b/packages/code/tests/workspace-orphan-recovery.test.ts @@ -5,6 +5,7 @@ import { tmpdir } from "os"; import { recoverOrphanedWorkspaceRuns } from "../src/lib/workspace/workspace-worker"; import { RunStore } from "../src/lib/run-recorder"; +import { ScheduledRetryStore } from "../src/lib/run-retry"; import { BASE_WORKTREE_NAME } from "../src/lib/workspace/repo-manager"; import type { WorkspaceConfig } from "../src/lib/workspace/config"; @@ -85,4 +86,33 @@ describe("recoverOrphanedWorkspaceRuns", () => { expect(store.getRun(done)?.status).toBe("succeeded"); expect(store.getRun(done)?.outcomeReason).toBeUndefined(); }); + + test("settles scheduled retries left running by the previous worker", async () => { + process.env.TASK_TRACKER = "jira"; + delete process.env.JIRA_BASE_URL; + + const retryStore = new ScheduledRetryStore(dbPath); + retryStore.schedule({ taskKey: "PROJ-1", actor: "sup@example.com" }); + retryStore.schedule({ taskKey: "PROJ-2", actor: "sup@example.com" }); + retryStore.claimNext(); // PROJ-1 → running + retryStore.claimNext(); // PROJ-2 → running + + await recoverOrphanedWorkspaceRuns({ + config: workspaceConfig(), + workspaceDir, + dbPath, + }); + + // Both rows were settled, so the dashboard's per-task guard unblocks and + // the operator can schedule again. + expect(retryStore.hasActive("PROJ-1")).toBe(false); + expect(retryStore.hasActive("PROJ-2")).toBe(false); + expect(retryStore.hasPending()).toBe(false); + + // Sanity: a fresh schedule is accepted after recovery. + expect(retryStore.schedule({ taskKey: "PROJ-1", actor: "sup@example.com" }).scheduled).toBe( + true, + ); + retryStore.close(); + }); }); diff --git a/packages/dashboard-ui/src/lib/api.ts b/packages/dashboard-ui/src/lib/api.ts index b45b8ee..2301475 100644 --- a/packages/dashboard-ui/src/lib/api.ts +++ b/packages/dashboard-ui/src/lib/api.ts @@ -62,6 +62,52 @@ export interface RunsResponse { export interface RunDetailResponse { run: RunRecord; stages: RunStageRecord[]; + /** Retry action metadata for this run (eligibility + audit trail). */ + retry: RetryInfo; +} + +/** One audited dashboard retry of a run. */ +export interface RetryAuditEntry { + id: number; + runId: number; + taskKey?: string; + actor: string; + action: "triggered" | "scheduled" | "failed"; + command?: string; + pid?: number; + message?: string; + createdAt: number; +} + +/** Retry metadata embedded in a run-detail response. */ +export interface RetryInfo { + eligible: boolean; + reason?: string; + /** Recent dashboard retries of this run, most recent first. */ + audit: RetryAuditEntry[]; +} + +/** + * Trigger a retry of a run: the fleet worker drains it through the normal + * pipeline (`schedule` mode) or the dashboard spawns `devintern + * --force` (`triggered`, standalone dashboard). Resolves with the parsed JSON + * body regardless of status; callers should branch on `response.ok`. + */ +export async function triggerRunRetry(runId: number): Promise<{ + ok: boolean; + body: { error?: string; status?: string; command?: string; pid?: number }; +}> { + const response = await fetch(`/api/runs/${runId}/retry`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + }); + const body = (await response.json().catch(() => ({}))) as { + error?: string; + status?: string; + command?: string; + pid?: number; + }; + return { ok: response.ok, body }; } export interface StatsResponse { diff --git a/packages/dashboard-ui/src/views/RunDetailView.tsx b/packages/dashboard-ui/src/views/RunDetailView.tsx index ff54fab..34bc964 100644 --- a/packages/dashboard-ui/src/views/RunDetailView.tsx +++ b/packages/dashboard-ui/src/views/RunDetailView.tsx @@ -1,5 +1,5 @@ import { useState } from "react"; -import { ArrowLeft, ChevronDown, ChevronRight } from "lucide-react"; +import { ArrowLeft, ChevronDown, ChevronRight, RefreshCcw } from "lucide-react"; import { RunResult } from "@/components/RunResult"; import { EmptyState, StageBadge, StatusBadge } from "@/components/shared"; @@ -9,11 +9,11 @@ import { TicketKey } from "@/components/TicketKey"; import { Button } from "@/components/ui/button"; import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; import { Markdown } from "@/lib/markdown"; -import { usePoll } from "@/lib/api"; -import type { RunDetailResponse, RunStageRecord } from "@/lib/api"; +import { triggerRunRetry, usePoll } from "@/lib/api"; +import type { RetryAuditEntry, RunDetailResponse, RunStageRecord } from "@/lib/api"; import { formatRunOrigin } from "@/lib/run-origin"; import { parseStageDetail } from "@/lib/stage-detail"; -import { formatDuration, formatTime } from "@/lib/utils"; +import { cn, formatDuration, formatTime } from "@/lib/utils"; const STAGE_LABELS: Record = { feasibility: "Feasibility check", @@ -101,9 +101,125 @@ function MetaItem({ label, value }: { label: string; value: React.ReactNode }) { ); } +/** Outcome banner after a retry attempt (success and failure are terminal). */ +function RetryFeedback({ kind, message }: { kind: "success" | "error"; message: string }) { + return ( +
+ {message} +
+ ); +} + +/** Inline confirmation strip shown between the first click and the POST. */ +function RetryConfirm({ + taskKey, + busy, + onCancel, + onConfirm, +}: { + taskKey?: string; + busy: boolean; + onCancel: () => void; + onConfirm: () => void; +}) { + return ( +
+

+ Schedule a fresh run of{" "} + {taskKey ? {taskKey} : "this task"} with{" "} + --force? The worker picks it up through its normal + pipeline (routing, worktree, per-repo environment) and skips the retry gate. +

+
+ + +
+
+ ); +} + +/** Audited retry history: who re-ran this run's task, when, and how. */ +function RetryAuditList({ entries }: { entries: RetryAuditEntry[] }) { + if (entries.length === 0) { + return null; + } + return ( + + Retry history + + {entries.map((entry) => ( +
+ + {entry.action} + + by {entry.actor} + + {formatTime(entry.createdAt)} + + {entry.pid !== undefined ? ( + pid {entry.pid} + ) : null} + {entry.message ? ( + {entry.message} + ) : null} +
+ ))} +
+
+ ); +} + /** One run: metadata card plus a stage-by-stage timeline. */ export function RunDetailView({ runId, onBack }: { runId: number; onBack: () => void }) { - const { data, error, loading } = usePoll(`/api/runs/${runId}`); + const { data, error, loading, refresh } = usePoll(`/api/runs/${runId}`); + const [confirming, setConfirming] = useState(false); + const [posting, setPosting] = useState(false); + const [feedback, setFeedback] = useState<{ kind: "success" | "error"; message: string } | null>( + null, + ); + + async function handleRetry() { + if (!data) { + return; + } + setPosting(true); + try { + const { ok, body } = await triggerRunRetry(data.run.id); + if (ok) { + const scheduled = body.status === "scheduled"; + const pidSuffix = body.pid !== undefined ? ` (pid ${body.pid})` : ""; + setFeedback({ + kind: "success", + message: scheduled + ? `Retry scheduled. The worker will start a fresh run for ${data.run.taskKey} shortly.` + : `Retry triggered${pidSuffix}. A fresh run for ${data.run.taskKey} will appear in the run list shortly.`, + }); + setConfirming(false); + refresh(); + } else { + setFeedback({ kind: "error", message: body.error ?? "Could not trigger the retry." }); + } + } catch (cause: unknown) { + setFeedback({ + kind: "error", + message: cause instanceof Error ? cause.message : String(cause), + }); + } finally { + setPosting(false); + } + } return (
@@ -131,9 +247,41 @@ export function RunDetailView({ runId, onBack }: { runId: number; onBack: () => /> + {data.retry.eligible && !confirming ? ( + + ) : null}
+ {!data.retry.eligible && data.retry.reason ? ( +

+ Not retriable: {data.retry.reason} +

+ ) : null} + {confirming ? ( +
+ setConfirming(false)} + onConfirm={() => void handleRetry()} + /> +
+ ) : null} + {feedback ? ( +
+ +
+ ) : null} {data.run.outcomeReason ? ( -

{data.run.outcomeReason}

+

{data.run.outcomeReason}

) : null} @@ -175,6 +323,8 @@ export function RunDetailView({ runId, onBack }: { runId: number; onBack: () => )} + +
{data.stages.map((stage, index) => (