From aac9c68178ad4d6283886b6b491509faeecba7c1 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Mon, 12 Oct 2026 18:16:36 +0200 Subject: [PATCH 01/18] feat(taxonomy): restack readout handler on DAG foundation Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 1cbbd5eb-61c6-4e6a-bb38-c377be5b2163 --- AGENTS.md | 2 + ENV_VARIABLES.md | 28 ++ .../openapi-snapshot.test.ts.snap | 63 +++++ apps/api/src/archive-har.test.ts | 42 ++- apps/api/src/archive-har.ts | 29 +- apps/api/src/index.ts | 2 + apps/api/src/routes/requests/taxonomy.test.ts | 107 ++++++++ apps/api/src/routes/requests/taxonomy.ts | 30 +++ apps/portal/src/components/TaxonomyTab.tsx | 248 ++++++++++++++++++ apps/portal/src/lib/api.ts | 10 + apps/portal/src/pages/RunDetail.tsx | 10 + apps/portal/src/types.ts | 1 + apps/scheduler/src/handler-dispatcher.test.ts | 22 +- apps/scheduler/src/notify-routes.test.ts | 8 +- apps/workers/taxonomy/Dockerfile | 50 ++++ apps/workers/taxonomy/handler.yaml | 11 + apps/workers/taxonomy/package.json | 24 ++ apps/workers/taxonomy/src/cli.ts | 206 +++++++++++++++ apps/workers/taxonomy/src/index.ts | 30 +++ apps/workers/taxonomy/src/prompt.ts | 88 +++++++ .../taxonomy/src/taxonomy-generator.ts | 202 ++++++++++++++ .../taxonomy/src/taxonomy-queue-processor.ts | 190 ++++++++++++++ apps/workers/taxonomy/src/tools.ts | 154 +++++++++++ apps/workers/taxonomy/tsconfig.json | 8 + docker-compose.dev.yml | 16 ++ docker-compose.yml | 46 ++++ docs/architecture/db-migrations.md | 2 +- docs/architecture/post-processing.md | 85 ++++-- package.json | 5 +- .../031-migrate-services-to-handler-dag.ts | 2 +- .../032-add-handler-status-index.ts | 8 + packages/shared/src/index.ts | 1 + packages/shared/src/schemas/index.ts | 1 + packages/shared/src/schemas/request.ts | 1 + packages/shared/src/schemas/taxonomy.test.ts | 180 +++++++++++++ packages/shared/src/schemas/taxonomy.ts | 104 ++++++++ .../src/scripts/register-handler.test.ts | 6 +- packages/shared/src/types/types.ts | 4 +- pnpm-lock.yaml | 40 ++- 39 files changed, 2025 insertions(+), 41 deletions(-) create mode 100644 apps/api/src/routes/requests/taxonomy.test.ts create mode 100644 apps/api/src/routes/requests/taxonomy.ts create mode 100644 apps/portal/src/components/TaxonomyTab.tsx create mode 100644 apps/workers/taxonomy/Dockerfile create mode 100644 apps/workers/taxonomy/handler.yaml create mode 100644 apps/workers/taxonomy/package.json create mode 100644 apps/workers/taxonomy/src/cli.ts create mode 100644 apps/workers/taxonomy/src/index.ts create mode 100644 apps/workers/taxonomy/src/prompt.ts create mode 100644 apps/workers/taxonomy/src/taxonomy-generator.ts create mode 100644 apps/workers/taxonomy/src/taxonomy-queue-processor.ts create mode 100644 apps/workers/taxonomy/src/tools.ts create mode 100644 apps/workers/taxonomy/tsconfig.json create mode 100644 packages/shared/src/schemas/taxonomy.test.ts create mode 100644 packages/shared/src/schemas/taxonomy.ts diff --git a/AGENTS.md b/AGENTS.md index 119d758fc..c3126c8a2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,6 +19,7 @@ apps/ coder-acp-copilot/ # GitHub Copilot worker (ACP SDK) coder-acp-claude-code/ # Claude Code worker (ACP SDK) report-generator/ # Post-run report generation + taxonomy/ # Post-run taxonomy generation model-scanners/ # Feature detection for Copilot and Anthropic models version-checkers/ # Poll for new agent/tool releases key-updaters/ # GitHub auth cookie management for VS Code Web @@ -69,6 +70,7 @@ Each worker implements the same queue-processor interface but adapts a different | `coder-acp-copilot` | GitHub Copilot | ACP SDK v0.14.1 | | `coder-acp-claude-code` | Claude Code | ACP SDK v0.13.1 | | `report-generator` | — | Copilot SDK | +| `taxonomy` | — | Copilot SDK | Workers consume tasks from Azure Storage Queues (named `queue-`) and write results to MongoDB and Blob Storage. Each has its own Dockerfile and docker-compose profile. diff --git a/ENV_VARIABLES.md b/ENV_VARIABLES.md index 4f20cc04b..503f16ff3 100644 --- a/ENV_VARIABLES.md +++ b/ENV_VARIABLES.md @@ -889,6 +889,34 @@ Timeout for the Copilot SDK session used by the report-generator worker. If the Git commit hash embedded in reporter metadata. Automatically set during CI/CD builds. Used to track which version of the report-generator produced a given report. +## Taxonomy Handler Configuration + +### AZURE_STORAGE_QUEUE_TAXONOMY +**Default:** `pp-taxonomy-queue` +**Type:** string + +Azure Storage Queue name for taxonomy-generation jobs. The scheduler enqueues `{ type: "taxonomy", requestId, runId }` messages here after the `pp-atif` dependency completes, and the taxonomy worker polls this queue. + +### TAXONOMY_MODEL +**Default:** `gpt-5.4` +**Type:** string + +The LLM model used by the taxonomy worker (via the Copilot SDK) to generate structured taxonomy JSON from run metadata and ATIF trajectories. + +> **Model choice matters for taxonomy quality.** A/B testing across nine models on a multi-iteration run with a real defect (agent lost context in iteration 2) showed the verdict ranged from 72 to 100 depending on the model. `gpt-4.1` (the previous default) missed action items and, on harder runs, produced the thinnest output; weaker models rubber-stamped a buggy run with a perfect score. `gpt-5.4` was the best value: it correctly caught the critical defect while using the fewest tokens and lowest cost of the models tested. Stronger alternatives (e.g. `claude-opus-4-7`) yield richer evidence at higher token cost; avoid `gpt-5.3-codex` as a default (≈1M tokens / ~7 min per run). + +### SCOPE_MT_API_URL +**Default:** `http://localhost:3001` (local), `http://api:80` (Docker) +**Type:** URL string + +Base URL of the Scope API. The taxonomy worker calls this to fetch the request document and active run state through its Copilot tools. + +### SESSION_TIMEOUT_MS +**Default:** `300000` (5 minutes) +**Type:** integer (milliseconds) + +Timeout for the Copilot SDK session used by the taxonomy worker. If the model does not finish generating a valid taxonomy JSON document within this window, the worker marks `run.handlerStatus.pp-taxonomy` as failed. + ## Scheduler Configuration ### SCHEDULER_POLL_INTERVAL_MS diff --git a/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap b/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap index 2b987b988..92da77346 100644 --- a/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap +++ b/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap @@ -4011,6 +4011,9 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` ], "type": "string", }, + "taxonomyUrl": { + "type": "string", + }, "tokenUsage": { "$ref": "#/components/schemas/TokenUsage", }, @@ -11441,6 +11444,40 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` ], }, }, + "/api/v1/requests/{id}/runs/{runId}/taxonomy": { + "get": { + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "type": "string", + }, + }, + { + "in": "path", + "name": "runId", + "required": true, + "schema": { + "type": "string", + }, + }, + ], + "responses": { + "200": { + "description": "Taxonomy v1 JSON document (scorecard, behavior analysis, action list)", + }, + "404": { + "description": "Not found — taxonomy not yet generated", + }, + }, + "summary": "Download taxonomy JSON for a specific attempt", + "tags": [ + "Requests", + ], + }, + }, "/api/v1/requests/{id}/runs/{runId}/tool-calls": { "get": { "parameters": [ @@ -11543,6 +11580,32 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` ], }, }, + "/api/v1/requests/{id}/taxonomy": { + "get": { + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "type": "string", + }, + }, + ], + "responses": { + "200": { + "description": "Taxonomy v1 JSON document (scorecard, behavior analysis, action list)", + }, + "404": { + "description": "Not found — taxonomy not yet generated", + }, + }, + "summary": "Download taxonomy JSON for the run", + "tags": [ + "Requests", + ], + }, + }, "/api/v1/requests/{id}/tool-calls": { "get": { "parameters": [ diff --git a/apps/api/src/archive-har.test.ts b/apps/api/src/archive-har.test.ts index 9d01fc261..af8205f2b 100644 --- a/apps/api/src/archive-har.test.ts +++ b/apps/api/src/archive-har.test.ts @@ -99,6 +99,17 @@ describe("rewriteHarUrlsForArchive", () => { expect(result.run!.turns![1].toolCallsUrl).toBe("iteration-2.tool-calls.jsonl"); expect(result.run!.turns![2].toolCallsUrl).toBeUndefined(); }); + + it("rewrites run.taxonomyUrl to taxonomy.json", () => { + const resource = { + run: { + taxonomyUrl: "https://storage.blob.core.windows.net/snapshots/abc123/runs/attempt-1/taxonomy.json", + turns: [], + }, + }; + const result = rewriteHarUrlsForArchive(resource); + expect(result.run!.taxonomyUrl).toBe("taxonomy.json"); + }); }); // --- blobNameFromSnapshotsUrl --- @@ -902,7 +913,36 @@ describe("packRunIntoTar", () => { await packRunIntoTar(p, run, container, "run-011", () => true); p.finalize(); await entriesPromise; - expect(run.codebase!.archiveUrl).toBe(blobUrl); }); + + it("bundles run-level taxonomy.json and rewrites taxonomyUrl", async () => { + const { pack } = await import("tar-stream"); + const p = pack(); + const taxonomyData = Buffer.from('{"topics":["agent"]}'); + const container = makeMockBlobContainer({ + "run-012/runs/attempt-1/taxonomy.json": { body: taxonomyData, length: taxonomyData.length }, + }); + + const run: ArchivableRun = { + _id: "run-012", + run: { + taxonomyUrl: "https://storage.blob.core.windows.net/snapshots/run-012/runs/attempt-1/taxonomy.json", + turns: [], + }, + }; + + const entriesPromise = collectPackEntries(p); + await packRunIntoTar(p, run, container, "run-012", () => true); + p.finalize(); + + const entries = await entriesPromise; + const names = entries.map(e => e.name); + expect(names).toContain("run-012/taxonomy.json"); + expect(entries.find(e => e.name === "run-012/taxonomy.json")!.data).toEqual(taxonomyData); + + const yaml = entries.find(e => e.name === "run-012/run.yaml")!.data.toString(); + expect(yaml).toContain("taxonomy.json"); + expect(yaml).not.toContain("blob.core.windows.net"); + }); }); diff --git a/apps/api/src/archive-har.ts b/apps/api/src/archive-har.ts index 204eaf124..c28658aea 100644 --- a/apps/api/src/archive-har.ts +++ b/apps/api/src/archive-har.ts @@ -41,11 +41,12 @@ export function blobNameFromLogsUrl(url: string): string | null { /** * Deep-clone a run resource and rewrite `harUrl`, `rawChatUrl`, - * `chatResultUrl`, and `toolCallsUrl` fields to relative archive paths. + * `taxonomyUrl`, `chatResultUrl`, and `toolCallsUrl` fields to relative archive paths. * * - `run.harUrl` → `"run.har"` * - Per-turn `harUrl` → `"iteration-{N}.har"` * - `run.rawChatUrl` → `"run.chat-export.json"` + * - `run.taxonomyUrl` → `"taxonomy.json"` * - Per-turn `rawChatUrl` → `"iteration-{N}.chat-export.json"` * - Per-turn `chatResultUrl` → `"iteration-{N}.chat-result.json"` * - Per-turn `toolCallsUrl` → `"iteration-{N}.tool-calls.jsonl"` @@ -54,6 +55,7 @@ export function rewriteHarUrlsForArchive; [key: string]: unknown; }; @@ -66,6 +68,9 @@ export function rewriteHarUrlsForArchive ({ + checkMigrations: vi.fn().mockResolvedValue({ ready: true, applied: ["001", "002"], pending: [] }), +})); + +vi.hoisted(() => { + process.env.STORAGE_CONNECTION_STRING = "DefaultEndpointsProtocol=https;AccountName=mockaccount;AccountKey=bW9jaw==;EndpointSuffix=core.windows.net"; +}); + +vi.mock("../../llm.js", () => ({ isLlmAvailable: vi.fn().mockReturnValue(false) })); +vi.mock("../../prompt-feature-llm.js", () => ({ isLlmAvailable: vi.fn().mockReturnValue(false) })); +vi.mock("../../task-prompt-llm.js", () => ({ isTaskPromptLlmAvailable: vi.fn().mockReturnValue(false) })); + +const { mockDownload, mockGetProperties, mockGetBlockBlobClient, mockGetBlobClient, mockGetContainerClient } = vi.hoisted(() => { + const mockDownload = vi.fn(); + const mockGetProperties = vi.fn(); + const mockGetBlockBlobClient = vi.fn().mockReturnValue({ download: mockDownload, getProperties: mockGetProperties }); + const mockGetBlobClient = vi.fn().mockReturnValue({ download: mockDownload, getProperties: mockGetProperties }); + const mockGetContainerClient = vi.fn().mockReturnValue({ getBlockBlobClient: mockGetBlockBlobClient, getBlobClient: mockGetBlobClient }); + return { mockDownload, mockGetProperties, mockGetBlockBlobClient, mockGetBlobClient, mockGetContainerClient }; +}); + +vi.mock("@azure/storage-blob", async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + BlobServiceClient: { fromConnectionString: vi.fn().mockReturnValue({ getContainerClient: mockGetContainerClient }) }, + }; +}); + +describe("taxonomy endpoints", () => { + let mocks: ReturnType; + + beforeAll(() => { + mocks = createAllMockDependencies(); + _injectTestDependencies(mocks); + }); + + beforeEach(() => { + vi.clearAllMocks(); + mocks = createAllMockDependencies(); + _injectTestDependencies(mocks); + rewireBlobMocks({ mockDownload, mockGetProperties, mockGetBlockBlobClient, mockGetBlobClient, mockGetContainerClient }); + mockDownload.mockResolvedValue({ readableStreamBody: readableFrom('{"status":"ok"}'), contentLength: 15 }); + mockGetProperties.mockResolvedValue({ contentLength: 15 }); + }); + + describe.each(VARIANTS)("$label", (variant) => { + const taxonomyUrl = (id: string) => variant.url(id) + "taxonomy"; + + it("returns 404 when request not found", async () => { + (mocks.collection.findOne as ReturnType).mockResolvedValue(null); + const res = await supertest(app).get(taxonomyUrl("missing")); + expect(res.status).toBe(404); + expect(res.body).toMatchObject({ error: "Request not found" }); + }); + + it("returns 404 when taxonomyUrl is not set", async () => { + variant.mockRequest(mocks, "req-1", { _id: "run-1", attemptNumber: 1, status: "done" }); + const res = await supertest(app).get(taxonomyUrl("req-1")); + expect(res.status).toBe(404); + expect(res.body).toMatchObject({ error: "Taxonomy not yet generated for this run" }); + }); + + it("returns taxonomy JSON when taxonomyUrl is set", async () => { + variant.mockRequest(mocks, "req-1", { + _id: "run-1", + attemptNumber: 1, + status: "done", + taxonomyUrl: blobUrl("req-1/taxonomy.json"), + }); + + const res = await supertest(app).get(taxonomyUrl("req-1")).buffer(true).parse(rawParser); + expect(res.status).toBe(200); + expect(res.headers["content-type"]).toMatch("application/json"); + expect(res.headers["content-disposition"]).toContain("req-1-taxonomy.json"); + expect(mockGetBlockBlobClient).toHaveBeenCalledWith("req-1/taxonomy.json"); + }); + }); + + it("works with per-run variant", async () => { + RUN_LEVEL.mockRequest(mocks, "req-1", { + _id: "run-1", + attemptNumber: 1, + status: "done", + taxonomyUrl: blobUrl("req-1/runs/run-1/taxonomy.json"), + }); + + const res = await supertest(app).get(RUN_LEVEL.url("req-1") + "taxonomy").buffer(true).parse(rawParser); + expect(res.status).toBe(200); + expect(res.headers["content-disposition"]).toContain("req-1-taxonomy.json"); + expect(mockGetBlockBlobClient).toHaveBeenCalledWith("req-1/runs/run-1/taxonomy.json"); + }); +}); diff --git a/apps/api/src/routes/requests/taxonomy.ts b/apps/api/src/routes/requests/taxonomy.ts new file mode 100644 index 000000000..e74d303f6 --- /dev/null +++ b/apps/api/src/routes/requests/taxonomy.ts @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { RouteContext } from "../../route-context.js"; +import { downloadBlobToResponse } from "./blob-helpers.js"; +import { registerArtifactRoutes } from "./artifact-route.js"; + +export function registerRequestsTaxonomyRoutes(ctx: RouteContext): void { + registerArtifactRoutes(ctx, { + path: "taxonomy", + summary: "Download taxonomy JSON for the run", + perRunSummary: "Download taxonomy JSON for a specific attempt", + responseDescription: "Taxonomy v1 JSON document (scorecard, behavior analysis, action list)", + errorResponses: { 404: { description: "Not found — taxonomy not yet generated" } }, + blobNotFoundMessage: "Taxonomy file not found — the blob may have been deleted or is no longer available", + handler: async (ctx, _req, res, targetRun, id) => { + const taxonomyUrl = targetRun.taxonomyUrl; + if (!taxonomyUrl) { + res.status(404).json({ error: "Taxonomy not yet generated for this run" }); + return; + } + + await downloadBlobToResponse(ctx, res, taxonomyUrl, { + contentType: "application/json", + filename: `${id}-taxonomy.json`, + label: "Taxonomy", + }); + }, + }); +} diff --git a/apps/portal/src/components/TaxonomyTab.tsx b/apps/portal/src/components/TaxonomyTab.tsx new file mode 100644 index 000000000..e97c8c8c4 --- /dev/null +++ b/apps/portal/src/components/TaxonomyTab.tsx @@ -0,0 +1,248 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { useQuery } from "@tanstack/react-query"; +import { api } from "@/lib/api"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Loader2, Download, ChevronDown, ChevronRight } from "lucide-react"; +import { useState } from "react"; + +interface TaxonomyDimension { + name: string; + score: number; + maxScore: number; + justification: string; +} + +interface BehaviorPattern { + id: string; + category: string; + observation: string; + evidence: string[]; + impact: "positive" | "negative" | "neutral"; +} + +interface ActionItem { + id: string; + priority: "high" | "medium" | "low"; + title: string; + description: string; + category: string; +} + +interface TaxonomyData { + meta?: { + schemaVersion?: string; + generatedAt?: string; + requestId?: string; + runId?: string; + model?: string; + }; + scorecard?: { + overall?: number; + dimensions?: TaxonomyDimension[]; + }; + behaviorAnalysis?: { + patterns?: BehaviorPattern[]; + }; + actionList?: { + items?: ActionItem[]; + }; +} + +function ScoreBar({ score, maxScore }: { score: number; maxScore: number }) { + const pct = Math.round((score / maxScore) * 100); + const color = pct >= 80 ? "bg-green-500" : pct >= 50 ? "bg-yellow-500" : "bg-red-500"; + return ( +
+
+
+
+ {score}/{maxScore} +
+ ); +} + +function ImpactBadge({ impact }: { impact: string }) { + const variant = impact === "positive" ? "default" : impact === "negative" ? "destructive" : "secondary"; + return {impact}; +} + +function PriorityBadge({ priority }: { priority: string }) { + const colors: Record = { + high: "bg-red-100 text-red-800 dark:bg-red-900/30 dark:text-red-300", + medium: "bg-yellow-100 text-yellow-800 dark:bg-yellow-900/30 dark:text-yellow-300", + low: "bg-blue-100 text-blue-800 dark:bg-blue-900/30 dark:text-blue-300", + }; + return {priority}; +} + +function CollapsibleSection({ title, defaultOpen = true, children }: { title: string; defaultOpen?: boolean; children: React.ReactNode }) { + const [open, setOpen] = useState(defaultOpen); + return ( +
+ + {open &&
{children}
} +
+ ); +} + +export function TaxonomyTab({ runId, attemptRunId }: { runId: string; attemptRunId?: string }) { + const url = attemptRunId ? api.runTaxonomyUrl(runId, attemptRunId) : api.taxonomyUrl(runId); + + const { data: taxonomy, isLoading, error } = useQuery({ + queryKey: ["run-taxonomy", runId, attemptRunId], + queryFn: async () => { + const res = await fetch(url); + if (!res.ok) { + if (res.status === 404) return null; + throw new Error(`Failed to fetch taxonomy: ${res.statusText}`); + } + return res.json(); + }, + }); + + if (isLoading) { + return ( +
+ + Loading taxonomy… +
+ ); + } + + if (error) { + return ( +
+

Failed to load taxonomy: {(error as Error).message}

+
+ ); + } + + if (!taxonomy) { + return ( +
+

Taxonomy not yet generated for this run.

+

It will appear here once post-processing completes.

+
+ ); + } + + const { meta, scorecard, behaviorAnalysis, actionList } = taxonomy; + + return ( +
+ {/* Header with download */} +
+
+

Run Taxonomy

+ {meta?.generatedAt && ( +

+ Generated {new Date(meta.generatedAt).toLocaleString()} + {meta.model && <> · Model: {meta.model}} +

+ )} +
+ +
+ + {/* Scorecard */} + {scorecard && ( + + + + Scorecard + {scorecard.overall !== undefined && ( + {scorecard.overall}% + )} + + + + {scorecard.dimensions?.map((dim) => ( +
+
+ {dim.name} +
+ +

{dim.justification}

+
+ ))} + + + )} + + {/* Behavior Analysis */} + {behaviorAnalysis?.patterns && behaviorAnalysis.patterns.length > 0 && ( + + + + Behavior Analysis ({behaviorAnalysis.patterns.length}) + + + + +
+ {behaviorAnalysis.patterns.map((pattern) => ( +
+
+ {pattern.category} + +
+

{pattern.observation}

+ {pattern.evidence.length > 0 && ( +
    + {pattern.evidence.map((e, i) =>
  • {e}
    • )} +
+ )} +
+ ))} +
+ + + + )} + + {/* Action List */} + {actionList?.items && actionList.items.length > 0 && ( + + + + Action Items ({actionList.items.length}) + + + + +
+ {actionList.items.map((item) => ( +
+ +
+

{item.title}

+

{item.description}

+ {item.category} +
+
+ ))} +
+ + + + )} + + {/* Raw JSON fallback — collapsed by default */} + + 
+          {JSON.stringify(taxonomy, null, 2)}
+
+ +
+ ); +} diff --git a/apps/portal/src/lib/api.ts b/apps/portal/src/lib/api.ts index 7f835054c..f264e4a9d 100644 --- a/apps/portal/src/lib/api.ts +++ b/apps/portal/src/lib/api.ts @@ -503,6 +503,16 @@ export const api = { return `${BASE}/requests/${id}/atif${qs}`; }, + /** Get taxonomy JSON URL for a request (latest run) */ + taxonomyUrl: (id: string): string => { + return `${BASE}/requests/${id}/taxonomy`; + }, + + /** Get taxonomy JSON URL for a specific attempt */ + runTaxonomyUrl: (requestId: string, runId: string): string => { + return `${BASE}/requests/${requestId}/runs/${runId}/taxonomy`; + }, + /** Download a batch archive of multiple runs as a single .tar.gz */ batchArchive: async (ids: string[]): Promise => { const resp = await apiClient(`${BASE}/requests/archive`, { diff --git a/apps/portal/src/pages/RunDetail.tsx b/apps/portal/src/pages/RunDetail.tsx index 81d3429bb..23067ffb3 100644 --- a/apps/portal/src/pages/RunDetail.tsx +++ b/apps/portal/src/pages/RunDetail.tsx @@ -24,6 +24,7 @@ import { CriteriaGraphView } from "@/components/CriteriaGraphView"; import { HarNetworkViewer } from "@/components/HarNetworkViewer"; import { ConversationView } from "@/components/ConversationView"; import { VideoPlayer } from "@/components/VideoPlayer"; +import { TaxonomyTab } from "@/components/TaxonomyTab"; import { useLogStream } from "@/hooks/use-log-stream"; import { useAllTurnsToolCalls } from "@/hooks/useHarExtraction"; import { MarkdownRenderer } from "@/components/MarkdownRenderer"; @@ -718,6 +719,7 @@ export function RunDetail() { } const hasHarData = !!(activeRun?.harUrl || activeRun?.turns?.some(t => t.harUrl)); + const hasTaxonomy = !!activeRun?.taxonomyUrl; const hasVideoData = !!(activeRun?.videoUrls?.length || activeRun?.setupVideoUrls?.length || activeRun?.turns?.some(t => t.videoUrls?.length)); const videoCount = (activeRun?.setupVideoUrls?.length ?? 0) + (activeRun?.videoUrls?.length ?? 0) @@ -1166,6 +1168,7 @@ export function RunDetail() { )} {hasHarData && Network} {hasHarData && Tool Calls} + {hasTaxonomy && Taxonomy} {hasVideoData && } {(hasConfiguredObservations || hasObservationResults) && ( @@ -1245,6 +1248,13 @@ export function RunDetail() { )} + {/* Taxonomy tab — structured scorecard + behavior analysis + action items */} + {hasTaxonomy && ( + + + + )} + {/* Video tab — session recording player */} {hasVideoData && ( diff --git a/apps/portal/src/types.ts b/apps/portal/src/types.ts index f510eee81..ea1654747 100644 --- a/apps/portal/src/types.ts +++ b/apps/portal/src/types.ts @@ -187,6 +187,7 @@ export interface RunState { * instead. Empty/absent when no run-subject observations were selected. */ observationResults?: CriterionResult[]; + taxonomyUrl?: string; } export interface Run { diff --git a/apps/scheduler/src/handler-dispatcher.test.ts b/apps/scheduler/src/handler-dispatcher.test.ts index 32d954243..847e6be90 100644 --- a/apps/scheduler/src/handler-dispatcher.test.ts +++ b/apps/scheduler/src/handler-dispatcher.test.ts @@ -16,11 +16,11 @@ const HANDLERS: HandlerServiceDocument[] = [ dependsOn: [], }, { - _id: "pp-example", + _id: "pp-taxonomy", type: "post-process-handler", version: 1, - queue: "pp-example-queue", - selector: "example", + queue: "pp-taxonomy-queue", + selector: "taxonomy", autoBackfill: false, dependsOn: ["pp-atif"], }, @@ -87,7 +87,7 @@ describe("HandlerDispatcher — drain detection", () => { it("is drained when all handlers are done", () => { expect( - isDrained(HANDLERS, { "pp-atif": { status: "done" }, "pp-example": { status: "done" } }), + isDrained(HANDLERS, { "pp-atif": { status: "done" }, "pp-taxonomy": { status: "done" } }), ).toBe(true); }); @@ -99,13 +99,13 @@ describe("HandlerDispatcher — drain detection", () => { expect( isDrained(HANDLERS, { "pp-atif": { status: "done" }, - "pp-example": { status: "processing" }, + "pp-taxonomy": { status: "processing" }, }), ).toBe(false); }); it("is drained when a failed dependency blocks its descendant", () => { - // pp-atif failed → pp-example can never run (blocked) → terminal. + // pp-atif failed → pp-taxonomy can never run (blocked) → terminal. expect(isDrained(HANDLERS, { "pp-atif": { status: "failed" } })).toBe(true); }); @@ -113,7 +113,7 @@ describe("HandlerDispatcher — drain detection", () => { expect( isDrained(HANDLERS, { "pp-atif": { status: "done" }, - "pp-example": { status: "failed" }, + "pp-taxonomy": { status: "failed" }, }), ).toBe(true); }); @@ -143,7 +143,7 @@ describe("HandlerDispatcher — maybeTriggerReports", () => { run: { _id: "run-1", status: "done", - handlerStatus: { "pp-atif": { status: "done" }, "pp-example": { status: "done" } }, + handlerStatus: { "pp-atif": { status: "done" }, "pp-taxonomy": { status: "done" } }, }, }); collection.findOneAndUpdate.mockResolvedValue({ _id: "req-1" }); // claim succeeds @@ -203,7 +203,7 @@ describe("HandlerDispatcher — maybeTriggerReports", () => { run: { _id: "run-2", status: "done", - handlerStatus: { "pp-atif": { status: "done" }, "pp-example": { status: "done" } }, + handlerStatus: { "pp-atif": { status: "done" }, "pp-taxonomy": { status: "done" } }, }, }); @@ -220,7 +220,7 @@ describe("HandlerDispatcher — maybeTriggerReports", () => { run: { _id: "run-1", status: "done", - handlerStatus: { "pp-atif": { status: "done" }, "pp-example": { status: "done" } }, + handlerStatus: { "pp-atif": { status: "done" }, "pp-taxonomy": { status: "done" } }, }, }); collection.findOneAndUpdate.mockResolvedValue(null); // another path claimed it first @@ -237,7 +237,7 @@ describe("HandlerDispatcher — maybeTriggerReports", () => { run: { _id: "run-1", status: "done", - handlerStatus: { "pp-atif": { status: "done" }, "pp-example": { status: "done" } }, + handlerStatus: { "pp-atif": { status: "done" }, "pp-taxonomy": { status: "done" } }, }, }); collection.findOneAndUpdate.mockResolvedValue({ _id: "req-1" }); diff --git a/apps/scheduler/src/notify-routes.test.ts b/apps/scheduler/src/notify-routes.test.ts index 284bf081a..38a852ae1 100644 --- a/apps/scheduler/src/notify-routes.test.ts +++ b/apps/scheduler/src/notify-routes.test.ts @@ -328,11 +328,11 @@ describe("createHttpServer", () => { servers.push(server); const doc = { - _id: "pp-example", + _id: "pp-taxonomy", type: "post-process-handler", version: 1, - queue: "pp-example-queue", - selector: "example", + queue: "pp-taxonomy-queue", + selector: "taxonomy", autoBackfill: false, dependsOn: ["pp-atif"], }; @@ -344,7 +344,7 @@ describe("createHttpServer", () => { }); expect(response.status).toBe(200); - expect(await response.json()).toEqual({ ok: true, handlerId: "pp-example" }); + expect(await response.json()).toEqual({ ok: true, handlerId: "pp-taxonomy" }); expect(handler.registerHandler).toHaveBeenCalledWith(doc); }); diff --git a/apps/workers/taxonomy/Dockerfile b/apps/workers/taxonomy/Dockerfile new file mode 100644 index 000000000..b6be8f2ad --- /dev/null +++ b/apps/workers/taxonomy/Dockerfile @@ -0,0 +1,50 @@ +FROM node:22.22.0-slim AS builder + +RUN corepack enable && corepack prepare pnpm@10.29.1 --activate + +WORKDIR /app + +COPY package.json pnpm-workspace.yaml pnpm-lock.yaml ./ +COPY tsconfig.base.json ./ +COPY packages/shared/package.json ./packages/shared/ +COPY apps/workers/taxonomy/package.json ./apps/workers/taxonomy/ + +RUN pnpm install --frozen-lockfile + +COPY packages/shared/tsconfig.json ./packages/shared/ +COPY packages/shared/src ./packages/shared/src/ +COPY apps/workers/taxonomy/tsconfig.json ./apps/workers/taxonomy/ +COPY apps/workers/taxonomy/src ./apps/workers/taxonomy/src/ +COPY apps/workers/taxonomy/handler.yaml ./apps/workers/taxonomy/ + +RUN pnpm --filter shared build && pnpm --filter taxonomy-handler build + +# --- Dev stage: hot reload with tsx watch --- +FROM builder AS dev +ENV NODE_ENV=development +ENV SERVICE_DIR=workers/taxonomy +COPY scripts/dev-entrypoint.sh /app/scripts/dev-entrypoint.sh +EXPOSE 80 +ENV PORT=80 +CMD ["/app/scripts/dev-entrypoint.sh"] + +# --- Production stage --- +FROM node:22.22.0-slim + +RUN corepack enable && corepack prepare pnpm@10.29.1 --activate + +WORKDIR /app + +COPY package.json pnpm-workspace.yaml pnpm-lock.yaml ./ +COPY packages/shared/package.json ./packages/shared/ +COPY apps/workers/taxonomy/package.json ./apps/workers/taxonomy/ + +RUN pnpm install --frozen-lockfile --prod + +COPY --from=builder /app/packages/shared/dist ./packages/shared/dist/ +COPY --from=builder /app/apps/workers/taxonomy/dist ./apps/workers/taxonomy/dist/ +COPY --from=builder /app/apps/workers/taxonomy/handler.yaml ./apps/workers/taxonomy/handler.yaml + +WORKDIR /app/apps/workers/taxonomy + +CMD ["node", "dist/index.js"] diff --git a/apps/workers/taxonomy/handler.yaml b/apps/workers/taxonomy/handler.yaml new file mode 100644 index 000000000..b479f905e --- /dev/null +++ b/apps/workers/taxonomy/handler.yaml @@ -0,0 +1,11 @@ +# Post-process handler registration metadata. +# Read by the generic register-handler script and POSTed to the +# scheduler's POST /handlers/register endpoint on deploy. +_id: pp-taxonomy +type: post-process-handler +version: 1 +queue: pp-taxonomy-queue +selector: taxonomy +autoBackfill: false +dependsOn: + - pp-atif diff --git a/apps/workers/taxonomy/package.json b/apps/workers/taxonomy/package.json new file mode 100644 index 000000000..52cdd6b2f --- /dev/null +++ b/apps/workers/taxonomy/package.json @@ -0,0 +1,24 @@ +{ + "name": "taxonomy-handler", + "version": "1.0.0", + "type": "module", + "main": "dist/index.js", + "scripts": { + "build": "tsc", + "dev": "tsx watch src/index.ts", + "start": "node dist/index.js", + "taxonomy": "tsx src/cli.ts", + "lint": "eslint src/" + }, + "dependencies": { + "shared": "workspace:*", + "@azure/storage-queue": "^12.25.0", + "@github/copilot-sdk": "^0.1.17", + "dotenv": "^16.4.5" + }, + "devDependencies": { + "@types/node": "^22.7.0", + "tsx": "^4.19.0", + "typescript": "^5.6.2" + } +} diff --git a/apps/workers/taxonomy/src/cli.ts b/apps/workers/taxonomy/src/cli.ts new file mode 100644 index 000000000..a5f51f600 --- /dev/null +++ b/apps/workers/taxonomy/src/cli.ts @@ -0,0 +1,206 @@ +#!/usr/bin/env npx tsx +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * Taxonomy post-processor developer CLI. + * + * Invokes the taxonomy generation loop directly against an existing requestId, + * streams the Copilot SDK session events live, and saves the validated taxonomy + * to a local JSON file. This is a developer/troubleshooting tool only — it never + * writes to Mongo (handlerStatus) or blob storage (taxonomy.json) and does not + * notify the scheduler. + * + * Usage: + * npx tsx src/cli.ts [options] + * pnpm taxonomy [options] + * + * Options: + * --run, -r Run id for the prompt/logging (default: active run from API). + * --model Override TAXONOMY_MODEL (e.g. a reasoning model). + * --timeout Override SESSION_TIMEOUT_MS (per-attempt session timeout). + * --output, -o Output file path (default: taxonomy-.json). + * --stdout Also print the taxonomy JSON to stdout. + * --quiet Suppress live session-event streaming on stderr. + * --help, -h Show this help. + */ +import "dotenv/config"; +import { writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { SessionEvent } from "@github/copilot-sdk"; +import { TokenManagerClient, type LogEvent, type RequestDocument } from "shared"; +import { generateTaxonomy } from "./taxonomy-generator.js"; + +interface CliArgs { + requestId: string; + runId?: string; + model?: string; + timeoutMs?: number; + output?: string; + stdout: boolean; + quiet: boolean; +} + +function printHelp(): void { + process.stderr.write( + [ + "Taxonomy post-processor developer CLI", + "", + "Usage: npx tsx src/cli.ts [options]", + "", + "Options:", + " --run, -r Run id for the prompt/logging (default: active run from API)", + " --model Override TAXONOMY_MODEL", + " --timeout Override SESSION_TIMEOUT_MS (per-attempt session timeout)", + " --output, -o Output file path (default: taxonomy-.json)", + " --stdout Also print the taxonomy JSON to stdout", + " --quiet Suppress live session-event streaming on stderr", + " --help, -h Show this help", + "", + ].join("\n"), + ); +} + +function parseArgs(argv: string[]): CliArgs | undefined { + const args: Partial & { stdout: boolean; quiet: boolean } = { stdout: false, quiet: false }; + const positionals: string[] = []; + + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + switch (arg) { + case "--help": + case "-h": + printHelp(); + return undefined; + case "--run": + case "-r": + args.runId = argv[++i]; + break; + case "--model": + args.model = argv[++i]; + break; + case "--timeout": + args.timeoutMs = Number(argv[++i]); + break; + case "--output": + case "-o": + args.output = argv[++i]; + break; + case "--stdout": + args.stdout = true; + break; + case "--quiet": + args.quiet = true; + break; + default: + if (arg.startsWith("-")) { + throw new Error(`Unknown option: ${arg}`); + } + positionals.push(arg); + } + } + + const requestId = positionals[0]; + if (!requestId) { + throw new Error("Missing required argument"); + } + if (args.timeoutMs !== undefined && Number.isNaN(args.timeoutMs)) { + throw new Error("--timeout must be a number (milliseconds)"); + } + + return { ...args, requestId }; +} + +async function resolveRunId(apiBaseUrl: string, requestId: string, explicit?: string): Promise { + if (explicit) return explicit; + try { + const response = await fetch(`${apiBaseUrl}/api/v1/requests/${requestId}`); + if (response.ok) { + const request: RequestDocument = await response.json(); + if (request.run?._id) return request.run._id; + } + } catch { + // fall through to requestId fallback + } + return requestId; +} + +function summarizeEvent(event: SessionEvent): string | undefined { + switch (event.type) { + case "session.start": + return `session.start model=${event.data.selectedModel}`; + case "session.error": + return `session.error ${event.data.errorType}: ${event.data.message}`; + case "tool.execution_start": + return `tool.start ${event.data.toolName} ${JSON.stringify(event.data.arguments ?? {})}`; + case "tool.execution_complete": + return `tool.done ${event.data.success ? "ok" : "FAILED"}${event.data.error ? ` ${event.data.error.message}` : ""}`; + case "assistant.message": + return `assistant.message (${event.data.content.length} chars)`; + default: + return undefined; + } +} + +async function main(): Promise { + let parsed: CliArgs | undefined; + try { + parsed = parseArgs(process.argv.slice(2)); + } catch (error) { + process.stderr.write(`Error: ${error instanceof Error ? error.message : String(error)}\n\n`); + printHelp(); + process.exit(2); + } + if (!parsed) return; // help shown + + const apiBaseUrl = process.env.SCOPE_MT_API_URL || "http://localhost:3001"; + const taxonomyModel = parsed.model || process.env.TAXONOMY_MODEL || "gpt-5.4"; + const sessionTimeoutMs = parsed.timeoutMs ?? (Number(process.env.SESSION_TIMEOUT_MS) || 5 * 60 * 1000); + const outputPath = resolve(parsed.output ?? `taxonomy-${parsed.requestId}.json`); + + const githubToken = process.env.GITHUB_TOKEN + ? process.env.GITHUB_TOKEN + : await new TokenManagerClient(process.env.TOKEN_MANAGER_URL).acquireToken("copilot-sdk"); + + const runId = await resolveRunId(apiBaseUrl, parsed.requestId, parsed.runId); + + const log = (level: LogEvent["level"], msg: string, data?: Record): void => { + if (parsed.quiet) return; + const suffix = data && Object.keys(data).length > 0 ? ` ${JSON.stringify(data)}` : ""; + process.stderr.write(`[${level}] ${msg}${suffix}\n`); + }; + + process.stderr.write( + `Generating taxonomy: requestId=${parsed.requestId} runId=${runId} model=${taxonomyModel} timeout=${sessionTimeoutMs}ms\n`, + ); + + const onEvent = parsed.quiet + ? undefined + : (event: SessionEvent): void => { + const summary = summarizeEvent(event); + if (summary) process.stderr.write(` · ${summary}\n`); + }; + + const taxonomy = await generateTaxonomy({ + requestId: parsed.requestId, + runId, + apiBaseUrl, + taxonomyModel, + githubToken, + sessionTimeoutMs, + log, + onEvent, + }); + + const json = JSON.stringify(taxonomy, null, 2); + await writeFile(outputPath, `${json}\n`, "utf-8"); + process.stderr.write(`\nTaxonomy written to ${outputPath}\n`); + if (parsed.stdout) { + process.stdout.write(`${json}\n`); + } +} + +main().catch((error) => { + process.stderr.write(`\nTaxonomy generation failed: ${error instanceof Error ? error.message : String(error)}\n`); + process.exit(1); +}); diff --git a/apps/workers/taxonomy/src/index.ts b/apps/workers/taxonomy/src/index.ts new file mode 100644 index 000000000..b1a4f322b --- /dev/null +++ b/apps/workers/taxonomy/src/index.ts @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import "dotenv/config"; +import { TaxonomyQueueProcessor, type TaxonomyQueueProcessorConfig } from "./taxonomy-queue-processor.js"; + +const config: TaxonomyQueueProcessorConfig = { + mongoUri: process.env.MONGO_CONNECTION_STRING || process.env.AZURE_COSMOS_CONNECTION_STRING || "mongodb://localhost:27017", + mongoDatabase: process.env.MONGO_DATABASE || "requests-db", + mongoCollection: "requests", + storageAccountName: process.env.AZURE_STORAGE_ACCOUNT_NAME || "devstoreaccount1", + storageConnectionString: process.env.STORAGE_CONNECTION_STRING || process.env.AZURE_STORAGE_CONNECTION_STRING || undefined, + queueName: process.env.AZURE_STORAGE_QUEUE_TAXONOMY || "pp-taxonomy-queue", + batchSize: Number(process.env.BATCH_SIZE) || 1, + pollIntervalMs: Number(process.env.POLL_INTERVAL_MS) || 5000, + redisHost: process.env.REDIS_HOST || "localhost", + redisPort: Number(process.env.REDIS_PORT) || 6379, + redisPassword: process.env.REDIS_PASSWORD || "", + taxonomyModel: process.env.TAXONOMY_MODEL || "gpt-5.4", + apiBaseUrl: process.env.SCOPE_MT_API_URL || "http://localhost:3001", + sessionTimeoutMs: Number(process.env.SESSION_TIMEOUT_MS) || 5 * 60 * 1000, + schedulerUrl: process.env.SCHEDULER_URL, + tokenManagerUrl: process.env.TOKEN_MANAGER_URL, +}; + +const processor = new TaxonomyQueueProcessor(config); +processor.start().catch((err) => { + console.error("[taxonomy-handler] Fatal error:", err); + process.exit(1); +}); diff --git a/apps/workers/taxonomy/src/prompt.ts b/apps/workers/taxonomy/src/prompt.ts new file mode 100644 index 000000000..47ea28e4d --- /dev/null +++ b/apps/workers/taxonomy/src/prompt.ts @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +export const TAXONOMY_SYSTEM_PROMPT = `You generate benchmark taxonomy documents for completed Scope runs. + +Your job is to inspect the run metadata and ATIF trajectories, then return a taxonomy JSON document that conforms exactly to the taxonomy Zod schema enforced by the worker. + +Required JSON shape: +{ + "meta": { + "generatedAt": "ISO-8601 datetime with offset", + "schemaVersion": "string", + "runId": "string", + "requestId": "string" + }, + "scorecard": { + "summary": "string", + "profileResults": [ + { + "profileId": "string", + "select": "best|median|worst", + "taskOutcome": "success|partial|failure", + "overallScore": 0-100, + "dimensions": [ + { + "name": "string", + "score": 0-100, + "rationale": "string", + "evidence": ["string"] + } + ] + } + ], + "crossProfileInsights": ["string"] + }, + "behaviorAnalysis": { + "patterns": [ + { + "id": "string", + "category": "strength|weakness|inconsistency|regression", + "title": "string", + "description": "string", + "frequency": 0-1, + "impact": "critical|high|medium|low", + "evidence": [ + { + "runId": "string", + "iteration": 0-or-greater integer, + "excerpt": "string" + } + ] + } + ], + "agentPersonality": { + "communicationStyle": "string", + "problemSolvingApproach": "string", + "errorRecoveryBehavior": "string" + } + }, + "actionList": { + "fix": [actionItem], + "improve": [actionItem], + "investigate": [actionItem] + } +} + +Each actionItem must include: +- id +- title +- priority: critical|high|medium|low +- description +- ownership: agent-team|platform-team|prompt-engineer|scenario-author +- relatedPatterns: string[] +- suggestedApproach + +Output rules: +- Return raw JSON only. +- Do not wrap the JSON in markdown fences. +- Do not include commentary before or after the JSON. +- Every field must be valid for the schema. +- Preserve the 3-layer structure: scorecard, behaviorAnalysis, actionList. +- If a value is unknown, use the schema-compatible empty/null/default value rather than inventing facts. + +Use the available tools to gather evidence before answering: +- get_run_data for the request, scenario, run, turns, and criteria outcomes. +- get_atif_trajectory for ATIF evidence from individual iterations. + +Ground every classification in the actual run evidence. Prefer precise, stable labels over verbose prose. If validation feedback is provided later in the conversation, correct the JSON and return a full replacement document.`; diff --git a/apps/workers/taxonomy/src/taxonomy-generator.ts b/apps/workers/taxonomy/src/taxonomy-generator.ts new file mode 100644 index 000000000..ada150f2c --- /dev/null +++ b/apps/workers/taxonomy/src/taxonomy-generator.ts @@ -0,0 +1,202 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { CopilotClient, type PermissionHandler, SessionEvent } from "@github/copilot-sdk"; +import { + type LogEvent, + taxonomySchema, + type TaxonomyDocument, +} from "shared"; +import { TAXONOMY_SYSTEM_PROMPT } from "./prompt.js"; +import { createTaxonomyTools } from "./tools.js"; + +export const DEFAULT_SESSION_TIMEOUT_MS = 5 * 60 * 1000; +const DELTA_LOG_INTERVAL = 2_000; +const approveTaxonomyTools: PermissionHandler = request => + request.kind === "custom-tool" + ? { kind: "approved" } + : { kind: "denied-by-rules", rules: [] }; + +type TaxonomyLogFn = ( + level: LogEvent["level"], + msg: string, + data?: Record, +) => Promise | void; + +export interface GenerateTaxonomyParams { + requestId: string; + runId: string; + apiBaseUrl: string; + taxonomyModel: string; + githubToken: string; + sessionTimeoutMs?: number; + log: TaxonomyLogFn; + /** Optional raw session-event hook (used by the dev CLI to render live progress). */ + onEvent?: (event: SessionEvent) => void; +} + +/** + * Runs the Copilot SDK taxonomy generation loop and returns a schema-validated + * taxonomy document. This is side-effect-free with respect to Mongo and blob + * storage (it only reads run data and ATIF trajectories through the API); + * persistence is the caller's responsibility. + */ +export async function generateTaxonomy(params: GenerateTaxonomyParams): Promise { + const { requestId, runId, apiBaseUrl, taxonomyModel, githubToken, log, onEvent } = params; + const client = new CopilotClient({ githubToken }); + const tools = createTaxonomyTools(apiBaseUrl, requestId); + const timeoutMs = params.sessionTimeoutMs ?? DEFAULT_SESSION_TIMEOUT_MS; + + let streamedResponse = ""; + let latestMessage = ""; + let lastLoggedCharCount = 0; + const toolNames = new Map(); + + try { + const session = await client.createSession({ + model: taxonomyModel, + streaming: true, + tools, + availableTools: [], + excludedTools: ["builtin:*", "mcp:*"], + onPermissionRequest: approveTaxonomyTools, + systemMessage: { mode: "replace", content: TAXONOMY_SYSTEM_PROMPT }, + }); + + session.on((event: SessionEvent) => { + onEvent?.(event); + switch (event.type) { + case "session.start": + void log("info", "Copilot session started", { + sessionId: event.data.sessionId, + model: event.data.selectedModel, + }); + break; + case "session.error": + void log("error", `Session error: ${event.data.message}`, { errorType: event.data.errorType }); + break; + case "tool.execution_start": + toolNames.set(event.data.toolCallId, event.data.toolName); + void log("info", `Tool call: ${event.data.toolName}`, { + toolCallId: event.data.toolCallId, + arguments: event.data.arguments as Record | undefined, + }); + break; + case "tool.execution_complete": { + const toolName = toolNames.get(event.data.toolCallId) ?? "unknown"; + toolNames.delete(event.data.toolCallId); + void log("info", `Tool result: ${toolName} (${event.data.success ? "success" : "failed"})`, { + toolCallId: event.data.toolCallId, + error: event.data.error?.message, + }); + break; + } + case "assistant.message_delta": + streamedResponse += event.data.deltaContent; + if (streamedResponse.length - lastLoggedCharCount >= DELTA_LOG_INTERVAL) { + lastLoggedCharCount = streamedResponse.length; + void log("info", `Generating taxonomy… (${streamedResponse.length} chars so far)`); + } + break; + case "assistant.message": + latestMessage = event.data.content; + break; + } + }); + + let attempts = 0; + const maxAttempts = 3; + let prompt = [ + `Generate taxonomy JSON for request ${requestId} and run ${runId}.`, + "Use get_run_data first, then inspect ATIF trajectories that materially affect the classification.", + "Return only raw JSON that satisfies the taxonomy schema.", + ].join(" "); + let lastJson: string | undefined; + + while (attempts < maxAttempts) { + attempts += 1; + streamedResponse = ""; + latestMessage = ""; + lastLoggedCharCount = 0; + + await log("info", `Sending taxonomy prompt to Copilot SDK (attempt ${attempts}/${maxAttempts})`, { + requestId, + runId, + timeoutMs, + }); + + const response = await session.sendAndWait({ prompt }, timeoutMs); + const responseText = response?.data.content?.trim() || latestMessage.trim() || streamedResponse.trim(); + + if (!responseText) { + throw new Error("Copilot SDK returned an empty taxonomy response"); + } + + const jsonText = extractJson(responseText); + lastJson = jsonText; + + let parsedJson: unknown; + try { + parsedJson = JSON.parse(jsonText) as unknown; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + if (attempts >= maxAttempts) { + throw new Error(`Taxonomy response was not valid JSON after ${maxAttempts} attempts: ${errorMessage}`); + } + + prompt = [ + "Your previous response was not valid JSON.", + `JSON parse error: ${errorMessage}.`, + "Return the full taxonomy document again as raw JSON only, with no markdown fences or commentary.", + "Replace the entire response with corrected JSON.", + ].join(" "); + continue; + } + + const validation = taxonomySchema.safeParse(parsedJson); + if (validation.success) { + return validation.data; + } + + const validationErrors = formatValidationErrors(validation.error.issues); + await log("warn", `Taxonomy validation failed on attempt ${attempts}`, { + issues: validationErrors, + }); + + if (attempts >= maxAttempts) { + throw new Error( + `Taxonomy schema validation failed after ${maxAttempts} attempts: ${validationErrors.join(" | ")}`, + ); + } + + prompt = [ + "Your previous taxonomy JSON did not satisfy the schema.", + "Correct the document and return the entire replacement JSON object.", + "Do not include markdown fences or commentary.", + `Validation errors: ${validationErrors.join("; ")}`, + lastJson ? `Previous JSON: ${lastJson}` : "", + ].filter(Boolean).join(" "); + } + + throw new Error("Taxonomy generation ended without a valid document"); + } finally { + await client.stop().catch(() => undefined); + } +} + +function extractJson(responseText: string): string { + const trimmed = responseText.trim(); + if (!trimmed.startsWith("```")) { + return trimmed; + } + + const fenceMatch = trimmed.match(/^```(?:json)?\s*([\s\S]*?)\s*```$/i); + return fenceMatch?.[1]?.trim() ?? trimmed; +} + +function formatValidationErrors(issues: Array<{ path: PropertyKey[]; message: string }>): string[] { + return issues.map((issue) => { + const path = issue.path.length > 0 ? issue.path.map((segment) => String(segment)).join(".") : ""; + return `${path}: ${issue.message}`; + }); +} diff --git a/apps/workers/taxonomy/src/taxonomy-queue-processor.ts b/apps/workers/taxonomy/src/taxonomy-queue-processor.ts new file mode 100644 index 000000000..1d8160db0 --- /dev/null +++ b/apps/workers/taxonomy/src/taxonomy-queue-processor.ts @@ -0,0 +1,190 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { DequeuedMessageItem } from "@azure/storage-queue"; +import { + BaseQueueProcessor, + BlobStorage, + type BaseQueueProcessorConfig, + type LogEvent, + type RequestDocument, + TokenManagerClient, + type TaxonomyDocument, + type VisibilityHeartbeat, + withRetry, +} from "shared"; +import { generateTaxonomy } from "./taxonomy-generator.js"; + +const HANDLER_ID = "pp-taxonomy"; +const HANDLER_VERSION = 1; +const DEFAULT_SESSION_TIMEOUT_MS = 5 * 60 * 1000; + +interface TaxonomyQueueMessage { + type: "taxonomy"; + requestId: string; + runId?: string; +} + +export interface TaxonomyQueueProcessorConfig extends BaseQueueProcessorConfig { + taxonomyModel: string; + apiBaseUrl: string; + sessionTimeoutMs?: number; + schedulerUrl?: string; + tokenManagerUrl?: string; +} + +export class TaxonomyQueueProcessor extends BaseQueueProcessor { + private taxonomyConfig: TaxonomyQueueProcessorConfig; + private tokenClient: TokenManagerClient; + private blobStorage: BlobStorage; + + constructor(config: TaxonomyQueueProcessorConfig) { + super(config, "taxonomy-handler"); + this.taxonomyConfig = config; + this.tokenClient = new TokenManagerClient(config.tokenManagerUrl); + this.blobStorage = new BlobStorage({ + storageAccountName: config.storageAccountName, + storageConnectionString: config.storageConnectionString, + }); + } + + protected override async handleRequest( + doc: RequestDocument, + message: DequeuedMessageItem, + heartbeat: VisibilityHeartbeat, + log: (level: LogEvent["level"], msg: string, data?: Record) => Promise, + payload?: Record, + ): Promise { + const queueMessage = payload as TaxonomyQueueMessage | undefined; + const requestId = doc._id; + const activeRunId = doc.run?._id; + const runId = queueMessage?.runId ?? activeRunId ?? requestId; + + if (!queueMessage || queueMessage.type !== "taxonomy") { + await log("warn", "Discarding message with unsupported taxonomy payload", { payload }); + await this.safeDeleteMessage(message.messageId, heartbeat.stop()); + return; + } + + if (queueMessage.runId && activeRunId && queueMessage.runId !== activeRunId) { + await log("warn", "Discarding stale taxonomy message for superseded run", { + requestId, + queuedRunId: queueMessage.runId, + activeRunId, + }); + await this.safeDeleteMessage(message.messageId, heartbeat.stop()); + return; + } + + if (!doc.run) { + throw new Error(`Request ${requestId} has no active run to classify`); + } + + await withRetry(() => this.collection.updateOne( + { _id: requestId } as never, + { + $set: { + [`run.handlerStatus.${HANDLER_ID}.status`]: "processing", + [`run.handlerStatus.${HANDLER_ID}.version`]: HANDLER_VERSION, + [`run.handlerStatus.${HANDLER_ID}.updatedAt`]: new Date(), + [`run.handlerStatus.${HANDLER_ID}.error`]: null, + }, + } as never, + )); + + await log("info", "Starting taxonomy generation", { requestId, runId, model: this.taxonomyConfig.taxonomyModel }); + + try { + const taxonomy = await this.generateTaxonomy(requestId, runId, log); + const blobName = `${requestId}/runs/${runId}/taxonomy.json`; + const taxonomyUrl = await this.blobStorage.uploadJson(blobName, taxonomy); + + await withRetry(() => this.collection.updateOne( + { _id: requestId } as never, + { + $set: { + [`run.taxonomyUrl`]: taxonomyUrl, + [`run.handlerStatus.${HANDLER_ID}.status`]: "done", + [`run.handlerStatus.${HANDLER_ID}.version`]: HANDLER_VERSION, + [`run.handlerStatus.${HANDLER_ID}.updatedAt`]: new Date(), + }, + $unset: { + [`run.handlerStatus.${HANDLER_ID}.error`]: "", + }, + } as never, + )); + + await log("info", "Taxonomy generation completed", { + requestId, + runId, + blobName, + taxonomyUrl, + }); + + await this.notifyHandlerComplete(requestId, runId, "done"); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + + await withRetry(() => this.collection.updateOne( + { _id: requestId } as never, + { + $set: { + [`run.handlerStatus.${HANDLER_ID}.status`]: "failed", + [`run.handlerStatus.${HANDLER_ID}.version`]: HANDLER_VERSION, + [`run.handlerStatus.${HANDLER_ID}.updatedAt`]: new Date(), + [`run.handlerStatus.${HANDLER_ID}.error`]: errorMessage, + }, + } as never, + )); + + await log("error", `Taxonomy generation failed: ${errorMessage}`, { requestId, runId }); + await this.notifyHandlerComplete(requestId, runId, "failed"); + throw error; + } + + const finalPopReceipt = heartbeat.stop(); + await this.safeDeleteMessage(message.messageId, finalPopReceipt); + } + + private async generateTaxonomy( + requestId: string, + runId: string, + log: (level: LogEvent["level"], msg: string, data?: Record) => Promise, + ): Promise { + const githubToken = await this.tokenClient.acquireToken("copilot-sdk"); + await log("info", "Acquired Copilot SDK token"); + + return generateTaxonomy({ + requestId, + runId, + apiBaseUrl: this.taxonomyConfig.apiBaseUrl, + taxonomyModel: this.taxonomyConfig.taxonomyModel, + githubToken, + sessionTimeoutMs: this.taxonomyConfig.sessionTimeoutMs ?? DEFAULT_SESSION_TIMEOUT_MS, + log, + }); + } + + private async notifyHandlerComplete( + requestId: string, + runId: string, + status: "done" | "failed", + ): Promise { + const schedulerUrl = this.taxonomyConfig.schedulerUrl; + if (!schedulerUrl) return; + + try { + const response = await fetch(`${schedulerUrl}/notify/handler-complete`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ requestId, runId, handlerId: HANDLER_ID, status }), + }); + + if (!response.ok) { + console.warn(`[taxonomy-handler] Scheduler notify failed: ${response.status}`); + } + } catch (error) { + console.warn(`[taxonomy-handler] Scheduler notify error for ${requestId}/${runId}:`, error); + } + } +} diff --git a/apps/workers/taxonomy/src/tools.ts b/apps/workers/taxonomy/src/tools.ts new file mode 100644 index 000000000..ea4eced32 --- /dev/null +++ b/apps/workers/taxonomy/src/tools.ts @@ -0,0 +1,154 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { defineTool } from "@github/copilot-sdk"; +import { type ConversationTurn, type RequestDocument } from "shared"; + +interface GetAtifTrajectoryArgs { + iteration?: number; +} + +function selectTurnWithAtif(turns: ConversationTurn[] | undefined, requestedIteration?: number): ConversationTurn | undefined { + if (!turns?.length) return undefined; + + if (requestedIteration !== undefined) { + return turns.find((turn) => turn.iteration === requestedIteration && typeof turn.atifUrl === "string"); + } + + return [...turns] + .filter((turn) => typeof turn.atifUrl === "string") + .sort((a, b) => b.iteration - a.iteration)[0]; +} + +export function createTaxonomyTools( + apiBaseUrl: string, + requestId: string, +) { + const getRunData = defineTool("get_run_data", { + description: + "Fetch the full request document plus the active run state, including scenario, persona, turns, criteria results, and handler status.", + parameters: { + type: "object", + properties: {}, + }, + handler: async () => { + try { + const response = await fetch(`${apiBaseUrl}/api/v1/requests/${requestId}`); + if (!response.ok) { + return { error: `Failed to fetch request: ${response.status} ${response.statusText}` }; + } + + const request: RequestDocument = await response.json(); + const run = request.run; + + return { + request: { + id: request._id, + workerType: request.workerType, + model: request.model, + reasoningEffort: request.reasoningEffort, + maxIterations: request.maxIterations, + createdAt: request.createdAt, + updatedAt: request.updatedAt, + persona: request.persona, + personaInstructions: request.personaInstructions, + mcpServers: request.mcpServers, + skillRevisions: request.skillRevisions, + extensions: request.extensions, + priority: request.priority, + scenario: request.scenario, + }, + run: run ? { + id: run._id, + attemptNumber: run.attemptNumber, + status: run.status, + outcome: run.outcome, + result: run.result, + error: run.error, + startedAt: run.startedAt, + finishedAt: run.finishedAt, + updatedAt: run.updatedAt, + aiCallCount: run.aiCallCount, + tokenUsage: run.tokenUsage, + handlerStatus: run.handlerStatus, + turns: run.turns?.map((turn) => ({ + iteration: turn.iteration, + passed: turn.passed, + timestamp: turn.timestamp, + startedAt: turn.startedAt, + durationMs: turn.durationMs, + judgeFeedback: turn.judgeFeedback, + codingAgentResponse: turn.codingAgentResponse, + aiCallCount: turn.aiCallCount, + tokenUsage: turn.tokenUsage, + snapshotUrl: turn.snapshotUrl, + rawChatUrl: turn.rawChatUrl, + rawChatFormat: turn.rawChatFormat, + toolCallsUrl: turn.toolCallsUrl, + toolCallCount: turn.toolCallCount, + atifUrl: turn.atifUrl, + criteriaResults: turn.criteriaResults, + })) ?? [], + } : null, + }; + } catch (err) { + return { error: `Failed to fetch run data: ${err}` }; + } + }, + }); + + const getAtifTrajectory = defineTool("get_atif_trajectory", { + description: + "Fetch an ATIF trajectory JSON document for a specific iteration, or the latest available iteration when omitted.", + parameters: { + type: "object", + properties: { + iteration: { + type: "number", + description: "Optional 1-based iteration number. If omitted, the latest available ATIF trajectory is returned.", + }, + }, + }, + handler: async (args: GetAtifTrajectoryArgs) => { + try { + const response = await fetch(`${apiBaseUrl}/api/v1/requests/${requestId}`); + if (!response.ok) { + return { error: `Failed to fetch request: ${response.status} ${response.statusText}` }; + } + + const request: RequestDocument = await response.json(); + const selectedTurn = selectTurnWithAtif(request.run?.turns, args.iteration); + + if (!selectedTurn?.atifUrl) { + return { + error: args.iteration !== undefined + ? `No ATIF trajectory available for iteration ${args.iteration}` + : "No ATIF trajectory is available for this run", + }; + } + + const runId = request.run?._id; + const iteration = selectedTurn.iteration; + const atifUrl = runId + ? `${apiBaseUrl}/api/v1/requests/${requestId}/runs/${runId}/atif?iteration=${iteration}` + : `${apiBaseUrl}/api/v1/requests/${requestId}/atif?iteration=${iteration}`; + + const atifResponse = await fetch(atifUrl); + if (!atifResponse.ok) { + return { error: `Failed to fetch ATIF trajectory: ${atifResponse.status} ${atifResponse.statusText}` }; + } + + const trajectory: unknown = await atifResponse.json(); + + return { + iteration, + trajectory, + }; + } catch (err) { + return { error: `Failed to fetch ATIF trajectory: ${err}` }; + } + }, + }); + + return [getRunData, getAtifTrajectory]; +} diff --git a/apps/workers/taxonomy/tsconfig.json b/apps/workers/taxonomy/tsconfig.json new file mode 100644 index 000000000..bf012ecbb --- /dev/null +++ b/apps/workers/taxonomy/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] +} diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 411c8fc2e..f5e309b30 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -217,6 +217,22 @@ services: - action: rebuild path: ./pnpm-lock.yaml + taxonomy-handler: + build: + target: dev + develop: + watch: + - action: sync + path: ./apps/workers/taxonomy/src + target: /app/apps/workers/taxonomy/src + - action: sync + path: ./packages/shared/src + target: /app/packages/shared/src + - action: rebuild + path: ./apps/workers/taxonomy/package.json + - action: rebuild + path: ./pnpm-lock.yaml + register-handler-post-processor: build: target: dev diff --git a/docker-compose.yml b/docker-compose.yml index 027888cd2..05f510d37 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,7 @@ # docker compose --profile vscode-web up # + VS Code Web worker # docker compose --profile claude-code --profile copilot up # multiple workers # docker compose --profile report up # + Report Generator worker +# docker compose --profile taxonomy up # + Taxonomy handler worker # docker compose up mongodb redis azurite # infra only (run app services natively) # # Host-native debugging: @@ -869,6 +870,51 @@ services: api: condition: service_started + taxonomy-handler: + build: + context: . + dockerfile: apps/workers/taxonomy/Dockerfile + profiles: [taxonomy, all] + environment: + <<: *common-env + AZURE_STORAGE_QUEUE_TAXONOMY: pp-taxonomy-queue + TAXONOMY_MODEL: ${TAXONOMY_MODEL:-gpt-5.4} + SCOPE_MT_API_URL: http://api:80 + SCHEDULER_URL: http://scheduler:8080 + GITHUB_TOKEN: ${GITHUB_TOKEN:-} + TOKEN_MANAGER_URL: http://token-manager:80 + GIT_COMMIT: ${GIT_COMMIT:-development} + SESSION_TIMEOUT_MS: ${SESSION_TIMEOUT_MS:-300000} + depends_on: + mongodb: + condition: service_healthy + required: false + redis: + condition: service_healthy + azurite: + condition: service_healthy + api: + condition: service_started + scheduler: + condition: service_started + register-handler-taxonomy: + condition: service_completed_successfully + + register-handler-taxonomy: + build: + context: . + dockerfile: apps/workers/taxonomy/Dockerfile + profiles: [taxonomy, all] + environment: + <<: *common-env + HANDLER_YAML_PATH: apps/workers/taxonomy/handler.yaml + SCHEDULER_URL: http://scheduler:8080 + depends_on: + scheduler: + condition: service_started + restart: "no" + entrypoint: ["npx", "tsx", "packages/shared/src/scripts/register-handler.ts"] + register-handler-post-processor: build: context: . diff --git a/docs/architecture/db-migrations.md b/docs/architecture/db-migrations.md index fc7cfd9a8..c3d8baf6c 100644 --- a/docs/architecture/db-migrations.md +++ b/docs/architecture/db-migrations.md @@ -116,7 +116,7 @@ When you run a command: | `029-create-users-collection` | Creates the database-enforced unique identity index for authentication; uses Cosmos collection-creation extensions for continuous-backup accounts and preserves native MongoDB index creation (see below) | | `030-create-resource-indexes` | Creates project-scoped resource indexes | | `031-migrate-services-to-handler-dag` | Replaces the legacy single `post-processor` service row with per-handler service registrations and dependencies | -| `032-add-handler-status-index` | Adds a handler-specific dispatch index for `pp-atif` polling | +| `032-add-handler-status-index` | Adds handler-specific dispatch indexes for `pp-atif`, `pp-taxonomy`, and `pp-report` polling | | `033-backfill-handler-status` | Backfills `run.handlerStatus.pp-atif` from deprecated `run.postProcessorStatus` / `run.postProcessorVersion` fields | | `034-create-experiment-storage-indexes` | Adds the project paging and immutable execution-history indexes for canonical Experiment storage; request payload chunks remain point-read by deterministic `_id` only | diff --git a/docs/architecture/post-processing.md b/docs/architecture/post-processing.md index cf00f2bb2..315f474b1 100644 --- a/docs/architecture/post-processing.md +++ b/docs/architecture/post-processing.md @@ -34,7 +34,23 @@ Two paths trigger post-processing: - Updates `run.turns.$.atifUrl` in the request document 4. On success, stamps `run.handlerStatus["pp-atif"] = { status: "done", version: N }` (while legacy `run.postProcessorVersion` / `run.postProcessorStatus` may still be mirrored during the transition) -5. Triggers any downstream handlers once the scheduler sees the dependency graph is satisfied. (Reports are **not** a DAG handler — see [Report Triggering](#report-triggering-on-dag-drain).) +5. Triggers the downstream taxonomy handler once the scheduler sees the dependency graph is satisfied. (Reports are **not** a DAG handler — see [Report Triggering](#report-triggering-on-dag-drain).) + +## Taxonomy Generation + +The taxonomy handler (`apps/workers/taxonomy/`) consumes messages from the dedicated `pp-taxonomy-queue` and produces a structured JSON artifact for a completed run. It uses the GitHub Copilot SDK with a tool-backed session to fetch run metadata and ATIF trajectories from the API, then validates the generated JSON against the shared taxonomy Zod schema. + +### Flow + +1. Queue message arrives: `{ type: "taxonomy", requestId, runId }` +2. Worker fetches the request document from MongoDB and marks `run.handlerStatus["pp-taxonomy"].status = "processing"` +3. Copilot SDK session calls: + - `get_run_data` to fetch the request + active run from the API + - `get_atif_trajectory` to fetch per-iteration ATIF JSON from the API (`GET /api/v1/requests/:id/runs/:runId/atif?iteration=N`) +4. The worker validates the assistant response with `taxonomySchema.safeParse()` +5. If validation fails, the worker feeds the schema errors back into the same session and retries up to 3 total attempts +6. On success, the worker uploads `taxonomy.json` to `{requestId}/runs/{runId}/taxonomy.json` +7. The worker stamps `run.handlerStatus["pp-taxonomy"] = { status: "done", version: 1 }` and calls `POST /notify/handler-complete` ### Blob Storage Layout @@ -45,16 +61,52 @@ Two paths trigger post-processing: atif.trajectory.json iteration-2/ atif.trajectory.json + taxonomy.json ``` +The completed run stores the taxonomy blob URL on `run.taxonomyUrl`. + ### API Endpoints | Endpoint | Description | | --- | --- | | `GET /api/v1/requests/:id/atif?iteration=N` | Download ATIF for the latest run | | `GET /api/v1/requests/:id/runs/:runId/atif?iteration=N` | Download ATIF for a specific run | +| `GET /api/v1/requests/:id/taxonomy` | Download taxonomy JSON for the latest run | +| `GET /api/v1/requests/:id/runs/:runId/taxonomy` | Download taxonomy JSON for a specific run | + +The `iteration` query parameter is mandatory only for the ATIF endpoints. ATIF files are also included in archive exports as `iteration-{N}.atif.trajectory.json`. + +### Local troubleshooting: taxonomy CLI + +The Copilot SDK generation loop lives in `apps/workers/taxonomy/src/taxonomy-generator.ts` (`generateTaxonomy()`), shared by both the queue processor and a standalone developer CLI (`src/cli.ts`). The CLI invokes generation directly against an existing `requestId`, streams the session events live, and saves the validated taxonomy to a local file — **it never writes to MongoDB, blob storage, or the scheduler** (read-only). It is a developer tool and is intentionally not part of the Scope CLI. + +Because both taxonomy tools (`get_run_data` and `get_atif_trajectory`) read through the API, the CLI needs only an API URL and a Copilot token — no storage connection string. This lets it run against any environment (local or a remote deployment such as integration) given a reachable API. + +```bash +# Local stack (use your worktree's host-mapped API port): +SCOPE_MT_API_URL=http://localhost:3001 \ +GITHUB_TOKEN=$(gh auth token) \ +pnpm taxonomy [--run ] [--model ] [--timeout ] [-o out.json] [--stdout] [--quiet] + +# Integration (read-only against the deployed API): +SCOPE_MT_API_URL=https://msscope-int.azurewebsites.net \ +GITHUB_TOKEN=$(gh auth token) \ +pnpm taxonomy -o taxonomy-int.json +``` + +The root `pnpm taxonomy` script proxies to `pnpm --filter taxonomy-handler taxonomy` (i.e. `tsx src/cli.ts`). + +| Option | Description | +| --- | --- | +| `--run, -r ` | Run id used in the prompt/logging (default: active run resolved from the API) | +| `--model ` | Override `TAXONOMY_MODEL` (e.g. A/B a reasoning model vs the `gpt-5.4` default) | +| `--timeout ` | Override `SESSION_TIMEOUT_MS` (per-attempt session timeout) | +| `--output, -o ` | Output file path (default: `taxonomy-.json` in the cwd) | +| `--stdout` | Also print the taxonomy JSON to stdout | +| `--quiet` | Suppress live session-event streaming on stderr | -The `iteration` query parameter is mandatory for the ATIF endpoints. ATIF files are also included in archive exports as `iteration-{N}.atif.trajectory.json`. +Live progress (tool calls, errors, deltas) is written to stderr so the saved JSON file stays clean. The CLI exits non-zero on failure (empty response, JSON/schema validation failure after 3 attempts, timeout, or model rate limit) and surfaces the underlying error message — making it well suited to diagnosing the oversized-`get_atif_trajectory` paging loop and Copilot model rate limits. ## Taxonomy Observations (`pp-taxonomy`) @@ -165,7 +217,7 @@ The `AtifHandler` has two complementary test suites: The post-processor uses a version-based re-processing scheme: -- Each handler is registered independently in the `services` collection using a `post-process-handler` document keyed by handler ID (for example `pp-atif`) +- Each handler is registered independently in the `services` collection using a `post-process-handler` document keyed by handler ID (for example `pp-atif`, `pp-taxonomy`) - Each registration stores the handler `version`, queue name, selector, `autoBackfill` behavior, and `dependsOn` DAG edges - The scheduler compares each request's per-handler status/version state (for example `run.handlerStatus["pp-atif"]`) against the registered handler entry for that selector - Requests with missing or outdated handler state are re-dispatched for that specific handler once its dependencies are satisfied @@ -174,17 +226,18 @@ This allows deploying handler improvements and having them automatically applied ## Handler Registration -Each post-process handler declares its DAG metadata in a `handler.yaml` file colocated with the worker (e.g. `apps/workers/post-processor/handler.yaml`). This mirrors the coding-agent `agent.yaml` registration pattern. +Each post-process handler declares its DAG metadata in a `handler.yaml` file colocated with the worker (e.g. `apps/workers/taxonomy/handler.yaml`). This mirrors the coding-agent `agent.yaml` registration pattern. ```yaml -# apps/workers/post-processor/handler.yaml -_id: pp-atif +# apps/workers/taxonomy/handler.yaml +_id: pp-taxonomy type: post-process-handler version: 1 -queue: post-processor-queue -selector: atif -autoBackfill: true -dependsOn: [] +queue: pp-taxonomy-queue +selector: taxonomy +autoBackfill: false +dependsOn: + - pp-atif ``` On deploy, a generic registration script — `packages/shared/src/scripts/register-handler.ts` — reads the `handler.yaml` pointed to by `HANDLER_YAML_PATH`, waits for the scheduler's `/health`, and POSTs the parsed document to `${SCHEDULER_URL}/handlers/register` (with retry). The scheduler validates the document, runs a cycle check against the existing topology, and upserts it into the `services` collection via `HandlerDispatcher.registerHandler`. **Workers never write to the `services` collection directly** — the scheduler is the sole owner. @@ -233,7 +286,7 @@ active template's trigger and creates the properly-templated `ReportDocument`(s) ```mermaid flowchart LR - RT["run done"] --> DAG["scheduler dispatches DAG
(pp-atif → …)"] + RT["run done"] --> DAG["scheduler dispatches DAG
(pp-atif → pp-taxonomy)"] DAG -->|"all handlers terminal"| DR{DAG drained?} DR -->|yes| TRIG["POST /api/v1/reports/trigger
{ requestId }"] TRIG --> API["API evaluates each template trigger
→ ReportDocument + report-queue {reportId}"] @@ -246,7 +299,7 @@ flowchart LR registered handler is terminal. A run with **zero** registered handlers is vacuously drained, so reports trigger immediately on run completion. -**Trigger regardless of handler success.** A flaky handler must not block report +**Trigger regardless of handler success.** A flaky taxonomy must not block report generation (reports analyze the run/ATIF), so the scheduler triggers reports whenever the DAG drains — not only when every handler succeeded. @@ -290,18 +343,20 @@ stateDiagram-v2 | Environment Variable | Default | Description | | --- | --- | --- | | `AZURE_STORAGE_QUEUE_POSTPROCESSOR` | `post-processor-queue` | Queue name for ATIF post-processor messages | +| `AZURE_STORAGE_QUEUE_TAXONOMY` | `pp-taxonomy-queue` | Queue name for taxonomy-generation jobs | | `SCHEDULER_PP_POLL_INTERVAL_MS` | `30000` | Scheduler backfill polling interval | | `BATCH_SIZE` | `1` | Messages to process per poll (worker-side) | | `POLL_INTERVAL_MS` | `5000` | Worker queue polling interval | -| `SCOPE_MT_API_URL` / `API_BASE_URL` | `http://localhost:3001` | API URL used by report Copilot tools | -| `SESSION_TIMEOUT_MS` | `300000` | Timeout for report Copilot SDK sessions | +| `SCOPE_MT_API_URL` / `API_BASE_URL` | `http://localhost:3001` | API URL used by report/taxonomy Copilot tools | +| `TAXONOMY_MODEL` | `gpt-5.4` | Copilot SDK model used by the taxonomy handler | +| `SESSION_TIMEOUT_MS` | `300000` | Timeout for report/taxonomy Copilot SDK sessions | | `SCHEDULER_URL` | _(unset)_ | Scheduler base URL — used by handlers to notify completion and by the register-handler script to reach `/handlers/register` | | `API_URL` | `http://api:80` | API base URL the **scheduler** uses to call `POST /api/v1/reports/trigger` when a run's handler DAG drains | | `HANDLER_YAML_PATH` | _(unset)_ | Path to a handler's `handler.yaml`, read by the generic `register-handler` script | ## Infrastructure -- **Queues:** Azure Storage Queues (`post-processor-queue` for ATIF, `report-queue` for reports) +- **Queues:** Azure Storage Queues (`post-processor-queue` for ATIF, `pp-taxonomy-queue` for taxonomy, `report-queue` for reports) - **Blob Storage:** `snapshots` container (shared with HAR, tool-calls, and other artifacts) - **Database:** MongoDB `requests` collection (`run.handlerStatus` map keyed by handler ID, with deprecated `run.postProcessorStatus` / `run.postProcessorVersion` compatibility fields during migration) - **KEDA:** ScaledObject scales the worker to zero when queue is empty diff --git a/package.json b/package.json index 635bb66ae..a98a960c8 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "dev:coder-acp-claude-code": "pnpm --filter coder-acp-claude-code dev", "dev:coder-acp-copilot": "pnpm --filter coder-acp-copilot dev", "dev:report-generator": "pnpm --filter report-generator dev", + "taxonomy": "pnpm --filter taxonomy-handler taxonomy", "docker:up": "scripts/worktree-env.sh && scripts/dev-compose.sh up --build", "docker:up:claude-code": "scripts/worktree-env.sh && SCOPE_REGISTER_CLAUDE_CODE_AGENT=true scripts/dev-compose.sh --env-file .env --env-file apps/workers/coder-acp-claude-code/versions.env --profile claude-code --profile report up --build", "docker:up:copilot": "scripts/worktree-env.sh && SCOPE_REGISTER_COPILOT_AGENT=true scripts/dev-compose.sh --env-file .env --env-file apps/workers/coder-acp-copilot/versions.env --profile copilot --profile report up --build", @@ -26,9 +27,9 @@ "docker:up:portal": "scripts/worktree-env.sh && scripts/dev-compose.sh --profile auth --profile portal up --build", "docker:dev": "scripts/worktree-env.sh && COMPOSE_EXPERIMENTAL_WATCH_POLL=1 scripts/dev-compose.sh -f docker-compose.yml -f docker-compose.dev.yml up --build --watch", "docker:dev:claude-code": "scripts/worktree-env.sh && SCOPE_REGISTER_CLAUDE_CODE_AGENT=true GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) COMPOSE_EXPERIMENTAL_WATCH_POLL=1 scripts/dev-compose.sh --env-file .env --env-file apps/workers/coder-acp-claude-code/versions.env -f docker-compose.yml -f docker-compose.dev.yml --profile claude-code --profile report --profile post-processor --profile auth --profile portal up --build --watch", - "docker:dev:copilot": "scripts/worktree-env.sh && SCOPE_REGISTER_COPILOT_AGENT=true GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) COMPOSE_EXPERIMENTAL_WATCH_POLL=1 scripts/dev-compose.sh --env-file .env --env-file apps/workers/coder-acp-copilot/versions.env -f docker-compose.yml -f docker-compose.dev.yml --profile copilot --profile report --profile post-processor --profile auth --profile portal up --build --watch", + "docker:dev:copilot": "scripts/worktree-env.sh && SCOPE_REGISTER_COPILOT_AGENT=true GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) COMPOSE_EXPERIMENTAL_WATCH_POLL=1 scripts/dev-compose.sh --env-file .env --env-file apps/workers/coder-acp-copilot/versions.env -f docker-compose.yml -f docker-compose.dev.yml --profile copilot --profile report --profile post-processor --profile taxonomy --profile auth --profile portal up --build --watch", "docker:dev:portal": "scripts/worktree-env.sh && GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) COMPOSE_EXPERIMENTAL_WATCH_POLL=1 scripts/dev-compose.sh -f docker-compose.yml -f docker-compose.dev.yml --profile auth --profile portal up --build --watch", - "docker:dev:all": "scripts/worktree-env.sh && SCOPE_REGISTER_COPILOT_AGENT=true SCOPE_REGISTER_CLAUDE_CODE_AGENT=true GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) COMPOSE_EXPERIMENTAL_WATCH_POLL=1 scripts/dev-compose.sh --env-file .env --env-file apps/workers/coder-acp-copilot/versions.env --env-file apps/workers/coder-acp-claude-code/versions.env -f docker-compose.yml -f docker-compose.dev.yml --profile claude-code --profile copilot --profile report --profile post-processor --profile auth --profile portal up --build --watch", + "docker:dev:all": "scripts/worktree-env.sh && SCOPE_REGISTER_COPILOT_AGENT=true SCOPE_REGISTER_CLAUDE_CODE_AGENT=true GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) COMPOSE_EXPERIMENTAL_WATCH_POLL=1 scripts/dev-compose.sh --env-file .env --env-file apps/workers/coder-acp-copilot/versions.env --env-file apps/workers/coder-acp-claude-code/versions.env -f docker-compose.yml -f docker-compose.dev.yml --profile claude-code --profile copilot --profile report --profile post-processor --profile taxonomy --profile auth --profile portal up --build --watch", "docker:dev:build": "scripts/worktree-env.sh && GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) scripts/dev-compose.sh --env-file .env --env-file apps/workers/coder-acp-copilot/versions.env --env-file apps/workers/coder-acp-claude-code/versions.env -f docker-compose.yml -f docker-compose.dev.yml --profile claude-code --profile copilot --profile report --profile auth --profile portal build", "docker:down": "scripts/worktree-env.sh && scripts/dev-compose.sh --env-file .env --env-file apps/workers/coder-acp-copilot/versions.env --env-file apps/workers/coder-acp-claude-code/versions.env --profile claude-code --profile copilot --profile report --profile portal down --volumes", "docker:build": "scripts/worktree-env.sh && scripts/dev-compose.sh --env-file .env --env-file apps/workers/coder-acp-copilot/versions.env --env-file apps/workers/coder-acp-claude-code/versions.env --profile claude-code --profile copilot --profile report build", diff --git a/packages/db-migrations/src/migrations/031-migrate-services-to-handler-dag.ts b/packages/db-migrations/src/migrations/031-migrate-services-to-handler-dag.ts index 570b12e40..87fba84c8 100644 --- a/packages/db-migrations/src/migrations/031-migrate-services-to-handler-dag.ts +++ b/packages/db-migrations/src/migrations/031-migrate-services-to-handler-dag.ts @@ -7,7 +7,7 @@ import type { MigrationInterface } from "mongo-migrate-ts"; /** * Migration 031: Remove the legacy "post-processor" service entry. * - * Handler service registration (e.g. pp-atif) is now owned + * Handler service registration (pp-atif, pp-taxonomy) is now owned * by each worker's handler.yaml + the generic register-handler script, * which POSTs to the scheduler's POST /handlers/register endpoint on deploy. * Reports are not a DAG handler — the scheduler triggers report generation diff --git a/packages/db-migrations/src/migrations/032-add-handler-status-index.ts b/packages/db-migrations/src/migrations/032-add-handler-status-index.ts index 44d7a448a..cf69fb7c0 100644 --- a/packages/db-migrations/src/migrations/032-add-handler-status-index.ts +++ b/packages/db-migrations/src/migrations/032-add-handler-status-index.ts @@ -13,6 +13,14 @@ const HANDLER_STATUS_INDEXES = [ updatedAt: -1 as const, }, }, + { + name: "idx_handler_dispatch_taxonomy", + key: { + "run.handlerStatus.pp-taxonomy.status": 1 as const, + "run.status": 1 as const, + updatedAt: -1 as const, + }, + }, ] as const; export class AddHandlerStatusIndex implements MigrationInterface { diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 94bfa7c85..6abb0ed5b 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -29,5 +29,6 @@ export * from "./har/index.js"; export * from "./utils/index.js"; export * from "./schemas/index.js"; export * from "./auth/index.js"; +export * from "./schemas/taxonomy.js"; export * from "./cursor.js"; export * from "./run-duration.js"; diff --git a/packages/shared/src/schemas/index.ts b/packages/shared/src/schemas/index.ts index cb1df8b4c..a1e4d9761 100644 --- a/packages/shared/src/schemas/index.ts +++ b/packages/shared/src/schemas/index.ts @@ -22,3 +22,4 @@ export * from "./account.js"; export * from "./feature-flag.js"; export * from "./profile.js"; export * from "./project.js"; +export * from "./taxonomy.js"; diff --git a/packages/shared/src/schemas/request.ts b/packages/shared/src/schemas/request.ts index bf4c75743..8e72d5765 100644 --- a/packages/shared/src/schemas/request.ts +++ b/packages/shared/src/schemas/request.ts @@ -248,6 +248,7 @@ export const RunStateSchema = z podName: z.string().optional(), }).optional(), harUrl: z.string().optional(), + taxonomyUrl: z.string().optional(), videoUrls: z.array(z.string()).optional(), setupVideoUrls: z.array(z.string()).optional(), tokenUsage: TokenUsageSchema.optional(), diff --git a/packages/shared/src/schemas/taxonomy.test.ts b/packages/shared/src/schemas/taxonomy.test.ts new file mode 100644 index 000000000..320f89ceb --- /dev/null +++ b/packages/shared/src/schemas/taxonomy.test.ts @@ -0,0 +1,180 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, expect, it } from "vitest"; +import { taxonomySchema } from "./taxonomy.js"; + +describe("taxonomySchema", () => { + const validFixture = { + meta: { + generatedAt: "2025-02-01T12:00:00Z", + schemaVersion: "1.0.0", + runId: "run-123", + requestId: "req-456", + }, + scorecard: { + summary: "The agent usually succeeds but struggles with recovery quality.", + profileResults: [ + { + profileId: "profile-best", + select: "best", + taskOutcome: "success", + overallScore: 92, + dimensions: [ + { + name: "Correctness", + score: 95, + rationale: "Implemented the requested behavior fully.", + evidence: ["All acceptance criteria passed", "No regressions observed"], + }, + ], + }, + ], + crossProfileInsights: [ + "The agent communicates clearly across profiles.", + "Stress conditions reduce recovery quality.", + ], + }, + behaviorAnalysis: { + patterns: [ + { + id: "pattern-1", + category: "strength", + title: "Consistent planning", + description: "The agent outlines a clear plan before making changes.", + frequency: 0.75, + impact: "high", + evidence: [ + { + runId: "run-123", + iteration: 1, + excerpt: "Plan: inspect the failing endpoint, then patch the handler.", + }, + ], + }, + ], + agentPersonality: { + communicationStyle: "Concise and structured", + problemSolvingApproach: "Hypothesis-driven debugging", + errorRecoveryBehavior: "Retries with targeted fixes after inspecting evidence", + }, + }, + actionList: { + fix: [ + { + id: "action-1", + title: "Stabilize recovery prompts", + priority: "critical", + description: "Reduce failures when the first attempt is rejected.", + ownership: "prompt-engineer", + relatedPatterns: ["pattern-1"], + suggestedApproach: "Add explicit fallback guidance for failed first passes.", + }, + ], + improve: [ + { + id: "action-2", + title: "Expand planning examples", + priority: "medium", + description: "Reinforce successful planning behavior.", + ownership: "agent-team", + relatedPatterns: ["pattern-1"], + suggestedApproach: "Fine-tune on traces with strong upfront decomposition.", + }, + ], + investigate: [ + { + id: "action-3", + title: "Review profile variance", + priority: "low", + description: "Understand why profile-specific regressions appear sporadically.", + ownership: "scenario-author", + relatedPatterns: ["pattern-1"], + suggestedApproach: "Compare scenario wording against affected profile runs.", + }, + ], + }, + }; + + it("accepts a valid taxonomy fixture", () => { + const result = taxonomySchema.safeParse(validFixture); + expect(result.success).toBe(true); + }); + + it("rejects fixtures missing required fields with precise paths", () => { + const invalidFixture = { + ...validFixture, + scorecard: { + ...validFixture.scorecard, + profileResults: [ + { + ...validFixture.scorecard.profileResults[0], + dimensions: [ + { + name: "Correctness", + score: 95, + evidence: ["All acceptance criteria passed"], + }, + ], + }, + ], + }, + behaviorAnalysis: { + ...validFixture.behaviorAnalysis, + agentPersonality: { + communicationStyle: "Concise and structured", + errorRecoveryBehavior: "Retries with targeted fixes after inspecting evidence", + }, + }, + }; + + const result = taxonomySchema.safeParse(invalidFixture); + expect(result.success).toBe(false); + if (result.success) { + throw new Error("Expected invalid taxonomy fixture to fail parsing"); + } + + const issuePaths = result.error.issues.map((issue) => issue.path); + expect(issuePaths).toContainEqual([ + "scorecard", + "profileResults", + 0, + "dimensions", + 0, + "rationale", + ]); + expect(issuePaths).toContainEqual([ + "behaviorAnalysis", + "agentPersonality", + "problemSolvingApproach", + ]); + }); + + it("enforces enum values", () => { + const invalidFixture = { + ...validFixture, + actionList: { + ...validFixture.actionList, + fix: [ + { + ...validFixture.actionList.fix[0], + priority: "urgent", + }, + ], + }, + }; + + const result = taxonomySchema.safeParse(invalidFixture); + expect(result.success).toBe(false); + if (result.success) { + throw new Error("Expected invalid enum fixture to fail parsing"); + } + + expect(result.error.issues.map((issue) => issue.path)).toContainEqual([ + "actionList", + "fix", + 0, + "priority", + ]); + }); +}); diff --git a/packages/shared/src/schemas/taxonomy.ts b/packages/shared/src/schemas/taxonomy.ts new file mode 100644 index 000000000..a837d4f91 --- /dev/null +++ b/packages/shared/src/schemas/taxonomy.ts @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { z } from "zod"; + +const scoreSchema = z.number().min(0).max(100); +const frequencySchema = z.number().min(0).max(1); + +/** Taxonomy document metadata. */ +export const taxonomyMetaSchema = z.object({ + generatedAt: z.string().datetime({ offset: true }), + schemaVersion: z.string().min(1), + runId: z.string().min(1), + requestId: z.string().min(1), +}); + +/** Scorecard dimension result for a single profile. */ +export const scorecardDimensionSchema = z.object({ + name: z.string().min(1), + score: scoreSchema, + rationale: z.string().min(1), + evidence: z.array(z.string().min(1)), +}); + +/** Scorecard result for a specific profile slice. */ +export const profileResultSchema = z.object({ + profileId: z.string().min(1), + select: z.enum(["best", "median", "worst"]), + taskOutcome: z.enum(["success", "partial", "failure"]), + overallScore: scoreSchema, + dimensions: z.array(scorecardDimensionSchema), +}); + +/** Top-level scorecard section summarizing profile results. */ +export const scorecardSchema = z.object({ + summary: z.string().min(1), + profileResults: z.array(profileResultSchema), + crossProfileInsights: z.array(z.string().min(1)), +}); + +/** Evidence excerpt supporting an observed behavior pattern. */ +export const behaviorEvidenceSchema = z.object({ + runId: z.string().min(1), + iteration: z.number().int().min(0), + excerpt: z.string().min(1), +}); + +/** Cross-run behavioral pattern identified by the taxonomy analysis. */ +export const behaviorPatternSchema = z.object({ + id: z.string().min(1), + category: z.enum(["strength", "weakness", "inconsistency", "regression"]), + title: z.string().min(1), + description: z.string().min(1), + frequency: frequencySchema, + impact: z.enum(["critical", "high", "medium", "low"]), + evidence: z.array(behaviorEvidenceSchema), +}); + +/** Qualitative description of the agent's personality traits. */ +export const agentPersonalitySchema = z.object({ + communicationStyle: z.string().min(1), + problemSolvingApproach: z.string().min(1), + errorRecoveryBehavior: z.string().min(1), +}); + +/** Behavior analysis section describing patterns and personality. */ +export const behaviorAnalysisSchema = z.object({ + patterns: z.array(behaviorPatternSchema), + agentPersonality: agentPersonalitySchema, +}); + +/** Action item emitted by the taxonomy analysis. */ +export const actionItemSchema = z.object({ + id: z.string().min(1), + title: z.string().min(1), + priority: z.enum(["critical", "high", "medium", "low"]), + description: z.string().min(1), + ownership: z.enum([ + "agent-team", + "platform-team", + "prompt-engineer", + "scenario-author", + ]), + relatedPatterns: z.array(z.string().min(1)), + suggestedApproach: z.string().min(1), +}); + +/** Prioritized action lists grouped by remediation intent. */ +export const actionListSchema = z.object({ + fix: z.array(actionItemSchema), + improve: z.array(actionItemSchema), + investigate: z.array(actionItemSchema), +}); + +/** Complete taxonomy document produced by the taxonomy post-processor. */ +export const taxonomySchema = z.object({ + meta: taxonomyMetaSchema, + scorecard: scorecardSchema, + behaviorAnalysis: behaviorAnalysisSchema, + actionList: actionListSchema, +}); + +/** Inferred TypeScript type for the taxonomy document. */ +export type TaxonomyDocument = z.infer; diff --git a/packages/shared/src/scripts/register-handler.test.ts b/packages/shared/src/scripts/register-handler.test.ts index 162916404..0c3858b58 100644 --- a/packages/shared/src/scripts/register-handler.test.ts +++ b/packages/shared/src/scripts/register-handler.test.ts @@ -6,11 +6,11 @@ import { parseHandlerDocument } from "./register-handler.js"; describe("parseHandlerDocument", () => { const valid = { - _id: "pp-example", + _id: "pp-taxonomy", type: "post-process-handler", version: 1, - queue: "pp-example-queue", - selector: "example", + queue: "pp-taxonomy-queue", + selector: "taxonomy", autoBackfill: false, dependsOn: ["pp-atif"], }; diff --git a/packages/shared/src/types/types.ts b/packages/shared/src/types/types.ts index 5d34c7363..eb0b6a1a6 100644 --- a/packages/shared/src/types/types.ts +++ b/packages/shared/src/types/types.ts @@ -535,13 +535,14 @@ export interface RunState { podName?: string; }; harUrl?: string; + taxonomyUrl?: string; videoUrls?: string[]; setupVideoUrls?: string[]; tokenUsage?: TokenUsage; aiCallCount?: number; rawChatUrl?: string; rawChatFormat?: string; - /** Per-handler post-processing status. Keys are handler IDs (e.g. "pp-atif"). */ + /** Per-handler post-processing status. Keys are handler IDs (e.g. "pp-atif", "pp-taxonomy"). */ handlerStatus?: Record; /** Whole-run observation results — one entry per selected `subject:"run"` * observation criterion, evaluated once against the final snapshot + the @@ -622,6 +623,7 @@ export const RUN_STATE_FIELD_NAMES = [ "lastHeartbeatAt", "worker", "harUrl", + "taxonomyUrl", "videoUrls", "setupVideoUrls", "tokenUsage", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d4a29c36a..44a7b86bc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -787,6 +787,31 @@ importers: specifier: ^5.6.2 version: 5.9.3 + apps/workers/taxonomy: + dependencies: + '@azure/storage-queue': + specifier: ^12.25.0 + version: 12.29.0 + '@github/copilot-sdk': + specifier: ^0.1.17 + version: 0.1.32(typescript@5.9.3) + dotenv: + specifier: 16.6.1 + version: 16.6.1 + shared: + specifier: workspace:* + version: link:../../../packages/shared + devDependencies: + '@types/node': + specifier: ^22.7.0 + version: 22.19.10 + tsx: + specifier: ^4.19.0 + version: 4.21.0 + typescript: + specifier: ^5.6.2 + version: 5.9.3 + evaluations/static-prompts: dependencies: shared: @@ -798,6 +823,7 @@ importers: yaml: specifier: ^2.8.1 version: 2.8.3 + devDependencies: '@types/node': specifier: ^22.18.6 @@ -1558,6 +1584,10 @@ packages: libc: [musl] hasBin: true + '@github/copilot-sdk@0.1.32': + resolution: {integrity: sha1-mkVfe7Zkq80wJuGboVpfCcAwg4E=, tarball: https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@github/copilot-sdk/-/copilot-sdk-0.1.32.tgz} + engines: {node: '>=20.0.0'} + '@github/copilot-sdk@1.0.1': resolution: {integrity: sha512-w6AaS0WqqTE/3iyUrZznvgCLQhsUF7ZmEVCneacuHCfOzlH0r6ww9WUmyA0zgqmXO75V0IYrkIcnFke/qJkkDg==} engines: {node: ^20.19.0 || >=22.12.0} @@ -4782,7 +4812,7 @@ packages: engines: {node: '>=12'} dotenv@16.6.1: - resolution: {integrity: sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==} + resolution: {integrity: sha1-dz8OaVJ6gxXHKF1e5zxEWdIKgCA=, tarball: https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/dotenv/-/dotenv-16.6.1.tgz} engines: {node: '>=12'} dunder-proto@1.0.1: @@ -7699,6 +7729,14 @@ snapshots: '@github/copilot-linuxmusl-x64@1.0.63': optional: true + '@github/copilot-sdk@0.1.32(typescript@5.9.3)': + dependencies: + '@github/copilot': 1.0.63(typescript@5.9.3) + vscode-jsonrpc: 8.2.1 + zod: 4.3.6 + transitivePeerDependencies: + - typescript + '@github/copilot-sdk@1.0.1(typescript@5.9.3)': dependencies: '@github/copilot': 1.0.63(typescript@5.9.3) From e459f275e53f91c149d1bc5df8bed9af79bde580 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Tue, 13 Oct 2026 16:47:46 +0200 Subject: [PATCH 02/18] feat(readout): generate submission-level reports Add the pp-readout submission handler, exact-vector evidence aggregation, structured artifacts, report APIs, and stale-publication safeguards. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 1cbbd5eb-61c6-4e6a-bb38-c377be5b2163 --- .../openapi-snapshot.test.ts.snap | 1164 +++++++++++++---- apps/api/src/archive-har.test.ts | 40 - apps/api/src/archive-har.ts | 29 +- apps/api/src/index.ts | 2 - .../api/src/routes/reports-structured.test.ts | 100 ++ apps/api/src/routes/reports.ts | 61 +- apps/api/src/routes/requests/taxonomy.test.ts | 107 -- apps/api/src/routes/requests/taxonomy.ts | 30 - apps/scheduler/src/handler-dispatcher.test.ts | 22 +- apps/scheduler/src/notify-routes.test.ts | 8 +- apps/workers/{taxonomy => readout}/Dockerfile | 20 +- apps/workers/readout/handler.yaml | 12 + .../{taxonomy => readout}/package.json | 8 +- apps/workers/readout/src/index.ts | 39 + apps/workers/readout/src/prompt.ts | 63 + .../readout/src/readout-builder.test.ts | 191 +++ apps/workers/readout/src/readout-builder.ts | 303 +++++ apps/workers/readout/src/readout-generator.ts | 98 ++ .../src/readout-queue-processor.test.ts | 274 ++++ .../readout/src/readout-queue-processor.ts | 303 +++++ .../{taxonomy => readout}/tsconfig.json | 0 .../src/report-queue-processor.ts | 5 + apps/workers/taxonomy/handler.yaml | 11 - apps/workers/taxonomy/src/cli.ts | 206 --- apps/workers/taxonomy/src/index.ts | 30 - apps/workers/taxonomy/src/prompt.ts | 88 -- .../taxonomy/src/taxonomy-generator.ts | 202 --- .../taxonomy/src/taxonomy-queue-processor.ts | 190 --- apps/workers/taxonomy/src/tools.ts | 154 --- docker-compose.dev.yml | 10 +- docker-compose.yml | 28 +- package.json | 6 +- .../031-migrate-services-to-handler-dag.ts | 2 +- .../032-add-handler-status-index.ts | 8 - packages/shared/src/index.ts | 2 +- packages/shared/src/schemas/index.ts | 1 + packages/shared/src/schemas/readout.test.ts | 48 + packages/shared/src/schemas/readout.ts | 84 ++ packages/shared/src/schemas/report.ts | 14 +- packages/shared/src/schemas/request.ts | 1 - packages/shared/src/schemas/taxonomy.test.ts | 180 --- packages/shared/src/schemas/taxonomy.ts | 104 -- .../src/scripts/register-handler.test.ts | 20 +- packages/shared/src/types/types.ts | 16 +- pnpm-lock.yaml | 30 +- 45 files changed, 2620 insertions(+), 1694 deletions(-) create mode 100644 apps/api/src/routes/reports-structured.test.ts delete mode 100644 apps/api/src/routes/requests/taxonomy.test.ts delete mode 100644 apps/api/src/routes/requests/taxonomy.ts rename apps/workers/{taxonomy => readout}/Dockerfile (60%) create mode 100644 apps/workers/readout/handler.yaml rename apps/workers/{taxonomy => readout}/package.json (76%) create mode 100644 apps/workers/readout/src/index.ts create mode 100644 apps/workers/readout/src/prompt.ts create mode 100644 apps/workers/readout/src/readout-builder.test.ts create mode 100644 apps/workers/readout/src/readout-builder.ts create mode 100644 apps/workers/readout/src/readout-generator.ts create mode 100644 apps/workers/readout/src/readout-queue-processor.test.ts create mode 100644 apps/workers/readout/src/readout-queue-processor.ts rename apps/workers/{taxonomy => readout}/tsconfig.json (100%) delete mode 100644 apps/workers/taxonomy/handler.yaml delete mode 100644 apps/workers/taxonomy/src/cli.ts delete mode 100644 apps/workers/taxonomy/src/index.ts delete mode 100644 apps/workers/taxonomy/src/prompt.ts delete mode 100644 apps/workers/taxonomy/src/taxonomy-generator.ts delete mode 100644 apps/workers/taxonomy/src/taxonomy-queue-processor.ts delete mode 100644 apps/workers/taxonomy/src/tools.ts create mode 100644 packages/shared/src/schemas/readout.test.ts create mode 100644 packages/shared/src/schemas/readout.ts delete mode 100644 packages/shared/src/schemas/taxonomy.test.ts delete mode 100644 packages/shared/src/schemas/taxonomy.ts diff --git a/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap b/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap index 92da77346..df289f3ed 100644 --- a/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap +++ b/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap @@ -3022,6 +3022,15 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` ], "type": "string", }, + "structuredArtifact": { + "$ref": "#/components/schemas/StructuredReportArtifact", + }, + "submissionId": { + "type": "string", + }, + "submissionRunId": { + "type": "string", + }, "templateId": { "type": "string", }, @@ -3035,7 +3044,6 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, "required": [ "_id", - "requestId", "status", "createdAt", "projectId", @@ -3410,9 +3418,27 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, "type": "array", }, + "submissionExpectedRequestCount": { + "minimum": 1, + "type": "integer", + }, + "submissionFinalizedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, "submissionId": { "type": "string", }, + "submissionInitialRunId": { + "type": "string", + }, + "submissionRequestIndex": { + "minimum": 0, + "type": "integer", + }, "taskPromptId": { "type": "string", }, @@ -4011,9 +4037,6 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` ], "type": "string", }, - "taxonomyUrl": { - "type": "string", - }, "tokenUsage": { "$ref": "#/components/schemas/TokenUsage", }, @@ -4311,246 +4334,704 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` ], "type": "object", }, - "SuggestedPromptFeature": { + "StructuredReportArtifact": { "properties": { - "behavior": { + "contentType": { + "enum": [ + "application/json", + ], "type": "string", }, - "prompt": { + "schemaId": { "type": "string", }, - "suggestedId": { + "schemaVersion": { + "type": "string", + }, + "url": { "type": "string", }, }, "required": [ - "suggestedId", - "behavior", - "prompt", + "schemaId", + "schemaVersion", + "contentType", + "url", ], "type": "object", }, - "TaskPromptResponse": { + "SubmissionBlockingDetail": { "properties": { - "_id": { + "error": { "type": "string", }, - "contentBlobUrl": { + "handlerId": { "type": "string", }, - "createdAt": { - "format": "date-time", - "type": [ - "string", - "null", + "reason": { + "enum": [ + "request-missing", + "run-changed", + "dependency-not-ready", + "dependency-failed", + "dependency-blocked", ], + "type": "string", }, - "deletedAt": { - "format": "date-time", - "type": [ - "string", - "null", - ], + "requestId": { + "type": "string", }, - "features": { - "items": { - "$ref": "#/components/schemas/PromptFeatureResult", - }, - "type": "array", + "runId": { + "type": "string", }, - "featuresExtractedAt": { - "format": "date-time", - "type": [ - "string", - "null", - ], + }, + "required": [ + "requestId", + "runId", + "handlerId", + "reason", + ], + "type": "object", + }, + "SubmissionDependencyProjection": { + "properties": { + "error": { + "type": "string", }, - "keyId": { + "handlerId": { "type": "string", }, - "projectId": { + "requestId": { "type": "string", }, - "text": { + "runId": { "type": "string", }, - "type": { - "$ref": "#/components/schemas/PromptType", + "state": { + "enum": [ + "waiting", + "ready", + "failed", + "blocked", + ], + "type": "string", + }, + "updatedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "version": { + "type": "number", }, }, "required": [ - "_id", - "keyId", - "createdAt", - "projectId", + "requestId", + "runId", + "handlerId", + "state", ], "type": "object", }, - "TokenUsage": { + "SubmissionHandlerStatus": { "properties": { - "completionTokens": { - "type": "number", + "error": { + "type": "string", }, - "promptTokens": { - "type": "number", + "status": { + "enum": [ + "queued", + "processing", + "done", + "failed", + ], + "type": "string", }, - "totalTokens": { + "updatedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "version": { "type": "number", }, }, "required": [ - "promptTokens", - "completionTokens", - "totalTokens", + "status", ], "type": "object", }, - "TriggerReportsInput": { + "SubmissionListResponse": { "properties": { - "requestId": { - "type": "string", + "submissions": { + "items": { + "$ref": "#/components/schemas/SubmissionResponse", + }, + "type": "array", }, }, "required": [ - "requestId", + "submissions", ], "type": "object", }, - "UpdateAgentInput": { + "SubmissionReportPointer": { "properties": { - "available": { - "type": "boolean", - }, - "capabilities": { - "$ref": "#/components/schemas/AgentCapabilities", - }, - "defaultModel": { - "type": "string", - }, - "description": { + "reportId": { "type": "string", }, - "modelProvider": { - "type": "string", + "updatedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], }, - "name": { + "url": { "type": "string", }, - "supportedModels": { - "items": { - "type": "string", - }, - "type": "array", - }, }, "type": "object", }, - "UpdateCodebaseInput": { + "SubmissionRequestRef": { "properties": { - "defaultBranch": { - "type": "string", - }, - "description": { + "requestId": { "type": "string", }, - "name": { - "minLength": 1, + "runId": { "type": "string", }, }, + "required": [ + "requestId", + "runId", + ], "type": "object", }, - "UpdateCriteriaInput": { + "SubmissionRequestSummary": { "properties": { - "dependsOn": { - "items": { - "type": "string", - }, - "type": "array", - }, - "gates": { - "items": { - "enum": [ - "select", - "build", - "test", - "run", - "deploy", - ], - "type": "string", - }, - "type": "array", + "model": { + "type": "string", }, - "kind": { - "enum": [ - "gate", - "observation", - ], + "outcome": { "type": "string", }, - "prompt": { + "profileId": { "type": "string", }, - "subject": { - "enum": [ - "run", - "iteration", - ], + "requestId": { "type": "string", }, - "taxonomyElementId": { - "enum": [ - "dimension:idiomatic-use", - "dimension:dependency-currency", - "dimension:configuration-correctness", - ], + "runId": { "type": "string", }, - }, - "type": "object", - }, - "UpdateExtensionInput": { - "properties": { - "description": { + "status": { "type": "string", }, - "name": { + "task": { "type": "string", }, - }, - "type": "object", - }, - "UpdateFeatureFlagInput": { - "properties": { - "enabled": { - "type": "boolean", + "workerType": { + "type": "string", }, }, "required": [ - "enabled", + "requestId", + "runId", + "task", + "status", + "workerType", ], "type": "object", }, - "UpdateInsightInput": { + "SubmissionResponse": { "properties": { - "category": { - "type": "string", - }, - "description": { - "type": "string", + "attemptNumber": { + "type": "integer", }, - "tags": { + "blocking": { "items": { - "type": "string", + "$ref": "#/components/schemas/SubmissionBlockingDetail", }, "type": "array", }, - "title": { - "type": "string", - }, - }, - "type": "object", - }, + "createdAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "dependencyProjection": { + "items": { + "$ref": "#/components/schemas/SubmissionDependencyProjection", + }, + "type": "array", + }, + "handlerProgress": { + "additionalProperties": { + "$ref": "#/components/schemas/SubmissionHandlerStatus", + }, + "type": "object", + }, + "projectId": { + "type": "string", + }, + "readiness": { + "enum": [ + "waiting", + "ready", + "blocked", + "processing", + "done", + "failed", + ], + "type": "string", + }, + "report": { + "$ref": "#/components/schemas/SubmissionReportPointer", + }, + "requestIds": { + "items": { + "type": "string", + }, + "type": "array", + }, + "requests": { + "items": { + "$ref": "#/components/schemas/SubmissionRequestSummary", + }, + "type": "array", + }, + "submissionId": { + "type": "string", + }, + "submissionRunId": { + "type": "string", + }, + "supersededAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "supersededBySubmissionRunId": { + "type": "string", + }, + "supersededReason": { + "enum": [ + "request-vector-changed", + "manual-retry", + ], + "type": "string", + }, + "updatedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + }, + "required": [ + "submissionId", + "projectId", + "requestIds", + "createdAt", + "submissionRunId", + "attemptNumber", + "readiness", + "requests", + "dependencyProjection", + "blocking", + ], + "type": "object", + }, + "SubmissionRetryResponse": { + "properties": { + "attemptNumber": { + "type": "integer", + }, + "blocking": { + "items": { + "$ref": "#/components/schemas/SubmissionBlockingDetail", + }, + "type": "array", + }, + "readiness": { + "enum": [ + "waiting", + "ready", + "blocked", + "processing", + "done", + "failed", + ], + "type": "string", + }, + "submissionId": { + "type": "string", + }, + "submissionRunId": { + "type": "string", + }, + }, + "required": [ + "submissionId", + "submissionRunId", + "attemptNumber", + "readiness", + "blocking", + ], + "type": "object", + }, + "SubmissionRunResponse": { + "properties": { + "_id": { + "type": "string", + }, + "attemptNumber": { + "minimum": 1, + "type": "integer", + }, + "blocking": { + "default": [], + "items": { + "$ref": "#/components/schemas/SubmissionBlockingDetail", + }, + "type": "array", + }, + "createdAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "dependencyProjection": { + "default": [], + "items": { + "$ref": "#/components/schemas/SubmissionDependencyProjection", + }, + "type": "array", + }, + "finishedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "handlerStatus": { + "additionalProperties": { + "$ref": "#/components/schemas/SubmissionHandlerStatus", + }, + "type": "object", + }, + "projectId": { + "type": "string", + }, + "report": { + "$ref": "#/components/schemas/SubmissionReportPointer", + }, + "requests": { + "items": { + "$ref": "#/components/schemas/SubmissionRequestRef", + }, + "type": "array", + }, + "startedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "submissionId": { + "type": "string", + }, + "supersededAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "supersededBySubmissionRunId": { + "type": "string", + }, + "supersededReason": { + "enum": [ + "request-vector-changed", + "manual-retry", + ], + "type": "string", + }, + "updatedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "vectorFingerprint": { + "type": "string", + }, + }, + "required": [ + "_id", + "attemptNumber", + "vectorFingerprint", + "requests", + "createdAt", + ], + "type": "object", + }, + "SuggestedPromptFeature": { + "properties": { + "behavior": { + "type": "string", + }, + "prompt": { + "type": "string", + }, + "suggestedId": { + "type": "string", + }, + }, + "required": [ + "suggestedId", + "behavior", + "prompt", + ], + "type": "object", + }, + "TaskPromptResponse": { + "properties": { + "_id": { + "type": "string", + }, + "contentBlobUrl": { + "type": "string", + }, + "createdAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "deletedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "features": { + "items": { + "$ref": "#/components/schemas/PromptFeatureResult", + }, + "type": "array", + }, + "featuresExtractedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "keyId": { + "type": "string", + }, + "projectId": { + "type": "string", + }, + "text": { + "type": "string", + }, + "type": { + "$ref": "#/components/schemas/PromptType", + }, + }, + "required": [ + "_id", + "keyId", + "createdAt", + "projectId", + ], + "type": "object", + }, + "TokenUsage": { + "properties": { + "completionTokens": { + "type": "number", + }, + "promptTokens": { + "type": "number", + }, + "totalTokens": { + "type": "number", + }, + }, + "required": [ + "promptTokens", + "completionTokens", + "totalTokens", + ], + "type": "object", + }, + "TriggerReportsInput": { + "properties": { + "requestId": { + "type": "string", + }, + }, + "required": [ + "requestId", + ], + "type": "object", + }, + "UpdateAgentInput": { + "properties": { + "available": { + "type": "boolean", + }, + "capabilities": { + "$ref": "#/components/schemas/AgentCapabilities", + }, + "defaultModel": { + "type": "string", + }, + "description": { + "type": "string", + }, + "modelProvider": { + "type": "string", + }, + "name": { + "type": "string", + }, + "supportedModels": { + "items": { + "type": "string", + }, + "type": "array", + }, + }, + "type": "object", + }, + "UpdateCodebaseInput": { + "properties": { + "defaultBranch": { + "type": "string", + }, + "description": { + "type": "string", + }, + "name": { + "minLength": 1, + "type": "string", + }, + }, + "type": "object", + }, + "UpdateCriteriaInput": { + "properties": { + "dependsOn": { + "items": { + "type": "string", + }, + "type": "array", + }, + "gates": { + "items": { + "enum": [ + "select", + "build", + "test", + "run", + "deploy", + ], + "type": "string", + }, + "type": "array", + }, + "kind": { + "enum": [ + "gate", + "observation", + ], + "type": "string", + }, + "prompt": { + "type": "string", + }, + "subject": { + "enum": [ + "run", + "iteration", + ], + "type": "string", + }, + "taxonomyElementId": { + "enum": [ + "dimension:idiomatic-use", + "dimension:dependency-currency", + "dimension:configuration-correctness", + ], + "type": "string", + }, + }, + "type": "object", + }, + "UpdateExtensionInput": { + "properties": { + "description": { + "type": "string", + }, + "name": { + "type": "string", + }, + }, + "type": "object", + }, + "UpdateFeatureFlagInput": { + "properties": { + "enabled": { + "type": "boolean", + }, + }, + "required": [ + "enabled", + ], + "type": "object", + }, + "UpdateInsightInput": { + "properties": { + "category": { + "type": "string", + }, + "description": { + "type": "string", + }, + "tags": { + "items": { + "type": "string", + }, + "type": "array", + }, + "title": { + "type": "string", + }, + }, + "type": "object", + }, "UpdateMcpServerInput": { "properties": { "args": { @@ -9344,6 +9825,22 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "type": "string", }, }, + { + "in": "query", + "name": "submissionId", + "required": false, + "schema": { + "type": "string", + }, + }, + { + "in": "query", + "name": "submissionRunId", + "required": false, + "schema": { + "type": "string", + }, + }, { "in": "query", "name": "projectId", @@ -9786,6 +10283,32 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` ], }, }, + "/api/v1/reports/{id}/structured": { + "get": { + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "type": "string", + }, + }, + ], + "responses": { + "200": { + "description": "Success", + }, + "404": { + "description": "Report or structured artifact not found", + }, + }, + "summary": "Download a report's structured JSON artifact", + "tags": [ + "Reports", + ], + }, + }, "/api/v1/requests": { "get": { "parameters": [ @@ -11373,49 +11896,7 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "parameters": [ { "in": "path", - "name": "id", - "required": true, - "schema": { - "type": "string", - }, - }, - { - "in": "path", - "name": "runId", - "required": true, - "schema": { - "type": "string", - }, - }, - ], - "responses": { - "200": { - "description": "Server-sent event stream of log entries", - }, - "404": { - "description": "Not found", - }, - }, - "summary": "Stream logs for a specific attempt (SSE)", - "tags": [ - "Requests", - ], - }, - }, - "/api/v1/requests/{id}/runs/{runId}/snapshots/{iteration}": { - "get": { - "parameters": [ - { - "in": "path", - "name": "id", - "required": true, - "schema": { - "type": "string", - }, - }, - { - "in": "path", - "name": "runId", + "name": "id", "required": true, "schema": { "type": "string", @@ -11423,7 +11904,7 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, { "in": "path", - "name": "iteration", + "name": "runId", "required": true, "schema": { "type": "string", @@ -11432,19 +11913,19 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` ], "responses": { "200": { - "description": "Gzipped snapshot archive", + "description": "Server-sent event stream of log entries", }, "404": { "description": "Not found", }, }, - "summary": "Download iteration snapshot for a specific attempt", + "summary": "Stream logs for a specific attempt (SSE)", "tags": [ "Requests", ], }, }, - "/api/v1/requests/{id}/runs/{runId}/taxonomy": { + "/api/v1/requests/{id}/runs/{runId}/snapshots/{iteration}": { "get": { "parameters": [ { @@ -11463,16 +11944,24 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "type": "string", }, }, + { + "in": "path", + "name": "iteration", + "required": true, + "schema": { + "type": "string", + }, + }, ], "responses": { "200": { - "description": "Taxonomy v1 JSON document (scorecard, behavior analysis, action list)", + "description": "Gzipped snapshot archive", }, "404": { - "description": "Not found — taxonomy not yet generated", + "description": "Not found", }, }, - "summary": "Download taxonomy JSON for a specific attempt", + "summary": "Download iteration snapshot for a specific attempt", "tags": [ "Requests", ], @@ -11580,32 +12069,6 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` ], }, }, - "/api/v1/requests/{id}/taxonomy": { - "get": { - "parameters": [ - { - "in": "path", - "name": "id", - "required": true, - "schema": { - "type": "string", - }, - }, - ], - "responses": { - "200": { - "description": "Taxonomy v1 JSON document (scorecard, behavior analysis, action list)", - }, - "404": { - "description": "Not found — taxonomy not yet generated", - }, - }, - "summary": "Download taxonomy JSON for the run", - "tags": [ - "Requests", - ], - }, - }, "/api/v1/requests/{id}/tool-calls": { "get": { "parameters": [ @@ -12661,6 +13124,255 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` ], }, }, + "/api/v1/submissions": { + "get": { + "parameters": [ + { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "in": "query", + "name": "projectId", + "required": true, + "schema": { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, + "type": "string", + }, + }, + { + "in": "query", + "name": "limit", + "required": false, + "schema": { + "default": 50, + "maximum": 100, + "minimum": 1, + "type": "integer", + }, + }, + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SubmissionListResponse", + }, + }, + }, + "description": "Success", + }, + }, + "summary": "List submissions", + "tags": [ + "Submissions", + ], + }, + }, + "/api/v1/submissions/{id}": { + "get": { + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "type": "string", + }, + }, + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SubmissionResponse", + }, + }, + }, + "description": "Success", + }, + "404": { + "description": "Submission not found", + }, + }, + "summary": "Get a submission", + "tags": [ + "Submissions", + ], + }, + }, + "/api/v1/submissions/{id}/readout": { + "get": { + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "type": "string", + }, + }, + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ReportResponse", + }, + }, + }, + "description": "Success", + }, + "404": { + "description": "Current readout not found", + }, + }, + "summary": "Get the current submission readout report", + "tags": [ + "Submissions", + ], + }, + }, + "/api/v1/submissions/{id}/retry": { + "post": { + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "type": "string", + }, + }, + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "submissionRunId": { + "type": "string", + }, + }, + "required": [ + "submissionRunId", + ], + "type": "object", + }, + }, + }, + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SubmissionRetryResponse", + }, + }, + }, + "description": "Success", + }, + "404": { + "description": "Submission not found", + }, + "409": { + "description": "Submission attempt is stale or dependencies are blocked", + }, + }, + "summary": "Retry current submission processing", + "tags": [ + "Submissions", + ], + }, + }, + "/api/v1/submissions/{id}/runs": { + "get": { + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "type": "string", + }, + }, + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "runs": { + "items": { + "$ref": "#/components/schemas/SubmissionRunResponse", + }, + "type": "array", + }, + }, + "required": [ + "runs", + ], + "type": "object", + }, + }, + }, + "description": "Success", + }, + "404": { + "description": "Submission not found", + }, + }, + "summary": "List current and historical submission attempts", + "tags": [ + "Submissions", + ], + }, + }, + "/api/v1/submissions/{id}/runs/{submissionRunId}": { + "get": { + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "type": "string", + }, + }, + { + "in": "path", + "name": "submissionRunId", + "required": true, + "schema": { + "type": "string", + }, + }, + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SubmissionRunResponse", + }, + }, + }, + "description": "Success", + }, + "404": { + "description": "Submission attempt not found", + }, + }, + "summary": "Get one submission attempt", + "tags": [ + "Submissions", + ], + }, + }, "/api/v1/task-prompts": { "get": { "parameters": [ diff --git a/apps/api/src/archive-har.test.ts b/apps/api/src/archive-har.test.ts index af8205f2b..f123b4b8c 100644 --- a/apps/api/src/archive-har.test.ts +++ b/apps/api/src/archive-har.test.ts @@ -100,16 +100,6 @@ describe("rewriteHarUrlsForArchive", () => { expect(result.run!.turns![2].toolCallsUrl).toBeUndefined(); }); - it("rewrites run.taxonomyUrl to taxonomy.json", () => { - const resource = { - run: { - taxonomyUrl: "https://storage.blob.core.windows.net/snapshots/abc123/runs/attempt-1/taxonomy.json", - turns: [], - }, - }; - const result = rewriteHarUrlsForArchive(resource); - expect(result.run!.taxonomyUrl).toBe("taxonomy.json"); - }); }); // --- blobNameFromSnapshotsUrl --- @@ -915,34 +905,4 @@ describe("packRunIntoTar", () => { await entriesPromise; expect(run.codebase!.archiveUrl).toBe(blobUrl); }); - - it("bundles run-level taxonomy.json and rewrites taxonomyUrl", async () => { - const { pack } = await import("tar-stream"); - const p = pack(); - const taxonomyData = Buffer.from('{"topics":["agent"]}'); - const container = makeMockBlobContainer({ - "run-012/runs/attempt-1/taxonomy.json": { body: taxonomyData, length: taxonomyData.length }, - }); - - const run: ArchivableRun = { - _id: "run-012", - run: { - taxonomyUrl: "https://storage.blob.core.windows.net/snapshots/run-012/runs/attempt-1/taxonomy.json", - turns: [], - }, - }; - - const entriesPromise = collectPackEntries(p); - await packRunIntoTar(p, run, container, "run-012", () => true); - p.finalize(); - - const entries = await entriesPromise; - const names = entries.map(e => e.name); - expect(names).toContain("run-012/taxonomy.json"); - expect(entries.find(e => e.name === "run-012/taxonomy.json")!.data).toEqual(taxonomyData); - - const yaml = entries.find(e => e.name === "run-012/run.yaml")!.data.toString(); - expect(yaml).toContain("taxonomy.json"); - expect(yaml).not.toContain("blob.core.windows.net"); - }); }); diff --git a/apps/api/src/archive-har.ts b/apps/api/src/archive-har.ts index c28658aea..204eaf124 100644 --- a/apps/api/src/archive-har.ts +++ b/apps/api/src/archive-har.ts @@ -41,12 +41,11 @@ export function blobNameFromLogsUrl(url: string): string | null { /** * Deep-clone a run resource and rewrite `harUrl`, `rawChatUrl`, - * `taxonomyUrl`, `chatResultUrl`, and `toolCallsUrl` fields to relative archive paths. + * `chatResultUrl`, and `toolCallsUrl` fields to relative archive paths. * * - `run.harUrl` → `"run.har"` * - Per-turn `harUrl` → `"iteration-{N}.har"` * - `run.rawChatUrl` → `"run.chat-export.json"` - * - `run.taxonomyUrl` → `"taxonomy.json"` * - Per-turn `rawChatUrl` → `"iteration-{N}.chat-export.json"` * - Per-turn `chatResultUrl` → `"iteration-{N}.chat-result.json"` * - Per-turn `toolCallsUrl` → `"iteration-{N}.tool-calls.jsonl"` @@ -55,7 +54,6 @@ export function rewriteHarUrlsForArchive; [key: string]: unknown; }; @@ -68,9 +66,6 @@ export function rewriteHarUrlsForArchive { + it("proxies the stored JSON artifact through the API", async () => { + const app = express(); + app.use(express.json()); + const downloadJson = vi.fn().mockResolvedValue({ + meta: { submissionRunId: "submission-run-1" }, + }); + const ctx = { + app, + registry: new OpenAPIRegistry(), + reportCollection: { + findOne: vi.fn().mockResolvedValue({ + _id: "report-1", + projectId: "project-1", + submissionId: "submission-1", + submissionRunId: "submission-run-1", + status: "completed", + structuredArtifact: { + schemaId: "scope.submission-readout", + schemaVersion: "1.0", + contentType: "application/json", + url: "https://storage.example/structured.json", + }, + createdAt: new Date(), + }), + }, + blobStorage: { downloadJson }, + } as unknown as RouteContext; + registerReportsRoutes(ctx); + app.use(( + error: Error, + _req: express.Request, + res: express.Response, + _next: express.NextFunction, + ) => res.status(500).json({ error: error.message })); + + const response = await request(app).get("/api/v1/reports/report-1/structured"); + + expect(response.status).toBe(200); + expect(response.headers["content-disposition"]).toContain( + "report-1.structured.json", + ); + expect(response.body).toEqual({ + meta: { submissionRunId: "submission-run-1" }, + }); + expect(downloadJson).toHaveBeenCalledWith( + "https://storage.example/structured.json", + ); + }); + + it("filters submission reports by the exact submission run", async () => { + const app = express(); + app.use(express.json()); + const toArray = vi.fn().mockResolvedValue([{ + _id: "report-1", + projectId: "project-1", + submissionId: "submission-1", + submissionRunId: "submission-run-1", + status: "completed", + createdAt: new Date("2026-10-13T12:00:00.000Z"), + }]); + const find = vi.fn(() => ({ + sort: vi.fn(() => ({ toArray })), + })); + const ctx = { + app, + registry: new OpenAPIRegistry(), + reportCollection: { find }, + requestCollection: { + find: vi.fn(() => ({ toArray: vi.fn().mockResolvedValue([]) })), + }, + } as unknown as RouteContext; + registerReportsRoutes(ctx); + + const response = await request(app).get( + "/api/v1/reports?submissionId=submission-1&submissionRunId=submission-run-1", + ); + + expect(response.status).toBe(200); + expect(find).toHaveBeenCalledWith({ + submissionId: "submission-1", + submissionRunId: "submission-run-1", + }); + expect(response.body[0]).toMatchObject({ + id: "report-1", + submissionId: "submission-1", + submissionRunId: "submission-run-1", + }); + }); +}); diff --git a/apps/api/src/routes/reports.ts b/apps/api/src/routes/reports.ts index d6dd7aa32..381600660 100644 --- a/apps/api/src/routes/reports.ts +++ b/apps/api/src/routes/reports.ts @@ -15,6 +15,7 @@ import { ReportResponseSchema, TriggerReportsInputSchema, evaluateTrigger, + withRetry, } from "shared"; import type { TaskPromptDocument } from "shared"; import { apiRoute } from "../openapi/api-route.js"; @@ -108,18 +109,29 @@ apiRoute(ctx.app, ctx.registry, { path: "/api/v1/reports", tags: ["Reports"], summary: "List reports", - query: z.object({ requestId: z.string().optional(), projectId: z.string().optional() }), + query: z.object({ + requestId: z.string().optional(), + submissionId: z.string().optional(), + submissionRunId: z.string().optional(), + projectId: z.string().optional(), + }), response: z.array(ReportResponseSchema), handler: async (req, res, next) => { try { const requestIdFilter = req.query.requestId as string; + const submissionIdFilter = req.query.submissionId as string; + const submissionRunIdFilter = req.query.submissionRunId as string; const filter: Record = {}; if (requestIdFilter) { // Parent-scoped: the run id already pins the project. filter.requestId = requestIdFilter; + } else if (submissionIdFilter) { + filter.submissionId = submissionIdFilter; + if (submissionRunIdFilter) filter.submissionRunId = submissionRunIdFilter; } else { // Top-level reports list requires an explicit project (400 if absent). filter.projectId = getQueryProjectId(req); + if (submissionRunIdFilter) filter.submissionRunId = submissionRunIdFilter; } const reports = await ctx.reportCollection @@ -128,13 +140,21 @@ apiRoute(ctx.app, ctx.registry, { .toArray(); // Enrich reports with the task from their associated run - const requestIds = [...new Set(reports.map(r => r.requestId))]; + const requestIds = [ + ...new Set(reports.flatMap((report) => report.requestId ? [report.requestId] : [])), + ]; const runs = requestIds.length > 0 ? await ctx.requestCollection.find({ _id: { $in: requestIds } as any }, { projection: { _id: 1, "scenario.task": 1 } }).toArray() : []; const taskByRequestId = new Map(runs.map(r => [r._id, r.scenario?.task])); - res.json(reports.map(r => ({ ...r, id: r._id, task: taskByRequestId.get(r.requestId) }))); + res.json(reports.map((report) => ({ + ...report, + id: report._id, + ...(report.requestId + ? { task: taskByRequestId.get(report.requestId) } + : {}), + }))); } catch (error) { next(error); } @@ -201,6 +221,39 @@ apiRoute(ctx.app, ctx.registry, { }, }); +apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/reports/:id/structured", + tags: ["Reports"], + summary: "Download a report's structured JSON artifact", + params: z.object({ id: z.string() }), + response: z.any(), + rawResponse: true, + errorResponses: { + 404: { description: "Report or structured artifact not found" }, + }, + handler: async (req, res, next) => { + try { + const report = await ctx.reportCollection.findOne({ _id: req.params.id }); + if (!report?.structuredArtifact) { + res.status(404).json({ error: "Structured report artifact not found" }); + return; + } + const structured = await withRetry(() => + ctx.blobStorage.downloadJson(report.structuredArtifact!.url) + ); + res.setHeader("Content-Type", "application/json"); + res.setHeader( + "Content-Disposition", + `attachment; filename="${req.params.id}.structured.json"`, + ); + res.json(structured); + } catch (error) { + next(error); + } + }, +}); + apiRoute(ctx.app, ctx.registry, { method: "post", path: "/api/v1/reports/bulk-status", @@ -229,7 +282,7 @@ apiRoute(ctx.app, ctx.registry, { // Build a map of requestId → latest report status const statusMap: Record = {}; for (const report of reports) { - if (!statusMap[report.requestId]) { + if (report.requestId && !statusMap[report.requestId]) { statusMap[report.requestId] = { reportId: report._id, status: report.status, diff --git a/apps/api/src/routes/requests/taxonomy.test.ts b/apps/api/src/routes/requests/taxonomy.test.ts deleted file mode 100644 index 3909df826..000000000 --- a/apps/api/src/routes/requests/taxonomy.test.ts +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -/** - * Unit tests for taxonomy download endpoints: - * - GET /api/v1/requests/:id/taxonomy - * - GET /api/v1/requests/:id/runs/:runId/taxonomy - */ -import { describe, it, expect, beforeAll, beforeEach, vi } from "vitest"; -import supertest from "supertest"; -import { app, _injectTestDependencies } from "../../index.js"; -import { createAllMockDependencies } from "../../test-helpers.js"; -import { readableFrom, rawParser, blobUrl, VARIANTS, RUN_LEVEL, rewireBlobMocks } from "./test-blob-helpers.js"; - -vi.mock("db-migrations/check-migrations", () => ({ - checkMigrations: vi.fn().mockResolvedValue({ ready: true, applied: ["001", "002"], pending: [] }), -})); - -vi.hoisted(() => { - process.env.STORAGE_CONNECTION_STRING = "DefaultEndpointsProtocol=https;AccountName=mockaccount;AccountKey=bW9jaw==;EndpointSuffix=core.windows.net"; -}); - -vi.mock("../../llm.js", () => ({ isLlmAvailable: vi.fn().mockReturnValue(false) })); -vi.mock("../../prompt-feature-llm.js", () => ({ isLlmAvailable: vi.fn().mockReturnValue(false) })); -vi.mock("../../task-prompt-llm.js", () => ({ isTaskPromptLlmAvailable: vi.fn().mockReturnValue(false) })); - -const { mockDownload, mockGetProperties, mockGetBlockBlobClient, mockGetBlobClient, mockGetContainerClient } = vi.hoisted(() => { - const mockDownload = vi.fn(); - const mockGetProperties = vi.fn(); - const mockGetBlockBlobClient = vi.fn().mockReturnValue({ download: mockDownload, getProperties: mockGetProperties }); - const mockGetBlobClient = vi.fn().mockReturnValue({ download: mockDownload, getProperties: mockGetProperties }); - const mockGetContainerClient = vi.fn().mockReturnValue({ getBlockBlobClient: mockGetBlockBlobClient, getBlobClient: mockGetBlobClient }); - return { mockDownload, mockGetProperties, mockGetBlockBlobClient, mockGetBlobClient, mockGetContainerClient }; -}); - -vi.mock("@azure/storage-blob", async (importOriginal) => { - const original = await importOriginal(); - return { - ...original, - BlobServiceClient: { fromConnectionString: vi.fn().mockReturnValue({ getContainerClient: mockGetContainerClient }) }, - }; -}); - -describe("taxonomy endpoints", () => { - let mocks: ReturnType; - - beforeAll(() => { - mocks = createAllMockDependencies(); - _injectTestDependencies(mocks); - }); - - beforeEach(() => { - vi.clearAllMocks(); - mocks = createAllMockDependencies(); - _injectTestDependencies(mocks); - rewireBlobMocks({ mockDownload, mockGetProperties, mockGetBlockBlobClient, mockGetBlobClient, mockGetContainerClient }); - mockDownload.mockResolvedValue({ readableStreamBody: readableFrom('{"status":"ok"}'), contentLength: 15 }); - mockGetProperties.mockResolvedValue({ contentLength: 15 }); - }); - - describe.each(VARIANTS)("$label", (variant) => { - const taxonomyUrl = (id: string) => variant.url(id) + "taxonomy"; - - it("returns 404 when request not found", async () => { - (mocks.collection.findOne as ReturnType).mockResolvedValue(null); - const res = await supertest(app).get(taxonomyUrl("missing")); - expect(res.status).toBe(404); - expect(res.body).toMatchObject({ error: "Request not found" }); - }); - - it("returns 404 when taxonomyUrl is not set", async () => { - variant.mockRequest(mocks, "req-1", { _id: "run-1", attemptNumber: 1, status: "done" }); - const res = await supertest(app).get(taxonomyUrl("req-1")); - expect(res.status).toBe(404); - expect(res.body).toMatchObject({ error: "Taxonomy not yet generated for this run" }); - }); - - it("returns taxonomy JSON when taxonomyUrl is set", async () => { - variant.mockRequest(mocks, "req-1", { - _id: "run-1", - attemptNumber: 1, - status: "done", - taxonomyUrl: blobUrl("req-1/taxonomy.json"), - }); - - const res = await supertest(app).get(taxonomyUrl("req-1")).buffer(true).parse(rawParser); - expect(res.status).toBe(200); - expect(res.headers["content-type"]).toMatch("application/json"); - expect(res.headers["content-disposition"]).toContain("req-1-taxonomy.json"); - expect(mockGetBlockBlobClient).toHaveBeenCalledWith("req-1/taxonomy.json"); - }); - }); - - it("works with per-run variant", async () => { - RUN_LEVEL.mockRequest(mocks, "req-1", { - _id: "run-1", - attemptNumber: 1, - status: "done", - taxonomyUrl: blobUrl("req-1/runs/run-1/taxonomy.json"), - }); - - const res = await supertest(app).get(RUN_LEVEL.url("req-1") + "taxonomy").buffer(true).parse(rawParser); - expect(res.status).toBe(200); - expect(res.headers["content-disposition"]).toContain("req-1-taxonomy.json"); - expect(mockGetBlockBlobClient).toHaveBeenCalledWith("req-1/runs/run-1/taxonomy.json"); - }); -}); diff --git a/apps/api/src/routes/requests/taxonomy.ts b/apps/api/src/routes/requests/taxonomy.ts deleted file mode 100644 index e74d303f6..000000000 --- a/apps/api/src/routes/requests/taxonomy.ts +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -import type { RouteContext } from "../../route-context.js"; -import { downloadBlobToResponse } from "./blob-helpers.js"; -import { registerArtifactRoutes } from "./artifact-route.js"; - -export function registerRequestsTaxonomyRoutes(ctx: RouteContext): void { - registerArtifactRoutes(ctx, { - path: "taxonomy", - summary: "Download taxonomy JSON for the run", - perRunSummary: "Download taxonomy JSON for a specific attempt", - responseDescription: "Taxonomy v1 JSON document (scorecard, behavior analysis, action list)", - errorResponses: { 404: { description: "Not found — taxonomy not yet generated" } }, - blobNotFoundMessage: "Taxonomy file not found — the blob may have been deleted or is no longer available", - handler: async (ctx, _req, res, targetRun, id) => { - const taxonomyUrl = targetRun.taxonomyUrl; - if (!taxonomyUrl) { - res.status(404).json({ error: "Taxonomy not yet generated for this run" }); - return; - } - - await downloadBlobToResponse(ctx, res, taxonomyUrl, { - contentType: "application/json", - filename: `${id}-taxonomy.json`, - label: "Taxonomy", - }); - }, - }); -} diff --git a/apps/scheduler/src/handler-dispatcher.test.ts b/apps/scheduler/src/handler-dispatcher.test.ts index 847e6be90..32d954243 100644 --- a/apps/scheduler/src/handler-dispatcher.test.ts +++ b/apps/scheduler/src/handler-dispatcher.test.ts @@ -16,11 +16,11 @@ const HANDLERS: HandlerServiceDocument[] = [ dependsOn: [], }, { - _id: "pp-taxonomy", + _id: "pp-example", type: "post-process-handler", version: 1, - queue: "pp-taxonomy-queue", - selector: "taxonomy", + queue: "pp-example-queue", + selector: "example", autoBackfill: false, dependsOn: ["pp-atif"], }, @@ -87,7 +87,7 @@ describe("HandlerDispatcher — drain detection", () => { it("is drained when all handlers are done", () => { expect( - isDrained(HANDLERS, { "pp-atif": { status: "done" }, "pp-taxonomy": { status: "done" } }), + isDrained(HANDLERS, { "pp-atif": { status: "done" }, "pp-example": { status: "done" } }), ).toBe(true); }); @@ -99,13 +99,13 @@ describe("HandlerDispatcher — drain detection", () => { expect( isDrained(HANDLERS, { "pp-atif": { status: "done" }, - "pp-taxonomy": { status: "processing" }, + "pp-example": { status: "processing" }, }), ).toBe(false); }); it("is drained when a failed dependency blocks its descendant", () => { - // pp-atif failed → pp-taxonomy can never run (blocked) → terminal. + // pp-atif failed → pp-example can never run (blocked) → terminal. expect(isDrained(HANDLERS, { "pp-atif": { status: "failed" } })).toBe(true); }); @@ -113,7 +113,7 @@ describe("HandlerDispatcher — drain detection", () => { expect( isDrained(HANDLERS, { "pp-atif": { status: "done" }, - "pp-taxonomy": { status: "failed" }, + "pp-example": { status: "failed" }, }), ).toBe(true); }); @@ -143,7 +143,7 @@ describe("HandlerDispatcher — maybeTriggerReports", () => { run: { _id: "run-1", status: "done", - handlerStatus: { "pp-atif": { status: "done" }, "pp-taxonomy": { status: "done" } }, + handlerStatus: { "pp-atif": { status: "done" }, "pp-example": { status: "done" } }, }, }); collection.findOneAndUpdate.mockResolvedValue({ _id: "req-1" }); // claim succeeds @@ -203,7 +203,7 @@ describe("HandlerDispatcher — maybeTriggerReports", () => { run: { _id: "run-2", status: "done", - handlerStatus: { "pp-atif": { status: "done" }, "pp-taxonomy": { status: "done" } }, + handlerStatus: { "pp-atif": { status: "done" }, "pp-example": { status: "done" } }, }, }); @@ -220,7 +220,7 @@ describe("HandlerDispatcher — maybeTriggerReports", () => { run: { _id: "run-1", status: "done", - handlerStatus: { "pp-atif": { status: "done" }, "pp-taxonomy": { status: "done" } }, + handlerStatus: { "pp-atif": { status: "done" }, "pp-example": { status: "done" } }, }, }); collection.findOneAndUpdate.mockResolvedValue(null); // another path claimed it first @@ -237,7 +237,7 @@ describe("HandlerDispatcher — maybeTriggerReports", () => { run: { _id: "run-1", status: "done", - handlerStatus: { "pp-atif": { status: "done" }, "pp-taxonomy": { status: "done" } }, + handlerStatus: { "pp-atif": { status: "done" }, "pp-example": { status: "done" } }, }, }); collection.findOneAndUpdate.mockResolvedValue({ _id: "req-1" }); diff --git a/apps/scheduler/src/notify-routes.test.ts b/apps/scheduler/src/notify-routes.test.ts index 38a852ae1..284bf081a 100644 --- a/apps/scheduler/src/notify-routes.test.ts +++ b/apps/scheduler/src/notify-routes.test.ts @@ -328,11 +328,11 @@ describe("createHttpServer", () => { servers.push(server); const doc = { - _id: "pp-taxonomy", + _id: "pp-example", type: "post-process-handler", version: 1, - queue: "pp-taxonomy-queue", - selector: "taxonomy", + queue: "pp-example-queue", + selector: "example", autoBackfill: false, dependsOn: ["pp-atif"], }; @@ -344,7 +344,7 @@ describe("createHttpServer", () => { }); expect(response.status).toBe(200); - expect(await response.json()).toEqual({ ok: true, handlerId: "pp-taxonomy" }); + expect(await response.json()).toEqual({ ok: true, handlerId: "pp-example" }); expect(handler.registerHandler).toHaveBeenCalledWith(doc); }); diff --git a/apps/workers/taxonomy/Dockerfile b/apps/workers/readout/Dockerfile similarity index 60% rename from apps/workers/taxonomy/Dockerfile rename to apps/workers/readout/Dockerfile index b6be8f2ad..dc615f72c 100644 --- a/apps/workers/taxonomy/Dockerfile +++ b/apps/workers/readout/Dockerfile @@ -7,22 +7,22 @@ WORKDIR /app COPY package.json pnpm-workspace.yaml pnpm-lock.yaml ./ COPY tsconfig.base.json ./ COPY packages/shared/package.json ./packages/shared/ -COPY apps/workers/taxonomy/package.json ./apps/workers/taxonomy/ +COPY apps/workers/readout/package.json ./apps/workers/readout/ RUN pnpm install --frozen-lockfile COPY packages/shared/tsconfig.json ./packages/shared/ COPY packages/shared/src ./packages/shared/src/ -COPY apps/workers/taxonomy/tsconfig.json ./apps/workers/taxonomy/ -COPY apps/workers/taxonomy/src ./apps/workers/taxonomy/src/ -COPY apps/workers/taxonomy/handler.yaml ./apps/workers/taxonomy/ +COPY apps/workers/readout/tsconfig.json ./apps/workers/readout/ +COPY apps/workers/readout/src ./apps/workers/readout/src/ +COPY apps/workers/readout/handler.yaml ./apps/workers/readout/ -RUN pnpm --filter shared build && pnpm --filter taxonomy-handler build +RUN pnpm --filter shared build && pnpm --filter readout-handler build # --- Dev stage: hot reload with tsx watch --- FROM builder AS dev ENV NODE_ENV=development -ENV SERVICE_DIR=workers/taxonomy +ENV SERVICE_DIR=workers/readout COPY scripts/dev-entrypoint.sh /app/scripts/dev-entrypoint.sh EXPOSE 80 ENV PORT=80 @@ -37,14 +37,14 @@ WORKDIR /app COPY package.json pnpm-workspace.yaml pnpm-lock.yaml ./ COPY packages/shared/package.json ./packages/shared/ -COPY apps/workers/taxonomy/package.json ./apps/workers/taxonomy/ +COPY apps/workers/readout/package.json ./apps/workers/readout/ RUN pnpm install --frozen-lockfile --prod COPY --from=builder /app/packages/shared/dist ./packages/shared/dist/ -COPY --from=builder /app/apps/workers/taxonomy/dist ./apps/workers/taxonomy/dist/ -COPY --from=builder /app/apps/workers/taxonomy/handler.yaml ./apps/workers/taxonomy/handler.yaml +COPY --from=builder /app/apps/workers/readout/dist ./apps/workers/readout/dist/ +COPY --from=builder /app/apps/workers/readout/handler.yaml ./apps/workers/readout/handler.yaml -WORKDIR /app/apps/workers/taxonomy +WORKDIR /app/apps/workers/readout CMD ["node", "dist/index.js"] diff --git a/apps/workers/readout/handler.yaml b/apps/workers/readout/handler.yaml new file mode 100644 index 000000000..967490565 --- /dev/null +++ b/apps/workers/readout/handler.yaml @@ -0,0 +1,12 @@ +# Submission handler registration metadata. +# Read by the generic register-handler script and POSTed to the +# scheduler's POST /handlers/register endpoint on deploy. +_id: pp-readout +type: submission-handler +version: 1 +queue: pp-readout-queue +selector: readout +autoBackfill: false +dependsOn: + - pp-observations +dependencyMode: all-runs diff --git a/apps/workers/taxonomy/package.json b/apps/workers/readout/package.json similarity index 76% rename from apps/workers/taxonomy/package.json rename to apps/workers/readout/package.json index 52cdd6b2f..8e4b87b7f 100644 --- a/apps/workers/taxonomy/package.json +++ b/apps/workers/readout/package.json @@ -1,5 +1,5 @@ { - "name": "taxonomy-handler", + "name": "readout-handler", "version": "1.0.0", "type": "module", "main": "dist/index.js", @@ -7,14 +7,14 @@ "build": "tsc", "dev": "tsx watch src/index.ts", "start": "node dist/index.js", - "taxonomy": "tsx src/cli.ts", "lint": "eslint src/" }, "dependencies": { "shared": "workspace:*", "@azure/storage-queue": "^12.25.0", - "@github/copilot-sdk": "^0.1.17", - "dotenv": "^16.4.5" + "@github/copilot-sdk": "1.0.1", + "dotenv": "^16.4.5", + "zod": "^4.3.6" }, "devDependencies": { "@types/node": "^22.7.0", diff --git a/apps/workers/readout/src/index.ts b/apps/workers/readout/src/index.ts new file mode 100644 index 000000000..e9ed516eb --- /dev/null +++ b/apps/workers/readout/src/index.ts @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import "dotenv/config"; +import { + ReadoutQueueProcessor, + type ReadoutQueueProcessorConfig, +} from "./readout-queue-processor.js"; + +const config: ReadoutQueueProcessorConfig = { + mongoUri: + process.env.MONGO_CONNECTION_STRING || + process.env.AZURE_COSMOS_CONNECTION_STRING || + "mongodb://localhost:27017", + mongoDatabase: process.env.MONGO_DATABASE || "requests-db", + mongoCollection: "submissions", + storageAccountName: + process.env.AZURE_STORAGE_ACCOUNT_NAME || "devstoreaccount1", + storageConnectionString: + process.env.STORAGE_CONNECTION_STRING || + process.env.AZURE_STORAGE_CONNECTION_STRING || + undefined, + queueName: process.env.AZURE_STORAGE_QUEUE_READOUT || "pp-readout-queue", + batchSize: Number(process.env.BATCH_SIZE) || 1, + pollIntervalMs: Number(process.env.POLL_INTERVAL_MS) || 5000, + redisHost: process.env.REDIS_HOST || "localhost", + redisPort: Number(process.env.REDIS_PORT) || 6379, + redisPassword: process.env.REDIS_PASSWORD || "", + model: process.env.READOUT_MODEL || "gpt-5.4", + sessionTimeoutMs: + Number(process.env.READOUT_SESSION_TIMEOUT_MS) || 5 * 60 * 1000, + tokenManagerUrl: process.env.TOKEN_MANAGER_URL, +}; + +const processor = new ReadoutQueueProcessor(config); +processor.start().catch((error) => { + console.error("[readout-handler] Fatal error:", error); + process.exit(1); +}); diff --git a/apps/workers/readout/src/prompt.ts b/apps/workers/readout/src/prompt.ts new file mode 100644 index 000000000..0841817b8 --- /dev/null +++ b/apps/workers/readout/src/prompt.ts @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { SubmissionEvidence } from "./readout-builder.js"; + +export const READOUT_SYSTEM_PROMPT = `You synthesize a submission-level benchmark readout from an exact immutable vector of Scope runs. + +The user message contains JSON evidence for every request/run pair in that vector: request metadata, coding outcome, profile metadata, criteria definitions, run-level and per-iteration observations, and ATIF trajectories. + +Return raw JSON only with this shape: +{ + "title": "string", + "executiveSummary": "string", + "profiles": [ + { "profileId": "optional string", "requestIds": ["string"], "summary": "string" } + ], + "runs": [ + { "requestId": "string", "runId": "string", "trajectoryHighlights": ["string"] } + ], + "findings": [ + { + "id": "string", + "title": "string", + "description": "string", + "severity": "critical|high|medium|low|info", + "evidence": [ + { + "requestId": "string", + "runId": "string", + "iteration": 0, + "criterionId": "optional string", + "excerpt": "string" + } + ] + } + ], + "actions": [ + { + "id": "string", + "title": "string", + "priority": "critical|high|medium|low", + "description": "string", + "relatedFindingIds": ["string"] + } + ] +} + +Rules: +- Use only requestId, runId, iteration, criterionId, and profileId values present in the evidence. +- Treat failed coding outcomes as valid evidence. +- Treat observation results with passed:false as evidence, not as missing data. +- An empty selected-observation set is valid; summarize trajectory and outcome evidence without inventing observations. +- Include every exact-vector run once in runs and every request once in a profile group. +- Preserve disagreements and mixed results instead of averaging them away. +- Ground every finding in at least one evidence reference. +- Return no Markdown fences or commentary.`; + +export function composeReadoutUserPrompt(evidence: SubmissionEvidence): string { + return [ + "Generate the submission readout synthesis for this exact immutable evidence vector.", + JSON.stringify(evidence), + ].join("\n\n"); +} diff --git a/apps/workers/readout/src/readout-builder.test.ts b/apps/workers/readout/src/readout-builder.test.ts new file mode 100644 index 000000000..fa377becb --- /dev/null +++ b/apps/workers/readout/src/readout-builder.test.ts @@ -0,0 +1,191 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, expect, it } from "vitest"; +import type { + CriteriaDocument, + RequestDocument, + RunState, +} from "shared"; +import { + composeSubmissionReadout, + renderSubmissionReadoutMarkdown, + type SubmissionEvidence, +} from "./readout-builder.js"; + +function evidence(): SubmissionEvidence { + const criterion: CriteriaDocument = { + id: "uses-current-api", + projectId: "project-1", + prompt: "Uses the current API", + kind: "observation", + subject: "run", + taxonomyElementId: "dimension:dependency-currency", + createdAt: new Date(), + }; + const request = { + _id: "request-1", + projectId: "project-1", + workerType: "copilot", + profileId: "profile-1", + profileVersionId: "profile-version-1", + observations: [criterion.id], + scenario: { task: "Update the API", criteria: [] }, + createdAt: new Date(), + priority: 0, + } satisfies RequestDocument; + const run = { + _id: "run-1", + attemptNumber: 1, + status: "done", + outcome: "failed", + observationResults: [{ + criterionId: criterion.id, + passed: false, + feedback: "The deprecated API remained.", + evaluated: true, + }], + turns: [{ + iteration: 1, + judgeFeedback: "", + snapshotUrl: "https://example.test/snapshot", + passed: false, + timestamp: new Date("2026-10-13T12:00:00.000Z"), + observationResults: [{ + criterionId: criterion.id, + passed: true, + feedback: "The first iteration used the current API.", + evaluated: true, + }], + }], + } satisfies RunState; + return { + submissionId: "submission-1", + submissionRunId: "submission-run-1", + runs: [{ request, run, criteria: [criterion], trajectories: [] }], + }; +} + +describe("submission readout composition", () => { + it("preserves mixed run/iteration observations, profiles, and provenance", () => { + const readout = composeSubmissionReadout(evidence(), { + title: "Submission readout", + executiveSummary: "The run failed after mixed observation evidence.", + profiles: [{ + profileId: "profile-1", + requestIds: ["request-1"], + summary: "One profiled run.", + }], + runs: [{ + requestId: "request-1", + runId: "run-1", + trajectoryHighlights: ["The agent changed approach between iterations."], + }], + findings: [{ + id: "finding-1", + title: "Regression", + description: "The final state regressed.", + severity: "high", + evidence: [{ + requestId: "request-1", + runId: "run-1", + iteration: 1, + criterionId: "uses-current-api", + excerpt: "The deprecated API remained.", + }], + }], + actions: [{ + id: "action-1", + title: "Prevent regression", + priority: "high", + description: "Add a final-state check.", + relatedFindingIds: ["finding-1"], + }], + }, new Date("2026-10-13T12:00:00.000Z")); + + expect(readout.runs[0].observations).toEqual([ + expect.objectContaining({ subject: "run", passed: false }), + expect.objectContaining({ subject: "iteration", passed: true }), + ]); + expect(readout.findings[0].evidence[0]).toMatchObject({ + submissionRunId: "submission-run-1", + requestId: "request-1", + runId: "run-1", + }); + expect(readout.profiles).toEqual([{ + profileId: "profile-1", + requestIds: ["request-1"], + summary: "One profiled run.", + }]); + expect(readout.summary.failedRuns).toBe(1); + expect(readout.summary.passedObservations).toBe(1); + expect(readout.summary.failedObservations).toBe(1); + expect(renderSubmissionReadoutMarkdown(readout)).toContain( + "Schema: `scope.submission-readout@1.0`", + ); + }); + + it("rejects evidence outside the exact vector", () => { + expect(() => composeSubmissionReadout(evidence(), { + title: "Submission readout", + executiveSummary: "Invalid evidence.", + profiles: [{ requestIds: ["request-1"], summary: "One run." }], + runs: [{ + requestId: "request-1", + runId: "run-1", + trajectoryHighlights: [], + }], + findings: [{ + id: "finding-1", + title: "Invalid", + description: "References a newer run.", + severity: "high", + evidence: [{ + requestId: "request-1", + runId: "run-newer", + excerpt: "Not in the vector.", + }], + }], + actions: [], + })).toThrow("non-vector run"); + }); + + it("rejects findings that cite a missing iteration", () => { + expect(() => composeSubmissionReadout(evidence(), { + title: "Submission readout", + executiveSummary: "Invalid iteration evidence.", + profiles: [], + runs: [], + findings: [{ + id: "finding-1", + title: "Invalid", + description: "References a missing iteration.", + severity: "high", + evidence: [{ + requestId: "request-1", + runId: "run-1", + iteration: 2, + excerpt: "Not in the vector.", + }], + }], + actions: [], + })).toThrow("unknown iteration 2"); + }); + + it("canonically retains every exact-vector profile when synthesis omits it", () => { + const readout = composeSubmissionReadout(evidence(), { + title: "Submission readout", + executiveSummary: "Profile synthesis was omitted.", + profiles: [], + runs: [], + findings: [], + actions: [], + }); + + expect(readout.profiles).toEqual([{ + profileId: "profile-1", + requestIds: ["request-1"], + summary: "1 exact-vector run.", + }]); + }); +}); diff --git a/apps/workers/readout/src/readout-builder.ts b/apps/workers/readout/src/readout-builder.ts new file mode 100644 index 000000000..0143188b9 --- /dev/null +++ b/apps/workers/readout/src/readout-builder.ts @@ -0,0 +1,303 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { z } from "zod"; +import { + SUBMISSION_READOUT_SCHEMA_ID, + SUBMISSION_READOUT_SCHEMA_VERSION, + submissionReadoutSchema, + type CriteriaDocument, + type CriterionResult, + type RequestDocument, + type RunState, + type SubmissionReadout, +} from "shared"; + +export interface ExactRunEvidence { + request: RequestDocument; + run: RunState; + criteria: CriteriaDocument[]; + trajectories: Array<{ iteration: number; trajectory: unknown }>; +} + +export interface SubmissionEvidence { + submissionId: string; + submissionRunId: string; + runs: ExactRunEvidence[]; +} + +const evidenceReferenceSchema = z.object({ + requestId: z.string().min(1), + runId: z.string().min(1), + iteration: z.number().int().min(0).optional(), + criterionId: z.string().min(1).optional(), + excerpt: z.string().min(1), +}); + +export const readoutSynthesisSchema = z.object({ + title: z.string().min(1), + executiveSummary: z.string().min(1), + profiles: z.array(z.object({ + profileId: z.string().optional(), + requestIds: z.array(z.string().min(1)).min(1), + summary: z.string().min(1), + })), + runs: z.array(z.object({ + requestId: z.string().min(1), + runId: z.string().min(1), + trajectoryHighlights: z.array(z.string().min(1)), + })), + findings: z.array(z.object({ + id: z.string().min(1), + title: z.string().min(1), + description: z.string().min(1), + severity: z.enum(["critical", "high", "medium", "low", "info"]), + evidence: z.array(evidenceReferenceSchema).min(1), + })), + actions: z.array(z.object({ + id: z.string().min(1), + title: z.string().min(1), + priority: z.enum(["critical", "high", "medium", "low"]), + description: z.string().min(1), + relatedFindingIds: z.array(z.string().min(1)), + })), +}); + +export type ReadoutSynthesis = z.infer; + +function observationsForRun(evidence: ExactRunEvidence) { + const criteriaById = new Map(evidence.criteria.map((criterion) => [criterion.id, criterion])); + const mapResult = ( + result: CriterionResult, + subject: "run" | "iteration", + ) => { + const criterion = criteriaById.get(result.criterionId); + return { + criterionId: result.criterionId, + ...(criterion?.taxonomyElementId + ? { taxonomyElementId: criterion.taxonomyElementId } + : {}), + subject, + passed: result.passed, + feedback: result.feedback, + }; + }; + return [ + ...(evidence.run.observationResults ?? []).map((result) => + mapResult(result, "run") + ), + ...(evidence.run.turns ?? []).flatMap((turn) => + (turn.observationResults ?? []).map((result) => + mapResult(result, "iteration") + ) + ), + ]; +} + +export function composeSubmissionReadout( + evidence: SubmissionEvidence, + synthesis: ReadoutSynthesis, + generatedAt = new Date(), +): SubmissionReadout { + const exactRuns = new Map( + evidence.runs.map((entry) => [ + `${entry.request._id}:${entry.run._id}`, + entry, + ]), + ); + const highlights = new Map( + synthesis.runs.map((entry) => [ + `${entry.requestId}:${entry.runId}`, + entry.trajectoryHighlights, + ]), + ); + for (const key of highlights.keys()) { + if (!exactRuns.has(key)) { + throw new Error(`Readout synthesis referenced non-vector run ${key}`); + } + } + + const criterionIds = new Set( + evidence.runs.flatMap((entry) => entry.criteria.map((criterion) => criterion.id)), + ); + for (const finding of synthesis.findings) { + for (const reference of finding.evidence) { + const key = `${reference.requestId}:${reference.runId}`; + const exactRun = exactRuns.get(key); + if (!exactRun) { + throw new Error(`Readout finding ${finding.id} referenced non-vector run ${key}`); + } + if ( + reference.iteration !== undefined && + !(exactRun.run.turns ?? []).some( + (turn) => turn.iteration === reference.iteration, + ) + ) { + throw new Error( + `Readout finding ${finding.id} referenced unknown iteration ${reference.iteration} for ${key}`, + ); + } + if (reference.criterionId && !criterionIds.has(reference.criterionId)) { + throw new Error( + `Readout finding ${finding.id} referenced unknown criterion ${reference.criterionId}`, + ); + } + } + } + + const runs = evidence.runs.map((entry) => ({ + requestId: entry.request._id, + runId: entry.run._id, + task: entry.request.scenario.task, + workerType: entry.request.workerType, + ...(entry.request.model ? { model: entry.request.model } : {}), + ...(entry.request.profileId ? { profileId: entry.request.profileId } : {}), + ...(entry.request.profileVersionId + ? { profileVersionId: entry.request.profileVersionId } + : {}), + status: entry.run.status, + ...(entry.run.outcome ? { outcome: entry.run.outcome } : {}), + observations: observationsForRun(entry), + trajectoryHighlights: + highlights.get(`${entry.request._id}:${entry.run._id}`) ?? [], + })); + const allObservations = runs.flatMap((run) => run.observations); + const synthesisProfiles = new Map( + synthesis.profiles.map((profile) => [profile.profileId ?? "", profile]), + ); + const requestIdsByProfile = new Map(); + for (const entry of evidence.runs) { + const key = entry.request.profileId ?? ""; + requestIdsByProfile.set(key, [ + ...(requestIdsByProfile.get(key) ?? []), + entry.request._id, + ]); + } + const profiles = [...requestIdsByProfile].map(([profileId, requestIds]) => ({ + ...(profileId ? { profileId } : {}), + requestIds, + summary: + synthesisProfiles.get(profileId)?.summary ?? + `${requestIds.length} exact-vector run${requestIds.length === 1 ? "" : "s"}.`, + })); + const findingIds = new Set(synthesis.findings.map((finding) => finding.id)); + for (const action of synthesis.actions) { + const unknown = action.relatedFindingIds.find((id) => !findingIds.has(id)); + if (unknown) { + throw new Error(`Readout action ${action.id} referenced unknown finding ${unknown}`); + } + } + + return submissionReadoutSchema.parse({ + meta: { + schemaId: SUBMISSION_READOUT_SCHEMA_ID, + schemaVersion: SUBMISSION_READOUT_SCHEMA_VERSION, + generatedAt: generatedAt.toISOString(), + submissionId: evidence.submissionId, + submissionRunId: evidence.submissionRunId, + }, + summary: { + title: synthesis.title, + executiveSummary: synthesis.executiveSummary, + totalRuns: runs.length, + succeededRuns: runs.filter((run) => run.outcome === "succeeded").length, + failedRuns: runs.filter((run) => run.outcome === "failed").length, + passedObservations: allObservations.filter((result) => result.passed).length, + failedObservations: allObservations.filter((result) => !result.passed).length, + }, + profiles, + runs, + findings: synthesis.findings.map((finding) => ({ + ...finding, + evidence: finding.evidence.map((reference) => ({ + submissionRunId: evidence.submissionRunId, + ...reference, + })), + })), + actions: synthesis.actions, + }); +} + +export function renderSubmissionReadoutMarkdown(readout: SubmissionReadout): string { + const lines = [ + `# ${readout.summary.title}`, + "", + readout.summary.executiveSummary, + "", + "## Summary", + "", + `- Runs: ${readout.summary.totalRuns}`, + `- Succeeded: ${readout.summary.succeededRuns}`, + `- Failed: ${readout.summary.failedRuns}`, + `- Passed observations: ${readout.summary.passedObservations}`, + `- Failed observations: ${readout.summary.failedObservations}`, + "", + "## Runs", + "", + ]; + for (const run of readout.runs) { + lines.push( + `### ${run.task || run.requestId}`, + "", + `- Request: \`${run.requestId}\``, + `- Run: \`${run.runId}\``, + `- Status: ${run.status}${run.outcome ? ` / ${run.outcome}` : ""}`, + `- Profile: ${run.profileId ?? "none"}`, + "", + ); + if (run.observations.length > 0) { + lines.push("| Observation | Subject | Result | Evidence |", "| --- | --- | --- | --- |"); + for (const observation of run.observations) { + lines.push( + `| \`${observation.criterionId}\` | ${observation.subject} | ${observation.passed ? "Passed" : "Failed"} | ${observation.feedback.replaceAll("|", "\\|")} |`, + ); + } + lines.push(""); + } + if (run.trajectoryHighlights.length > 0) { + lines.push( + "**Trajectory highlights**", + "", + ...run.trajectoryHighlights.map((highlight) => `- ${highlight}`), + "", + ); + } + } + lines.push("## Findings", ""); + if (readout.findings.length === 0) { + lines.push("No cross-run findings were identified.", ""); + } else { + for (const finding of readout.findings) { + lines.push( + `### ${finding.title}`, + "", + `**Severity:** ${finding.severity}`, + "", + finding.description, + "", + ...finding.evidence.map( + (evidence) => + `- \`${evidence.requestId}/${evidence.runId}\`${evidence.iteration === undefined ? "" : ` iteration ${evidence.iteration}`}: ${evidence.excerpt}`, + ), + "", + ); + } + } + lines.push("## Actions", ""); + if (readout.actions.length === 0) { + lines.push("No follow-up actions were generated.", ""); + } else { + for (const action of readout.actions) { + lines.push( + `- **${action.title}** (${action.priority}): ${action.description}`, + ); + } + lines.push(""); + } + lines.push( + "---", + `Schema: \`${readout.meta.schemaId}@${readout.meta.schemaVersion}\``, + `Submission run: \`${readout.meta.submissionRunId}\``, + ); + return lines.join("\n"); +} diff --git a/apps/workers/readout/src/readout-generator.ts b/apps/workers/readout/src/readout-generator.ts new file mode 100644 index 000000000..51cc320d0 --- /dev/null +++ b/apps/workers/readout/src/readout-generator.ts @@ -0,0 +1,98 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { CopilotClient, type PermissionHandler, SessionEvent } from "@github/copilot-sdk"; +import type { LogEvent } from "shared"; +import { + composeSubmissionReadout, + readoutSynthesisSchema, + type SubmissionEvidence, +} from "./readout-builder.js"; +import { + READOUT_SYSTEM_PROMPT, + composeReadoutUserPrompt, +} from "./prompt.js"; + +const approveNoTools: PermissionHandler = () => ({ + kind: "denied-by-rules", + rules: [], +}); + +type ReadoutLog = ( + level: LogEvent["level"], + message: string, + data?: Record, +) => Promise | void; + +export async function generateSubmissionReadout(input: { + evidence: SubmissionEvidence; + model: string; + githubToken: string; + timeoutMs: number; + log: ReadoutLog; +}) { + const client = new CopilotClient({ gitHubToken: input.githubToken }); + let latestMessage = ""; + let streamedResponse = ""; + try { + const session = await client.createSession({ + model: input.model, + streaming: true, + availableTools: [], + excludedTools: ["builtin:*", "mcp:*"], + onPermissionRequest: approveNoTools, + systemMessage: { mode: "replace", content: READOUT_SYSTEM_PROMPT }, + }); + session.on((event: SessionEvent) => { + if (event.type === "assistant.message_delta") { + streamedResponse += event.data.deltaContent; + } else if (event.type === "assistant.message") { + latestMessage = event.data.content; + } else if (event.type === "session.error") { + void input.log("error", event.data.message, { + errorType: event.data.errorType, + }); + } + }); + + let prompt = composeReadoutUserPrompt(input.evidence); + for (let attempt = 1; attempt <= 3; attempt += 1) { + streamedResponse = ""; + latestMessage = ""; + await input.log("info", `Generating submission readout (${attempt}/3)`); + const response = await session.sendAndWait({ prompt }, input.timeoutMs); + const text = + response?.data.content?.trim() || + latestMessage.trim() || + streamedResponse.trim(); + if (!text) throw new Error("Copilot SDK returned an empty readout response"); + const jsonText = text.replace(/^```(?:json)?\s*/i, "").replace(/\s*```$/, ""); + let parsed: unknown; + try { + parsed = JSON.parse(jsonText) as unknown; + } catch (error) { + if (attempt === 3) { + throw new Error(`Readout response was not valid JSON: ${String(error)}`); + } + prompt = "Return the complete corrected readout synthesis as raw JSON only."; + continue; + } + const result = readoutSynthesisSchema.safeParse(parsed); + if (result.success) { + return composeSubmissionReadout(input.evidence, result.data); + } + if (attempt === 3) { + throw new Error( + `Readout synthesis failed schema validation: ${result.error.issues.map((issue) => `${issue.path.join(".")}: ${issue.message}`).join("; ")}`, + ); + } + prompt = [ + "Return the complete corrected readout synthesis as raw JSON only.", + `Validation errors: ${result.error.issues.map((issue) => `${issue.path.join(".")}: ${issue.message}`).join("; ")}`, + ].join("\n"); + } + throw new Error("Readout generation ended without a valid result"); + } finally { + await client.stop().catch(() => undefined); + } +} diff --git a/apps/workers/readout/src/readout-queue-processor.test.ts b/apps/workers/readout/src/readout-queue-processor.test.ts new file mode 100644 index 000000000..6d842073a --- /dev/null +++ b/apps/workers/readout/src/readout-queue-processor.test.ts @@ -0,0 +1,274 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { DequeuedMessageItem } from "@azure/storage-queue"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { SubmissionDocument, VisibilityHeartbeat } from "shared"; +import type { RequestDocument, RunHistoryDocument } from "shared"; + +const coordination = vi.hoisted(() => ({ + claim: vi.fn(), + complete: vi.fn(), + isCurrent: vi.fn(), +})); +const generate = vi.hoisted(() => vi.fn()); + +vi.mock("shared", async (importOriginal) => ({ + ...(await importOriginal()), + claimSubmissionHandler: coordination.claim, + completeSubmissionHandler: coordination.complete, + isCurrentSubmissionRun: coordination.isCurrent, +})); +vi.mock("./readout-generator.js", () => ({ + generateSubmissionReadout: generate, +})); + +import { ReadoutQueueProcessor } from "./readout-queue-processor.js"; + +const submission: SubmissionDocument = { + _id: "submission-1", + submissionId: "submission-1", + projectId: "project-1", + requestIds: ["request-1"], + run: { + _id: "submission-run-1", + attemptNumber: 1, + vectorFingerprint: "request-1:run-1", + requests: [{ requestId: "request-1", runId: "run-1" }], + dependencyProjection: [], + handlerStatus: { + "pp-readout": { + version: 1, + status: "processing", + updatedAt: new Date("2026-10-13T12:00:00.000Z"), + }, + }, + blocking: [], + createdAt: new Date("2026-10-13T12:00:00.000Z"), + startedAt: new Date("2026-10-13T12:00:00.000Z"), + updatedAt: new Date("2026-10-13T12:00:00.000Z"), + }, + createdAt: new Date("2026-10-13T12:00:00.000Z"), + updatedAt: new Date("2026-10-13T12:00:00.000Z"), +}; + +function processor() { + const instance = new ReadoutQueueProcessor({ + mongoUri: "mongodb://localhost:27017", + mongoDatabase: "scope", + mongoCollection: "submissions", + storageAccountName: "devstoreaccount1", + storageConnectionString: "UseDevelopmentStorage=true", + queueName: "pp-readout-queue", + batchSize: 1, + pollIntervalMs: 100, + redisHost: "localhost", + redisPort: 6379, + redisPassword: "", + model: "test-model", + sessionTimeoutMs: 1_000, + }); + const insertOne = vi.fn().mockResolvedValue({ acknowledged: true }); + Object.assign(instance as object, { + collection: {}, + db: { + collection: vi.fn((name: string) => + name === "reports" ? { insertOne } : {} + ), + }, + loadEvidence: vi.fn().mockResolvedValue({ + submissionId: submission._id, + submissionRunId: submission.run._id, + runs: [], + }), + tokenClient: { + acquireToken: vi.fn().mockResolvedValue("token"), + }, + blobStorage: { + uploadJson: vi.fn().mockImplementation((path: string) => + Promise.resolve(`https://storage.example/${path}`) + ), + }, + safeDeleteMessage: vi.fn().mockResolvedValue(undefined), + }); + return { instance, insertOne }; +} + +const message = { messageId: "message-1" } as DequeuedMessageItem; +const heartbeat = { + stop: vi.fn().mockResolvedValue(undefined), +} as unknown as VisibilityHeartbeat; +const payload = { + type: "readout", + submissionId: submission._id, + submissionRunId: submission.run._id, +}; +type LogFn = ReturnType; +type ReadoutPayload = typeof payload; + +async function handle(instance: ReadoutQueueProcessor, log: LogFn = vi.fn()) { + await (instance as unknown as { + handleRequest( + submission: SubmissionDocument, + message: DequeuedMessageItem, + heartbeat: VisibilityHeartbeat, + log: LogFn, + payload: ReadoutPayload, + ): Promise; + }).handleRequest(submission, message, heartbeat, log, payload); +} + +describe("readout queue orchestration", () => { + beforeEach(() => { + vi.clearAllMocks(); + coordination.isCurrent.mockResolvedValue(true); + generate.mockResolvedValue({ + meta: { + schemaId: "scope.submission-readout", + schemaVersion: "1.0", + generatedAt: "2026-10-13T12:00:00.000Z", + submissionId: submission._id, + submissionRunId: submission.run._id, + }, + summary: { + title: "Submission readout", + executiveSummary: "Summary", + totalRuns: 0, + succeededRuns: 0, + failedRuns: 0, + passedObservations: 0, + failedObservations: 0, + }, + profiles: [], + runs: [], + findings: [], + actions: [], + }); + }); + + it("discards stale or duplicate exact-run messages before generation", async () => { + coordination.claim.mockResolvedValue(null); + const { instance, insertOne } = processor(); + + await handle(instance); + + expect(generate).not.toHaveBeenCalled(); + expect(insertOne).not.toHaveBeenCalled(); + expect((instance as unknown as { safeDeleteMessage: ReturnType }) + .safeDeleteMessage).toHaveBeenCalledOnce(); + }); + + it("keeps superseded completions historical and uses new report ids on retry", async () => { + coordination.claim.mockResolvedValue(submission); + coordination.complete.mockResolvedValue(false); + const { instance, insertOne } = processor(); + const log = vi.fn().mockResolvedValue(undefined); + + await handle(instance, log); + await handle(instance, log); + + expect(insertOne).toHaveBeenCalledTimes(2); + const reports = insertOne.mock.calls.map(([report]) => report); + expect(reports[0]._id).not.toBe(reports[1]._id); + expect(reports).toEqual([ + expect.objectContaining({ + submissionId: submission._id, + submissionRunId: submission.run._id, + }), + expect.objectContaining({ + submissionId: submission._id, + submissionRunId: submission.run._id, + }), + ]); + expect(coordination.complete).toHaveBeenCalledTimes(2); + expect(log).toHaveBeenCalledWith( + "warn", + "Readout report remained historical after supersession", + expect.objectContaining({ + submissionId: submission._id, + submissionRunId: submission.run._id, + }), + ); + }); + + it("does not publish when the submission is superseded during upload", async () => { + coordination.claim.mockResolvedValue(submission); + coordination.isCurrent + .mockResolvedValueOnce(true) + .mockResolvedValueOnce(true) + .mockResolvedValueOnce(true) + .mockResolvedValueOnce(true) + .mockResolvedValueOnce(false); + const { instance, insertOne } = processor(); + + await handle(instance); + + expect(generate).toHaveBeenCalledOnce(); + expect(insertOne).not.toHaveBeenCalled(); + expect(coordination.complete).not.toHaveBeenCalled(); + }); + + it("loads the archived run named by the immutable exact vector", async () => { + const { instance } = processor(); + delete (instance as unknown as Record).loadEvidence; + const currentRequest = { + _id: "request-1", + projectId: "project-1", + workerType: "coder-acp-copilot", + observations: [], + scenario: { task: "Task", criteria: [] }, + createdAt: new Date("2026-10-13T12:00:00.000Z"), + priority: 0, + run: { + _id: "run-newer", + attemptNumber: 2, + status: "done", + outcome: "succeeded", + }, + } satisfies RequestDocument; + const archivedRun = { + _id: "run-1", + requestId: "request-1", + projectId: "project-1", + attemptNumber: 1, + status: "done", + outcome: "failed", + } satisfies RunHistoryDocument; + const historyFindOne = vi.fn().mockResolvedValue(archivedRun); + Object.assign(instance as object, { + db: { + collection: vi.fn((name: string) => { + if (name === "requests") { + return { + find: vi.fn(() => ({ + toArray: vi.fn().mockResolvedValue([currentRequest]), + })), + }; + } + if (name === "runs") return { findOne: historyFindOne }; + if (name === "criteria") { + return { + find: vi.fn(() => ({ + toArray: vi.fn().mockResolvedValue([]), + })), + }; + } + return {}; + }), + }, + }); + + const evidence = await (instance as unknown as { + loadEvidence(value: SubmissionDocument): Promise<{ + runs: Array<{ run: RunHistoryDocument | RequestDocument["run"] }>; + }>; + }).loadEvidence(submission); + + expect(historyFindOne).toHaveBeenCalledWith({ + _id: "run-1", + requestId: "request-1", + projectId: "project-1", + }); + expect(evidence.runs[0].run?._id).toBe("run-1"); + }); +}); diff --git a/apps/workers/readout/src/readout-queue-processor.ts b/apps/workers/readout/src/readout-queue-processor.ts new file mode 100644 index 000000000..3d0f52fd6 --- /dev/null +++ b/apps/workers/readout/src/readout-queue-processor.ts @@ -0,0 +1,303 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { DequeuedMessageItem } from "@azure/storage-queue"; +import { randomUUID } from "node:crypto"; +import { + BaseQueueProcessor, + BlobStorage, + type BaseQueueProcessorConfig, + type CriteriaDocument, + type LogEvent, + type ReportDocument, + type RequestDocument, + type RunHistoryDocument, + type SubmissionDocument, + type SubmissionQueueMessagePayload, + TokenManagerClient, + claimSubmissionHandler, + completeSubmissionHandler, + isCurrentSubmissionRun, + SUBMISSION_READOUT_SCHEMA_ID, + SUBMISSION_READOUT_SCHEMA_VERSION, + type VisibilityHeartbeat, + withRetry, +} from "shared"; +import { + renderSubmissionReadoutMarkdown, + type ExactRunEvidence, + type SubmissionEvidence, +} from "./readout-builder.js"; +import { generateSubmissionReadout } from "./readout-generator.js"; + +const HANDLER_ID = "pp-readout"; +const HANDLER_VERSION = 1; + +export interface ReadoutQueueProcessorConfig extends BaseQueueProcessorConfig { + model: string; + sessionTimeoutMs: number; + tokenManagerUrl?: string; +} + +function isReadoutMessage( + payload: Record | undefined, +): payload is Record & SubmissionQueueMessagePayload { + return payload?.type === "readout" && + typeof payload.submissionId === "string" && + typeof payload.submissionRunId === "string"; +} + +export class ReadoutQueueProcessor extends BaseQueueProcessor { + private readonly readoutConfig: ReadoutQueueProcessorConfig; + private readonly tokenClient: TokenManagerClient; + private readonly blobStorage: BlobStorage; + + constructor(config: ReadoutQueueProcessorConfig) { + super(config, "readout-handler"); + this.readoutConfig = config; + this.tokenClient = new TokenManagerClient(config.tokenManagerUrl); + this.blobStorage = new BlobStorage({ + storageAccountName: config.storageAccountName, + storageConnectionString: config.storageConnectionString, + }); + } + + protected override extractDocumentId(payload: Record): string { + return typeof payload.submissionId === "string" ? payload.submissionId : ""; + } + + protected override async handleRequest( + _submission: SubmissionDocument, + message: DequeuedMessageItem, + heartbeat: VisibilityHeartbeat, + log: ( + level: LogEvent["level"], + message: string, + data?: Record, + ) => Promise, + payload?: Record, + ): Promise { + if (!isReadoutMessage(payload)) { + await log("warn", "Discarding unsupported readout queue payload", { payload }); + await this.safeDeleteMessage(message.messageId, heartbeat.stop()); + return; + } + const { submissionId, submissionRunId } = payload; + const claimed = await withRetry(() => + claimSubmissionHandler({ + submissions: this.collection, + submissionId, + submissionRunId, + handlerId: HANDLER_ID, + version: HANDLER_VERSION, + }) + ); + if (!claimed) { + await log("info", "Discarding stale or duplicate readout message", { + submissionId, + submissionRunId, + }); + await this.safeDeleteMessage(message.messageId, heartbeat.stop()); + return; + } + + try { + if (!(await this.isCurrent(submissionId, submissionRunId))) return; + const evidence = await this.loadEvidence(claimed); + + if (!(await this.isCurrent(submissionId, submissionRunId))) return; + const githubToken = await this.tokenClient.acquireToken("copilot-sdk"); + + if (!(await this.isCurrent(submissionId, submissionRunId))) return; + const readout = await generateSubmissionReadout({ + evidence, + model: this.readoutConfig.model, + githubToken, + timeoutMs: this.readoutConfig.sessionTimeoutMs, + log, + }); + const markdown = renderSubmissionReadoutMarkdown(readout); + + if (!(await this.isCurrent(submissionId, submissionRunId))) return; + const reportId = randomUUID(); + const structuredUrl = await this.blobStorage.uploadJson( + `reports/${reportId}/structured.json`, + readout, + ); + + if (!(await this.isCurrent(submissionId, submissionRunId))) return; + const now = new Date(); + const report: ReportDocument = { + _id: reportId, + projectId: claimed.projectId, + submissionId, + submissionRunId, + reporter: { + id: HANDLER_ID, + name: "Submission Readout", + gitHash: process.env.GIT_COMMIT || "unknown", + model: this.readoutConfig.model, + agentId: "copilot-sdk", + agentVersion: process.env.COPILOT_SDK_VERSION || "unknown", + }, + content: markdown, + structuredArtifact: { + schemaId: SUBMISSION_READOUT_SCHEMA_ID, + schemaVersion: SUBMISSION_READOUT_SCHEMA_VERSION, + contentType: "application/json", + url: structuredUrl, + }, + status: "completed", + createdAt: now, + updatedAt: now, + }; + await withRetry(() => + this.db.collection("reports").insertOne(report) + ); + + const completed = await withRetry(() => + completeSubmissionHandler({ + submissions: this.collection, + submissionId, + submissionRunId, + handlerId: HANDLER_ID, + version: HANDLER_VERSION, + status: "done", + report: { + reportId, + url: `/api/v1/reports/${reportId}`, + updatedAt: now, + }, + }) + ); + if (!completed) { + await log("warn", "Readout report remained historical after supersession", { + submissionId, + submissionRunId, + reportId, + }); + } else { + await log("info", "Submission readout completed", { + submissionId, + submissionRunId, + reportId, + }); + } + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + if (await this.isCurrent(submissionId, submissionRunId)) { + await withRetry(() => + completeSubmissionHandler({ + submissions: this.collection, + submissionId, + submissionRunId, + handlerId: HANDLER_ID, + version: HANDLER_VERSION, + status: "failed", + error: errorMessage, + }) + ); + } + await log("error", `Submission readout failed: ${errorMessage}`, { + submissionId, + submissionRunId, + }); + } finally { + await this.safeDeleteMessage(message.messageId, heartbeat.stop()); + } + } + + private isCurrent( + submissionId: string, + submissionRunId: string, + ): Promise { + return withRetry(() => + isCurrentSubmissionRun({ + submissions: this.collection, + submissionId, + submissionRunId, + }) + ); + } + + private async loadEvidence( + submission: SubmissionDocument, + ): Promise { + const requests = await withRetry(() => + this.db.collection("requests").find({ + _id: { $in: submission.run.requests.map((ref) => ref.requestId) }, + projectId: submission.projectId, + }).toArray() + ); + const requestById = new Map(requests.map((request) => [request._id, request])); + const history = this.db.collection("runs"); + const exactRuns: Array<{ request: RequestDocument; run: RunHistoryDocument | RequestDocument["run"] }> = []; + for (const ref of submission.run.requests) { + const request = requestById.get(ref.requestId); + if (!request) throw new Error(`Exact-vector request ${ref.requestId} was not found`); + const run = request.run?._id === ref.runId + ? request.run + : await withRetry(() => history.findOne({ + _id: ref.runId, + requestId: ref.requestId, + projectId: submission.projectId, + })); + if (!run) { + throw new Error(`Exact-vector run ${ref.requestId}/${ref.runId} was not found`); + } + exactRuns.push({ request, run }); + } + + const criterionIds = [ + ...new Set( + exactRuns.flatMap(({ request, run }) => [ + ...(request.observations ?? []), + ...(run?.observationResults ?? []).map((result) => result.criterionId), + ...(run?.turns ?? []).flatMap((turn) => + (turn.observationResults ?? []).map((result) => result.criterionId) + ), + ]), + ), + ]; + const criteria = criterionIds.length === 0 + ? [] + : await withRetry(() => + this.db.collection("criteria").find({ + projectId: submission.projectId, + id: { $in: criterionIds }, + }).toArray() + ); + const criteriaById = new Map(criteria.map((criterion) => [criterion.id, criterion])); + + const runs: ExactRunEvidence[] = []; + for (const { request, run } of exactRuns) { + if (!run) continue; + const trajectories = await Promise.all( + (run.turns ?? []).flatMap((turn) => + turn.atifUrl + ? [withRetry(() => + this.blobStorage.downloadJson(turn.atifUrl!) + ).then((trajectory) => ({ + iteration: turn.iteration, + trajectory, + }))] + : [], + ), + ); + runs.push({ + request, + run, + criteria: (request.observations ?? []).flatMap((id) => { + const criterion = criteriaById.get(id); + return criterion ? [criterion] : []; + }), + trajectories, + }); + } + return { + submissionId: submission._id, + submissionRunId: submission.run._id, + runs, + }; + } +} diff --git a/apps/workers/taxonomy/tsconfig.json b/apps/workers/readout/tsconfig.json similarity index 100% rename from apps/workers/taxonomy/tsconfig.json rename to apps/workers/readout/tsconfig.json diff --git a/apps/workers/report-generator/src/report-queue-processor.ts b/apps/workers/report-generator/src/report-queue-processor.ts index 7ecaac8f8..1a0523a97 100644 --- a/apps/workers/report-generator/src/report-queue-processor.ts +++ b/apps/workers/report-generator/src/report-queue-processor.ts @@ -88,6 +88,11 @@ export class ReportQueueProcessor extends BaseQueueProcessor { const generationStart = Date.now(); const reportId = doc._id; const requestId = doc.requestId; + if (!requestId) { + throw new Error( + `Report ${reportId} has no requestId and cannot be processed by report-generator`, + ); + } if (!ReportQueueProcessor.coldStartTracked) { ReportQueueProcessor.coldStartTracked = true; diff --git a/apps/workers/taxonomy/handler.yaml b/apps/workers/taxonomy/handler.yaml deleted file mode 100644 index b479f905e..000000000 --- a/apps/workers/taxonomy/handler.yaml +++ /dev/null @@ -1,11 +0,0 @@ -# Post-process handler registration metadata. -# Read by the generic register-handler script and POSTed to the -# scheduler's POST /handlers/register endpoint on deploy. -_id: pp-taxonomy -type: post-process-handler -version: 1 -queue: pp-taxonomy-queue -selector: taxonomy -autoBackfill: false -dependsOn: - - pp-atif diff --git a/apps/workers/taxonomy/src/cli.ts b/apps/workers/taxonomy/src/cli.ts deleted file mode 100644 index a5f51f600..000000000 --- a/apps/workers/taxonomy/src/cli.ts +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/env npx tsx -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -/** - * Taxonomy post-processor developer CLI. - * - * Invokes the taxonomy generation loop directly against an existing requestId, - * streams the Copilot SDK session events live, and saves the validated taxonomy - * to a local JSON file. This is a developer/troubleshooting tool only — it never - * writes to Mongo (handlerStatus) or blob storage (taxonomy.json) and does not - * notify the scheduler. - * - * Usage: - * npx tsx src/cli.ts [options] - * pnpm taxonomy [options] - * - * Options: - * --run, -r Run id for the prompt/logging (default: active run from API). - * --model Override TAXONOMY_MODEL (e.g. a reasoning model). - * --timeout Override SESSION_TIMEOUT_MS (per-attempt session timeout). - * --output, -o Output file path (default: taxonomy-.json). - * --stdout Also print the taxonomy JSON to stdout. - * --quiet Suppress live session-event streaming on stderr. - * --help, -h Show this help. - */ -import "dotenv/config"; -import { writeFile } from "node:fs/promises"; -import { resolve } from "node:path"; -import { SessionEvent } from "@github/copilot-sdk"; -import { TokenManagerClient, type LogEvent, type RequestDocument } from "shared"; -import { generateTaxonomy } from "./taxonomy-generator.js"; - -interface CliArgs { - requestId: string; - runId?: string; - model?: string; - timeoutMs?: number; - output?: string; - stdout: boolean; - quiet: boolean; -} - -function printHelp(): void { - process.stderr.write( - [ - "Taxonomy post-processor developer CLI", - "", - "Usage: npx tsx src/cli.ts [options]", - "", - "Options:", - " --run, -r Run id for the prompt/logging (default: active run from API)", - " --model Override TAXONOMY_MODEL", - " --timeout Override SESSION_TIMEOUT_MS (per-attempt session timeout)", - " --output, -o Output file path (default: taxonomy-.json)", - " --stdout Also print the taxonomy JSON to stdout", - " --quiet Suppress live session-event streaming on stderr", - " --help, -h Show this help", - "", - ].join("\n"), - ); -} - -function parseArgs(argv: string[]): CliArgs | undefined { - const args: Partial & { stdout: boolean; quiet: boolean } = { stdout: false, quiet: false }; - const positionals: string[] = []; - - for (let i = 0; i < argv.length; i += 1) { - const arg = argv[i]; - switch (arg) { - case "--help": - case "-h": - printHelp(); - return undefined; - case "--run": - case "-r": - args.runId = argv[++i]; - break; - case "--model": - args.model = argv[++i]; - break; - case "--timeout": - args.timeoutMs = Number(argv[++i]); - break; - case "--output": - case "-o": - args.output = argv[++i]; - break; - case "--stdout": - args.stdout = true; - break; - case "--quiet": - args.quiet = true; - break; - default: - if (arg.startsWith("-")) { - throw new Error(`Unknown option: ${arg}`); - } - positionals.push(arg); - } - } - - const requestId = positionals[0]; - if (!requestId) { - throw new Error("Missing required argument"); - } - if (args.timeoutMs !== undefined && Number.isNaN(args.timeoutMs)) { - throw new Error("--timeout must be a number (milliseconds)"); - } - - return { ...args, requestId }; -} - -async function resolveRunId(apiBaseUrl: string, requestId: string, explicit?: string): Promise { - if (explicit) return explicit; - try { - const response = await fetch(`${apiBaseUrl}/api/v1/requests/${requestId}`); - if (response.ok) { - const request: RequestDocument = await response.json(); - if (request.run?._id) return request.run._id; - } - } catch { - // fall through to requestId fallback - } - return requestId; -} - -function summarizeEvent(event: SessionEvent): string | undefined { - switch (event.type) { - case "session.start": - return `session.start model=${event.data.selectedModel}`; - case "session.error": - return `session.error ${event.data.errorType}: ${event.data.message}`; - case "tool.execution_start": - return `tool.start ${event.data.toolName} ${JSON.stringify(event.data.arguments ?? {})}`; - case "tool.execution_complete": - return `tool.done ${event.data.success ? "ok" : "FAILED"}${event.data.error ? ` ${event.data.error.message}` : ""}`; - case "assistant.message": - return `assistant.message (${event.data.content.length} chars)`; - default: - return undefined; - } -} - -async function main(): Promise { - let parsed: CliArgs | undefined; - try { - parsed = parseArgs(process.argv.slice(2)); - } catch (error) { - process.stderr.write(`Error: ${error instanceof Error ? error.message : String(error)}\n\n`); - printHelp(); - process.exit(2); - } - if (!parsed) return; // help shown - - const apiBaseUrl = process.env.SCOPE_MT_API_URL || "http://localhost:3001"; - const taxonomyModel = parsed.model || process.env.TAXONOMY_MODEL || "gpt-5.4"; - const sessionTimeoutMs = parsed.timeoutMs ?? (Number(process.env.SESSION_TIMEOUT_MS) || 5 * 60 * 1000); - const outputPath = resolve(parsed.output ?? `taxonomy-${parsed.requestId}.json`); - - const githubToken = process.env.GITHUB_TOKEN - ? process.env.GITHUB_TOKEN - : await new TokenManagerClient(process.env.TOKEN_MANAGER_URL).acquireToken("copilot-sdk"); - - const runId = await resolveRunId(apiBaseUrl, parsed.requestId, parsed.runId); - - const log = (level: LogEvent["level"], msg: string, data?: Record): void => { - if (parsed.quiet) return; - const suffix = data && Object.keys(data).length > 0 ? ` ${JSON.stringify(data)}` : ""; - process.stderr.write(`[${level}] ${msg}${suffix}\n`); - }; - - process.stderr.write( - `Generating taxonomy: requestId=${parsed.requestId} runId=${runId} model=${taxonomyModel} timeout=${sessionTimeoutMs}ms\n`, - ); - - const onEvent = parsed.quiet - ? undefined - : (event: SessionEvent): void => { - const summary = summarizeEvent(event); - if (summary) process.stderr.write(` · ${summary}\n`); - }; - - const taxonomy = await generateTaxonomy({ - requestId: parsed.requestId, - runId, - apiBaseUrl, - taxonomyModel, - githubToken, - sessionTimeoutMs, - log, - onEvent, - }); - - const json = JSON.stringify(taxonomy, null, 2); - await writeFile(outputPath, `${json}\n`, "utf-8"); - process.stderr.write(`\nTaxonomy written to ${outputPath}\n`); - if (parsed.stdout) { - process.stdout.write(`${json}\n`); - } -} - -main().catch((error) => { - process.stderr.write(`\nTaxonomy generation failed: ${error instanceof Error ? error.message : String(error)}\n`); - process.exit(1); -}); diff --git a/apps/workers/taxonomy/src/index.ts b/apps/workers/taxonomy/src/index.ts deleted file mode 100644 index b1a4f322b..000000000 --- a/apps/workers/taxonomy/src/index.ts +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -import "dotenv/config"; -import { TaxonomyQueueProcessor, type TaxonomyQueueProcessorConfig } from "./taxonomy-queue-processor.js"; - -const config: TaxonomyQueueProcessorConfig = { - mongoUri: process.env.MONGO_CONNECTION_STRING || process.env.AZURE_COSMOS_CONNECTION_STRING || "mongodb://localhost:27017", - mongoDatabase: process.env.MONGO_DATABASE || "requests-db", - mongoCollection: "requests", - storageAccountName: process.env.AZURE_STORAGE_ACCOUNT_NAME || "devstoreaccount1", - storageConnectionString: process.env.STORAGE_CONNECTION_STRING || process.env.AZURE_STORAGE_CONNECTION_STRING || undefined, - queueName: process.env.AZURE_STORAGE_QUEUE_TAXONOMY || "pp-taxonomy-queue", - batchSize: Number(process.env.BATCH_SIZE) || 1, - pollIntervalMs: Number(process.env.POLL_INTERVAL_MS) || 5000, - redisHost: process.env.REDIS_HOST || "localhost", - redisPort: Number(process.env.REDIS_PORT) || 6379, - redisPassword: process.env.REDIS_PASSWORD || "", - taxonomyModel: process.env.TAXONOMY_MODEL || "gpt-5.4", - apiBaseUrl: process.env.SCOPE_MT_API_URL || "http://localhost:3001", - sessionTimeoutMs: Number(process.env.SESSION_TIMEOUT_MS) || 5 * 60 * 1000, - schedulerUrl: process.env.SCHEDULER_URL, - tokenManagerUrl: process.env.TOKEN_MANAGER_URL, -}; - -const processor = new TaxonomyQueueProcessor(config); -processor.start().catch((err) => { - console.error("[taxonomy-handler] Fatal error:", err); - process.exit(1); -}); diff --git a/apps/workers/taxonomy/src/prompt.ts b/apps/workers/taxonomy/src/prompt.ts deleted file mode 100644 index 47ea28e4d..000000000 --- a/apps/workers/taxonomy/src/prompt.ts +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -export const TAXONOMY_SYSTEM_PROMPT = `You generate benchmark taxonomy documents for completed Scope runs. - -Your job is to inspect the run metadata and ATIF trajectories, then return a taxonomy JSON document that conforms exactly to the taxonomy Zod schema enforced by the worker. - -Required JSON shape: -{ - "meta": { - "generatedAt": "ISO-8601 datetime with offset", - "schemaVersion": "string", - "runId": "string", - "requestId": "string" - }, - "scorecard": { - "summary": "string", - "profileResults": [ - { - "profileId": "string", - "select": "best|median|worst", - "taskOutcome": "success|partial|failure", - "overallScore": 0-100, - "dimensions": [ - { - "name": "string", - "score": 0-100, - "rationale": "string", - "evidence": ["string"] - } - ] - } - ], - "crossProfileInsights": ["string"] - }, - "behaviorAnalysis": { - "patterns": [ - { - "id": "string", - "category": "strength|weakness|inconsistency|regression", - "title": "string", - "description": "string", - "frequency": 0-1, - "impact": "critical|high|medium|low", - "evidence": [ - { - "runId": "string", - "iteration": 0-or-greater integer, - "excerpt": "string" - } - ] - } - ], - "agentPersonality": { - "communicationStyle": "string", - "problemSolvingApproach": "string", - "errorRecoveryBehavior": "string" - } - }, - "actionList": { - "fix": [actionItem], - "improve": [actionItem], - "investigate": [actionItem] - } -} - -Each actionItem must include: -- id -- title -- priority: critical|high|medium|low -- description -- ownership: agent-team|platform-team|prompt-engineer|scenario-author -- relatedPatterns: string[] -- suggestedApproach - -Output rules: -- Return raw JSON only. -- Do not wrap the JSON in markdown fences. -- Do not include commentary before or after the JSON. -- Every field must be valid for the schema. -- Preserve the 3-layer structure: scorecard, behaviorAnalysis, actionList. -- If a value is unknown, use the schema-compatible empty/null/default value rather than inventing facts. - -Use the available tools to gather evidence before answering: -- get_run_data for the request, scenario, run, turns, and criteria outcomes. -- get_atif_trajectory for ATIF evidence from individual iterations. - -Ground every classification in the actual run evidence. Prefer precise, stable labels over verbose prose. If validation feedback is provided later in the conversation, correct the JSON and return a full replacement document.`; diff --git a/apps/workers/taxonomy/src/taxonomy-generator.ts b/apps/workers/taxonomy/src/taxonomy-generator.ts deleted file mode 100644 index ada150f2c..000000000 --- a/apps/workers/taxonomy/src/taxonomy-generator.ts +++ /dev/null @@ -1,202 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -import { CopilotClient, type PermissionHandler, SessionEvent } from "@github/copilot-sdk"; -import { - type LogEvent, - taxonomySchema, - type TaxonomyDocument, -} from "shared"; -import { TAXONOMY_SYSTEM_PROMPT } from "./prompt.js"; -import { createTaxonomyTools } from "./tools.js"; - -export const DEFAULT_SESSION_TIMEOUT_MS = 5 * 60 * 1000; -const DELTA_LOG_INTERVAL = 2_000; -const approveTaxonomyTools: PermissionHandler = request => - request.kind === "custom-tool" - ? { kind: "approved" } - : { kind: "denied-by-rules", rules: [] }; - -type TaxonomyLogFn = ( - level: LogEvent["level"], - msg: string, - data?: Record, -) => Promise | void; - -export interface GenerateTaxonomyParams { - requestId: string; - runId: string; - apiBaseUrl: string; - taxonomyModel: string; - githubToken: string; - sessionTimeoutMs?: number; - log: TaxonomyLogFn; - /** Optional raw session-event hook (used by the dev CLI to render live progress). */ - onEvent?: (event: SessionEvent) => void; -} - -/** - * Runs the Copilot SDK taxonomy generation loop and returns a schema-validated - * taxonomy document. This is side-effect-free with respect to Mongo and blob - * storage (it only reads run data and ATIF trajectories through the API); - * persistence is the caller's responsibility. - */ -export async function generateTaxonomy(params: GenerateTaxonomyParams): Promise { - const { requestId, runId, apiBaseUrl, taxonomyModel, githubToken, log, onEvent } = params; - const client = new CopilotClient({ githubToken }); - const tools = createTaxonomyTools(apiBaseUrl, requestId); - const timeoutMs = params.sessionTimeoutMs ?? DEFAULT_SESSION_TIMEOUT_MS; - - let streamedResponse = ""; - let latestMessage = ""; - let lastLoggedCharCount = 0; - const toolNames = new Map(); - - try { - const session = await client.createSession({ - model: taxonomyModel, - streaming: true, - tools, - availableTools: [], - excludedTools: ["builtin:*", "mcp:*"], - onPermissionRequest: approveTaxonomyTools, - systemMessage: { mode: "replace", content: TAXONOMY_SYSTEM_PROMPT }, - }); - - session.on((event: SessionEvent) => { - onEvent?.(event); - switch (event.type) { - case "session.start": - void log("info", "Copilot session started", { - sessionId: event.data.sessionId, - model: event.data.selectedModel, - }); - break; - case "session.error": - void log("error", `Session error: ${event.data.message}`, { errorType: event.data.errorType }); - break; - case "tool.execution_start": - toolNames.set(event.data.toolCallId, event.data.toolName); - void log("info", `Tool call: ${event.data.toolName}`, { - toolCallId: event.data.toolCallId, - arguments: event.data.arguments as Record | undefined, - }); - break; - case "tool.execution_complete": { - const toolName = toolNames.get(event.data.toolCallId) ?? "unknown"; - toolNames.delete(event.data.toolCallId); - void log("info", `Tool result: ${toolName} (${event.data.success ? "success" : "failed"})`, { - toolCallId: event.data.toolCallId, - error: event.data.error?.message, - }); - break; - } - case "assistant.message_delta": - streamedResponse += event.data.deltaContent; - if (streamedResponse.length - lastLoggedCharCount >= DELTA_LOG_INTERVAL) { - lastLoggedCharCount = streamedResponse.length; - void log("info", `Generating taxonomy… (${streamedResponse.length} chars so far)`); - } - break; - case "assistant.message": - latestMessage = event.data.content; - break; - } - }); - - let attempts = 0; - const maxAttempts = 3; - let prompt = [ - `Generate taxonomy JSON for request ${requestId} and run ${runId}.`, - "Use get_run_data first, then inspect ATIF trajectories that materially affect the classification.", - "Return only raw JSON that satisfies the taxonomy schema.", - ].join(" "); - let lastJson: string | undefined; - - while (attempts < maxAttempts) { - attempts += 1; - streamedResponse = ""; - latestMessage = ""; - lastLoggedCharCount = 0; - - await log("info", `Sending taxonomy prompt to Copilot SDK (attempt ${attempts}/${maxAttempts})`, { - requestId, - runId, - timeoutMs, - }); - - const response = await session.sendAndWait({ prompt }, timeoutMs); - const responseText = response?.data.content?.trim() || latestMessage.trim() || streamedResponse.trim(); - - if (!responseText) { - throw new Error("Copilot SDK returned an empty taxonomy response"); - } - - const jsonText = extractJson(responseText); - lastJson = jsonText; - - let parsedJson: unknown; - try { - parsedJson = JSON.parse(jsonText) as unknown; - } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error); - if (attempts >= maxAttempts) { - throw new Error(`Taxonomy response was not valid JSON after ${maxAttempts} attempts: ${errorMessage}`); - } - - prompt = [ - "Your previous response was not valid JSON.", - `JSON parse error: ${errorMessage}.`, - "Return the full taxonomy document again as raw JSON only, with no markdown fences or commentary.", - "Replace the entire response with corrected JSON.", - ].join(" "); - continue; - } - - const validation = taxonomySchema.safeParse(parsedJson); - if (validation.success) { - return validation.data; - } - - const validationErrors = formatValidationErrors(validation.error.issues); - await log("warn", `Taxonomy validation failed on attempt ${attempts}`, { - issues: validationErrors, - }); - - if (attempts >= maxAttempts) { - throw new Error( - `Taxonomy schema validation failed after ${maxAttempts} attempts: ${validationErrors.join(" | ")}`, - ); - } - - prompt = [ - "Your previous taxonomy JSON did not satisfy the schema.", - "Correct the document and return the entire replacement JSON object.", - "Do not include markdown fences or commentary.", - `Validation errors: ${validationErrors.join("; ")}`, - lastJson ? `Previous JSON: ${lastJson}` : "", - ].filter(Boolean).join(" "); - } - - throw new Error("Taxonomy generation ended without a valid document"); - } finally { - await client.stop().catch(() => undefined); - } -} - -function extractJson(responseText: string): string { - const trimmed = responseText.trim(); - if (!trimmed.startsWith("```")) { - return trimmed; - } - - const fenceMatch = trimmed.match(/^```(?:json)?\s*([\s\S]*?)\s*```$/i); - return fenceMatch?.[1]?.trim() ?? trimmed; -} - -function formatValidationErrors(issues: Array<{ path: PropertyKey[]; message: string }>): string[] { - return issues.map((issue) => { - const path = issue.path.length > 0 ? issue.path.map((segment) => String(segment)).join(".") : ""; - return `${path}: ${issue.message}`; - }); -} diff --git a/apps/workers/taxonomy/src/taxonomy-queue-processor.ts b/apps/workers/taxonomy/src/taxonomy-queue-processor.ts deleted file mode 100644 index 1d8160db0..000000000 --- a/apps/workers/taxonomy/src/taxonomy-queue-processor.ts +++ /dev/null @@ -1,190 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -import { DequeuedMessageItem } from "@azure/storage-queue"; -import { - BaseQueueProcessor, - BlobStorage, - type BaseQueueProcessorConfig, - type LogEvent, - type RequestDocument, - TokenManagerClient, - type TaxonomyDocument, - type VisibilityHeartbeat, - withRetry, -} from "shared"; -import { generateTaxonomy } from "./taxonomy-generator.js"; - -const HANDLER_ID = "pp-taxonomy"; -const HANDLER_VERSION = 1; -const DEFAULT_SESSION_TIMEOUT_MS = 5 * 60 * 1000; - -interface TaxonomyQueueMessage { - type: "taxonomy"; - requestId: string; - runId?: string; -} - -export interface TaxonomyQueueProcessorConfig extends BaseQueueProcessorConfig { - taxonomyModel: string; - apiBaseUrl: string; - sessionTimeoutMs?: number; - schedulerUrl?: string; - tokenManagerUrl?: string; -} - -export class TaxonomyQueueProcessor extends BaseQueueProcessor { - private taxonomyConfig: TaxonomyQueueProcessorConfig; - private tokenClient: TokenManagerClient; - private blobStorage: BlobStorage; - - constructor(config: TaxonomyQueueProcessorConfig) { - super(config, "taxonomy-handler"); - this.taxonomyConfig = config; - this.tokenClient = new TokenManagerClient(config.tokenManagerUrl); - this.blobStorage = new BlobStorage({ - storageAccountName: config.storageAccountName, - storageConnectionString: config.storageConnectionString, - }); - } - - protected override async handleRequest( - doc: RequestDocument, - message: DequeuedMessageItem, - heartbeat: VisibilityHeartbeat, - log: (level: LogEvent["level"], msg: string, data?: Record) => Promise, - payload?: Record, - ): Promise { - const queueMessage = payload as TaxonomyQueueMessage | undefined; - const requestId = doc._id; - const activeRunId = doc.run?._id; - const runId = queueMessage?.runId ?? activeRunId ?? requestId; - - if (!queueMessage || queueMessage.type !== "taxonomy") { - await log("warn", "Discarding message with unsupported taxonomy payload", { payload }); - await this.safeDeleteMessage(message.messageId, heartbeat.stop()); - return; - } - - if (queueMessage.runId && activeRunId && queueMessage.runId !== activeRunId) { - await log("warn", "Discarding stale taxonomy message for superseded run", { - requestId, - queuedRunId: queueMessage.runId, - activeRunId, - }); - await this.safeDeleteMessage(message.messageId, heartbeat.stop()); - return; - } - - if (!doc.run) { - throw new Error(`Request ${requestId} has no active run to classify`); - } - - await withRetry(() => this.collection.updateOne( - { _id: requestId } as never, - { - $set: { - [`run.handlerStatus.${HANDLER_ID}.status`]: "processing", - [`run.handlerStatus.${HANDLER_ID}.version`]: HANDLER_VERSION, - [`run.handlerStatus.${HANDLER_ID}.updatedAt`]: new Date(), - [`run.handlerStatus.${HANDLER_ID}.error`]: null, - }, - } as never, - )); - - await log("info", "Starting taxonomy generation", { requestId, runId, model: this.taxonomyConfig.taxonomyModel }); - - try { - const taxonomy = await this.generateTaxonomy(requestId, runId, log); - const blobName = `${requestId}/runs/${runId}/taxonomy.json`; - const taxonomyUrl = await this.blobStorage.uploadJson(blobName, taxonomy); - - await withRetry(() => this.collection.updateOne( - { _id: requestId } as never, - { - $set: { - [`run.taxonomyUrl`]: taxonomyUrl, - [`run.handlerStatus.${HANDLER_ID}.status`]: "done", - [`run.handlerStatus.${HANDLER_ID}.version`]: HANDLER_VERSION, - [`run.handlerStatus.${HANDLER_ID}.updatedAt`]: new Date(), - }, - $unset: { - [`run.handlerStatus.${HANDLER_ID}.error`]: "", - }, - } as never, - )); - - await log("info", "Taxonomy generation completed", { - requestId, - runId, - blobName, - taxonomyUrl, - }); - - await this.notifyHandlerComplete(requestId, runId, "done"); - } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error); - - await withRetry(() => this.collection.updateOne( - { _id: requestId } as never, - { - $set: { - [`run.handlerStatus.${HANDLER_ID}.status`]: "failed", - [`run.handlerStatus.${HANDLER_ID}.version`]: HANDLER_VERSION, - [`run.handlerStatus.${HANDLER_ID}.updatedAt`]: new Date(), - [`run.handlerStatus.${HANDLER_ID}.error`]: errorMessage, - }, - } as never, - )); - - await log("error", `Taxonomy generation failed: ${errorMessage}`, { requestId, runId }); - await this.notifyHandlerComplete(requestId, runId, "failed"); - throw error; - } - - const finalPopReceipt = heartbeat.stop(); - await this.safeDeleteMessage(message.messageId, finalPopReceipt); - } - - private async generateTaxonomy( - requestId: string, - runId: string, - log: (level: LogEvent["level"], msg: string, data?: Record) => Promise, - ): Promise { - const githubToken = await this.tokenClient.acquireToken("copilot-sdk"); - await log("info", "Acquired Copilot SDK token"); - - return generateTaxonomy({ - requestId, - runId, - apiBaseUrl: this.taxonomyConfig.apiBaseUrl, - taxonomyModel: this.taxonomyConfig.taxonomyModel, - githubToken, - sessionTimeoutMs: this.taxonomyConfig.sessionTimeoutMs ?? DEFAULT_SESSION_TIMEOUT_MS, - log, - }); - } - - private async notifyHandlerComplete( - requestId: string, - runId: string, - status: "done" | "failed", - ): Promise { - const schedulerUrl = this.taxonomyConfig.schedulerUrl; - if (!schedulerUrl) return; - - try { - const response = await fetch(`${schedulerUrl}/notify/handler-complete`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ requestId, runId, handlerId: HANDLER_ID, status }), - }); - - if (!response.ok) { - console.warn(`[taxonomy-handler] Scheduler notify failed: ${response.status}`); - } - } catch (error) { - console.warn(`[taxonomy-handler] Scheduler notify error for ${requestId}/${runId}:`, error); - } - } -} diff --git a/apps/workers/taxonomy/src/tools.ts b/apps/workers/taxonomy/src/tools.ts deleted file mode 100644 index ea4eced32..000000000 --- a/apps/workers/taxonomy/src/tools.ts +++ /dev/null @@ -1,154 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -import { defineTool } from "@github/copilot-sdk"; -import { type ConversationTurn, type RequestDocument } from "shared"; - -interface GetAtifTrajectoryArgs { - iteration?: number; -} - -function selectTurnWithAtif(turns: ConversationTurn[] | undefined, requestedIteration?: number): ConversationTurn | undefined { - if (!turns?.length) return undefined; - - if (requestedIteration !== undefined) { - return turns.find((turn) => turn.iteration === requestedIteration && typeof turn.atifUrl === "string"); - } - - return [...turns] - .filter((turn) => typeof turn.atifUrl === "string") - .sort((a, b) => b.iteration - a.iteration)[0]; -} - -export function createTaxonomyTools( - apiBaseUrl: string, - requestId: string, -) { - const getRunData = defineTool("get_run_data", { - description: - "Fetch the full request document plus the active run state, including scenario, persona, turns, criteria results, and handler status.", - parameters: { - type: "object", - properties: {}, - }, - handler: async () => { - try { - const response = await fetch(`${apiBaseUrl}/api/v1/requests/${requestId}`); - if (!response.ok) { - return { error: `Failed to fetch request: ${response.status} ${response.statusText}` }; - } - - const request: RequestDocument = await response.json(); - const run = request.run; - - return { - request: { - id: request._id, - workerType: request.workerType, - model: request.model, - reasoningEffort: request.reasoningEffort, - maxIterations: request.maxIterations, - createdAt: request.createdAt, - updatedAt: request.updatedAt, - persona: request.persona, - personaInstructions: request.personaInstructions, - mcpServers: request.mcpServers, - skillRevisions: request.skillRevisions, - extensions: request.extensions, - priority: request.priority, - scenario: request.scenario, - }, - run: run ? { - id: run._id, - attemptNumber: run.attemptNumber, - status: run.status, - outcome: run.outcome, - result: run.result, - error: run.error, - startedAt: run.startedAt, - finishedAt: run.finishedAt, - updatedAt: run.updatedAt, - aiCallCount: run.aiCallCount, - tokenUsage: run.tokenUsage, - handlerStatus: run.handlerStatus, - turns: run.turns?.map((turn) => ({ - iteration: turn.iteration, - passed: turn.passed, - timestamp: turn.timestamp, - startedAt: turn.startedAt, - durationMs: turn.durationMs, - judgeFeedback: turn.judgeFeedback, - codingAgentResponse: turn.codingAgentResponse, - aiCallCount: turn.aiCallCount, - tokenUsage: turn.tokenUsage, - snapshotUrl: turn.snapshotUrl, - rawChatUrl: turn.rawChatUrl, - rawChatFormat: turn.rawChatFormat, - toolCallsUrl: turn.toolCallsUrl, - toolCallCount: turn.toolCallCount, - atifUrl: turn.atifUrl, - criteriaResults: turn.criteriaResults, - })) ?? [], - } : null, - }; - } catch (err) { - return { error: `Failed to fetch run data: ${err}` }; - } - }, - }); - - const getAtifTrajectory = defineTool("get_atif_trajectory", { - description: - "Fetch an ATIF trajectory JSON document for a specific iteration, or the latest available iteration when omitted.", - parameters: { - type: "object", - properties: { - iteration: { - type: "number", - description: "Optional 1-based iteration number. If omitted, the latest available ATIF trajectory is returned.", - }, - }, - }, - handler: async (args: GetAtifTrajectoryArgs) => { - try { - const response = await fetch(`${apiBaseUrl}/api/v1/requests/${requestId}`); - if (!response.ok) { - return { error: `Failed to fetch request: ${response.status} ${response.statusText}` }; - } - - const request: RequestDocument = await response.json(); - const selectedTurn = selectTurnWithAtif(request.run?.turns, args.iteration); - - if (!selectedTurn?.atifUrl) { - return { - error: args.iteration !== undefined - ? `No ATIF trajectory available for iteration ${args.iteration}` - : "No ATIF trajectory is available for this run", - }; - } - - const runId = request.run?._id; - const iteration = selectedTurn.iteration; - const atifUrl = runId - ? `${apiBaseUrl}/api/v1/requests/${requestId}/runs/${runId}/atif?iteration=${iteration}` - : `${apiBaseUrl}/api/v1/requests/${requestId}/atif?iteration=${iteration}`; - - const atifResponse = await fetch(atifUrl); - if (!atifResponse.ok) { - return { error: `Failed to fetch ATIF trajectory: ${atifResponse.status} ${atifResponse.statusText}` }; - } - - const trajectory: unknown = await atifResponse.json(); - - return { - iteration, - trajectory, - }; - } catch (err) { - return { error: `Failed to fetch ATIF trajectory: ${err}` }; - } - }, - }); - - return [getRunData, getAtifTrajectory]; -} diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index f5e309b30..49c6443e8 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -217,19 +217,19 @@ services: - action: rebuild path: ./pnpm-lock.yaml - taxonomy-handler: + readout-handler: build: target: dev develop: watch: - action: sync - path: ./apps/workers/taxonomy/src - target: /app/apps/workers/taxonomy/src + path: ./apps/workers/readout/src + target: /app/apps/workers/readout/src - action: sync path: ./packages/shared/src target: /app/packages/shared/src - action: rebuild - path: ./apps/workers/taxonomy/package.json + path: ./apps/workers/readout/package.json - action: rebuild path: ./pnpm-lock.yaml @@ -237,7 +237,7 @@ services: build: target: dev - register-handler-taxonomy: + register-handler-readout: build: target: dev diff --git a/docker-compose.yml b/docker-compose.yml index 05f510d37..2b4c91d91 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,7 +9,7 @@ # docker compose --profile vscode-web up # + VS Code Web worker # docker compose --profile claude-code --profile copilot up # multiple workers # docker compose --profile report up # + Report Generator worker -# docker compose --profile taxonomy up # + Taxonomy handler worker +# docker compose --profile readout up # + Submission readout worker # docker compose up mongodb redis azurite # infra only (run app services natively) # # Host-native debugging: @@ -870,21 +870,19 @@ services: api: condition: service_started - taxonomy-handler: + readout-handler: build: context: . - dockerfile: apps/workers/taxonomy/Dockerfile - profiles: [taxonomy, all] + dockerfile: apps/workers/readout/Dockerfile + profiles: [readout, all] environment: <<: *common-env - AZURE_STORAGE_QUEUE_TAXONOMY: pp-taxonomy-queue - TAXONOMY_MODEL: ${TAXONOMY_MODEL:-gpt-5.4} - SCOPE_MT_API_URL: http://api:80 - SCHEDULER_URL: http://scheduler:8080 + AZURE_STORAGE_QUEUE_READOUT: pp-readout-queue + READOUT_MODEL: ${READOUT_MODEL:-gpt-5.4} GITHUB_TOKEN: ${GITHUB_TOKEN:-} TOKEN_MANAGER_URL: http://token-manager:80 GIT_COMMIT: ${GIT_COMMIT:-development} - SESSION_TIMEOUT_MS: ${SESSION_TIMEOUT_MS:-300000} + READOUT_SESSION_TIMEOUT_MS: ${READOUT_SESSION_TIMEOUT_MS:-300000} depends_on: mongodb: condition: service_healthy @@ -893,21 +891,19 @@ services: condition: service_healthy azurite: condition: service_healthy - api: - condition: service_started scheduler: condition: service_started - register-handler-taxonomy: + register-handler-readout: condition: service_completed_successfully - register-handler-taxonomy: + register-handler-readout: build: context: . - dockerfile: apps/workers/taxonomy/Dockerfile - profiles: [taxonomy, all] + dockerfile: apps/workers/readout/Dockerfile + profiles: [readout, all] environment: <<: *common-env - HANDLER_YAML_PATH: apps/workers/taxonomy/handler.yaml + HANDLER_YAML_PATH: apps/workers/readout/handler.yaml SCHEDULER_URL: http://scheduler:8080 depends_on: scheduler: diff --git a/package.json b/package.json index a98a960c8..d12b00825 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "dev:coder-acp-claude-code": "pnpm --filter coder-acp-claude-code dev", "dev:coder-acp-copilot": "pnpm --filter coder-acp-copilot dev", "dev:report-generator": "pnpm --filter report-generator dev", - "taxonomy": "pnpm --filter taxonomy-handler taxonomy", + "dev:readout": "pnpm --filter readout-handler dev", "docker:up": "scripts/worktree-env.sh && scripts/dev-compose.sh up --build", "docker:up:claude-code": "scripts/worktree-env.sh && SCOPE_REGISTER_CLAUDE_CODE_AGENT=true scripts/dev-compose.sh --env-file .env --env-file apps/workers/coder-acp-claude-code/versions.env --profile claude-code --profile report up --build", "docker:up:copilot": "scripts/worktree-env.sh && SCOPE_REGISTER_COPILOT_AGENT=true scripts/dev-compose.sh --env-file .env --env-file apps/workers/coder-acp-copilot/versions.env --profile copilot --profile report up --build", @@ -27,9 +27,9 @@ "docker:up:portal": "scripts/worktree-env.sh && scripts/dev-compose.sh --profile auth --profile portal up --build", "docker:dev": "scripts/worktree-env.sh && COMPOSE_EXPERIMENTAL_WATCH_POLL=1 scripts/dev-compose.sh -f docker-compose.yml -f docker-compose.dev.yml up --build --watch", "docker:dev:claude-code": "scripts/worktree-env.sh && SCOPE_REGISTER_CLAUDE_CODE_AGENT=true GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) COMPOSE_EXPERIMENTAL_WATCH_POLL=1 scripts/dev-compose.sh --env-file .env --env-file apps/workers/coder-acp-claude-code/versions.env -f docker-compose.yml -f docker-compose.dev.yml --profile claude-code --profile report --profile post-processor --profile auth --profile portal up --build --watch", - "docker:dev:copilot": "scripts/worktree-env.sh && SCOPE_REGISTER_COPILOT_AGENT=true GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) COMPOSE_EXPERIMENTAL_WATCH_POLL=1 scripts/dev-compose.sh --env-file .env --env-file apps/workers/coder-acp-copilot/versions.env -f docker-compose.yml -f docker-compose.dev.yml --profile copilot --profile report --profile post-processor --profile taxonomy --profile auth --profile portal up --build --watch", + "docker:dev:copilot": "scripts/worktree-env.sh && SCOPE_REGISTER_COPILOT_AGENT=true GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) COMPOSE_EXPERIMENTAL_WATCH_POLL=1 scripts/dev-compose.sh --env-file .env --env-file apps/workers/coder-acp-copilot/versions.env -f docker-compose.yml -f docker-compose.dev.yml --profile copilot --profile report --profile post-processor --profile readout --profile auth --profile portal up --build --watch", "docker:dev:portal": "scripts/worktree-env.sh && GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) COMPOSE_EXPERIMENTAL_WATCH_POLL=1 scripts/dev-compose.sh -f docker-compose.yml -f docker-compose.dev.yml --profile auth --profile portal up --build --watch", - "docker:dev:all": "scripts/worktree-env.sh && SCOPE_REGISTER_COPILOT_AGENT=true SCOPE_REGISTER_CLAUDE_CODE_AGENT=true GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) COMPOSE_EXPERIMENTAL_WATCH_POLL=1 scripts/dev-compose.sh --env-file .env --env-file apps/workers/coder-acp-copilot/versions.env --env-file apps/workers/coder-acp-claude-code/versions.env -f docker-compose.yml -f docker-compose.dev.yml --profile claude-code --profile copilot --profile report --profile post-processor --profile taxonomy --profile auth --profile portal up --build --watch", + "docker:dev:all": "scripts/worktree-env.sh && SCOPE_REGISTER_COPILOT_AGENT=true SCOPE_REGISTER_CLAUDE_CODE_AGENT=true GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) COMPOSE_EXPERIMENTAL_WATCH_POLL=1 scripts/dev-compose.sh --env-file .env --env-file apps/workers/coder-acp-copilot/versions.env --env-file apps/workers/coder-acp-claude-code/versions.env -f docker-compose.yml -f docker-compose.dev.yml --profile claude-code --profile copilot --profile report --profile post-processor --profile readout --profile auth --profile portal up --build --watch", "docker:dev:build": "scripts/worktree-env.sh && GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) scripts/dev-compose.sh --env-file .env --env-file apps/workers/coder-acp-copilot/versions.env --env-file apps/workers/coder-acp-claude-code/versions.env -f docker-compose.yml -f docker-compose.dev.yml --profile claude-code --profile copilot --profile report --profile auth --profile portal build", "docker:down": "scripts/worktree-env.sh && scripts/dev-compose.sh --env-file .env --env-file apps/workers/coder-acp-copilot/versions.env --env-file apps/workers/coder-acp-claude-code/versions.env --profile claude-code --profile copilot --profile report --profile portal down --volumes", "docker:build": "scripts/worktree-env.sh && scripts/dev-compose.sh --env-file .env --env-file apps/workers/coder-acp-copilot/versions.env --env-file apps/workers/coder-acp-claude-code/versions.env --profile claude-code --profile copilot --profile report build", diff --git a/packages/db-migrations/src/migrations/031-migrate-services-to-handler-dag.ts b/packages/db-migrations/src/migrations/031-migrate-services-to-handler-dag.ts index 87fba84c8..570b12e40 100644 --- a/packages/db-migrations/src/migrations/031-migrate-services-to-handler-dag.ts +++ b/packages/db-migrations/src/migrations/031-migrate-services-to-handler-dag.ts @@ -7,7 +7,7 @@ import type { MigrationInterface } from "mongo-migrate-ts"; /** * Migration 031: Remove the legacy "post-processor" service entry. * - * Handler service registration (pp-atif, pp-taxonomy) is now owned + * Handler service registration (e.g. pp-atif) is now owned * by each worker's handler.yaml + the generic register-handler script, * which POSTs to the scheduler's POST /handlers/register endpoint on deploy. * Reports are not a DAG handler — the scheduler triggers report generation diff --git a/packages/db-migrations/src/migrations/032-add-handler-status-index.ts b/packages/db-migrations/src/migrations/032-add-handler-status-index.ts index cf69fb7c0..44d7a448a 100644 --- a/packages/db-migrations/src/migrations/032-add-handler-status-index.ts +++ b/packages/db-migrations/src/migrations/032-add-handler-status-index.ts @@ -13,14 +13,6 @@ const HANDLER_STATUS_INDEXES = [ updatedAt: -1 as const, }, }, - { - name: "idx_handler_dispatch_taxonomy", - key: { - "run.handlerStatus.pp-taxonomy.status": 1 as const, - "run.status": 1 as const, - updatedAt: -1 as const, - }, - }, ] as const; export class AddHandlerStatusIndex implements MigrationInterface { diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 6abb0ed5b..1b6abd4b1 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -29,6 +29,6 @@ export * from "./har/index.js"; export * from "./utils/index.js"; export * from "./schemas/index.js"; export * from "./auth/index.js"; -export * from "./schemas/taxonomy.js"; +export * from "./schemas/readout.js"; export * from "./cursor.js"; export * from "./run-duration.js"; diff --git a/packages/shared/src/schemas/index.ts b/packages/shared/src/schemas/index.ts index a1e4d9761..85ec7dd34 100644 --- a/packages/shared/src/schemas/index.ts +++ b/packages/shared/src/schemas/index.ts @@ -23,3 +23,4 @@ export * from "./feature-flag.js"; export * from "./profile.js"; export * from "./project.js"; export * from "./taxonomy.js"; +export * from "./readout.js"; diff --git a/packages/shared/src/schemas/readout.test.ts b/packages/shared/src/schemas/readout.test.ts new file mode 100644 index 000000000..503dcfc8c --- /dev/null +++ b/packages/shared/src/schemas/readout.test.ts @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, expect, it } from "vitest"; +import { + SUBMISSION_READOUT_SCHEMA_ID, + SUBMISSION_READOUT_SCHEMA_VERSION, + submissionReadoutSchema, +} from "./readout.js"; + +describe("submissionReadoutSchema", () => { + it("accepts a one-request readout with empty selected observations", () => { + expect(submissionReadoutSchema.parse({ + meta: { + schemaId: SUBMISSION_READOUT_SCHEMA_ID, + schemaVersion: SUBMISSION_READOUT_SCHEMA_VERSION, + generatedAt: "2026-10-13T12:00:00.000Z", + submissionId: "submission-1", + submissionRunId: "submission-run-1", + }, + summary: { + title: "Submission readout", + executiveSummary: "The single run completed without selected observations.", + totalRuns: 1, + succeededRuns: 1, + failedRuns: 0, + passedObservations: 0, + failedObservations: 0, + }, + profiles: [{ + requestIds: ["request-1"], + summary: "One unprofiled request.", + }], + runs: [{ + requestId: "request-1", + runId: "run-1", + task: "Update the endpoint", + workerType: "copilot", + status: "done", + outcome: "succeeded", + observations: [], + trajectoryHighlights: [], + }], + findings: [], + actions: [], + })).toBeTruthy(); + }); +}); diff --git a/packages/shared/src/schemas/readout.ts b/packages/shared/src/schemas/readout.ts new file mode 100644 index 000000000..f8e85ca7e --- /dev/null +++ b/packages/shared/src/schemas/readout.ts @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { z } from "zod"; + +export const SUBMISSION_READOUT_SCHEMA_ID = "scope.submission-readout"; +export const SUBMISSION_READOUT_SCHEMA_VERSION = "1.0"; + +export const readoutObservationSchema = z.object({ + criterionId: z.string().min(1), + taxonomyElementId: z.string().optional(), + subject: z.enum(["run", "iteration"]), + passed: z.boolean(), + feedback: z.string(), +}); + +export const readoutEvidenceSchema = z.object({ + submissionRunId: z.string().min(1), + requestId: z.string().min(1), + runId: z.string().min(1), + iteration: z.number().int().min(0).optional(), + criterionId: z.string().min(1).optional(), + excerpt: z.string().min(1), +}); + +export const readoutRunSchema = z.object({ + requestId: z.string().min(1), + runId: z.string().min(1), + task: z.string(), + workerType: z.string().min(1), + model: z.string().optional(), + profileId: z.string().optional(), + profileVersionId: z.string().optional(), + status: z.string(), + outcome: z.string().optional(), + observations: z.array(readoutObservationSchema), + trajectoryHighlights: z.array(z.string().min(1)), +}); + +export const readoutFindingSchema = z.object({ + id: z.string().min(1), + title: z.string().min(1), + description: z.string().min(1), + severity: z.enum(["critical", "high", "medium", "low", "info"]), + evidence: z.array(readoutEvidenceSchema).min(1), +}); + +export const readoutActionSchema = z.object({ + id: z.string().min(1), + title: z.string().min(1), + priority: z.enum(["critical", "high", "medium", "low"]), + description: z.string().min(1), + relatedFindingIds: z.array(z.string().min(1)), +}); + +export const submissionReadoutSchema = z.object({ + meta: z.object({ + schemaId: z.literal(SUBMISSION_READOUT_SCHEMA_ID), + schemaVersion: z.literal(SUBMISSION_READOUT_SCHEMA_VERSION), + generatedAt: z.string().datetime({ offset: true }), + submissionId: z.string().min(1), + submissionRunId: z.string().min(1), + }), + summary: z.object({ + title: z.string().min(1), + executiveSummary: z.string().min(1), + totalRuns: z.number().int().min(1), + succeededRuns: z.number().int().min(0), + failedRuns: z.number().int().min(0), + passedObservations: z.number().int().min(0), + failedObservations: z.number().int().min(0), + }), + profiles: z.array(z.object({ + profileId: z.string().optional(), + requestIds: z.array(z.string().min(1)).min(1), + summary: z.string().min(1), + })), + runs: z.array(readoutRunSchema).min(1), + findings: z.array(readoutFindingSchema), + actions: z.array(readoutActionSchema), +}); + +export type SubmissionReadout = z.infer; +export type ReadoutRun = z.infer; diff --git a/packages/shared/src/schemas/report.ts b/packages/shared/src/schemas/report.ts index c9aa697db..afeb0ac5d 100644 --- a/packages/shared/src/schemas/report.ts +++ b/packages/shared/src/schemas/report.ts @@ -33,13 +33,25 @@ export const ReportStatusSchema = z.enum([ "failed", ]); +export const StructuredReportArtifactSchema = z + .object({ + schemaId: z.string(), + schemaVersion: z.string(), + contentType: z.literal("application/json"), + url: z.string(), + }) + .openapi("StructuredReportArtifact"); + export const ReportResponseSchema = z .object({ _id: z.string(), - requestId: z.string(), + requestId: z.string().optional(), + submissionId: z.string().optional(), + submissionRunId: z.string().optional(), templateId: z.string().optional(), reporter: ReporterSchema.optional(), content: z.string().optional(), + structuredArtifact: StructuredReportArtifactSchema.optional(), status: ReportStatusSchema, error: z.string().optional(), insightReferences: z.array(InsightReferenceSchema).optional(), diff --git a/packages/shared/src/schemas/request.ts b/packages/shared/src/schemas/request.ts index 8e72d5765..bf4c75743 100644 --- a/packages/shared/src/schemas/request.ts +++ b/packages/shared/src/schemas/request.ts @@ -248,7 +248,6 @@ export const RunStateSchema = z podName: z.string().optional(), }).optional(), harUrl: z.string().optional(), - taxonomyUrl: z.string().optional(), videoUrls: z.array(z.string()).optional(), setupVideoUrls: z.array(z.string()).optional(), tokenUsage: TokenUsageSchema.optional(), diff --git a/packages/shared/src/schemas/taxonomy.test.ts b/packages/shared/src/schemas/taxonomy.test.ts deleted file mode 100644 index 320f89ceb..000000000 --- a/packages/shared/src/schemas/taxonomy.test.ts +++ /dev/null @@ -1,180 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -import { describe, expect, it } from "vitest"; -import { taxonomySchema } from "./taxonomy.js"; - -describe("taxonomySchema", () => { - const validFixture = { - meta: { - generatedAt: "2025-02-01T12:00:00Z", - schemaVersion: "1.0.0", - runId: "run-123", - requestId: "req-456", - }, - scorecard: { - summary: "The agent usually succeeds but struggles with recovery quality.", - profileResults: [ - { - profileId: "profile-best", - select: "best", - taskOutcome: "success", - overallScore: 92, - dimensions: [ - { - name: "Correctness", - score: 95, - rationale: "Implemented the requested behavior fully.", - evidence: ["All acceptance criteria passed", "No regressions observed"], - }, - ], - }, - ], - crossProfileInsights: [ - "The agent communicates clearly across profiles.", - "Stress conditions reduce recovery quality.", - ], - }, - behaviorAnalysis: { - patterns: [ - { - id: "pattern-1", - category: "strength", - title: "Consistent planning", - description: "The agent outlines a clear plan before making changes.", - frequency: 0.75, - impact: "high", - evidence: [ - { - runId: "run-123", - iteration: 1, - excerpt: "Plan: inspect the failing endpoint, then patch the handler.", - }, - ], - }, - ], - agentPersonality: { - communicationStyle: "Concise and structured", - problemSolvingApproach: "Hypothesis-driven debugging", - errorRecoveryBehavior: "Retries with targeted fixes after inspecting evidence", - }, - }, - actionList: { - fix: [ - { - id: "action-1", - title: "Stabilize recovery prompts", - priority: "critical", - description: "Reduce failures when the first attempt is rejected.", - ownership: "prompt-engineer", - relatedPatterns: ["pattern-1"], - suggestedApproach: "Add explicit fallback guidance for failed first passes.", - }, - ], - improve: [ - { - id: "action-2", - title: "Expand planning examples", - priority: "medium", - description: "Reinforce successful planning behavior.", - ownership: "agent-team", - relatedPatterns: ["pattern-1"], - suggestedApproach: "Fine-tune on traces with strong upfront decomposition.", - }, - ], - investigate: [ - { - id: "action-3", - title: "Review profile variance", - priority: "low", - description: "Understand why profile-specific regressions appear sporadically.", - ownership: "scenario-author", - relatedPatterns: ["pattern-1"], - suggestedApproach: "Compare scenario wording against affected profile runs.", - }, - ], - }, - }; - - it("accepts a valid taxonomy fixture", () => { - const result = taxonomySchema.safeParse(validFixture); - expect(result.success).toBe(true); - }); - - it("rejects fixtures missing required fields with precise paths", () => { - const invalidFixture = { - ...validFixture, - scorecard: { - ...validFixture.scorecard, - profileResults: [ - { - ...validFixture.scorecard.profileResults[0], - dimensions: [ - { - name: "Correctness", - score: 95, - evidence: ["All acceptance criteria passed"], - }, - ], - }, - ], - }, - behaviorAnalysis: { - ...validFixture.behaviorAnalysis, - agentPersonality: { - communicationStyle: "Concise and structured", - errorRecoveryBehavior: "Retries with targeted fixes after inspecting evidence", - }, - }, - }; - - const result = taxonomySchema.safeParse(invalidFixture); - expect(result.success).toBe(false); - if (result.success) { - throw new Error("Expected invalid taxonomy fixture to fail parsing"); - } - - const issuePaths = result.error.issues.map((issue) => issue.path); - expect(issuePaths).toContainEqual([ - "scorecard", - "profileResults", - 0, - "dimensions", - 0, - "rationale", - ]); - expect(issuePaths).toContainEqual([ - "behaviorAnalysis", - "agentPersonality", - "problemSolvingApproach", - ]); - }); - - it("enforces enum values", () => { - const invalidFixture = { - ...validFixture, - actionList: { - ...validFixture.actionList, - fix: [ - { - ...validFixture.actionList.fix[0], - priority: "urgent", - }, - ], - }, - }; - - const result = taxonomySchema.safeParse(invalidFixture); - expect(result.success).toBe(false); - if (result.success) { - throw new Error("Expected invalid enum fixture to fail parsing"); - } - - expect(result.error.issues.map((issue) => issue.path)).toContainEqual([ - "actionList", - "fix", - 0, - "priority", - ]); - }); -}); diff --git a/packages/shared/src/schemas/taxonomy.ts b/packages/shared/src/schemas/taxonomy.ts deleted file mode 100644 index a837d4f91..000000000 --- a/packages/shared/src/schemas/taxonomy.ts +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -import { z } from "zod"; - -const scoreSchema = z.number().min(0).max(100); -const frequencySchema = z.number().min(0).max(1); - -/** Taxonomy document metadata. */ -export const taxonomyMetaSchema = z.object({ - generatedAt: z.string().datetime({ offset: true }), - schemaVersion: z.string().min(1), - runId: z.string().min(1), - requestId: z.string().min(1), -}); - -/** Scorecard dimension result for a single profile. */ -export const scorecardDimensionSchema = z.object({ - name: z.string().min(1), - score: scoreSchema, - rationale: z.string().min(1), - evidence: z.array(z.string().min(1)), -}); - -/** Scorecard result for a specific profile slice. */ -export const profileResultSchema = z.object({ - profileId: z.string().min(1), - select: z.enum(["best", "median", "worst"]), - taskOutcome: z.enum(["success", "partial", "failure"]), - overallScore: scoreSchema, - dimensions: z.array(scorecardDimensionSchema), -}); - -/** Top-level scorecard section summarizing profile results. */ -export const scorecardSchema = z.object({ - summary: z.string().min(1), - profileResults: z.array(profileResultSchema), - crossProfileInsights: z.array(z.string().min(1)), -}); - -/** Evidence excerpt supporting an observed behavior pattern. */ -export const behaviorEvidenceSchema = z.object({ - runId: z.string().min(1), - iteration: z.number().int().min(0), - excerpt: z.string().min(1), -}); - -/** Cross-run behavioral pattern identified by the taxonomy analysis. */ -export const behaviorPatternSchema = z.object({ - id: z.string().min(1), - category: z.enum(["strength", "weakness", "inconsistency", "regression"]), - title: z.string().min(1), - description: z.string().min(1), - frequency: frequencySchema, - impact: z.enum(["critical", "high", "medium", "low"]), - evidence: z.array(behaviorEvidenceSchema), -}); - -/** Qualitative description of the agent's personality traits. */ -export const agentPersonalitySchema = z.object({ - communicationStyle: z.string().min(1), - problemSolvingApproach: z.string().min(1), - errorRecoveryBehavior: z.string().min(1), -}); - -/** Behavior analysis section describing patterns and personality. */ -export const behaviorAnalysisSchema = z.object({ - patterns: z.array(behaviorPatternSchema), - agentPersonality: agentPersonalitySchema, -}); - -/** Action item emitted by the taxonomy analysis. */ -export const actionItemSchema = z.object({ - id: z.string().min(1), - title: z.string().min(1), - priority: z.enum(["critical", "high", "medium", "low"]), - description: z.string().min(1), - ownership: z.enum([ - "agent-team", - "platform-team", - "prompt-engineer", - "scenario-author", - ]), - relatedPatterns: z.array(z.string().min(1)), - suggestedApproach: z.string().min(1), -}); - -/** Prioritized action lists grouped by remediation intent. */ -export const actionListSchema = z.object({ - fix: z.array(actionItemSchema), - improve: z.array(actionItemSchema), - investigate: z.array(actionItemSchema), -}); - -/** Complete taxonomy document produced by the taxonomy post-processor. */ -export const taxonomySchema = z.object({ - meta: taxonomyMetaSchema, - scorecard: scorecardSchema, - behaviorAnalysis: behaviorAnalysisSchema, - actionList: actionListSchema, -}); - -/** Inferred TypeScript type for the taxonomy document. */ -export type TaxonomyDocument = z.infer; diff --git a/packages/shared/src/scripts/register-handler.test.ts b/packages/shared/src/scripts/register-handler.test.ts index 0c3858b58..7d873e452 100644 --- a/packages/shared/src/scripts/register-handler.test.ts +++ b/packages/shared/src/scripts/register-handler.test.ts @@ -6,11 +6,11 @@ import { parseHandlerDocument } from "./register-handler.js"; describe("parseHandlerDocument", () => { const valid = { - _id: "pp-taxonomy", + _id: "pp-example", type: "post-process-handler", version: 1, - queue: "pp-taxonomy-queue", - selector: "taxonomy", + queue: "pp-example-queue", + selector: "example", autoBackfill: false, dependsOn: ["pp-atif"], }; @@ -51,4 +51,18 @@ describe("parseHandlerDocument", () => { it("rejects a non-object input", () => { expect(() => parseHandlerDocument("nope")).toThrow(/mapping/); }); + + it("accepts the pp-readout submission handler contract", () => { + const readout = { + _id: "pp-readout", + type: "submission-handler", + version: 1, + queue: "pp-readout-queue", + selector: "readout", + autoBackfill: false, + dependsOn: ["pp-observations"], + dependencyMode: "all-runs", + }; + expect(parseHandlerDocument(readout)).toEqual(readout); + }); }); diff --git a/packages/shared/src/types/types.ts b/packages/shared/src/types/types.ts index eb0b6a1a6..f98a33a61 100644 --- a/packages/shared/src/types/types.ts +++ b/packages/shared/src/types/types.ts @@ -535,14 +535,13 @@ export interface RunState { podName?: string; }; harUrl?: string; - taxonomyUrl?: string; videoUrls?: string[]; setupVideoUrls?: string[]; tokenUsage?: TokenUsage; aiCallCount?: number; rawChatUrl?: string; rawChatFormat?: string; - /** Per-handler post-processing status. Keys are handler IDs (e.g. "pp-atif", "pp-taxonomy"). */ + /** Per-handler post-processing status. Keys are handler IDs (e.g. "pp-atif"). */ handlerStatus?: Record; /** Whole-run observation results — one entry per selected `subject:"run"` * observation criterion, evaluated once against the final snapshot + the @@ -623,7 +622,6 @@ export const RUN_STATE_FIELD_NAMES = [ "lastHeartbeatAt", "worker", "harUrl", - "taxonomyUrl", "videoUrls", "setupVideoUrls", "tokenUsage", @@ -860,14 +858,24 @@ export interface Reporter { agentVersion: string; // @github/copilot-sdk package version } +export interface StructuredReportArtifact { + schemaId: string; + schemaVersion: string; + contentType: "application/json"; + url: string; +} + /** Report document stored in MongoDB */ export interface ReportDocument { _id: string; // UUID - requestId: string; // FK → RequestDocument._id + requestId?: string; // FK → RequestDocument._id for run reports + submissionId?: string; // FK → SubmissionDocument._id for submission reports + submissionRunId?: string; // Exact immutable submission attempt projectId: string; // FK → ProjectDocument._id (denormalized from request) templateId?: string; // FK → ReportTemplateDocument.id (slug of template that generated this report) reporter?: Reporter; // Set by the worker when it picks up the job content?: string; // Generated markdown report + structuredArtifact?: StructuredReportArtifact; status: ReportStatus; error?: string; insightReferences?: InsightReference[]; // Insights discovered/referenced by this report diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 44a7b86bc..75f306a3e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -759,7 +759,7 @@ importers: specifier: ^5.6.2 version: 5.9.3 - apps/workers/report-generator: + apps/workers/readout: dependencies: '@azure/storage-queue': specifier: ^12.25.0 @@ -773,9 +773,9 @@ importers: shared: specifier: workspace:* version: link:../../../packages/shared - telemetry: - specifier: workspace:* - version: link:../../../packages/telemetry + zod: + specifier: ^4.3.6 + version: 4.3.6 devDependencies: '@types/node': specifier: ^22.7.0 @@ -787,20 +787,23 @@ importers: specifier: ^5.6.2 version: 5.9.3 - apps/workers/taxonomy: + apps/workers/report-generator: dependencies: '@azure/storage-queue': specifier: ^12.25.0 version: 12.29.0 '@github/copilot-sdk': - specifier: ^0.1.17 - version: 0.1.32(typescript@5.9.3) + specifier: 1.0.1 + version: 1.0.1(typescript@5.9.3) dotenv: specifier: 16.6.1 version: 16.6.1 shared: specifier: workspace:* version: link:../../../packages/shared + telemetry: + specifier: workspace:* + version: link:../../../packages/telemetry devDependencies: '@types/node': specifier: ^22.7.0 @@ -823,7 +826,6 @@ importers: yaml: specifier: ^2.8.1 version: 2.8.3 - devDependencies: '@types/node': specifier: ^22.18.6 @@ -1584,10 +1586,6 @@ packages: libc: [musl] hasBin: true - '@github/copilot-sdk@0.1.32': - resolution: {integrity: sha1-mkVfe7Zkq80wJuGboVpfCcAwg4E=, tarball: https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@github/copilot-sdk/-/copilot-sdk-0.1.32.tgz} - engines: {node: '>=20.0.0'} - '@github/copilot-sdk@1.0.1': resolution: {integrity: sha512-w6AaS0WqqTE/3iyUrZznvgCLQhsUF7ZmEVCneacuHCfOzlH0r6ww9WUmyA0zgqmXO75V0IYrkIcnFke/qJkkDg==} engines: {node: ^20.19.0 || >=22.12.0} @@ -7729,14 +7727,6 @@ snapshots: '@github/copilot-linuxmusl-x64@1.0.63': optional: true - '@github/copilot-sdk@0.1.32(typescript@5.9.3)': - dependencies: - '@github/copilot': 1.0.63(typescript@5.9.3) - vscode-jsonrpc: 8.2.1 - zod: 4.3.6 - transitivePeerDependencies: - - typescript - '@github/copilot-sdk@1.0.1(typescript@5.9.3)': dependencies: '@github/copilot': 1.0.63(typescript@5.9.3) From be500328392561a2cff45a11737ea43a5a69fa46 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Tue, 13 Oct 2026 16:47:49 +0200 Subject: [PATCH 03/18] feat(readout): expose submission reports in UI and CLI Render current and historical submission readouts, support structured downloads, and add CLI access while removing the duplicated run taxonomy surface. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 1cbbd5eb-61c6-4e6a-bb38-c377be5b2163 --- apps/portal/src/components/TaxonomyTab.tsx | 248 ------------------- apps/portal/src/lib/api.ts | 13 +- apps/portal/src/pages/InsightDetail.tsx | 21 +- apps/portal/src/pages/ReportDetail.tsx | 61 +++-- apps/portal/src/pages/ReportPreviewPanel.tsx | 18 +- apps/portal/src/pages/ReportsList.tsx | 31 ++- apps/portal/src/pages/RunDetail.tsx | 10 - apps/portal/src/types.ts | 11 +- 8 files changed, 109 insertions(+), 304 deletions(-) delete mode 100644 apps/portal/src/components/TaxonomyTab.tsx diff --git a/apps/portal/src/components/TaxonomyTab.tsx b/apps/portal/src/components/TaxonomyTab.tsx deleted file mode 100644 index e97c8c8c4..000000000 --- a/apps/portal/src/components/TaxonomyTab.tsx +++ /dev/null @@ -1,248 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -import { useQuery } from "@tanstack/react-query"; -import { api } from "@/lib/api"; -import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; -import { Badge } from "@/components/ui/badge"; -import { Button } from "@/components/ui/button"; -import { Loader2, Download, ChevronDown, ChevronRight } from "lucide-react"; -import { useState } from "react"; - -interface TaxonomyDimension { - name: string; - score: number; - maxScore: number; - justification: string; -} - -interface BehaviorPattern { - id: string; - category: string; - observation: string; - evidence: string[]; - impact: "positive" | "negative" | "neutral"; -} - -interface ActionItem { - id: string; - priority: "high" | "medium" | "low"; - title: string; - description: string; - category: string; -} - -interface TaxonomyData { - meta?: { - schemaVersion?: string; - generatedAt?: string; - requestId?: string; - runId?: string; - model?: string; - }; - scorecard?: { - overall?: number; - dimensions?: TaxonomyDimension[]; - }; - behaviorAnalysis?: { - patterns?: BehaviorPattern[]; - }; - actionList?: { - items?: ActionItem[]; - }; -} - -function ScoreBar({ score, maxScore }: { score: number; maxScore: number }) { - const pct = Math.round((score / maxScore) * 100); - const color = pct >= 80 ? "bg-green-500" : pct >= 50 ? "bg-yellow-500" : "bg-red-500"; - return ( -
-
-
-
- {score}/{maxScore} -
- ); -} - -function ImpactBadge({ impact }: { impact: string }) { - const variant = impact === "positive" ? "default" : impact === "negative" ? "destructive" : "secondary"; - return {impact}; -} - -function PriorityBadge({ priority }: { priority: string }) { - const colors: Record = { - high: "bg-red-100 text-red-800 dark:bg-red-900/30 dark:text-red-300", - medium: "bg-yellow-100 text-yellow-800 dark:bg-yellow-900/30 dark:text-yellow-300", - low: "bg-blue-100 text-blue-800 dark:bg-blue-900/30 dark:text-blue-300", - }; - return {priority}; -} - -function CollapsibleSection({ title, defaultOpen = true, children }: { title: string; defaultOpen?: boolean; children: React.ReactNode }) { - const [open, setOpen] = useState(defaultOpen); - return ( -
- - {open &&
{children}
} -
- ); -} - -export function TaxonomyTab({ runId, attemptRunId }: { runId: string; attemptRunId?: string }) { - const url = attemptRunId ? api.runTaxonomyUrl(runId, attemptRunId) : api.taxonomyUrl(runId); - - const { data: taxonomy, isLoading, error } = useQuery({ - queryKey: ["run-taxonomy", runId, attemptRunId], - queryFn: async () => { - const res = await fetch(url); - if (!res.ok) { - if (res.status === 404) return null; - throw new Error(`Failed to fetch taxonomy: ${res.statusText}`); - } - return res.json(); - }, - }); - - if (isLoading) { - return ( -
- - Loading taxonomy… -
- ); - } - - if (error) { - return ( -
-

Failed to load taxonomy: {(error as Error).message}

-
- ); - } - - if (!taxonomy) { - return ( -
-

Taxonomy not yet generated for this run.

-

It will appear here once post-processing completes.

-
- ); - } - - const { meta, scorecard, behaviorAnalysis, actionList } = taxonomy; - - return ( -
- {/* Header with download */} -
-
-

Run Taxonomy

- {meta?.generatedAt && ( -

- Generated {new Date(meta.generatedAt).toLocaleString()} - {meta.model && <> · Model: {meta.model}} -

- )} -
- -
- - {/* Scorecard */} - {scorecard && ( - - - - Scorecard - {scorecard.overall !== undefined && ( - {scorecard.overall}% - )} - - - - {scorecard.dimensions?.map((dim) => ( -
-
- {dim.name} -
- -

{dim.justification}

-
- ))} - - - )} - - {/* Behavior Analysis */} - {behaviorAnalysis?.patterns && behaviorAnalysis.patterns.length > 0 && ( - - - - Behavior Analysis ({behaviorAnalysis.patterns.length}) - - - - -
- {behaviorAnalysis.patterns.map((pattern) => ( -
-
- {pattern.category} - -
-

{pattern.observation}

- {pattern.evidence.length > 0 && ( -
    - {pattern.evidence.map((e, i) =>
  • {e}
    • )} -
- )} -
- ))} -
- - - - )} - - {/* Action List */} - {actionList?.items && actionList.items.length > 0 && ( - - - - Action Items ({actionList.items.length}) - - - - -
- {actionList.items.map((item) => ( -
- -
-

{item.title}

-

{item.description}

- {item.category} -
-
- ))} -
- - - - )} - - {/* Raw JSON fallback — collapsed by default */} - - 
-          {JSON.stringify(taxonomy, null, 2)}
-
- -
- ); -} diff --git a/apps/portal/src/lib/api.ts b/apps/portal/src/lib/api.ts index f264e4a9d..d303dcf16 100644 --- a/apps/portal/src/lib/api.ts +++ b/apps/portal/src/lib/api.ts @@ -503,16 +503,6 @@ export const api = { return `${BASE}/requests/${id}/atif${qs}`; }, - /** Get taxonomy JSON URL for a request (latest run) */ - taxonomyUrl: (id: string): string => { - return `${BASE}/requests/${id}/taxonomy`; - }, - - /** Get taxonomy JSON URL for a specific attempt */ - runTaxonomyUrl: (requestId: string, runId: string): string => { - return `${BASE}/requests/${requestId}/runs/${runId}/taxonomy`; - }, - /** Download a batch archive of multiple runs as a single .tar.gz */ batchArchive: async (ids: string[]): Promise => { const resp = await apiClient(`${BASE}/requests/archive`, { @@ -918,6 +908,9 @@ export const api = { return request(`/reports/${id}`); }, + reportStructuredUrl: (id: string): string => + `${BASE}/reports/${encodeURIComponent(id)}/structured`, + /** Get reports for a specific run */ getRunReports: (requestId: string): Promise => { return request(`/requests/${requestId}/reports`); diff --git a/apps/portal/src/pages/InsightDetail.tsx b/apps/portal/src/pages/InsightDetail.tsx index 02792b53b..6c875853c 100644 --- a/apps/portal/src/pages/InsightDetail.tsx +++ b/apps/portal/src/pages/InsightDetail.tsx @@ -215,12 +215,21 @@ export function InsightDetail() { - - {formatId(report.requestId)} - + {report.submissionId ? ( + + {formatId(report.submissionId)} + + ) : report.requestId ? ( + + {formatId(report.requestId)} + + ) : "–"} diff --git a/apps/portal/src/pages/ReportDetail.tsx b/apps/portal/src/pages/ReportDetail.tsx index 685e69792..c65b9bbab 100644 --- a/apps/portal/src/pages/ReportDetail.tsx +++ b/apps/portal/src/pages/ReportDetail.tsx @@ -12,7 +12,7 @@ import { Badge } from "@/components/ui/badge"; import { ReportStatusBadge } from "@/components/ReportStatusBadge"; import { LogViewer } from "@/components/LogViewer"; import { useLogStream } from "@/hooks/use-log-stream"; -import { ArrowLeft, Copy, Check, ExternalLink, ClipboardCopy, Lightbulb } from "lucide-react"; +import { ArrowLeft, Copy, Check, Download, ExternalLink, ClipboardCopy, Lightbulb } from "lucide-react"; import { formatDate } from "@/lib/utils"; import { useEffect, useRef, useState } from "react"; import { MarkdownRenderer } from "@/components/MarkdownRenderer"; @@ -131,6 +131,14 @@ export function ReportDetail() {
+ {report.structuredArtifact && ( + + )} {report.templateId && ( )} - - Run: {report.requestId.slice(0, 8)}… - - + {report.submissionId ? ( + + + Submission: {report.submissionId.slice(0, 8)}… + + + + ) : report.requestId ? ( + + Run: {report.requestId.slice(0, 8)}… + + + ) : null} Created {formatDate(report.createdAt)}
@@ -330,13 +350,24 @@ export function ReportDetail() { {report._id}
- Run ID:{" "} - - {report.requestId} - + + {report.submissionId ? "Submission run:" : "Run ID:"} + {" "} + {report.submissionId ? ( + + {report.submissionRunId} + + ) : report.requestId ? ( + + {report.requestId} + + ) : "—"}
Created:{" "} diff --git a/apps/portal/src/pages/ReportPreviewPanel.tsx b/apps/portal/src/pages/ReportPreviewPanel.tsx index 477ca8331..f6ad08a20 100644 --- a/apps/portal/src/pages/ReportPreviewPanel.tsx +++ b/apps/portal/src/pages/ReportPreviewPanel.tsx @@ -57,11 +57,19 @@ export function ReportPreviewPanel() { onClose={closePanel} headerActions={
- - - + {report.submissionId ? ( + + + + ) : report.requestId ? ( + + + + ) : null}
@@ -804,9 +806,18 @@ export function RunDetail() {
{/* Back link + header */}
- + diff --git a/apps/portal/src/pages/SubmissionDetail.test.tsx b/apps/portal/src/pages/SubmissionDetail.test.tsx index 14f39774b..9481a6ff2 100644 --- a/apps/portal/src/pages/SubmissionDetail.test.tsx +++ b/apps/portal/src/pages/SubmissionDetail.test.tsx @@ -3,15 +3,20 @@ // @vitest-environment happy-dom import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; -import { cleanup, render, screen } from "@testing-library/react"; +import { cleanup, render, screen, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; import { MemoryRouter, Route, Routes } from "react-router-dom"; import { afterEach, describe, expect, it, vi } from "vitest"; -import type { Report, Submission } from "@/types"; +import type { + Report, + Submission, + SubmissionRunAttempt, +} from "@/types"; const submission: Submission = { submissionId: "submission-1", projectId: "project-1", - requestIds: ["request-1"], + requestIds: ["request-1", "request-2"], createdAt: "2026-01-01T00:00:00.000Z", submissionRunId: "submission-run-2", attemptNumber: 2, @@ -19,90 +24,298 @@ const submission: Submission = { requests: [ { requestId: "request-1", - runId: "run-1", + runId: "run-current-1", task: "Create an API", status: "done", outcome: "succeeded", workerType: "coder-acp-copilot", + model: "gpt-5.4", + }, + { + requestId: "request-2", + runId: "run-current-2", + task: "Document the API", + status: "done", + outcome: "succeeded", + workerType: "coder-acp-copilot", + model: "gpt-5.4", }, ], dependencyProjection: [ { requestId: "request-1", - runId: "run-1", + runId: "run-current-1", handlerId: "pp-observations", state: "ready", }, + { + requestId: "request-2", + runId: "run-current-2", + handlerId: "security-scan", + state: "ready", + }, ], blocking: [], - report: { reportId: "report-1" }, + handlerProgress: { + "pp-readout": { status: "done", version: 1 }, + }, + report: { reportId: "report-current" }, }; -const history = { - runs: [ - { - _id: "submission-run-2", - attemptNumber: 2, - requests: [{ requestId: "request-1", runId: "run-1" }], - report: { reportId: "report-1" }, - createdAt: "2026-01-01T00:00:00.000Z", - }, +const currentAttempt: SubmissionRunAttempt = { + _id: "submission-run-2", + attemptNumber: 2, + vectorFingerprint: "vector-current", + requests: [ + { requestId: "request-1", runId: "run-current-1" }, + { requestId: "request-2", runId: "run-current-2" }, ], + dependencyProjection: submission.dependencyProjection, + blocking: [], + handlerStatus: submission.handlerProgress, + report: { reportId: "report-current" }, + createdAt: "2026-01-02T00:00:00.000Z", }; -const report: Report = { - _id: "report-1", - id: "report-1", - submissionId: "submission-1", - submissionRunId: "submission-run-2", - status: "completed", - content: "# Generated report", - structuredArtifact: { - schemaId: "scope.submission-readout", - schemaVersion: "1.0", - contentType: "application/json", - url: "reports/report-1/structured.json", +const historicalAttempt: SubmissionRunAttempt = { + _id: "submission-run-1", + attemptNumber: 1, + vectorFingerprint: "vector-historical", + requests: [ + { requestId: "request-1", runId: "run-historical-1" }, + { requestId: "request-2", runId: "run-historical-2" }, + ], + dependencyProjection: [ + { + requestId: "request-1", + runId: "run-historical-1", + handlerId: "pp-observations", + state: "ready", + }, + { + requestId: "request-2", + runId: "run-historical-2", + handlerId: "security-scan", + state: "ready", + }, + ], + blocking: [], + handlerStatus: { + "pp-readout": { status: "done", version: 1 }, }, + report: { reportId: "report-historical" }, createdAt: "2026-01-01T00:00:00.000Z", + supersededAt: "2026-01-02T00:00:00.000Z", + supersededBySubmissionRunId: "submission-run-2", + supersededReason: "request-vector-changed", }; +const reports: Report[] = [ + { + _id: "report-current", + id: "report-current", + submissionId: "submission-1", + submissionRunId: "submission-run-2", + status: "completed", + content: "# Current generated report", + reporter: { + id: "pp-readout", + name: "Submission Readout", + gitHash: "abc", + model: "gpt-5.4", + agentId: "copilot", + agentVersion: "1.0.0", + }, + structuredArtifact: { + schemaId: "scope.submission-readout", + schemaVersion: "1.0", + contentType: "application/json", + url: "reports/report-current/structured.json", + }, + createdAt: "2026-01-02T00:00:00.000Z", + }, + { + _id: "report-historical", + id: "report-historical", + submissionId: "submission-1", + submissionRunId: "submission-run-1", + status: "completed", + content: "# Historical generated report", + createdAt: "2026-01-01T00:00:00.000Z", + }, +]; + vi.mock("@/lib/api", () => ({ api: { getSubmission: vi.fn(async () => submission), - getSubmissionRuns: vi.fn(async () => history), - getReport: vi.fn(async () => report), + getSubmissionRuns: vi.fn(async () => ({ + runs: [currentAttempt, historicalAttempt], + })), + getSubmissionReports: vi.fn(async () => reports), retrySubmission: vi.fn(), - reportStructuredUrl: vi.fn(() => "/api/v1/reports/report-1/structured"), + reportStructuredUrl: vi.fn( + (id: string) => `/api/v1/reports/${id}/structured`, + ), + submissionRunLogsUrl: vi.fn( + (_id: string, attemptId: string) => + `/api/v1/submissions/submission-1/runs/${attemptId}/logs`, + ), }, })); +vi.mock("@/hooks/use-log-stream", () => ({ + useLogStream: vi.fn(() => ({ + logs: [], + isConnected: false, + isDone: true, + error: null, + clear: vi.fn(), + })), +})); + import { SubmissionDetail } from "./SubmissionDetail"; +import { api } from "@/lib/api"; afterEach(() => { cleanup(); vi.clearAllMocks(); }); -describe("SubmissionDetail report terminology", () => { - it("uses generic report language for the current result and history", async () => { - const queryClient = new QueryClient({ - defaultOptions: { queries: { retry: false } }, - }); - render( - - - - } /> - - - , +function renderDetail(initialEntry = "/submissions/submission-1") { + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); + return render( + + + + } /> + Run detail destination
} /> + + + , + ); +} + +describe("SubmissionDetail", () => { + it("defaults to Reports and supports multiple current and historical reports", async () => { + renderDetail(); + + expect( + await screen.findByRole("heading", { name: "Current generated report" }), + ).toBeDefined(); + expect( + screen.getByRole("tab", { name: "Reports (2)" }).getAttribute("data-state"), + ).toBe("active"); + expect(screen.getAllByText("Current").length).toBeGreaterThan(0); + expect(screen.getByText("Historical")).toBeDefined(); + expect( + screen.getByRole("link", { name: "Structured JSON" }).getAttribute("href"), + ).toBe("/api/v1/reports/report-current/structured"); + + const historicalReportButton = screen + .getAllByRole("button") + .find( + (button) => + button.textContent?.includes("Historical") && + button.textContent?.includes("completed"), + ); + expect(historicalReportButton).toBeDefined(); + await userEvent.click(historicalReportButton!); + expect( + await screen.findByRole("heading", { + name: "Historical generated report", + }), + ).toBeDefined(); + expect( + screen.getByText(/Viewing attempt 1 \(historical and read-only\)/), + ).toBeDefined(); + }); + + it("renders the immutable member vector with the shared table and run navigation", async () => { + renderDetail(); + await screen.findByRole("heading", { name: "Current generated report" }); + + await userEvent.click(screen.getByRole("tab", { name: "Runs (2)" })); + + expect( + screen.getByRole("heading", { name: "Immutable member run vector" }), + ).toBeDefined(); + expect(screen.getByRole("columnheader", { name: "Request" })).toBeDefined(); + expect( + screen.getByRole("columnheader", { name: "Exact run attempt" }), + ).toBeDefined(); + expect( + screen.getByRole("columnheader", { name: "Dependencies" }), + ).toBeDefined(); + const requestLink = screen + .getAllByRole("link") + .find((link) => link.getAttribute("href")?.includes("run-current-1")); + expect(requestLink).toBeDefined(); + expect(requestLink!.getAttribute("href")).toContain( + "runId=run-current-1", + ); + expect(requestLink!.getAttribute("href")).toContain( + "fromSubmission=submission-1", + ); + }); + + it("switches attempts through the picker and scopes Processing to history", async () => { + renderDetail("/submissions/submission-1?tab=processing"); + await screen.findByText("Report ready"); + + const picker = screen.getByRole("combobox", { name: "Submission attempt" }); + await userEvent.click(picker); + const listbox = await screen.findByRole("listbox"); + await userEvent.click( + within(listbox).getByText(/Attempt 1 · Superseded/), ); - expect(await screen.findByText("Current report")).toBeDefined(); - expect(screen.getByText("Report generation")).toBeDefined(); - expect(screen.getByText("Report ready")).toBeDefined(); - expect(screen.getByRole("button", { name: "Retry report generation" })).toBeDefined(); - expect(await screen.findByRole("link", { name: "View report" })).toBeDefined(); - expect(await screen.findByRole("heading", { name: "Generated report" })).toBeDefined(); + expect( + await screen.findByText(/Viewing attempt 1 \(historical and read-only\)/), + ).toBeDefined(); + expect(screen.getByText("Dependencies ready")).toBeDefined(); + expect(screen.getByText("Observations 1/1")).toBeDefined(); + expect(screen.getByText("Security Scan 1/1")).toBeDefined(); + expect( + screen.getByRole("heading", { name: "Attempt processing logs" }), + ).toBeDefined(); + }); + + it("shows current attempt progress when only historical reports exist", async () => { + vi.mocked(api.getSubmission).mockResolvedValueOnce({ + ...submission, + readiness: "processing", + report: undefined, + handlerProgress: { + "pp-readout": { status: "processing", version: 1 }, + }, + }); + vi.mocked(api.getSubmissionRuns).mockResolvedValueOnce({ + runs: [ + { + ...currentAttempt, + report: undefined, + handlerStatus: { + "pp-readout": { status: "processing", version: 1 }, + }, + }, + historicalAttempt, + ], + }); + vi.mocked(api.getSubmissionReports).mockResolvedValueOnce([ + reports[1]!, + ]); + + renderDetail(); + + expect( + await screen.findByRole("tab", { name: "Reports (1)" }), + ).toBeDefined(); + expect(screen.getByText("Report not available")).toBeDefined(); + expect( + screen.getByText("A report is being generated for this attempt."), + ).toBeDefined(); + expect(screen.getByText("Historical")).toBeDefined(); }); }); From d1a195cd45d855143e65a1be427016fcf06fe0d6 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Tue, 13 Oct 2026 23:50:41 +0200 Subject: [PATCH 12/18] docs: describe attempt-scoped submission reports Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 1cbbd5eb-61c6-4e6a-bb38-c377be5b2163 --- docs/architecture/post-processing.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/architecture/post-processing.md b/docs/architecture/post-processing.md index c569a81d6..ff90331dc 100644 --- a/docs/architecture/post-processing.md +++ b/docs/architecture/post-processing.md @@ -144,7 +144,11 @@ Gate criteria are always `subject: "iteration"` (the API rejects `"run"` for gat - **Runs exactly once per run:** the scheduler dispatches `pp-taxonomy` a single time (it never sets `message.iteration`), so the run-subject branch needs no idempotency guard. 5. Stamps `run.handlerStatus["pp-taxonomy"] = { status: "done", version: N }` and notifies the scheduler. -Dimension grouping is **derived at read time** by joining each result's `criterionId` → criterion `taxonomyElementId` (the three R&A Readout dimensions; unset = unclassified). It never feeds back to the agent and never affects pass/fail. (#1053 later extends this worker with a richer scorecard artifact.) +Dimension grouping is **derived at read time** by joining each result's +`criterionId` → criterion `taxonomyElementId` (the three R&A Readout +dimensions; unset = unclassified). It never feeds back to the agent and never +affects pass/fail. Submission report synthesis consumes these classifications +after every exact-vector dependency is ready. ## Handler Interface @@ -277,6 +281,13 @@ report artifact proxy. Internal handler, queue, and schema identifiers retain `readout` because they identify this specific synthesis implementation rather than the public resource. +Submission-handler logs are persisted under the exact submission attempt and +streamed through +`GET /api/v1/submissions/:id/runs/:submissionRunId/logs`. Queue processors use +the message's `submissionRunId` as the log attempt key, so superseded attempts +remain independently replayable and cannot inherit logs from a newer current +attempt. + To **add or change a handler**, edit its `handler.yaml` (bump `version`, adjust `dependsOn`, etc.) — no code or migration changes are needed for registration. ## Report Triggering on DAG Drain From cd97f8f684767dc593f2509e40a9cf6be2e5d4c2 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Wed, 14 Oct 2026 00:54:20 +0200 Subject: [PATCH 13/18] fix: scope submission reports by type Replace report-version history with attempt-scoped horizontal report-type tabs and deterministic newest-per-type selection. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 1cbbd5eb-61c6-4e6a-bb38-c377be5b2163 --- .../components/SubmissionReports.stories.tsx | 113 +++++++++++++++++ .../src/components/ui/stories.play.test.tsx | 3 +- .../src/pages/SubmissionDetail.test.tsx | 119 ++++++++++++++---- 3 files changed, 213 insertions(+), 22 deletions(-) create mode 100644 apps/portal/src/components/SubmissionReports.stories.tsx diff --git a/apps/portal/src/components/SubmissionReports.stories.tsx b/apps/portal/src/components/SubmissionReports.stories.tsx new file mode 100644 index 000000000..d3762acbf --- /dev/null +++ b/apps/portal/src/components/SubmissionReports.stories.tsx @@ -0,0 +1,113 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { useState } from "react"; +import type { Meta, StoryObj } from "@storybook/react-vite"; +import { expect, userEvent, within } from "storybook/test"; +import { + SubmissionReports, + reportId, +} from "@/pages/SubmissionDetail"; +import type { + Report, + SubmissionRunAttempt, +} from "@/types"; + +const attempt: SubmissionRunAttempt = { + _id: "submission-run-current", + attemptNumber: 2, + vectorFingerprint: "vector-current", + requests: [{ requestId: "request-1", runId: "run-1" }], + dependencyProjection: [], + blocking: [], + handlerStatus: { + "pp-readout": { status: "done", version: 1 }, + }, + report: { reportId: "report-readout" }, + createdAt: "2026-01-02T00:00:00.000Z", +}; + +const readoutReport: Report = { + _id: "report-readout", + id: "report-readout", + submissionId: "submission-1", + submissionRunId: attempt._id, + status: "completed", + content: "# Submission report\n\nFive runs completed successfully.", + reporter: { + id: "pp-readout", + name: "Submission Readout", + gitHash: "abc", + model: "gpt-5.4", + agentId: "copilot", + agentVersion: "1.0.0", + }, + structuredArtifact: { + schemaId: "scope.submission-readout", + schemaVersion: "1.0", + contentType: "application/json", + url: "reports/report-readout/structured.json", + }, + createdAt: "2026-01-02T00:00:00.000Z", +}; + +const templateReport: Report = { + _id: "report-executive", + id: "report-executive", + submissionId: "submission-1", + submissionRunId: attempt._id, + templateId: "executive-summary", + status: "completed", + content: "# Executive summary\n\nA concise template-based view.", + createdAt: "2026-01-02T01:00:00.000Z", +}; + +function ReportTypesStory({ reports }: { reports: Report[] }) { + const [selected, setSelected] = useState(reports[0]); + return ( + undefined} + /> + ); +} + +const meta = { + title: "Submissions/Submission Reports", + component: ReportTypesStory, + parameters: { layout: "padded" }, +} satisfies Meta; + +export default meta; +type Story = StoryObj; + +export const OneReportType: Story = { + args: { reports: [readoutReport] }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await expect( + canvas.getByRole("tab", { name: "Submission Readout" }), + ).toBeVisible(); + await expect( + canvas.getByRole("heading", { name: "Submission report" }), + ).toBeVisible(); + }, +}; + +export const MultipleReportTypes: Story = { + args: { reports: [readoutReport, templateReport] }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await userEvent.click( + canvas.getByRole("tab", { name: "Executive Summary" }), + ); + await expect( + canvas.getByRole("heading", { name: "Executive summary" }), + ).toBeVisible(); + }, +}; diff --git a/apps/portal/src/components/ui/stories.play.test.tsx b/apps/portal/src/components/ui/stories.play.test.tsx index 7ca2290cc..5137d9b6d 100644 --- a/apps/portal/src/components/ui/stories.play.test.tsx +++ b/apps/portal/src/components/ui/stories.play.test.tsx @@ -26,11 +26,12 @@ import * as textarea from "./textarea.stories"; import * as tooltip from "./tooltip.stories"; import * as sonner from "./sonner.stories"; import * as submissionLink from "../SubmissionLink.stories"; +import * as submissionReports from "../SubmissionReports.stories"; const modules: Record> = { alert, alertDialog, checkbox, dialog, dropdownMenu, label, popover, scrollArea, select, separator, sheet, skeleton, switch: sw, table, tabs, textarea, - tooltip, sonner, submissionLink, + tooltip, sonner, submissionLink, submissionReports, }; afterEach(() => cleanup()); diff --git a/apps/portal/src/pages/SubmissionDetail.test.tsx b/apps/portal/src/pages/SubmissionDetail.test.tsx index 9481a6ff2..e2c825ccf 100644 --- a/apps/portal/src/pages/SubmissionDetail.test.tsx +++ b/apps/portal/src/pages/SubmissionDetail.test.tsx @@ -134,6 +134,33 @@ const reports: Report[] = [ }, createdAt: "2026-01-02T00:00:00.000Z", }, + { + _id: "report-current-older-duplicate", + id: "report-current-older-duplicate", + submissionId: "submission-1", + submissionRunId: "submission-run-2", + status: "completed", + content: "# Stale duplicate report", + reporter: { + id: "pp-readout", + name: "Submission Readout", + gitHash: "abc", + model: "gpt-5.4", + agentId: "copilot", + agentVersion: "1.0.0", + }, + createdAt: "2026-01-01T12:00:00.000Z", + }, + { + _id: "report-template-current", + id: "report-template-current", + submissionId: "submission-1", + submissionRunId: "submission-run-2", + templateId: "executive-summary", + status: "completed", + content: "# Executive summary report", + createdAt: "2026-01-02T01:00:00.000Z", + }, { _id: "report-historical", id: "report-historical", @@ -141,6 +168,14 @@ const reports: Report[] = [ submissionRunId: "submission-run-1", status: "completed", content: "# Historical generated report", + reporter: { + id: "pp-readout", + name: "Submission Readout", + gitHash: "abc", + model: "gpt-5.4", + agentId: "copilot", + agentVersion: "1.0.0", + }, createdAt: "2026-01-01T00:00:00.000Z", }, ]; @@ -198,7 +233,7 @@ function renderDetail(initialEntry = "/submissions/submission-1") { } describe("SubmissionDetail", () => { - it("defaults to Reports and supports multiple current and historical reports", async () => { + it("defaults to Reports and shows attempt-scoped report types", async () => { renderDetail(); expect( @@ -207,29 +242,35 @@ describe("SubmissionDetail", () => { expect( screen.getByRole("tab", { name: "Reports (2)" }).getAttribute("data-state"), ).toBe("active"); - expect(screen.getAllByText("Current").length).toBeGreaterThan(0); - expect(screen.getByText("Historical")).toBeDefined(); + expect( + screen.getByRole("tab", { name: "Submission Readout" }), + ).toBeDefined(); + expect( + screen.getByRole("tab", { name: "Executive Summary" }), + ).toBeDefined(); + expect(screen.getByText("Current")).toBeDefined(); expect( screen.getByRole("link", { name: "Structured JSON" }).getAttribute("href"), ).toBe("/api/v1/reports/report-current/structured"); - const historicalReportButton = screen - .getAllByRole("button") - .find( - (button) => - button.textContent?.includes("Historical") && - button.textContent?.includes("completed"), - ); - expect(historicalReportButton).toBeDefined(); - await userEvent.click(historicalReportButton!); + await userEvent.click( + screen.getByRole("tab", { name: "Executive Summary" }), + ); expect( - await screen.findByRole("heading", { - name: "Historical generated report", - }), + await screen.findByRole("heading", { name: "Executive summary report" }), ).toBeDefined(); + expect(screen.getByText("This attempt")).toBeDefined(); + }); + + it("selects the newest report when one attempt has duplicate report types", async () => { + renderDetail(); + expect( - screen.getByText(/Viewing attempt 1 \(historical and read-only\)/), + await screen.findByRole("heading", { name: "Current generated report" }), ).toBeDefined(); + expect( + screen.getByRole("link", { name: "Structured JSON" }).getAttribute("href"), + ).toBe("/api/v1/reports/report-current/structured"); }); it("renders the immutable member vector with the shared table and run navigation", async () => { @@ -282,6 +323,45 @@ describe("SubmissionDetail", () => { ).toBeDefined(); }); + it("scopes report types to the selected historical attempt", async () => { + renderDetail(); + await screen.findByRole("heading", { name: "Current generated report" }); + + const picker = screen.getByRole("combobox", { name: "Submission attempt" }); + await userEvent.click(picker); + const listbox = await screen.findByRole("listbox"); + await userEvent.click(within(listbox).getByText(/Attempt 1 · Superseded/)); + + expect( + await screen.findByRole("heading", { + name: "Historical generated report", + }), + ).toBeDefined(); + expect( + screen.getByRole("tab", { name: "Reports (1)" }), + ).toBeDefined(); + expect( + screen.getByRole("tab", { name: "Submission Readout" }), + ).toBeDefined(); + expect(screen.getByText("Historical")).toBeDefined(); + }); + + it("renders one report-type tab for a single template-less report", async () => { + vi.mocked(api.getSubmissionReports).mockResolvedValueOnce([reports[0]!]); + + renderDetail(); + + expect( + await screen.findByRole("heading", { name: "Current generated report" }), + ).toBeDefined(); + expect( + screen.getByRole("tab", { name: "Reports (1)" }), + ).toBeDefined(); + expect( + screen.getByRole("tab", { name: "Submission Readout" }), + ).toBeDefined(); + }); + it("shows current attempt progress when only historical reports exist", async () => { vi.mocked(api.getSubmission).mockResolvedValueOnce({ ...submission, @@ -303,19 +383,16 @@ describe("SubmissionDetail", () => { historicalAttempt, ], }); - vi.mocked(api.getSubmissionReports).mockResolvedValueOnce([ - reports[1]!, - ]); + vi.mocked(api.getSubmissionReports).mockResolvedValueOnce([reports[3]!]); renderDetail(); expect( - await screen.findByRole("tab", { name: "Reports (1)" }), + await screen.findByRole("tab", { name: "Reports" }), ).toBeDefined(); expect(screen.getByText("Report not available")).toBeDefined(); expect( screen.getByText("A report is being generated for this attempt."), ).toBeDefined(); - expect(screen.getByText("Historical")).toBeDefined(); }); }); From d373d4155d0ecdfafa4b926d4a844b5a0e30ff38 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Fri, 16 Oct 2026 05:51:22 -0700 Subject: [PATCH 14/18] feat: adapt readout stack to experiments Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4b4a63b3-29db-4d12-888f-051592682640 --- ENV_VARIABLES.md | 16 +- apps/api/src/experiment-coordinator.test.ts | 156 +++++++ apps/api/src/experiment-openapi.test.ts | 9 + apps/api/src/routes/experiments.test.ts | 114 ++++- apps/api/src/routes/experiments.ts | 285 ++++++++++++- .../api/src/routes/reports-structured.test.ts | 27 +- apps/api/src/routes/reports.ts | 27 +- apps/cli/src/commands/experiment.test.ts | 57 +++ apps/cli/src/commands/experiment.ts | 147 ++++++- apps/cli/src/commands/report.test.ts | 26 +- apps/cli/src/commands/report.ts | 38 +- apps/cli/src/commands/submission.test.ts | 78 ---- .../ExperimentReportCard.stories.tsx | 77 ++++ .../src/components/ExperimentReportCard.tsx | 97 +++++ .../src/components/SubmissionLink.stories.tsx | 32 -- .../src/components/SubmissionLink.test.tsx | 24 -- apps/portal/src/components/SubmissionLink.tsx | 33 -- .../components/SubmissionReports.stories.tsx | 113 ----- .../src/components/ui/stories.play.test.tsx | 5 +- apps/portal/src/lib/api.ts | 23 + apps/portal/src/lib/experiments.ts | 14 +- .../src/pages/ExperimentDetail.test.tsx | 172 ++++++++ apps/portal/src/pages/ExperimentDetail.tsx | 101 ++++- apps/portal/src/pages/InsightDetail.tsx | 6 +- apps/portal/src/pages/ReportDetail.tsx | 27 +- .../src/pages/ReportPreviewPanel.test.tsx | 12 +- apps/portal/src/pages/ReportPreviewPanel.tsx | 10 +- apps/portal/src/pages/ReportsList.tsx | 12 +- apps/portal/src/pages/RunDetail.tsx | 18 +- .../src/pages/SubmissionDetail.test.tsx | 398 ------------------ apps/portal/src/types.ts | 4 +- apps/workers/readout/handler.yaml | 4 +- apps/workers/readout/src/index.ts | 2 +- apps/workers/readout/src/prompt.ts | 8 +- .../readout/src/readout-builder.test.ts | 36 +- apps/workers/readout/src/readout-builder.ts | 36 +- apps/workers/readout/src/readout-generator.ts | 12 +- .../src/readout-queue-processor.test.ts | 301 ++++++------- .../readout/src/readout-queue-processor.ts | 281 ++++++++----- docker-compose.yml | 2 +- docs/architecture/experiments.md | 84 +++- docs/architecture/post-processing.md | 50 ++- docs/architecture/prompt-evaluations.md | 2 +- .../static-prompts/adapter/adapter.test.ts | 19 +- .../static-prompts/adapter/targets/reports.ts | 8 +- .../static-prompts/datasets/manifest.json | 2 +- .../static-prompts/datasets/v1/readouts.jsonl | 2 +- .../static-prompts/evaluation-manifest.yaml | 2 +- .../scripts/dataset-adapter-contract.test.ts | 4 +- .../scripts/validate-dataset.ts | 6 +- .../src/experiments/experiment-store.test.ts | 23 + .../src/experiments/experiment-store.ts | 26 ++ packages/shared/src/experiments/schemas.ts | 8 + packages/shared/src/experiments/types.ts | 7 + .../shared/src/queue/base-queue-processor.ts | 16 +- packages/shared/src/schemas/index.ts | 1 - packages/shared/src/schemas/readout.test.ts | 20 +- packages/shared/src/schemas/readout.ts | 18 +- packages/shared/src/schemas/report.ts | 4 +- packages/shared/src/types/types.ts | 4 +- 60 files changed, 1944 insertions(+), 1202 deletions(-) delete mode 100644 apps/cli/src/commands/submission.test.ts create mode 100644 apps/portal/src/components/ExperimentReportCard.stories.tsx create mode 100644 apps/portal/src/components/ExperimentReportCard.tsx delete mode 100644 apps/portal/src/components/SubmissionLink.stories.tsx delete mode 100644 apps/portal/src/components/SubmissionLink.test.tsx delete mode 100644 apps/portal/src/components/SubmissionLink.tsx delete mode 100644 apps/portal/src/components/SubmissionReports.stories.tsx create mode 100644 apps/portal/src/pages/ExperimentDetail.test.tsx delete mode 100644 apps/portal/src/pages/SubmissionDetail.test.tsx diff --git a/ENV_VARIABLES.md b/ENV_VARIABLES.md index 444397731..c79b005f5 100644 --- a/ENV_VARIABLES.md +++ b/ENV_VARIABLES.md @@ -889,23 +889,23 @@ Timeout for the Copilot SDK session used by the report-generator worker. If the Git commit hash embedded in reporter metadata. Automatically set during CI/CD builds. Used to track which version of the report-generator produced a given report. -## Submission Readout Handler Configuration +## Experiment Readout Handler Configuration ### AZURE_STORAGE_QUEUE_READOUT **Default:** `pp-readout-queue` **Type:** string -Azure Storage Queue name for submission readout jobs. The scheduler enqueues -`{ type: "readout", submissionId, submissionRunId }` only after -`pp-observations` is done for every exact request/run pair in the current -submission attempt. +Azure Storage Queue name for Experiment readout jobs. The coordinator enqueues +an exact execution message containing `projectId`, `experimentId`, +`experimentExecutionId`, handler identity/version, claim token, and execution +version only after `pp-observations` is done for every exact request/run pair. ### READOUT_MODEL **Default:** `gpt-5.4` **Type:** string The Copilot SDK model used by the dedicated `pp-readout` worker to synthesize a -schema-validated structured submission readout. Markdown is derived from the +schema-validated structured Experiment readout. Markdown is derived from the validated structured object. ### READOUT_SESSION_TIMEOUT_MS @@ -913,8 +913,8 @@ validated structured object. **Type:** integer (milliseconds) Timeout for one readout-generation attempt. Exhausted generation or schema -validation marks the exact current submission attempt's `pp-readout` handler as -failed; historical and superseded attempts remain immutable. +validation marks the exact current Experiment execution's `pp-readout` handler +as failed; historical and superseded executions remain immutable. ## Scheduler Configuration diff --git a/apps/api/src/experiment-coordinator.test.ts b/apps/api/src/experiment-coordinator.test.ts index 30a0afd09..5da0c8a9e 100644 --- a/apps/api/src/experiment-coordinator.test.ts +++ b/apps/api/src/experiment-coordinator.test.ts @@ -165,13 +165,26 @@ describe("ExperimentCoordinator", () => { _id: "run-1", attemptNumber: 1, status: "done", + outcome: "failed", logsUrl: "https://logs", + observationResults: [{ + criterionId: "criterion-1", + evaluated: true, + passed: false, + feedback: "Valid negative evidence", + }], handlerStatus: { "pp-observations": { status: "done", version: 3, updatedAt: now, }, + "unrelated-handler": { + status: "failed", + version: 1, + updatedAt: now, + error: "Unrelated failure", + }, }, }, }, @@ -318,6 +331,149 @@ describe("ExperimentCoordinator", () => { vi.mocked(ExperimentStore.prototype.casUpdateRequestStates).mock .calls[0]?.[0].requests, ).toHaveLength(40); + it.each([ + { + name: "failed observations", + services: [ + { + _id: "pp-observations", + type: "post-process-handler" as const, + version: 1, + queue: "queue-pp-observations", + selector: "observations", + autoBackfill: false, + dependsOn: [], + }, + ], + handlerStatus: { + "pp-observations": { + status: "failed" as const, + version: 1, + updatedAt: now, + error: "Observation processing failed", + }, + }, + expectedState: "failed", + expectedReason: "dependency-failed", + }, + { + name: "dependency-blocked observations", + services: [ + { + _id: "observation-prerequisite", + type: "post-process-handler" as const, + version: 1, + queue: "queue-observation-prerequisite", + selector: "observation-prerequisite", + autoBackfill: false, + dependsOn: [], + }, + { + _id: "pp-observations", + type: "post-process-handler" as const, + version: 1, + queue: "queue-pp-observations", + selector: "observations", + autoBackfill: false, + dependsOn: ["observation-prerequisite"], + }, + ], + handlerStatus: { + "observation-prerequisite": { + status: "failed" as const, + version: 1, + updatedAt: now, + error: "Prerequisite failed", + }, + }, + expectedState: "blocked", + expectedReason: "dependency-blocked", + }, + ])("does not dispatch when $name", async ({ + services, + handlerStatus, + expectedState, + expectedReason, + }) => { + const initialRequest: ExperimentRequestState = { + requestId: "request-1", + runId: "run-1", + dependencies: [], + blockers: [], + }; + const projectedRequest: ExperimentRequestState = { + ...initialRequest, + dependencies: [{ + handlerId: "pp-observations", + state: expectedState as "failed" | "blocked", + }], + blockers: [{ + requestId: "request-1", + runId: "run-1", + handlerId: "pp-observations", + reason: expectedReason as "dependency-failed" | "dependency-blocked", + ...(expectedReason === "dependency-failed" + ? { error: "Observation processing failed" } + : {}), + }], + }; + const initial = experiment([initialRequest]); + const projected = experiment([projectedRequest], { + version: 2, + blocking: projectedRequest.blockers, + }); + vi.spyOn(ExperimentStore.prototype, "get").mockResolvedValue(initial); + vi.spyOn(FixedPayloadStore.prototype, "readRequests").mockResolvedValue([ + initialRequest, + ]); + vi.spyOn( + ExperimentStore.prototype, + "casUpdateRequestStates", + ).mockResolvedValue({ + status: "applied", + experiment: projected, + requests: [projectedRequest], + }); + const enqueue = vi.fn().mockResolvedValue(undefined); + const coordinator = new ExperimentCoordinator( + collections({ + requests: [{ + _id: "request-1", + projectId: "project-1", + scenario: { task: "task", criteria: [] }, + workerType: "worker", + createdAt: now, + taskPromptId: "prompt-1", + priority: 0, + run: { + _id: "run-1", + attemptNumber: 1, + status: "done", + logsUrl: "https://logs", + handlerStatus, + }, + }], + services: [ + ...services, + { + _id: "pp-readout", + type: "experiment-handler", + version: 1, + queue: "queue-pp-readout", + selector: "readout", + autoBackfill: false, + dependsOn: ["pp-observations"], + dependencyMode: "all-runs", + }, + ], + }), + enqueue, + ); + + const result = await coordinator.reconcile("project-1", "experiment-1"); + + expect(result?.dispatched).toBe(false); + expect(enqueue).not.toHaveBeenCalled(); }); it("re-enqueues a dropped notice without changing its exact identity", async () => { diff --git a/apps/api/src/experiment-openapi.test.ts b/apps/api/src/experiment-openapi.test.ts index 7a6a1df3d..24bb7cab0 100644 --- a/apps/api/src/experiment-openapi.test.ts +++ b/apps/api/src/experiment-openapi.test.ts @@ -50,6 +50,15 @@ describe("Experiment OpenAPI contract", () => { expect(document.paths).toHaveProperty( "/api/v1/experiments/{id}/executions/{experimentExecutionId}", ); + expect(document.paths).toHaveProperty( + "/api/v1/experiments/{id}/executions/{experimentExecutionId}/reports", + ); + expect(document.paths).toHaveProperty( + "/api/v1/experiments/{id}/executions/{experimentExecutionId}/reports/{reportId}/structured", + ); + expect(document.paths).toHaveProperty( + "/api/v1/experiments/{id}/executions/{experimentExecutionId}/logs", + ); expect(document.paths).toHaveProperty("/api/v1/experiments/{id}/retry"); expect( Object.keys(document.paths ?? {}).filter((path) => diff --git a/apps/api/src/routes/experiments.test.ts b/apps/api/src/routes/experiments.test.ts index d778f48c2..7579f648a 100644 --- a/apps/api/src/routes/experiments.test.ts +++ b/apps/api/src/routes/experiments.test.ts @@ -11,7 +11,7 @@ import { import { describe, expect, it, vi } from "vitest"; import { z } from "zod"; import type { RouteContext } from "../route-context.js"; -import type { ExperimentDocument } from "shared"; +import { ExperimentStore, type ExperimentDocument } from "shared"; import { registerExperimentsRoutes } from "./experiments.js"; extendZodWithOpenApi(z); @@ -254,6 +254,9 @@ describe("Experiment API contract", () => { "/api/v1/experiments/{id}", "/api/v1/experiments/{id}/executions", "/api/v1/experiments/{id}/executions/{experimentExecutionId}", + "/api/v1/experiments/{id}/executions/{experimentExecutionId}/logs", + "/api/v1/experiments/{id}/executions/{experimentExecutionId}/reports", + "/api/v1/experiments/{id}/executions/{experimentExecutionId}/reports/{reportId}/structured", "/api/v1/experiments/{id}/retry", ]); expect(document.paths).not.toHaveProperty("/api/v1/submissions"); @@ -267,6 +270,7 @@ describe("Experiment API contract", () => { readiness: expect.any(Object), blockers: expect.any(Object), progress: expect.any(Object), + report: expect.any(Object), supersededByExperimentExecutionId: { type: "string" }, }, }); @@ -274,4 +278,112 @@ describe("Experiment API contract", () => { /claimToken|stateVersion|vectorFingerprint|storageProfile/, ); }); + + it("lists reports for one exact current or archived execution", async () => { + const execution = { + experimentExecutionId: "execution-2", + attemptNumber: 2, + version: 4, + requests: { + storage: "inline" as const, + entryCount: 0, + payloadFingerprint: "payload", + payloadVersion: 1, + entries: [], + }, + vectorFingerprint: "vector", + handlerSummary: {}, + blocking: [], + createdAt: new Date(), + updatedAt: new Date(), + }; + const getExecution = vi + .spyOn(ExperimentStore.prototype, "getExecution") + .mockResolvedValue(execution); + const toArray = vi.fn().mockResolvedValue([{ + _id: "report-1", + projectId: "project-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-2", + status: "completed", + createdAt: new Date(), + }]); + const find = vi.fn(() => ({ + sort: vi.fn(() => ({ toArray })), + })); + const { app } = harness({ + experimentCollection: {}, + experimentExecutionsCollection: {}, + experimentChunksCollection: {}, + reportCollection: { find }, + } as unknown as Partial); + + const response = await request(app).get( + "/api/v1/experiments/experiment-1/executions/execution-2/reports?projectId=project-1", + ); + + expect(response.status).toBe(200); + expect(getExecution).toHaveBeenCalledWith({ + projectId: "project-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-2", + }); + expect(find).toHaveBeenCalledWith({ + projectId: "project-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-2", + }); + expect(response.body).toEqual([ + expect.objectContaining({ + id: "report-1", + experimentExecutionId: "execution-2", + }), + ]); + }); + + it("retrieves structured JSON only through the exact execution identity", async () => { + const findOne = vi.fn().mockResolvedValue({ + _id: "report-1", + projectId: "project-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-2", + status: "completed", + structuredArtifact: { + schemaId: "scope.experiment-readout", + schemaVersion: "1.0", + contentType: "application/json", + url: "https://storage.invalid/readout.json", + }, + createdAt: new Date(), + }); + const downloadJson = vi.fn().mockResolvedValue({ + meta: { + experimentId: "experiment-1", + experimentExecutionId: "execution-2", + }, + }); + const { app } = harness({ + reportCollection: { findOne }, + blobStorage: { downloadJson }, + } as unknown as Partial); + + const response = await request(app).get( + "/api/v1/experiments/experiment-1/executions/execution-2/reports/report-1/structured?projectId=project-1", + ); + + expect(response.status).toBe(200); + expect(findOne).toHaveBeenCalledWith({ + _id: "report-1", + projectId: "project-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-2", + }); + expect(downloadJson).toHaveBeenCalledWith( + "https://storage.invalid/readout.json", + ); + expect(response.body.meta).toEqual({ + experimentId: "experiment-1", + experimentExecutionId: "execution-2", + }); + }); }); diff --git a/apps/api/src/routes/experiments.ts b/apps/api/src/routes/experiments.ts index 4739afe58..88b29539f 100644 --- a/apps/api/src/routes/experiments.ts +++ b/apps/api/src/routes/experiments.ts @@ -4,10 +4,13 @@ import { v4 as uuidv4 } from "uuid"; import { z } from "zod"; import type { Document } from "mongodb"; +import type { Response } from "express"; import { CreateRequestInputSchema, ExperimentStore, fingerprintExperimentPayload, + ReportResponseSchema, + withRetry, type ExperimentDocument, type ExperimentExecution, type ExperimentExecutionHistoryDocument, @@ -20,6 +23,8 @@ import { ProjectIdQuerySchema, getQueryProjectId, } from "../utils/project-scope.js"; +import { subscribeClient, unsubscribeClient } from "../utils/sse.js"; +import type { SSEClient } from "../utils/sse.js"; const CursorSchema = z.string().min(1).optional(); const PageQuerySchema = ProjectIdQuerySchema.extend({ @@ -82,6 +87,12 @@ const ReadinessSchema = z.enum([ "failed", ]); +const ExperimentReportReferenceSchema = z.object({ + reportId: z.string(), + url: z.string(), + updatedAt: z.coerce.date(), +}); + const ExperimentResponseSchema = z .object({ experimentId: z.string(), @@ -115,13 +126,7 @@ const ExperimentResponseSchema = z finishedAt: z.coerce.date().optional(), discardedAt: z.coerce.date().optional(), discardReason: z.string().optional(), - report: z - .object({ - reportId: z.string().optional(), - url: z.string().optional(), - updatedAt: z.coerce.date().optional(), - }) - .optional(), + report: ExperimentReportReferenceSchema.optional(), }), createdAt: z.coerce.date(), updatedAt: z.coerce.date(), @@ -157,6 +162,7 @@ const ExecutionResponseSchema = z supersededAt: z.coerce.date().optional(), supersededByExperimentExecutionId: z.string().optional(), supersededReason: z.enum(["rollover", "retry"]).optional(), + report: ExperimentReportReferenceSchema.optional(), }) .openapi("ExperimentExecutionResponse"); @@ -258,7 +264,154 @@ function executionResponse(execution: ExperimentExecution) { ...(execution.supersededReason ? { supersededReason: execution.supersededReason } : {}), + ...(execution.report ? { report: execution.report } : {}), + }; +} + +async function streamExperimentExecutionLogs( + ctx: RouteContext, + req: import("express").Request, + res: Response, + experiment: ExperimentDocument, + execution: ExperimentExecution, + fromStart: boolean, +): Promise { + const isCurrent = + experiment.execution.experimentExecutionId === + execution.experimentExecutionId; + res.setHeader("Content-Type", "text/event-stream"); + res.setHeader("Cache-Control", "no-cache"); + res.setHeader("Connection", "keep-alive"); + res.setHeader("X-Accel-Buffering", "no"); + res.flushHeaders(); + + if (fromStart) { + try { + const pastLogs = await ctx.blobStorage.getLogEvents( + experiment.experimentId, + execution.experimentExecutionId, + ); + for (const log of pastLogs) { + res.write(`data: ${JSON.stringify(log)}\n\n`); + } + } catch (error) { + console.error( + `Failed to replay Experiment logs for ${experiment.experimentId}/${execution.experimentExecutionId}:`, + error, + ); + res.write( + `event: error\ndata: ${JSON.stringify({ message: "Cannot connect to log storage" })}\n\n`, + ); + res.end(); + return; + } + } + + const executionReadiness = readiness(execution); + if ( + !isCurrent || + executionReadiness === "done" || + executionReadiness === "failed" || + executionReadiness === "blocked" + ) { + res.write( + `event: done\ndata: ${JSON.stringify({ status: executionReadiness })}\n\n`, + ); + res.end(); + return; + } + + let cleaned = false; + let redisSubscribed = false; + let changeStream: ReturnType | null = + null; + const heartbeat = setInterval(() => { + if (!cleaned) res.write(`:\n\n`); + }, 30_000); + const client: SSEClient = { + res, + accepts: (event) => + event.data?.runId === execution.experimentExecutionId, + cleanup: () => { + if (cleaned) return; + cleaned = true; + clearInterval(heartbeat); + if (changeStream) { + changeStream.close().catch((error: unknown) => { + console.error("Error closing Experiment log change stream:", error); + }); + } + if (redisSubscribed) { + unsubscribeClient(experiment.experimentId, client); + } + res.end(); + }, }; + + if (process.env.REDIS_HOST) { + try { + await subscribeClient(experiment.experimentId, client); + redisSubscribed = true; + } catch (error) { + console.error( + `Redis subscription failed for Experiment ${experiment.experimentId}:`, + error, + ); + } + } + try { + changeStream = ctx.experimentCollection.watch( + [ + { + $match: { + "documentKey._id": experiment.experimentId, + operationType: "update", + }, + }, + ], + { fullDocument: "updateLookup" }, + ); + changeStream.on("change", (change: { + operationType?: string; + fullDocument?: ExperimentDocument; + }) => { + const updated = change.fullDocument; + if (change.operationType !== "update" || !updated) return; + if ( + updated.execution.experimentExecutionId !== + execution.experimentExecutionId + ) { + res.write( + `event: done\ndata: ${JSON.stringify({ status: "superseded" })}\n\n`, + ); + client.cleanup(); + return; + } + const updatedReadiness = readiness(updated.execution); + if ( + updatedReadiness === "done" || + updatedReadiness === "failed" || + updatedReadiness === "blocked" + ) { + res.write( + `event: done\ndata: ${JSON.stringify({ status: updatedReadiness })}\n\n`, + ); + client.cleanup(); + } + }); + changeStream.on("error", (error: unknown) => { + console.error( + `Experiment log change stream failed for ${experiment.experimentId}:`, + error, + ); + }); + } catch (error) { + console.error( + `Failed to create Experiment log change stream for ${experiment.experimentId}:`, + error, + ); + } + req.on("close", () => client.cleanup()); } function experimentResponse(experiment: ExperimentDocument) { @@ -618,6 +771,124 @@ export function registerExperimentsRoutes(ctx: RouteContext): void { }, }); + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/experiments/:id/executions/:experimentExecutionId/reports", + tags: ["Reports"], + summary: "List reports for one exact Experiment execution", + params: z.object({ + id: z.string().min(1), + experimentExecutionId: z.string().min(1), + }), + query: ProjectIdQuerySchema, + response: z.array(ReportResponseSchema), + errorResponses: { 404: { description: "Experiment execution not found" } }, + handler: async (req, res) => { + const projectId = getQueryProjectId(req); + const execution = await store(ctx).getExecution({ + projectId, + experimentId: req.params.id, + experimentExecutionId: req.params.experimentExecutionId, + }); + if (!execution) { + res.status(404).json({ error: "Experiment execution not found" }); + return; + } + const reports = await ctx.reportCollection + .find({ + projectId, + experimentId: req.params.id, + experimentExecutionId: req.params.experimentExecutionId, + }) + .sort({ createdAt: -1 }) + .toArray(); + res.json(reports.map((report) => ({ ...report, id: report._id }))); + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/experiments/:id/executions/:experimentExecutionId/reports/:reportId/structured", + tags: ["Reports"], + summary: "Download structured JSON for an exact Experiment report", + params: z.object({ + id: z.string().min(1), + experimentExecutionId: z.string().min(1), + reportId: z.string().min(1), + }), + query: ProjectIdQuerySchema, + response: z.any(), + rawResponse: true, + errorResponses: { + 404: { description: "Experiment report or structured artifact not found" }, + }, + handler: async (req, res) => { + const projectId = getQueryProjectId(req); + const report = await ctx.reportCollection.findOne({ + _id: req.params.reportId, + projectId, + experimentId: req.params.id, + experimentExecutionId: req.params.experimentExecutionId, + }); + if (!report?.structuredArtifact) { + res.status(404).json({ + error: "Experiment report structured artifact not found", + }); + return; + } + const structured = await withRetry(() => + ctx.blobStorage.downloadJson(report.structuredArtifact!.url) + ); + res.setHeader("Content-Type", "application/json"); + res.setHeader( + "Content-Disposition", + `attachment; filename="${req.params.reportId}.structured.json"`, + ); + res.json(structured); + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/experiments/:id/executions/:experimentExecutionId/logs", + tags: ["Experiments"], + summary: "Stream logs for one exact Experiment execution", + params: z.object({ + id: z.string().min(1), + experimentExecutionId: z.string().min(1), + }), + query: ProjectIdQuerySchema.extend({ + fromStart: z.enum(["true", "false"]).optional(), + }), + response: z.any(), + rawResponse: true, + responseDescription: "Server-sent event stream of log entries", + errorResponses: { 404: { description: "Experiment execution not found" } }, + handler: async (req, res) => { + const projectId = getQueryProjectId(req); + const experiment = await store(ctx).get(projectId, req.params.id); + const execution = experiment + ? await store(ctx).getExecution({ + projectId, + experimentId: req.params.id, + experimentExecutionId: req.params.experimentExecutionId, + }) + : null; + if (!experiment || !execution) { + res.status(404).json({ error: "Experiment execution not found" }); + return; + } + await streamExperimentExecutionLogs( + ctx, + req, + res, + experiment, + execution, + req.query.fromStart === "true", + ); + }, + }); + apiRoute(ctx.app, ctx.registry, { method: "post", path: "/api/v1/experiments/:id/retry", diff --git a/apps/api/src/routes/reports-structured.test.ts b/apps/api/src/routes/reports-structured.test.ts index c2f45cef9..159613f8a 100644 --- a/apps/api/src/routes/reports-structured.test.ts +++ b/apps/api/src/routes/reports-structured.test.ts @@ -13,7 +13,7 @@ describe("structured report download", () => { const app = express(); app.use(express.json()); const downloadJson = vi.fn().mockResolvedValue({ - meta: { submissionRunId: "submission-run-1" }, + meta: { experimentExecutionId: "execution-1" }, }); const ctx = { app, @@ -22,11 +22,11 @@ describe("structured report download", () => { findOne: vi.fn().mockResolvedValue({ _id: "report-1", projectId: "project-1", - submissionId: "submission-1", - submissionRunId: "submission-run-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-1", status: "completed", structuredArtifact: { - schemaId: "scope.submission-readout", + schemaId: "scope.experiment-readout", schemaVersion: "1.0", contentType: "application/json", url: "https://storage.example/structured.json", @@ -51,21 +51,21 @@ describe("structured report download", () => { "report-1.structured.json", ); expect(response.body).toEqual({ - meta: { submissionRunId: "submission-run-1" }, + meta: { experimentExecutionId: "execution-1" }, }); expect(downloadJson).toHaveBeenCalledWith( "https://storage.example/structured.json", ); }); - it("filters submission reports by the exact submission run", async () => { + it("filters experiment reports by the exact execution and project", async () => { const app = express(); app.use(express.json()); const toArray = vi.fn().mockResolvedValue([{ _id: "report-1", projectId: "project-1", - submissionId: "submission-1", - submissionRunId: "submission-run-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-1", status: "completed", createdAt: new Date("2026-10-13T12:00:00.000Z"), }]); @@ -83,18 +83,19 @@ describe("structured report download", () => { registerReportsRoutes(ctx); const response = await request(app).get( - "/api/v1/reports?submissionId=submission-1&submissionRunId=submission-run-1", + "/api/v1/reports?projectId=project-1&experimentId=experiment-1&experimentExecutionId=execution-1", ); expect(response.status).toBe(200); expect(find).toHaveBeenCalledWith({ - submissionId: "submission-1", - submissionRunId: "submission-run-1", + projectId: "project-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-1", }); expect(response.body[0]).toMatchObject({ id: "report-1", - submissionId: "submission-1", - submissionRunId: "submission-run-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-1", }); }); }); diff --git a/apps/api/src/routes/reports.ts b/apps/api/src/routes/reports.ts index 381600660..5a2655a95 100644 --- a/apps/api/src/routes/reports.ts +++ b/apps/api/src/routes/reports.ts @@ -111,27 +111,38 @@ apiRoute(ctx.app, ctx.registry, { summary: "List reports", query: z.object({ requestId: z.string().optional(), - submissionId: z.string().optional(), - submissionRunId: z.string().optional(), + experimentId: z.string().optional(), + experimentExecutionId: z.string().optional(), projectId: z.string().optional(), + }).superRefine((value, ctx) => { + if (value.experimentExecutionId && !value.experimentId) { + ctx.addIssue({ + code: "custom", + path: ["experimentId"], + message: "experimentId is required with experimentExecutionId", + }); + } }), response: z.array(ReportResponseSchema), handler: async (req, res, next) => { try { const requestIdFilter = req.query.requestId as string; - const submissionIdFilter = req.query.submissionId as string; - const submissionRunIdFilter = req.query.submissionRunId as string; + const experimentIdFilter = req.query.experimentId as string; + const experimentExecutionIdFilter = + req.query.experimentExecutionId as string; const filter: Record = {}; if (requestIdFilter) { // Parent-scoped: the run id already pins the project. filter.requestId = requestIdFilter; - } else if (submissionIdFilter) { - filter.submissionId = submissionIdFilter; - if (submissionRunIdFilter) filter.submissionRunId = submissionRunIdFilter; + } else if (experimentIdFilter) { + filter.projectId = getQueryProjectId(req); + filter.experimentId = experimentIdFilter; + if (experimentExecutionIdFilter) { + filter.experimentExecutionId = experimentExecutionIdFilter; + } } else { // Top-level reports list requires an explicit project (400 if absent). filter.projectId = getQueryProjectId(req); - if (submissionRunIdFilter) filter.submissionRunId = submissionRunIdFilter; } const reports = await ctx.reportCollection diff --git a/apps/cli/src/commands/experiment.test.ts b/apps/cli/src/commands/experiment.test.ts index 949b81ea4..0df683064 100644 --- a/apps/cli/src/commands/experiment.test.ts +++ b/apps/cli/src/commands/experiment.test.ts @@ -198,4 +198,61 @@ describe("experiment commands", () => { "http://localhost:3100/api/v1/experiments/experiment-1/retry?projectId=project-1", ); }); + + it("retrieves structured JSON for one exact experiment execution", async () => { + const requests: CapturedRequest[] = []; + vi.stubGlobal("fetch", vi.fn(async (request: Request) => { + requests.push({ + url: request.url, + method: request.method, + body: await request.text(), + }); + const structured = request.url.endsWith( + "/structured?projectId=project-1", + ); + return new Response(JSON.stringify( + structured + ? { meta: { experimentExecutionId: "execution-1" } } + : [{ + id: "report-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-1", + status: "completed", + createdAt: "2026-01-01T00:00:00.000Z", + }], + ), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + })); + const output: string[] = []; + vi.spyOn(console, "log").mockImplementation((...parts: unknown[]) => { + output.push(parts.map(String).join(" ")); + }); + + await makeProgram().parseAsync( + [ + "experiment", + "reports", + "experiment-1", + "execution-1", + "--report", + "report-1", + "--structured", + "--project", + "project-1", + "--url", + "http://localhost:3100", + ], + { from: "user" }, + ); + + expect(requests.map((item) => item.url)).toEqual([ + "http://localhost:3100/api/v1/experiments/experiment-1/executions/execution-1/reports?projectId=project-1", + "http://localhost:3100/api/v1/experiments/experiment-1/executions/execution-1/reports/report-1/structured?projectId=project-1", + ]); + expect(output.join("\n")).toContain( + '"experimentExecutionId": "execution-1"', + ); + }); }); diff --git a/apps/cli/src/commands/experiment.ts b/apps/cli/src/commands/experiment.ts index fcc91de2a..84da5847a 100644 --- a/apps/cli/src/commands/experiment.ts +++ b/apps/cli/src/commands/experiment.ts @@ -4,9 +4,10 @@ import { readFileSync } from "node:fs"; import { resolve } from "node:path"; import { Command } from "commander"; +import EventSource from "eventsource"; import { stringify as stringifyYaml } from "yaml"; import { resolveScenarioAndPersona } from "../config-loader.js"; -import { apiFetch } from "../utils/api-client.js"; +import { apiFetch, getApiBasePath } from "../utils/api-client.js"; import { requireProjectId } from "../utils/config.js"; import { formatData, isMachineReadable } from "../utils/formatters.js"; import { parseGatesOption } from "../utils/gates.js"; @@ -14,6 +15,7 @@ import { configureHelp } from "../utils/helpFormatter.js"; import { buildResourceBindingSpecs, collectRepeatable } from "../utils/resources.js"; import { getDefaultApiUrl, + normalizeUrl, withOutputOption, withProjectOption, } from "../utils/shared.js"; @@ -35,10 +37,10 @@ interface ExperimentExecution { requests: ExperimentRequestSummary[]; handlerProgress: JsonRecord; blocking: ExperimentBlocker[]; - currentReport?: { - reportId?: string; - url?: string; - updatedAt?: string; + report?: { + reportId: string; + url: string; + updatedAt: string; }; createdAt: string; updatedAt?: string; @@ -189,9 +191,9 @@ function parseExecution(value: unknown): ExperimentExecution { {}, "experiment handler progress", ); - const report = execution.currentReport === undefined + const report = execution.report === undefined ? undefined - : asRecord(execution.currentReport, "experiment report"); + : asRecord(execution.report, "experiment report"); return { experimentExecutionId: stringField( execution, @@ -215,11 +217,11 @@ function parseExecution(value: unknown): ExperimentExecution { requests: requestVector.requests, handlerProgress: progress, blocking, - currentReport: report + report: report ? { - reportId: stringField(report, "reportId", "experiment report", true), - url: stringField(report, "url", "experiment report", true), - updatedAt: stringField(report, "updatedAt", "experiment report", true), + reportId: stringField(report, "reportId", "experiment report")!, + url: stringField(report, "url", "experiment report")!, + updatedAt: stringField(report, "updatedAt", "experiment report")!, } : undefined, createdAt: stringField( @@ -850,6 +852,129 @@ export function registerExperimentCommands(program: Command): void { } }); + withProjectOption( + withOutputOption( + experiment + .command("reports") + .description("List or retrieve reports for one exact experiment execution") + .argument("", "Experiment ID") + .argument("", "Experiment execution ID") + .option("--report ", "Select one report") + .option("--structured", "Print the selected report's structured JSON artifact") + .option("-u, --url ", "API base URL", getDefaultApiUrl()), + ["markdown"], + ), + ).action(async (id: string, executionId: string, options) => { + try { + const projectId = requireProjectId(options.project); + const path = + `/experiments/${encodeURIComponent(id)}/executions/${encodeURIComponent(executionId)}/reports`; + const response = await apiFetch(options.url, path, { projectId }); + if (!response.ok) await throwApiError(response); + const reportsValue = await json(response); + if (!Array.isArray(reportsValue)) { + throw new Error("Invalid experiment reports response from Scope"); + } + const reports = reportsValue.map((item) => + asRecord(item, "experiment report"), + ); + const selected = options.report + ? reports.find((item) => item.id === options.report) + : reports[0]; + if (options.report && !selected) { + throw new Error(`Report ${options.report} was not found for this execution`); + } + if (options.structured) { + if (!selected) { + throw new Error("No report is available for this execution"); + } + const reportId = stringField(selected, "id", "experiment report")!; + const structuredResponse = await apiFetch( + options.url, + `${path}/${encodeURIComponent(reportId)}/structured`, + { projectId }, + ); + if (!structuredResponse.ok) await throwApiError(structuredResponse); + console.log(JSON.stringify(await json(structuredResponse), null, 2)); + return; + } + const format = (options.output ?? "table") as OutputFormat; + if (format === "markdown") { + console.log(selected?.content ?? ""); + return; + } + console.log( + formatData( + options.report && selected ? [selected] : reports, + [ + { key: "id", label: "Report" }, + { key: "status", label: "Status" }, + { + key: "reporter", + label: "Reporter", + formatter: (row: JsonRecord) => + String((row.reporter as JsonRecord | undefined)?.name ?? ""), + }, + { key: "createdAt", label: "Created" }, + { key: "updatedAt", label: "Updated" }, + ], + format, + ), + ); + } catch (error) { + console.error( + errorText("Error:"), + error instanceof Error ? error.message : error, + ); + process.exitCode = 1; + } + }); + + withProjectOption( + experiment + .command("logs") + .description("Stream logs for one exact experiment execution") + .argument("", "Experiment ID") + .argument("", "Experiment execution ID") + .option("--from-start", "Include historical logs from the beginning", false) + .option("-u, --url ", "API base URL", getDefaultApiUrl()), + ).action(async (id: string, executionId: string, options) => { + const projectId = requireProjectId(options.project); + const params = new URLSearchParams({ projectId }); + if (options.fromStart) params.set("fromStart", "true"); + const eventSource = new EventSource( + `${normalizeUrl(options.url)}${getApiBasePath()}/experiments/${encodeURIComponent(id)}/executions/${encodeURIComponent(executionId)}/logs?${params}`, + ); + eventSource.onmessage = (event: MessageEvent) => { + try { + const log = JSON.parse(event.data) as { + timestamp: string; + level: string; + source?: string; + message: string; + }; + const source = log.source ? ` [${log.source}]` : ""; + console.log( + `${new Date(log.timestamp).toLocaleTimeString()} ${log.level}${source} ${log.message}`, + ); + } catch { + console.log(event.data); + } + }; + eventSource.addEventListener("done", () => { + eventSource.close(); + process.exit(0); + }); + eventSource.addEventListener("timeout", () => { + eventSource.close(); + process.exit(0); + }); + eventSource.onerror = () => { + eventSource.close(); + process.exit(1); + }; + }); + withProjectOption( experiment .command("retry") diff --git a/apps/cli/src/commands/report.test.ts b/apps/cli/src/commands/report.test.ts index 9fd85335b..c4d223593 100644 --- a/apps/cli/src/commands/report.test.ts +++ b/apps/cli/src/commands/report.test.ts @@ -20,7 +20,7 @@ describe("report get", () => { ok: true, status: 200, json: async () => ({ - meta: { schemaId: "scope.submission-readout" }, + meta: { schemaId: "scope.experiment-readout" }, }), }; })); @@ -47,17 +47,17 @@ describe("report get", () => { expect(urls).toEqual([ "http://localhost:3100/api/v1/reports/report-1/structured", ]); - expect(output.join("\n")).toContain('"schemaId": "scope.submission-readout"'); + expect(output.join("\n")).toContain('"schemaId": "scope.experiment-readout"'); }); - it("renders submission subject metadata through standard report retrieval", async () => { + it("renders experiment subject metadata through standard report retrieval", async () => { vi.stubGlobal("fetch", vi.fn(async () => ({ ok: true, status: 200, json: async () => ({ id: "report-1", - submissionId: "submission-1", - submissionRunId: "submission-run-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-1", status: "completed", createdAt: "2026-01-01T00:00:00.000Z", }), @@ -83,15 +83,15 @@ describe("report get", () => { { from: "user" }, ); - expect(output.join("\n")).toContain('"submissionId": "submission-1"'); + expect(output.join("\n")).toContain('"experimentId": "experiment-1"'); expect(output.join("\n")).toContain( - '"submissionRunId": "submission-run-1"', + '"experimentExecutionId": "execution-1"', ); }); }); describe("report list", () => { - it("passes submission filters to the global reports collection", async () => { + it("passes experiment filters to the global reports collection", async () => { const urls: string[] = []; vi.stubGlobal("fetch", vi.fn(async (request: Request) => { urls.push(request.url); @@ -109,10 +109,10 @@ describe("report list", () => { [ "report", "list", - "--submission", - "submission-1", - "--submission-run", - "submission-run-1", + "--experiment", + "experiment-1", + "--experiment-execution", + "execution-1", "--project", "project-1", "-o", @@ -124,7 +124,7 @@ describe("report list", () => { ); expect(urls).toEqual([ - "http://localhost:3100/api/v1/reports?submissionId=submission-1&submissionRunId=submission-run-1&projectId=project-1", + "http://localhost:3100/api/v1/reports?experimentId=experiment-1&experimentExecutionId=execution-1&projectId=project-1", ]); }); }); diff --git a/apps/cli/src/commands/report.ts b/apps/cli/src/commands/report.ts index 058f047ef..2a9807138 100644 --- a/apps/cli/src/commands/report.ts +++ b/apps/cli/src/commands/report.ts @@ -137,8 +137,8 @@ report const report = await response.json() as { id: string; requestId?: string; - submissionId?: string; - submissionRunId?: string; + experimentId?: string; + experimentExecutionId?: string; status: string; reporter?: { id: string; name: string; model: string; agentId: string; agentVersion: string }; content?: string; @@ -156,8 +156,8 @@ report const fields: DisplayField[] = [ { key: 'id', label: 'ID' }, { key: 'requestId', label: 'Run' }, - { key: 'submissionId', label: 'Submission' }, - { key: 'submissionRunId', label: 'Submission Run' }, + { key: 'experimentId', label: 'Experiment' }, + { key: 'experimentExecutionId', label: 'Experiment Execution' }, { key: 'status', label: 'Status' }, { key: 'reporter', label: 'Reporter', formatter: (r: any) => r.reporter ? `${r.reporter.name} (${r.reporter.agentId}@${r.reporter.agentVersion})` : '' }, { key: 'model', label: 'Model', formatter: (r: any) => r.reporter?.model || '' }, @@ -174,11 +174,11 @@ report if (report.requestId) { console.log(`${label('Run:')} ${value(report.requestId)}`); } - if (report.submissionId) { - console.log(`${label('Submission:')} ${value(report.submissionId)}`); + if (report.experimentId) { + console.log(`${label('Experiment:')} ${value(report.experimentId)}`); } - if (report.submissionRunId) { - console.log(`${label('Attempt:')} ${value(report.submissionRunId)}`); + if (report.experimentExecutionId) { + console.log(`${label('Execution:')} ${value(report.experimentExecutionId)}`); } console.log(`${label('Status:')} ${colorLevel(report.status === "completed" ? "info" : report.status === "failed" ? "error" : "warn")} ${report.status}`); console.log(`${label('Created:')} ${dimTimestamp(new Date(report.createdAt).toLocaleString())}`); @@ -210,8 +210,8 @@ report .command("list") .description("List all reports") .option("-r, --run ", "Filter by run ID") - .option("-s, --submission ", "Filter by submission ID") - .option("--submission-run ", "Filter by submission run ID") + .option("-e, --experiment ", "Filter by experiment ID") + .option("--experiment-execution ", "Filter by experiment execution ID") .option("-u, --url ", "API base URL", getDefaultApiUrl()) )) .action(async (options) => { @@ -220,9 +220,9 @@ report try { const params = new URLSearchParams(); if (options.run) params.set("requestId", options.run); - if (options.submission) params.set("submissionId", options.submission); - if (options.submissionRun) { - params.set("submissionRunId", options.submissionRun); + if (options.experiment) params.set("experimentId", options.experiment); + if (options.experimentExecution) { + params.set("experimentExecutionId", options.experimentExecution); } const qs = params.toString(); const response = await apiFetch(options.url, `/reports${qs ? `?${qs}` : ""}`, { projectId }); @@ -236,8 +236,8 @@ report const reports = await response.json() as Array<{ id: string; requestId?: string; - submissionId?: string; - submissionRunId?: string; + experimentId?: string; + experimentExecutionId?: string; status: string; reporter?: { model: string }; createdAt: string; @@ -259,11 +259,11 @@ report { key: 'requestId', label: 'Run ID', tableFormatter: (r: any) => r.requestId ? dimTimestamp(r.requestId) : 'N/A', }, - { key: 'submissionId', label: 'Submission', - tableFormatter: (r: any) => r.submissionId ? dimTimestamp(r.submissionId) : 'N/A', + { key: 'experimentId', label: 'Experiment', + tableFormatter: (r: any) => r.experimentId ? dimTimestamp(r.experimentId) : 'N/A', }, - { key: 'submissionRunId', label: 'Submission Run', - tableFormatter: (r: any) => r.submissionRunId ? dimTimestamp(r.submissionRunId) : 'N/A', + { key: 'experimentExecutionId', label: 'Experiment Execution', + tableFormatter: (r: any) => r.experimentExecutionId ? dimTimestamp(r.experimentExecutionId) : 'N/A', }, { key: 'status', label: 'Status', tableFormatter: (r: any) => { diff --git a/apps/cli/src/commands/submission.test.ts b/apps/cli/src/commands/submission.test.ts deleted file mode 100644 index c7a43069e..000000000 --- a/apps/cli/src/commands/submission.test.ts +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -import { afterEach, describe, expect, it, vi } from "vitest"; -import { Command } from "commander"; -import { registerSubmissionCommands } from "./submission.js"; - -interface MockResponse { - ok: boolean; - status: number; - json: () => Promise; -} - -function makeProgram(): Command { - const program = new Command(); - registerSubmissionCommands(program); - return program; -} - -function response(body: unknown): MockResponse { - return { - ok: true, - status: 200, - json: async () => body, - }; -} - -afterEach(() => { - vi.restoreAllMocks(); - vi.unstubAllGlobals(); - process.exitCode = undefined; -}); - -describe("submission report", () => { - it("lists current and historical reports through the plural resource", async () => { - const urls: string[] = []; - vi.stubGlobal("fetch", vi.fn(async (request: Request) => { - urls.push(request.url); - return response([ - { - id: "report-current", - submissionRunId: "submission-run-2", - status: "completed", - createdAt: "2026-01-02T00:00:00.000Z", - }, - { - id: "report-historical", - submissionRunId: "submission-run-1", - status: "failed", - createdAt: "2026-01-01T00:00:00.000Z", - }, - ]); - })); - const output: string[] = []; - vi.spyOn(console, "log").mockImplementation((...parts: unknown[]) => { - output.push(parts.map(String).join(" ")); - }); - - await makeProgram().parseAsync( - [ - "submission", - "reports", - "submission-1", - "-u", - "http://localhost:3100", - "-o", - "json", - ], - { from: "user" }, - ); - - expect(urls).toEqual([ - "http://localhost:3100/api/v1/submissions/submission-1/reports", - ]); - expect(output.join("\n")).toContain('"id": "report-current"'); - expect(output.join("\n")).toContain('"id": "report-historical"'); - }); -}); diff --git a/apps/portal/src/components/ExperimentReportCard.stories.tsx b/apps/portal/src/components/ExperimentReportCard.stories.tsx new file mode 100644 index 000000000..c47db044a --- /dev/null +++ b/apps/portal/src/components/ExperimentReportCard.stories.tsx @@ -0,0 +1,77 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { Meta, StoryObj } from "@storybook/react-vite"; +import { expect, within } from "storybook/test"; +import type { Report } from "@/types"; +import { ExperimentReportCard } from "./ExperimentReportCard"; + +const report: Report = { + _id: "report-1", + id: "report-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-2", + status: "completed", + content: "# Experiment readout\n\nFive exact-vector runs were analyzed.", + reporter: { + id: "pp-readout", + name: "Experiment Readout", + gitHash: "abc", + model: "gpt-5.4", + agentId: "copilot-sdk", + agentVersion: "1.0.0", + }, + structuredArtifact: { + schemaId: "scope.experiment-readout", + schemaVersion: "1.0", + contentType: "application/json", + url: "reports/report-1/structured.json", + }, + createdAt: "2026-01-02T00:00:00.000Z", +}; + +const meta = { + title: "Experiments/Reports", + component: ExperimentReportCard, + args: { + experimentId: "experiment-1", + experimentExecutionId: "execution-2", + attemptNumber: 2, + report, + structuredUrl: "/structured.json", + }, + parameters: { layout: "padded" }, +} satisfies Meta; + +export default meta; +type Story = StoryObj; + +export const Completed: Story = { + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await expect( + canvas.getByRole("heading", { name: "Experiment readout" }), + ).toBeVisible(); + await expect( + canvas.getByRole("link", { name: "Structured JSON" }), + ).toBeVisible(); + await expect(canvas.queryByRole("tab")).toBeNull(); + }, +}; + +export const Processing: Story = { + args: { + report: undefined, + structuredUrl: undefined, + isProcessing: true, + }, +}; + +export const ErrorWithRetry: Story = { + args: { + report: undefined, + structuredUrl: undefined, + error: "Report service unavailable", + onRetry: () => undefined, + }, +}; diff --git a/apps/portal/src/components/ExperimentReportCard.tsx b/apps/portal/src/components/ExperimentReportCard.tsx new file mode 100644 index 000000000..5d7ee6726 --- /dev/null +++ b/apps/portal/src/components/ExperimentReportCard.tsx @@ -0,0 +1,97 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { Download, ExternalLink } from "lucide-react"; +import { Link } from "react-router-dom"; +import { MarkdownRenderer } from "@/components/MarkdownRenderer"; +import { ReportStatusBadge } from "@/components/ReportStatusBadge"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import type { Report } from "@/types"; + +interface ExperimentReportCardProps { + experimentId: string; + experimentExecutionId: string; + attemptNumber?: number; + report?: Report; + structuredUrl?: string; + isLoading?: boolean; + error?: string; + isProcessing?: boolean; + onRetry?: () => void; +} + +export function ExperimentReportCard({ + experimentId, + experimentExecutionId, + attemptNumber, + report, + structuredUrl, + isLoading, + error, + isProcessing, + onRetry, +}: ExperimentReportCardProps) { + return ( + + +
+ Reports +

+ Attempt {attemptNumber ?? "…"} ·{" "} + {experimentExecutionId} +

+
+ {report && ( +
+ + {structuredUrl && ( + + )} + +
+ )} + + + {isLoading ? ( +

Loading report…

+ ) : error ? ( +
+

{error}

+ {onRetry && ( + + )} +
+ ) : report?.status === "failed" ? ( +

+ {report.error ?? "Readout generation failed."} +

+ ) : report?.content ? ( +
+ {report.content} +
+ ) : ( +

+ {isProcessing + ? "The report is being generated." + : "No built-in report is available for this execution."} +

+ )} + + + ); +} diff --git a/apps/portal/src/components/SubmissionLink.stories.tsx b/apps/portal/src/components/SubmissionLink.stories.tsx deleted file mode 100644 index 0df3ea96b..000000000 --- a/apps/portal/src/components/SubmissionLink.stories.tsx +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -import type { Meta, StoryObj } from "@storybook/react-vite"; -import { expect } from "storybook/test"; -import { SubmissionLink } from "./SubmissionLink"; - -const meta = { - title: "Submissions/SubmissionLink", - component: SubmissionLink, - args: { - submissionId: "submission-12345678", - }, -} satisfies Meta; - -export default meta; -type Story = StoryObj; - -export const Labeled: Story = { - play: async ({ canvas }) => { - await expect( - canvas.getByRole("link", { - name: "Open submission submission-12345678", - }), - ).toBeVisible(); - await expect(canvas.getByText("Submission")).toBeVisible(); - }, -}; - -export const Compact: Story = { - args: { compact: true }, -}; diff --git a/apps/portal/src/components/SubmissionLink.test.tsx b/apps/portal/src/components/SubmissionLink.test.tsx deleted file mode 100644 index 358959353..000000000 --- a/apps/portal/src/components/SubmissionLink.test.tsx +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -// @vitest-environment happy-dom -import { render, screen } from "@testing-library/react"; -import { MemoryRouter } from "react-router-dom"; -import { describe, expect, it } from "vitest"; -import { SubmissionLink } from "./SubmissionLink"; - -describe("SubmissionLink", () => { - it("provides a labeled, accessible path to Submission Detail", () => { - render( - - - , - ); - - const link = screen.getByRole("link", { - name: "Open submission submission-123", - }); - expect(link.getAttribute("href")).toBe("/submissions/submission-123"); - expect(screen.getByText("Submission")).toBeDefined(); - }); -}); diff --git a/apps/portal/src/components/SubmissionLink.tsx b/apps/portal/src/components/SubmissionLink.tsx deleted file mode 100644 index 243bc5268..000000000 --- a/apps/portal/src/components/SubmissionLink.tsx +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -import { Layers3 } from "lucide-react"; -import { Link } from "react-router-dom"; -import { ShortId } from "@/components/ShortId"; -import { cn } from "@/lib/utils"; - -export function SubmissionLink({ - submissionId, - className, - compact = false, -}: { - submissionId: string; - className?: string; - compact?: boolean; -}) { - return ( - event.stopPropagation()} - > - - {!compact && Submission} - - - ); -} diff --git a/apps/portal/src/components/SubmissionReports.stories.tsx b/apps/portal/src/components/SubmissionReports.stories.tsx deleted file mode 100644 index d3762acbf..000000000 --- a/apps/portal/src/components/SubmissionReports.stories.tsx +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -import { useState } from "react"; -import type { Meta, StoryObj } from "@storybook/react-vite"; -import { expect, userEvent, within } from "storybook/test"; -import { - SubmissionReports, - reportId, -} from "@/pages/SubmissionDetail"; -import type { - Report, - SubmissionRunAttempt, -} from "@/types"; - -const attempt: SubmissionRunAttempt = { - _id: "submission-run-current", - attemptNumber: 2, - vectorFingerprint: "vector-current", - requests: [{ requestId: "request-1", runId: "run-1" }], - dependencyProjection: [], - blocking: [], - handlerStatus: { - "pp-readout": { status: "done", version: 1 }, - }, - report: { reportId: "report-readout" }, - createdAt: "2026-01-02T00:00:00.000Z", -}; - -const readoutReport: Report = { - _id: "report-readout", - id: "report-readout", - submissionId: "submission-1", - submissionRunId: attempt._id, - status: "completed", - content: "# Submission report\n\nFive runs completed successfully.", - reporter: { - id: "pp-readout", - name: "Submission Readout", - gitHash: "abc", - model: "gpt-5.4", - agentId: "copilot", - agentVersion: "1.0.0", - }, - structuredArtifact: { - schemaId: "scope.submission-readout", - schemaVersion: "1.0", - contentType: "application/json", - url: "reports/report-readout/structured.json", - }, - createdAt: "2026-01-02T00:00:00.000Z", -}; - -const templateReport: Report = { - _id: "report-executive", - id: "report-executive", - submissionId: "submission-1", - submissionRunId: attempt._id, - templateId: "executive-summary", - status: "completed", - content: "# Executive summary\n\nA concise template-based view.", - createdAt: "2026-01-02T01:00:00.000Z", -}; - -function ReportTypesStory({ reports }: { reports: Report[] }) { - const [selected, setSelected] = useState(reports[0]); - return ( - undefined} - /> - ); -} - -const meta = { - title: "Submissions/Submission Reports", - component: ReportTypesStory, - parameters: { layout: "padded" }, -} satisfies Meta; - -export default meta; -type Story = StoryObj; - -export const OneReportType: Story = { - args: { reports: [readoutReport] }, - play: async ({ canvasElement }) => { - const canvas = within(canvasElement); - await expect( - canvas.getByRole("tab", { name: "Submission Readout" }), - ).toBeVisible(); - await expect( - canvas.getByRole("heading", { name: "Submission report" }), - ).toBeVisible(); - }, -}; - -export const MultipleReportTypes: Story = { - args: { reports: [readoutReport, templateReport] }, - play: async ({ canvasElement }) => { - const canvas = within(canvasElement); - await userEvent.click( - canvas.getByRole("tab", { name: "Executive Summary" }), - ); - await expect( - canvas.getByRole("heading", { name: "Executive summary" }), - ).toBeVisible(); - }, -}; diff --git a/apps/portal/src/components/ui/stories.play.test.tsx b/apps/portal/src/components/ui/stories.play.test.tsx index 5137d9b6d..06dbf5816 100644 --- a/apps/portal/src/components/ui/stories.play.test.tsx +++ b/apps/portal/src/components/ui/stories.play.test.tsx @@ -25,13 +25,12 @@ import * as tabs from "./tabs.stories"; import * as textarea from "./textarea.stories"; import * as tooltip from "./tooltip.stories"; import * as sonner from "./sonner.stories"; -import * as submissionLink from "../SubmissionLink.stories"; -import * as submissionReports from "../SubmissionReports.stories"; +import * as experimentReports from "../ExperimentReportCard.stories"; const modules: Record> = { alert, alertDialog, checkbox, dialog, dropdownMenu, label, popover, scrollArea, select, separator, sheet, skeleton, switch: sw, table, tabs, textarea, - tooltip, sonner, submissionLink, submissionReports, + tooltip, sonner, experimentReports, }; afterEach(() => cleanup()); diff --git a/apps/portal/src/lib/api.ts b/apps/portal/src/lib/api.ts index d303dcf16..2dcaf9b14 100644 --- a/apps/portal/src/lib/api.ts +++ b/apps/portal/src/lib/api.ts @@ -352,6 +352,29 @@ export const api = { return parseExperimentExecution(value); }, + listExperimentReports: ( + id: string, + experimentExecutionId: string, + ): Promise => + request( + `/experiments/${encodeURIComponent(id)}/executions/${encodeURIComponent(experimentExecutionId)}/reports`, + undefined, + { scoped: true }, + ), + + experimentReportStructuredUrl: ( + id: string, + experimentExecutionId: string, + reportId: string, + ): string => { + const projectId = getSelectedProjectId(); + if (!projectId) throw new ProjectRequiredError(); + return `${BASE}${withProjectId( + `/experiments/${encodeURIComponent(id)}/executions/${encodeURIComponent(experimentExecutionId)}/reports/${encodeURIComponent(reportId)}/structured`, + projectId, + )}`; + }, + retryExperiment: async ( id: string, experimentExecutionId: string, diff --git a/apps/portal/src/lib/experiments.ts b/apps/portal/src/lib/experiments.ts index 9d10275d2..9a7ea4fe5 100644 --- a/apps/portal/src/lib/experiments.ts +++ b/apps/portal/src/lib/experiments.ts @@ -48,7 +48,7 @@ export interface ExperimentExecutionView { requests: ExperimentRequestSummaryView[]; handlerProgress: Record; blocking: ExperimentBlockerView[]; - currentReport?: { reportId?: string; url?: string; updatedAt?: string }; + report?: { reportId: string; url: string; updatedAt: string }; createdAt: string; updatedAt?: string; startedAt?: string; @@ -184,16 +184,16 @@ export function parseExperimentExecution( execution.handlerSummary ?? {}; const handlerProgress = record(progressValue, "experiment handler progress"); - const reportValue = execution.currentReport; - const currentReport = + const reportValue = execution.report; + const report = reportValue === undefined ? undefined : (() => { const report = record(reportValue, "experiment report"); return { - reportId: optionalString(report, "reportId"), - url: optionalString(report, "url"), - updatedAt: optionalString(report, "updatedAt"), + reportId: requiredString(report, "reportId", "experiment report"), + url: requiredString(report, "url", "experiment report"), + updatedAt: requiredString(report, "updatedAt", "experiment report"), }; })(); @@ -215,7 +215,7 @@ export function parseExperimentExecution( requests: vector.requests, handlerProgress, blocking, - currentReport, + report, createdAt: requiredString(execution, "createdAt", "experiment execution"), updatedAt: optionalString(execution, "updatedAt"), startedAt: optionalString(execution, "startedAt"), diff --git a/apps/portal/src/pages/ExperimentDetail.test.tsx b/apps/portal/src/pages/ExperimentDetail.test.tsx new file mode 100644 index 000000000..aae87d1bb --- /dev/null +++ b/apps/portal/src/pages/ExperimentDetail.test.tsx @@ -0,0 +1,172 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// @vitest-environment happy-dom +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { MemoryRouter, Route, Routes } from "react-router-dom"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { ExperimentExecutionView, ExperimentView } from "@/lib/experiments"; +import type { Report } from "@/types"; + +const apiMocks = vi.hoisted(() => ({ + getExperiment: vi.fn(), + listExperimentExecutions: vi.fn(), + getExperimentExecution: vi.fn(), + listExperimentReports: vi.fn(), + retryExperiment: vi.fn(), + experimentReportStructuredUrl: vi.fn( + () => "/api/v1/experiments/experiment-1/executions/execution-1/reports/report-1/structured?projectId=project-1", + ), +})); + +vi.mock("@/lib/api", () => ({ api: apiMocks })); + +import { ExperimentDetail } from "./ExperimentDetail"; + +const execution = ( + experimentExecutionId: string, + attemptNumber: number, +): ExperimentExecutionView => ({ + experimentExecutionId, + attemptNumber, + requestCount: 1, + requests: [{ + requestId: `request-${attemptNumber}`, + runId: `run-${attemptNumber}`, + status: "done", + outcome: "failed", + }], + handlerProgress: { + "pp-readout": { status: "done", version: 1 }, + }, + blocking: [], + readiness: "done", + createdAt: `2026-01-0${attemptNumber}T00:00:00.000Z`, +}); + +const currentExecution = execution("execution-2", 2); +const historicalExecution = execution("execution-1", 1); +const experiment: ExperimentView = { + experimentId: "experiment-1", + projectId: "project-1", + metadata: { name: "Readout experiment" }, + revision: { + experimentRevisionId: "revision-1", + revisionNumber: 1, + memberCount: 1, + }, + execution: currentExecution, + readiness: "done", + blocking: [], + createdRequestIds: ["request-2"], + createdAt: "2026-01-01T00:00:00.000Z", +}; +const report: Report = { + _id: "report-1", + id: "report-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-1", + status: "completed", + content: "# Historical readout\n\nExact attempt evidence.", + reporter: { + id: "pp-readout", + name: "Experiment Readout", + gitHash: "abc", + model: "gpt-5.4", + agentId: "copilot-sdk", + agentVersion: "1.0.0", + }, + structuredArtifact: { + schemaId: "scope.experiment-readout", + schemaVersion: "1.0", + contentType: "application/json", + url: "reports/report-1/structured.json", + }, + createdAt: "2026-01-01T01:00:00.000Z", +}; + +function renderPage(initialEntry: string) { + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false }, mutations: { retry: false } }, + }); + return render( + + + + } /> + + + , + ); +} + +afterEach(() => { + cleanup(); + vi.clearAllMocks(); +}); + +describe("ExperimentDetail reports", () => { + it("deep-links to and renders the report for the exact selected execution", async () => { + apiMocks.getExperiment.mockResolvedValue(experiment); + apiMocks.listExperimentExecutions.mockResolvedValue({ + items: [currentExecution, historicalExecution], + }); + apiMocks.getExperimentExecution.mockResolvedValue(historicalExecution); + apiMocks.listExperimentReports.mockResolvedValue([report]); + + renderPage("/experiments/experiment-1?execution=execution-1"); + + expect( + await screen.findByRole("heading", { name: "Historical readout" }), + ).toBeDefined(); + expect(apiMocks.getExperimentExecution).toHaveBeenCalledWith( + "experiment-1", + "execution-1", + ); + expect(apiMocks.listExperimentReports).toHaveBeenCalledWith( + "experiment-1", + "execution-1", + ); + expect( + screen.getByRole("link", { name: "Structured JSON" }).getAttribute("href"), + ).toContain("/executions/execution-1/reports/report-1/structured"); + + let currentAttemptButton: HTMLElement | undefined; + await waitFor(() => { + currentAttemptButton = screen + .getAllByRole("button") + .find((button) => button.textContent?.includes("Attempt 2")); + expect(currentAttemptButton).toBeDefined(); + }); + fireEvent.click(currentAttemptButton!); + await waitFor(() => { + expect(apiMocks.listExperimentReports).toHaveBeenCalledWith( + "experiment-1", + "execution-2", + ); + }); + }); + + it("shows report load errors and retries the same exact execution", async () => { + apiMocks.getExperiment.mockResolvedValue(experiment); + apiMocks.listExperimentExecutions.mockResolvedValue({ + items: [currentExecution], + }); + apiMocks.getExperimentExecution.mockResolvedValue(currentExecution); + apiMocks.listExperimentReports + .mockRejectedValueOnce(new Error("Report service unavailable")) + .mockResolvedValueOnce([]); + + renderPage("/experiments/experiment-1?execution=execution-2"); + + expect(await screen.findByText("Report service unavailable")).toBeDefined(); + fireEvent.click(screen.getByRole("button", { name: "Retry" })); + await waitFor(() => { + expect(apiMocks.listExperimentReports).toHaveBeenCalledTimes(2); + }); + expect( + await screen.findByText("No built-in report is available for this execution."), + ).toBeDefined(); + }); +}); diff --git a/apps/portal/src/pages/ExperimentDetail.tsx b/apps/portal/src/pages/ExperimentDetail.tsx index 857633ba3..fbb1bdc2f 100644 --- a/apps/portal/src/pages/ExperimentDetail.tsx +++ b/apps/portal/src/pages/ExperimentDetail.tsx @@ -1,11 +1,11 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { useState } from "react"; import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { ArrowLeft, FileText, RotateCcw } from "lucide-react"; -import { Link, useParams } from "react-router-dom"; +import { Link, useParams, useSearchParams } from "react-router-dom"; import { toast } from "sonner"; +import { ExperimentReportCard } from "@/components/ExperimentReportCard"; import { ExperimentStatusCard } from "@/components/ExperimentStatusCard"; import { ShortId } from "@/components/ShortId"; import { Badge } from "@/components/ui/badge"; @@ -15,6 +15,18 @@ import { ApiError, api } from "@/lib/api"; import type { ExperimentExecutionView } from "@/lib/experiments"; import { formatDate } from "@/lib/utils"; +function handlerStatus(value: unknown): string | undefined { + if ( + value && + typeof value === "object" && + "status" in value && + typeof value.status === "string" + ) { + return value.status; + } + return undefined; +} + function ExecutionSummary({ execution, }: { @@ -51,7 +63,7 @@ function ExecutionSummary({ export function ExperimentDetail() { const { id } = useParams<{ id: string }>(); const queryClient = useQueryClient(); - const [selectedExecutionId, setSelectedExecutionId] = useState(); + const [searchParams, setSearchParams] = useSearchParams(); const experimentQuery = useQuery({ queryKey: ["experiment", id], queryFn: () => api.getExperiment(id!), @@ -70,11 +82,26 @@ export function ExperimentDetail() { queryFn: () => api.listExperimentExecutions(id!, { limit: 50 }), enabled: Boolean(id) && experimentQuery.isSuccess, }); + const selectedExecutionId = + searchParams.get("execution") ?? + experimentQuery.data?.execution.experimentExecutionId; const selectedExecutionQuery = useQuery({ queryKey: ["experiment-execution", id, selectedExecutionId], queryFn: () => api.getExperimentExecution(id!, selectedExecutionId!), enabled: Boolean(id && selectedExecutionId), }); + const reportsQuery = useQuery({ + queryKey: ["experiment-reports", id, selectedExecutionId], + queryFn: () => api.listExperimentReports(id!, selectedExecutionId!), + enabled: Boolean(id && selectedExecutionId), + refetchInterval: (query) => { + const report = query.state.data?.[0]; + if (report?.status === "completed" || report?.status === "failed") { + return false; + } + return 5_000; + }, + }); const retry = useMutation({ mutationFn: () => api.retryExperiment( @@ -87,6 +114,9 @@ export function ExperimentDetail() { void queryClient.invalidateQueries({ queryKey: ["experiment-executions", id], }); + void queryClient.invalidateQueries({ + queryKey: ["experiment-reports", id], + }); }, onError: (error) => { toast.error(error instanceof Error ? error.message : "Retry failed"); @@ -147,6 +177,14 @@ export function ExperimentDetail() { } const experiment = experimentQuery.data; + const selectedExecution = + selectedExecutionQuery.data ?? + (selectedExecutionId === experiment.execution.experimentExecutionId + ? experiment.execution + : undefined); + const readout = reportsQuery.data?.find( + (report) => report.reporter?.id === "pp-readout", + ); const canRetry = !["waiting", "processing"].includes( experiment.readiness ?? "", ); @@ -198,6 +236,40 @@ export function ExperimentDetail() { + void reportsQuery.refetch()} + /> + {experiment.blocking.length > 0 && ( @@ -233,17 +305,17 @@ export function ExperimentDetail() { Current runs - {experiment.execution.requests.length === 0 ? ( + {!selectedExecution || selectedExecution.requests.length === 0 ? (

{experiment.execution.requestCount === 0 ? "This experiment is empty." : `${experiment.execution.requestCount} linked runs are stored outside the inline response.`}

) : ( - experiment.execution.requests.map((request) => ( + selectedExecution.requests.map((request) => (
@@ -286,7 +358,18 @@ export function ExperimentDetail() { key={execution.experimentExecutionId} type="button" onClick={() => - setSelectedExecutionId(execution.experimentExecutionId) + setSearchParams( + (current) => { + const next = new URLSearchParams(current); + next.set( + "execution", + execution.experimentExecutionId, + ); + next.delete("report"); + return next; + }, + { replace: false }, + ) } className="flex w-full items-center justify-between rounded-md border p-3 text-left hover:bg-muted/40" > @@ -308,8 +391,8 @@ export function ExperimentDetail() { {selectedExecutionQuery.isLoading && (

Loading execution…

)} - {selectedExecutionQuery.data && ( - + {selectedExecution && ( + )}
); diff --git a/apps/portal/src/pages/InsightDetail.tsx b/apps/portal/src/pages/InsightDetail.tsx index 6c875853c..dd98f3110 100644 --- a/apps/portal/src/pages/InsightDetail.tsx +++ b/apps/portal/src/pages/InsightDetail.tsx @@ -215,12 +215,12 @@ export function InsightDetail() { - {report.submissionId ? ( + {report.experimentId && report.experimentExecutionId ? ( - {formatId(report.submissionId)} + {formatId(report.experimentId)} ) : report.requestId ? ( {report.structuredArtifact && (
- {report.submissionId ? "Submission run:" : "Run ID:"} + {report.experimentId ? "Experiment execution:" : "Run ID:"} {" "} - {report.submissionId ? ( + {report.experimentId && report.experimentExecutionId ? ( - {report.submissionRunId} + {report.experimentExecutionId} ) : report.requestId ? ( { vi.clearAllMocks(); }); -describe("ReportPreviewPanel submission reports", () => { - it("identifies a template-less report without using report-type terminology", async () => { +describe("ReportPreviewPanel experiment reports", () => { + it("identifies the built-in report without exposing its handler identifier", async () => { const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } }, }); @@ -54,7 +54,7 @@ describe("ReportPreviewPanel submission reports", () => { , ); - expect(await screen.findByText("template-less")).toBeDefined(); + expect(await screen.findByText("built-in")).toBeDefined(); expect(screen.queryByText("default template")).toBeNull(); }); }); diff --git a/apps/portal/src/pages/ReportPreviewPanel.tsx b/apps/portal/src/pages/ReportPreviewPanel.tsx index 9f682b906..9e31ae366 100644 --- a/apps/portal/src/pages/ReportPreviewPanel.tsx +++ b/apps/portal/src/pages/ReportPreviewPanel.tsx @@ -57,10 +57,12 @@ export function ReportPreviewPanel() { onClose={closePanel} headerActions={
- {report.submissionId ? ( - + {report.experimentId && report.experimentExecutionId ? ( + ) : report.requestId ? ( @@ -92,7 +94,7 @@ export function ReportPreviewPanel() { ) : ( - {report.submissionId ? "template-less" : "default template"} + {report.experimentId ? "built-in" : "default template"} )}
diff --git a/apps/portal/src/pages/ReportsList.tsx b/apps/portal/src/pages/ReportsList.tsx index d4b40f074..704474141 100644 --- a/apps/portal/src/pages/ReportsList.tsx +++ b/apps/portal/src/pages/ReportsList.tsx @@ -119,15 +119,17 @@ export function ReportsList() { header: "Subject", width: "120px", hidden: columnVisibility.isHidden("runId"), - cell: (r) => r.submissionId ? ( + cell: (r) => r.experimentId && r.experimentExecutionId ? ( { event.stopPropagation(); - navigate(`/submissions/${r.submissionId}`); + navigate( + `/experiments/${r.experimentId}?execution=${encodeURIComponent(r.experimentExecutionId!)}`, + ); }} > - Submission {formatId(r.submissionId)} + Experiment {formatId(r.experimentId)} ) : r.requestId ? ( ) : ( - {r.submissionId ? "template-less" : "default"} + {r.experimentId ? "built-in" : "default"} ), }, @@ -205,7 +207,7 @@ export function ReportsList() {
("grid"); const [reportsFilter, setReportsFilter] = useState<"latest" | "all">("latest"); @@ -706,10 +710,10 @@ export function RunDetail() { if (error || !run) { return (
- +
@@ -808,16 +812,12 @@ export function RunDetail() {
diff --git a/apps/portal/src/pages/SubmissionDetail.test.tsx b/apps/portal/src/pages/SubmissionDetail.test.tsx deleted file mode 100644 index e2c825ccf..000000000 --- a/apps/portal/src/pages/SubmissionDetail.test.tsx +++ /dev/null @@ -1,398 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -// @vitest-environment happy-dom -import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; -import { cleanup, render, screen, within } from "@testing-library/react"; -import userEvent from "@testing-library/user-event"; -import { MemoryRouter, Route, Routes } from "react-router-dom"; -import { afterEach, describe, expect, it, vi } from "vitest"; -import type { - Report, - Submission, - SubmissionRunAttempt, -} from "@/types"; - -const submission: Submission = { - submissionId: "submission-1", - projectId: "project-1", - requestIds: ["request-1", "request-2"], - createdAt: "2026-01-01T00:00:00.000Z", - submissionRunId: "submission-run-2", - attemptNumber: 2, - readiness: "done", - requests: [ - { - requestId: "request-1", - runId: "run-current-1", - task: "Create an API", - status: "done", - outcome: "succeeded", - workerType: "coder-acp-copilot", - model: "gpt-5.4", - }, - { - requestId: "request-2", - runId: "run-current-2", - task: "Document the API", - status: "done", - outcome: "succeeded", - workerType: "coder-acp-copilot", - model: "gpt-5.4", - }, - ], - dependencyProjection: [ - { - requestId: "request-1", - runId: "run-current-1", - handlerId: "pp-observations", - state: "ready", - }, - { - requestId: "request-2", - runId: "run-current-2", - handlerId: "security-scan", - state: "ready", - }, - ], - blocking: [], - handlerProgress: { - "pp-readout": { status: "done", version: 1 }, - }, - report: { reportId: "report-current" }, -}; - -const currentAttempt: SubmissionRunAttempt = { - _id: "submission-run-2", - attemptNumber: 2, - vectorFingerprint: "vector-current", - requests: [ - { requestId: "request-1", runId: "run-current-1" }, - { requestId: "request-2", runId: "run-current-2" }, - ], - dependencyProjection: submission.dependencyProjection, - blocking: [], - handlerStatus: submission.handlerProgress, - report: { reportId: "report-current" }, - createdAt: "2026-01-02T00:00:00.000Z", -}; - -const historicalAttempt: SubmissionRunAttempt = { - _id: "submission-run-1", - attemptNumber: 1, - vectorFingerprint: "vector-historical", - requests: [ - { requestId: "request-1", runId: "run-historical-1" }, - { requestId: "request-2", runId: "run-historical-2" }, - ], - dependencyProjection: [ - { - requestId: "request-1", - runId: "run-historical-1", - handlerId: "pp-observations", - state: "ready", - }, - { - requestId: "request-2", - runId: "run-historical-2", - handlerId: "security-scan", - state: "ready", - }, - ], - blocking: [], - handlerStatus: { - "pp-readout": { status: "done", version: 1 }, - }, - report: { reportId: "report-historical" }, - createdAt: "2026-01-01T00:00:00.000Z", - supersededAt: "2026-01-02T00:00:00.000Z", - supersededBySubmissionRunId: "submission-run-2", - supersededReason: "request-vector-changed", -}; - -const reports: Report[] = [ - { - _id: "report-current", - id: "report-current", - submissionId: "submission-1", - submissionRunId: "submission-run-2", - status: "completed", - content: "# Current generated report", - reporter: { - id: "pp-readout", - name: "Submission Readout", - gitHash: "abc", - model: "gpt-5.4", - agentId: "copilot", - agentVersion: "1.0.0", - }, - structuredArtifact: { - schemaId: "scope.submission-readout", - schemaVersion: "1.0", - contentType: "application/json", - url: "reports/report-current/structured.json", - }, - createdAt: "2026-01-02T00:00:00.000Z", - }, - { - _id: "report-current-older-duplicate", - id: "report-current-older-duplicate", - submissionId: "submission-1", - submissionRunId: "submission-run-2", - status: "completed", - content: "# Stale duplicate report", - reporter: { - id: "pp-readout", - name: "Submission Readout", - gitHash: "abc", - model: "gpt-5.4", - agentId: "copilot", - agentVersion: "1.0.0", - }, - createdAt: "2026-01-01T12:00:00.000Z", - }, - { - _id: "report-template-current", - id: "report-template-current", - submissionId: "submission-1", - submissionRunId: "submission-run-2", - templateId: "executive-summary", - status: "completed", - content: "# Executive summary report", - createdAt: "2026-01-02T01:00:00.000Z", - }, - { - _id: "report-historical", - id: "report-historical", - submissionId: "submission-1", - submissionRunId: "submission-run-1", - status: "completed", - content: "# Historical generated report", - reporter: { - id: "pp-readout", - name: "Submission Readout", - gitHash: "abc", - model: "gpt-5.4", - agentId: "copilot", - agentVersion: "1.0.0", - }, - createdAt: "2026-01-01T00:00:00.000Z", - }, -]; - -vi.mock("@/lib/api", () => ({ - api: { - getSubmission: vi.fn(async () => submission), - getSubmissionRuns: vi.fn(async () => ({ - runs: [currentAttempt, historicalAttempt], - })), - getSubmissionReports: vi.fn(async () => reports), - retrySubmission: vi.fn(), - reportStructuredUrl: vi.fn( - (id: string) => `/api/v1/reports/${id}/structured`, - ), - submissionRunLogsUrl: vi.fn( - (_id: string, attemptId: string) => - `/api/v1/submissions/submission-1/runs/${attemptId}/logs`, - ), - }, -})); - -vi.mock("@/hooks/use-log-stream", () => ({ - useLogStream: vi.fn(() => ({ - logs: [], - isConnected: false, - isDone: true, - error: null, - clear: vi.fn(), - })), -})); - -import { SubmissionDetail } from "./SubmissionDetail"; -import { api } from "@/lib/api"; - -afterEach(() => { - cleanup(); - vi.clearAllMocks(); -}); - -function renderDetail(initialEntry = "/submissions/submission-1") { - const queryClient = new QueryClient({ - defaultOptions: { queries: { retry: false } }, - }); - return render( - - - - } /> - Run detail destination
} /> - - - , - ); -} - -describe("SubmissionDetail", () => { - it("defaults to Reports and shows attempt-scoped report types", async () => { - renderDetail(); - - expect( - await screen.findByRole("heading", { name: "Current generated report" }), - ).toBeDefined(); - expect( - screen.getByRole("tab", { name: "Reports (2)" }).getAttribute("data-state"), - ).toBe("active"); - expect( - screen.getByRole("tab", { name: "Submission Readout" }), - ).toBeDefined(); - expect( - screen.getByRole("tab", { name: "Executive Summary" }), - ).toBeDefined(); - expect(screen.getByText("Current")).toBeDefined(); - expect( - screen.getByRole("link", { name: "Structured JSON" }).getAttribute("href"), - ).toBe("/api/v1/reports/report-current/structured"); - - await userEvent.click( - screen.getByRole("tab", { name: "Executive Summary" }), - ); - expect( - await screen.findByRole("heading", { name: "Executive summary report" }), - ).toBeDefined(); - expect(screen.getByText("This attempt")).toBeDefined(); - }); - - it("selects the newest report when one attempt has duplicate report types", async () => { - renderDetail(); - - expect( - await screen.findByRole("heading", { name: "Current generated report" }), - ).toBeDefined(); - expect( - screen.getByRole("link", { name: "Structured JSON" }).getAttribute("href"), - ).toBe("/api/v1/reports/report-current/structured"); - }); - - it("renders the immutable member vector with the shared table and run navigation", async () => { - renderDetail(); - await screen.findByRole("heading", { name: "Current generated report" }); - - await userEvent.click(screen.getByRole("tab", { name: "Runs (2)" })); - - expect( - screen.getByRole("heading", { name: "Immutable member run vector" }), - ).toBeDefined(); - expect(screen.getByRole("columnheader", { name: "Request" })).toBeDefined(); - expect( - screen.getByRole("columnheader", { name: "Exact run attempt" }), - ).toBeDefined(); - expect( - screen.getByRole("columnheader", { name: "Dependencies" }), - ).toBeDefined(); - const requestLink = screen - .getAllByRole("link") - .find((link) => link.getAttribute("href")?.includes("run-current-1")); - expect(requestLink).toBeDefined(); - expect(requestLink!.getAttribute("href")).toContain( - "runId=run-current-1", - ); - expect(requestLink!.getAttribute("href")).toContain( - "fromSubmission=submission-1", - ); - }); - - it("switches attempts through the picker and scopes Processing to history", async () => { - renderDetail("/submissions/submission-1?tab=processing"); - await screen.findByText("Report ready"); - - const picker = screen.getByRole("combobox", { name: "Submission attempt" }); - await userEvent.click(picker); - const listbox = await screen.findByRole("listbox"); - await userEvent.click( - within(listbox).getByText(/Attempt 1 · Superseded/), - ); - - expect( - await screen.findByText(/Viewing attempt 1 \(historical and read-only\)/), - ).toBeDefined(); - expect(screen.getByText("Dependencies ready")).toBeDefined(); - expect(screen.getByText("Observations 1/1")).toBeDefined(); - expect(screen.getByText("Security Scan 1/1")).toBeDefined(); - expect( - screen.getByRole("heading", { name: "Attempt processing logs" }), - ).toBeDefined(); - }); - - it("scopes report types to the selected historical attempt", async () => { - renderDetail(); - await screen.findByRole("heading", { name: "Current generated report" }); - - const picker = screen.getByRole("combobox", { name: "Submission attempt" }); - await userEvent.click(picker); - const listbox = await screen.findByRole("listbox"); - await userEvent.click(within(listbox).getByText(/Attempt 1 · Superseded/)); - - expect( - await screen.findByRole("heading", { - name: "Historical generated report", - }), - ).toBeDefined(); - expect( - screen.getByRole("tab", { name: "Reports (1)" }), - ).toBeDefined(); - expect( - screen.getByRole("tab", { name: "Submission Readout" }), - ).toBeDefined(); - expect(screen.getByText("Historical")).toBeDefined(); - }); - - it("renders one report-type tab for a single template-less report", async () => { - vi.mocked(api.getSubmissionReports).mockResolvedValueOnce([reports[0]!]); - - renderDetail(); - - expect( - await screen.findByRole("heading", { name: "Current generated report" }), - ).toBeDefined(); - expect( - screen.getByRole("tab", { name: "Reports (1)" }), - ).toBeDefined(); - expect( - screen.getByRole("tab", { name: "Submission Readout" }), - ).toBeDefined(); - }); - - it("shows current attempt progress when only historical reports exist", async () => { - vi.mocked(api.getSubmission).mockResolvedValueOnce({ - ...submission, - readiness: "processing", - report: undefined, - handlerProgress: { - "pp-readout": { status: "processing", version: 1 }, - }, - }); - vi.mocked(api.getSubmissionRuns).mockResolvedValueOnce({ - runs: [ - { - ...currentAttempt, - report: undefined, - handlerStatus: { - "pp-readout": { status: "processing", version: 1 }, - }, - }, - historicalAttempt, - ], - }); - vi.mocked(api.getSubmissionReports).mockResolvedValueOnce([reports[3]!]); - - renderDetail(); - - expect( - await screen.findByRole("tab", { name: "Reports" }), - ).toBeDefined(); - expect(screen.getByText("Report not available")).toBeDefined(); - expect( - screen.getByText("A report is being generated for this attempt."), - ).toBeDefined(); - }); -}); diff --git a/apps/portal/src/types.ts b/apps/portal/src/types.ts index d7f10794a..3ec950d5c 100644 --- a/apps/portal/src/types.ts +++ b/apps/portal/src/types.ts @@ -504,8 +504,8 @@ export interface Report { _id: string; id: string; requestId?: string; - submissionId?: string; - submissionRunId?: string; + experimentId?: string; + experimentExecutionId?: string; task?: string; reporter?: Reporter; content?: string; diff --git a/apps/workers/readout/handler.yaml b/apps/workers/readout/handler.yaml index 967490565..0c15c5000 100644 --- a/apps/workers/readout/handler.yaml +++ b/apps/workers/readout/handler.yaml @@ -1,8 +1,8 @@ -# Submission handler registration metadata. +# Experiment handler registration metadata. # Read by the generic register-handler script and POSTed to the # scheduler's POST /handlers/register endpoint on deploy. _id: pp-readout -type: submission-handler +type: experiment-handler version: 1 queue: pp-readout-queue selector: readout diff --git a/apps/workers/readout/src/index.ts b/apps/workers/readout/src/index.ts index e9ed516eb..d96b3221f 100644 --- a/apps/workers/readout/src/index.ts +++ b/apps/workers/readout/src/index.ts @@ -13,7 +13,7 @@ const config: ReadoutQueueProcessorConfig = { process.env.AZURE_COSMOS_CONNECTION_STRING || "mongodb://localhost:27017", mongoDatabase: process.env.MONGO_DATABASE || "requests-db", - mongoCollection: "submissions", + mongoCollection: "experiments", storageAccountName: process.env.AZURE_STORAGE_ACCOUNT_NAME || "devstoreaccount1", storageConnectionString: diff --git a/apps/workers/readout/src/prompt.ts b/apps/workers/readout/src/prompt.ts index 0841817b8..d60932615 100644 --- a/apps/workers/readout/src/prompt.ts +++ b/apps/workers/readout/src/prompt.ts @@ -1,9 +1,9 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import type { SubmissionEvidence } from "./readout-builder.js"; +import type { ExperimentEvidence } from "./readout-builder.js"; -export const READOUT_SYSTEM_PROMPT = `You synthesize a submission-level benchmark readout from an exact immutable vector of Scope runs. +export const READOUT_SYSTEM_PROMPT = `You synthesize an Experiment benchmark readout from an exact immutable execution vector of Scope runs. The user message contains JSON evidence for every request/run pair in that vector: request metadata, coding outcome, profile metadata, criteria definitions, run-level and per-iteration observations, and ATIF trajectories. @@ -55,9 +55,9 @@ Rules: - Ground every finding in at least one evidence reference. - Return no Markdown fences or commentary.`; -export function composeReadoutUserPrompt(evidence: SubmissionEvidence): string { +export function composeReadoutUserPrompt(evidence: ExperimentEvidence): string { return [ - "Generate the submission readout synthesis for this exact immutable evidence vector.", + "Generate the Experiment readout synthesis for this exact immutable execution evidence vector.", JSON.stringify(evidence), ].join("\n\n"); } diff --git a/apps/workers/readout/src/readout-builder.test.ts b/apps/workers/readout/src/readout-builder.test.ts index fa377becb..35379012b 100644 --- a/apps/workers/readout/src/readout-builder.test.ts +++ b/apps/workers/readout/src/readout-builder.test.ts @@ -8,12 +8,12 @@ import type { RunState, } from "shared"; import { - composeSubmissionReadout, - renderSubmissionReadoutMarkdown, - type SubmissionEvidence, + composeExperimentReadout, + renderExperimentReadoutMarkdown, + type ExperimentEvidence, } from "./readout-builder.js"; -function evidence(): SubmissionEvidence { +function evidence(): ExperimentEvidence { const criterion: CriteriaDocument = { id: "uses-current-api", projectId: "project-1", @@ -60,16 +60,16 @@ function evidence(): SubmissionEvidence { }], } satisfies RunState; return { - submissionId: "submission-1", - submissionRunId: "submission-run-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-1", runs: [{ request, run, criteria: [criterion], trajectories: [] }], }; } -describe("submission readout composition", () => { +describe("experiment readout composition", () => { it("preserves mixed run/iteration observations, profiles, and provenance", () => { - const readout = composeSubmissionReadout(evidence(), { - title: "Submission readout", + const readout = composeExperimentReadout(evidence(), { + title: "Experiment readout", executiveSummary: "The run failed after mixed observation evidence.", profiles: [{ profileId: "profile-1", @@ -108,7 +108,7 @@ describe("submission readout composition", () => { expect.objectContaining({ subject: "iteration", passed: true }), ]); expect(readout.findings[0].evidence[0]).toMatchObject({ - submissionRunId: "submission-run-1", + experimentExecutionId: "execution-1", requestId: "request-1", runId: "run-1", }); @@ -120,14 +120,14 @@ describe("submission readout composition", () => { expect(readout.summary.failedRuns).toBe(1); expect(readout.summary.passedObservations).toBe(1); expect(readout.summary.failedObservations).toBe(1); - expect(renderSubmissionReadoutMarkdown(readout)).toContain( - "Schema: `scope.submission-readout@1.0`", + expect(renderExperimentReadoutMarkdown(readout)).toContain( + "Schema: `scope.experiment-readout@1.0`", ); }); it("rejects evidence outside the exact vector", () => { - expect(() => composeSubmissionReadout(evidence(), { - title: "Submission readout", + expect(() => composeExperimentReadout(evidence(), { + title: "Experiment readout", executiveSummary: "Invalid evidence.", profiles: [{ requestIds: ["request-1"], summary: "One run." }], runs: [{ @@ -151,8 +151,8 @@ describe("submission readout composition", () => { }); it("rejects findings that cite a missing iteration", () => { - expect(() => composeSubmissionReadout(evidence(), { - title: "Submission readout", + expect(() => composeExperimentReadout(evidence(), { + title: "Experiment readout", executiveSummary: "Invalid iteration evidence.", profiles: [], runs: [], @@ -173,8 +173,8 @@ describe("submission readout composition", () => { }); it("canonically retains every exact-vector profile when synthesis omits it", () => { - const readout = composeSubmissionReadout(evidence(), { - title: "Submission readout", + const readout = composeExperimentReadout(evidence(), { + title: "Experiment readout", executiveSummary: "Profile synthesis was omitted.", profiles: [], runs: [], diff --git a/apps/workers/readout/src/readout-builder.ts b/apps/workers/readout/src/readout-builder.ts index 0143188b9..aab24a996 100644 --- a/apps/workers/readout/src/readout-builder.ts +++ b/apps/workers/readout/src/readout-builder.ts @@ -3,14 +3,14 @@ import { z } from "zod"; import { - SUBMISSION_READOUT_SCHEMA_ID, - SUBMISSION_READOUT_SCHEMA_VERSION, - submissionReadoutSchema, + EXPERIMENT_READOUT_SCHEMA_ID, + EXPERIMENT_READOUT_SCHEMA_VERSION, + experimentReadoutSchema, type CriteriaDocument, type CriterionResult, + type ExperimentReadout, type RequestDocument, type RunState, - type SubmissionReadout, } from "shared"; export interface ExactRunEvidence { @@ -20,9 +20,9 @@ export interface ExactRunEvidence { trajectories: Array<{ iteration: number; trajectory: unknown }>; } -export interface SubmissionEvidence { - submissionId: string; - submissionRunId: string; +export interface ExperimentEvidence { + experimentId: string; + experimentExecutionId: string; runs: ExactRunEvidence[]; } @@ -94,11 +94,11 @@ function observationsForRun(evidence: ExactRunEvidence) { ]; } -export function composeSubmissionReadout( - evidence: SubmissionEvidence, +export function composeExperimentReadout( + evidence: ExperimentEvidence, synthesis: ReadoutSynthesis, generatedAt = new Date(), -): SubmissionReadout { +): ExperimentReadout { const exactRuns = new Map( evidence.runs.map((entry) => [ `${entry.request._id}:${entry.run._id}`, @@ -188,13 +188,13 @@ export function composeSubmissionReadout( } } - return submissionReadoutSchema.parse({ + return experimentReadoutSchema.parse({ meta: { - schemaId: SUBMISSION_READOUT_SCHEMA_ID, - schemaVersion: SUBMISSION_READOUT_SCHEMA_VERSION, + schemaId: EXPERIMENT_READOUT_SCHEMA_ID, + schemaVersion: EXPERIMENT_READOUT_SCHEMA_VERSION, generatedAt: generatedAt.toISOString(), - submissionId: evidence.submissionId, - submissionRunId: evidence.submissionRunId, + experimentId: evidence.experimentId, + experimentExecutionId: evidence.experimentExecutionId, }, summary: { title: synthesis.title, @@ -210,7 +210,7 @@ export function composeSubmissionReadout( findings: synthesis.findings.map((finding) => ({ ...finding, evidence: finding.evidence.map((reference) => ({ - submissionRunId: evidence.submissionRunId, + experimentExecutionId: evidence.experimentExecutionId, ...reference, })), })), @@ -218,7 +218,7 @@ export function composeSubmissionReadout( }); } -export function renderSubmissionReadoutMarkdown(readout: SubmissionReadout): string { +export function renderExperimentReadoutMarkdown(readout: ExperimentReadout): string { const lines = [ `# ${readout.summary.title}`, "", @@ -297,7 +297,7 @@ export function renderSubmissionReadoutMarkdown(readout: SubmissionReadout): str lines.push( "---", `Schema: \`${readout.meta.schemaId}@${readout.meta.schemaVersion}\``, - `Submission run: \`${readout.meta.submissionRunId}\``, + `Experiment execution: \`${readout.meta.experimentExecutionId}\``, ); return lines.join("\n"); } diff --git a/apps/workers/readout/src/readout-generator.ts b/apps/workers/readout/src/readout-generator.ts index c30f2d76b..6ca486d2a 100644 --- a/apps/workers/readout/src/readout-generator.ts +++ b/apps/workers/readout/src/readout-generator.ts @@ -4,9 +4,9 @@ import { CopilotClient, type PermissionHandler, SessionEvent } from "@github/copilot-sdk"; import type { LogEvent } from "shared"; import { - composeSubmissionReadout, + composeExperimentReadout, readoutSynthesisSchema, - type SubmissionEvidence, + type ExperimentEvidence, } from "./readout-builder.js"; import { READOUT_SYSTEM_PROMPT, @@ -24,8 +24,8 @@ type ReadoutLog = ( data?: Record, ) => Promise | void; -export async function generateSubmissionReadout(input: { - evidence: SubmissionEvidence; +export async function generateExperimentReadout(input: { + evidence: ExperimentEvidence; model: string; githubToken: string; timeoutMs: number; @@ -59,7 +59,7 @@ export async function generateSubmissionReadout(input: { for (let attempt = 1; attempt <= 3; attempt += 1) { streamedResponse = ""; latestMessage = ""; - await input.log("info", `Generating submission readout (${attempt}/3)`); + await input.log("info", `Generating Experiment readout (${attempt}/3)`); const response = await session.sendAndWait({ prompt }, input.timeoutMs); const text = response?.data.content?.trim() || @@ -79,7 +79,7 @@ export async function generateSubmissionReadout(input: { } const result = readoutSynthesisSchema.safeParse(parsed); if (result.success) { - return composeSubmissionReadout(input.evidence, result.data); + return composeExperimentReadout(input.evidence, result.data); } if (attempt === 3) { throw new Error( diff --git a/apps/workers/readout/src/readout-queue-processor.test.ts b/apps/workers/readout/src/readout-queue-processor.test.ts index ee5c1ab91..7f2b4194e 100644 --- a/apps/workers/readout/src/readout-queue-processor.test.ts +++ b/apps/workers/readout/src/readout-queue-processor.test.ts @@ -3,64 +3,94 @@ import type { DequeuedMessageItem } from "@azure/storage-queue"; import { beforeEach, describe, expect, it, vi } from "vitest"; -import type { - CriteriaDocument, - SubmissionDocument, - VisibilityHeartbeat, +import { + EXPERIMENT_STORAGE_PROFILE_VERSION, + ExperimentStore, + type CriteriaDocument, + type ExperimentDocument, + type RequestDocument, + type RunHistoryDocument, + type VisibilityHeartbeat, } from "shared"; -import type { RequestDocument, RunHistoryDocument } from "shared"; -const coordination = vi.hoisted(() => ({ - claim: vi.fn(), - complete: vi.fn(), - isCurrent: vi.fn(), -})); const generate = vi.hoisted(() => vi.fn()); - -vi.mock("shared", async (importOriginal) => ({ - ...(await importOriginal()), - claimSubmissionHandler: coordination.claim, - completeSubmissionHandler: coordination.complete, - isCurrentSubmissionRun: coordination.isCurrent, -})); vi.mock("./readout-generator.js", () => ({ - generateSubmissionReadout: generate, + generateExperimentReadout: generate, })); import { ReadoutQueueProcessor } from "./readout-queue-processor.js"; -const submission: SubmissionDocument = { - _id: "submission-1", - submissionId: "submission-1", +const now = new Date("2026-10-13T12:00:00.000Z"); +const experiment: ExperimentDocument = { + _id: "experiment-1", + experimentId: "experiment-1", projectId: "project-1", - requestIds: ["request-1"], - run: { - _id: "submission-run-1", + metadata: { name: "Experiment" }, + storageProfileVersion: EXPERIMENT_STORAGE_PROFILE_VERSION, + stateVersion: 1, + revision: { + experimentRevisionId: "revision-1", + revisionNumber: 1, + memberCount: 1, + membershipFingerprint: "membership", + definitionFingerprint: "definition", + createdAt: now, + }, + execution: { + experimentExecutionId: "execution-1", attemptNumber: 1, - vectorFingerprint: "request-1:run-1", - requests: [{ requestId: "request-1", runId: "run-1" }], - dependencyProjection: [], - handlerStatus: { + version: 2, + requests: { + storage: "inline", + entryCount: 1, + payloadFingerprint: "payload", + payloadVersion: 1, + entries: [{ + requestId: "request-1", + runId: "run-1", + dependencies: [{ handlerId: "pp-observations", state: "ready" }], + blockers: [], + }], + }, + vectorFingerprint: "vector", + handlerSummary: { "pp-readout": { - version: 1, status: "processing", - updatedAt: new Date("2026-10-13T12:00:00.000Z"), + version: 1, + claimToken: "claim-1", + updatedAt: now, }, }, blocking: [], - createdAt: new Date("2026-10-13T12:00:00.000Z"), - startedAt: new Date("2026-10-13T12:00:00.000Z"), - updatedAt: new Date("2026-10-13T12:00:00.000Z"), + createdAt: now, + updatedAt: now, }, - createdAt: new Date("2026-10-13T12:00:00.000Z"), - updatedAt: new Date("2026-10-13T12:00:00.000Z"), + createdAt: now, + updatedAt: now, }; +const payload = { + type: "readout", + projectId: experiment.projectId, + experimentId: experiment.experimentId, + experimentExecutionId: experiment.execution.experimentExecutionId, + handlerId: "pp-readout", + handlerVersion: 1, + claimToken: "claim-1", + executionVersion: 1, +}; +type ReadoutPayload = typeof payload; + +const message = { messageId: "message-1" } as DequeuedMessageItem; +const heartbeat = { + stop: vi.fn().mockResolvedValue(undefined), +} as unknown as VisibilityHeartbeat; + function processor() { const instance = new ReadoutQueueProcessor({ mongoUri: "mongodb://localhost:27017", mongoDatabase: "scope", - mongoCollection: "submissions", + mongoCollection: "experiments", storageAccountName: "devstoreaccount1", storageConnectionString: "UseDevelopmentStorage=true", queueName: "pp-readout-queue", @@ -73,6 +103,10 @@ function processor() { sessionTimeoutMs: 1_000, }); const insertOne = vi.fn().mockResolvedValue({ acknowledged: true }); + const uploadJson = vi.fn().mockResolvedValue( + "https://storage.example/reports/report/structured.json", + ); + const safeDeleteMessage = vi.fn().mockResolvedValue(undefined); Object.assign(instance as object, { collection: {}, db: { @@ -81,67 +115,51 @@ function processor() { ), }, loadEvidence: vi.fn().mockResolvedValue({ - submissionId: submission._id, - submissionRunId: submission.run._id, + experimentId: experiment.experimentId, + experimentExecutionId: experiment.execution.experimentExecutionId, runs: [], }), tokenClient: { acquireToken: vi.fn().mockResolvedValue("token"), }, - blobStorage: { - uploadJson: vi.fn().mockImplementation((path: string) => - Promise.resolve(`https://storage.example/${path}`) - ), - }, - safeDeleteMessage: vi.fn().mockResolvedValue(undefined), + blobStorage: { uploadJson }, + safeDeleteMessage, }); - return { instance, insertOne }; + return { instance, insertOne, uploadJson, safeDeleteMessage }; } -const message = { messageId: "message-1" } as DequeuedMessageItem; -const heartbeat = { - stop: vi.fn().mockResolvedValue(undefined), -} as unknown as VisibilityHeartbeat; -const payload = { - type: "readout", - submissionId: submission._id, - submissionRunId: submission.run._id, -}; -type LogFn = ReturnType; -type ReadoutPayload = typeof payload; - -async function handle(instance: ReadoutQueueProcessor, log: LogFn = vi.fn()) { +async function handle(instance: ReadoutQueueProcessor, log = vi.fn()) { await (instance as unknown as { handleRequest( - submission: SubmissionDocument, + experiment: ExperimentDocument, message: DequeuedMessageItem, heartbeat: VisibilityHeartbeat, - log: LogFn, + log: ReturnType, payload: ReadoutPayload, ): Promise; - }).handleRequest(submission, message, heartbeat, log, payload); + }).handleRequest(experiment, message, heartbeat, log, payload); } describe("readout queue orchestration", () => { beforeEach(() => { - vi.clearAllMocks(); - coordination.isCurrent.mockResolvedValue(true); + vi.restoreAllMocks(); + generate.mockReset(); generate.mockResolvedValue({ meta: { - schemaId: "scope.submission-readout", + schemaId: "scope.experiment-readout", schemaVersion: "1.0", - generatedAt: "2026-10-13T12:00:00.000Z", - submissionId: submission._id, - submissionRunId: submission.run._id, + generatedAt: now.toISOString(), + experimentId: experiment.experimentId, + experimentExecutionId: experiment.execution.experimentExecutionId, }, summary: { - title: "Submission readout", + title: "Experiment readout", executiveSummary: "Summary", - totalRuns: 0, + totalRuns: 1, succeededRuns: 0, - failedRuns: 0, + failedRuns: 1, passedObservations: 0, - failedObservations: 0, + failedObservations: 1, }, profiles: [], runs: [], @@ -150,78 +168,86 @@ describe("readout queue orchestration", () => { }); }); - it("discards stale or duplicate exact-run messages before generation", async () => { - coordination.claim.mockResolvedValue(null); - const { instance, insertOne } = processor(); + it("discards stale or duplicate exact-execution messages before generation", async () => { + vi.spyOn(ExperimentStore.prototype, "claimHandler").mockResolvedValue(null); + const { instance, insertOne, safeDeleteMessage } = processor(); await handle(instance); expect(generate).not.toHaveBeenCalled(); expect(insertOne).not.toHaveBeenCalled(); - expect((instance as unknown as { safeDeleteMessage: ReturnType }) - .safeDeleteMessage).toHaveBeenCalledOnce(); + expect(safeDeleteMessage).toHaveBeenCalledOnce(); + }); + + it("publishes the report reference with final handler completion in one CAS", async () => { + vi.spyOn(ExperimentStore.prototype, "claimHandler").mockResolvedValue(experiment); + vi.spyOn(ExperimentStore.prototype, "get").mockResolvedValue(experiment); + const complete = vi + .spyOn(ExperimentStore.prototype, "completeHandler") + .mockResolvedValue(experiment); + const { instance, insertOne } = processor(); + + await handle(instance); + + expect(insertOne).toHaveBeenCalledWith(expect.objectContaining({ + experimentId: experiment.experimentId, + experimentExecutionId: experiment.execution.experimentExecutionId, + status: "completed", + })); + expect(complete).toHaveBeenCalledOnce(); + expect(complete).toHaveBeenCalledWith(expect.objectContaining({ + expectedExecutionVersion: experiment.execution.version, + status: "done", + report: expect.objectContaining({ + reportId: expect.any(String), + }), + })); }); - it("keeps superseded completions historical and uses new report ids on retry", async () => { - coordination.claim.mockResolvedValue(submission); - coordination.complete.mockResolvedValue(false); + it("keeps an inserted report historical when work is superseded in flight", async () => { + vi.spyOn(ExperimentStore.prototype, "claimHandler").mockResolvedValue(experiment); + vi.spyOn(ExperimentStore.prototype, "get") + .mockResolvedValueOnce(experiment) + .mockResolvedValueOnce(experiment) + .mockResolvedValueOnce(experiment) + .mockResolvedValueOnce(experiment) + .mockResolvedValueOnce(experiment) + .mockResolvedValueOnce(null); + const complete = vi.spyOn(ExperimentStore.prototype, "completeHandler"); const { instance, insertOne } = processor(); const log = vi.fn().mockResolvedValue(undefined); - await handle(instance, log); await handle(instance, log); - expect(insertOne).toHaveBeenCalledTimes(2); - const reports = insertOne.mock.calls.map(([report]) => report); - expect(reports[0]._id).not.toBe(reports[1]._id); - expect(reports).toEqual([ - expect.objectContaining({ - submissionId: submission._id, - submissionRunId: submission.run._id, - }), - expect.objectContaining({ - submissionId: submission._id, - submissionRunId: submission.run._id, - }), - ]); - expect(coordination.complete).toHaveBeenCalledTimes(2); + expect(insertOne).toHaveBeenCalledOnce(); + expect(complete).not.toHaveBeenCalled(); expect(log).toHaveBeenCalledWith( "warn", "Readout report remained historical after supersession", expect.objectContaining({ - submissionId: submission._id, - submissionRunId: submission.run._id, + experimentId: experiment.experimentId, + experimentExecutionId: experiment.execution.experimentExecutionId, }), ); }); - it("does not publish when the submission is superseded during upload", async () => { - coordination.claim.mockResolvedValue(submission); - coordination.isCurrent - .mockResolvedValueOnce(true) - .mockResolvedValueOnce(true) - .mockResolvedValueOnce(true) - .mockResolvedValueOnce(true) - .mockResolvedValueOnce(false); - const { instance, insertOne } = processor(); - - await handle(instance); - - expect(generate).toHaveBeenCalledOnce(); - expect(insertOne).not.toHaveBeenCalled(); - expect(coordination.complete).not.toHaveBeenCalled(); - }); - - it("loads the archived run named by the immutable exact vector", async () => { + it("loads the archived run named by the immutable exact execution vector", async () => { const { instance } = processor(); delete (instance as unknown as Record).loadEvidence; - const currentRequest = { + vi.spyOn(ExperimentStore.prototype, "readExecutionRequests") + .mockResolvedValue([{ + requestId: "request-1", + runId: "run-1", + dependencies: [{ handlerId: "pp-observations", state: "ready" }], + blockers: [], + }]); + const request = { _id: "request-1", projectId: "project-1", workerType: "coder-acp-copilot", observations: ["observation-criterion"], scenario: { task: "Task", criteria: ["task-criterion"] }, - createdAt: new Date("2026-10-13T12:00:00.000Z"), + createdAt: now, priority: 0, run: { _id: "run-newer", @@ -244,27 +270,27 @@ describe("readout queue orchestration", () => { feedback: "Archived evidence", }], } satisfies RunHistoryDocument; - const historyFindOne = vi.fn().mockResolvedValue(archivedRun); const criteria = [ { id: "task-criterion", prompt: "Task criterion", projectId: "project-1", - createdAt: new Date("2026-10-13T12:00:00.000Z"), + createdAt: now, }, { id: "observation-criterion", prompt: "Configured observation", projectId: "project-1", - createdAt: new Date("2026-10-13T12:00:00.000Z"), + createdAt: now, }, { id: "result-criterion", prompt: "Archived observation result", projectId: "project-1", - createdAt: new Date("2026-10-13T12:00:00.000Z"), + createdAt: now, }, ] satisfies CriteriaDocument[]; + const historyFindOne = vi.fn().mockResolvedValue(archivedRun); const criteriaFind = vi.fn(() => ({ toArray: vi.fn().mockResolvedValue(criteria), })); @@ -274,48 +300,33 @@ describe("readout queue orchestration", () => { if (name === "requests") { return { find: vi.fn(() => ({ - toArray: vi.fn().mockResolvedValue([currentRequest]), + toArray: vi.fn().mockResolvedValue([request]), })), }; } if (name === "runs") return { findOne: historyFindOne }; - if (name === "criteria") { - return { find: criteriaFind }; - } + if (name === "criteria") return { find: criteriaFind }; return {}; }), }, }); const evidence = await (instance as unknown as { - loadEvidence(value: SubmissionDocument): Promise<{ - runs: Array<{ - run: RunHistoryDocument | RequestDocument["run"]; - criteria: CriteriaDocument[]; - }>; + loadEvidence(value: typeof payload): Promise<{ + runs: Array<{ run: RunHistoryDocument }>; }>; - }).loadEvidence(submission); + }).loadEvidence(payload); expect(historyFindOne).toHaveBeenCalledWith({ _id: "run-1", requestId: "request-1", projectId: "project-1", }); - expect(evidence.runs[0].run?._id).toBe("run-1"); - expect(evidence.runs[0].criteria.map((criterion) => criterion.id)).toEqual([ - "task-criterion", - "observation-criterion", - "result-criterion", - ]); - expect(criteriaFind).toHaveBeenCalledWith({ - projectId: "project-1", - id: { - $in: expect.arrayContaining([ - "task-criterion", - "observation-criterion", - "result-criterion", - ]), - }, + expect(evidence.runs[0]?.run._id).toBe("run-1"); + expect(evidence.runs[0]?.run.outcome).toBe("failed"); + expect(evidence.runs[0]?.run.observationResults?.[0]).toMatchObject({ + passed: false, + feedback: "Archived evidence", }); }); }); diff --git a/apps/workers/readout/src/readout-queue-processor.ts b/apps/workers/readout/src/readout-queue-processor.ts index a33d260e3..9b73497d4 100644 --- a/apps/workers/readout/src/readout-queue-processor.ts +++ b/apps/workers/readout/src/readout-queue-processor.ts @@ -8,27 +8,27 @@ import { BlobStorage, type BaseQueueProcessorConfig, type CriteriaDocument, + EXPERIMENT_READOUT_SCHEMA_ID, + EXPERIMENT_READOUT_SCHEMA_VERSION, + type ExperimentDocument, + type ExperimentExecutionHistoryDocument, + type ExperimentQueueMessagePayload, + ExperimentStore, type LogEvent, type ReportDocument, type RequestDocument, type RunHistoryDocument, - type SubmissionDocument, - type SubmissionQueueMessagePayload, + type RunState, TokenManagerClient, - claimSubmissionHandler, - completeSubmissionHandler, - isCurrentSubmissionRun, - SUBMISSION_READOUT_SCHEMA_ID, - SUBMISSION_READOUT_SCHEMA_VERSION, type VisibilityHeartbeat, withRetry, } from "shared"; import { - renderSubmissionReadoutMarkdown, + renderExperimentReadoutMarkdown, type ExactRunEvidence, - type SubmissionEvidence, + type ExperimentEvidence, } from "./readout-builder.js"; -import { generateSubmissionReadout } from "./readout-generator.js"; +import { generateExperimentReadout } from "./readout-generator.js"; const HANDLER_ID = "pp-readout"; const HANDLER_VERSION = 1; @@ -39,15 +39,29 @@ export interface ReadoutQueueProcessorConfig extends BaseQueueProcessorConfig { tokenManagerUrl?: string; } +interface ReadoutQueueMessage extends ExperimentQueueMessagePayload { + type: "readout"; + projectId: string; + handlerId: string; + handlerVersion: number; + claimToken: string; + executionVersion: number; +} + function isReadoutMessage( payload: Record | undefined, -): payload is Record & SubmissionQueueMessagePayload { +): payload is Record & ReadoutQueueMessage { return payload?.type === "readout" && - typeof payload.submissionId === "string" && - typeof payload.submissionRunId === "string"; + typeof payload.projectId === "string" && + typeof payload.experimentId === "string" && + typeof payload.experimentExecutionId === "string" && + payload.handlerId === HANDLER_ID && + payload.handlerVersion === HANDLER_VERSION && + typeof payload.claimToken === "string" && + typeof payload.executionVersion === "number"; } -export class ReadoutQueueProcessor extends BaseQueueProcessor { +export class ReadoutQueueProcessor extends BaseQueueProcessor { private readonly readoutConfig: ReadoutQueueProcessorConfig; private readonly tokenClient: TokenManagerClient; private readonly blobStorage: BlobStorage; @@ -63,11 +77,11 @@ export class ReadoutQueueProcessor extends BaseQueueProcessor): string { - return typeof payload.submissionId === "string" ? payload.submissionId : ""; + return typeof payload.experimentId === "string" ? payload.experimentId : ""; } protected override async handleRequest( - _submission: SubmissionDocument, + _experiment: ExperimentDocument, message: DequeuedMessageItem, heartbeat: VisibilityHeartbeat, log: ( @@ -78,63 +92,73 @@ export class ReadoutQueueProcessor extends BaseQueueProcessor, ): Promise { if (!isReadoutMessage(payload)) { - await log("warn", "Discarding unsupported readout queue payload", { payload }); + await log("warn", "Discarding unsupported readout queue payload", { + payload, + }); await this.safeDeleteMessage(message.messageId, heartbeat.stop()); return; } - const { submissionId, submissionRunId } = payload; + + const store = this.experimentStore(); const claimed = await withRetry(() => - claimSubmissionHandler({ - submissions: this.collection, - submissionId, - submissionRunId, - handlerId: HANDLER_ID, - version: HANDLER_VERSION, + store.claimHandler({ + projectId: payload.projectId, + experimentId: payload.experimentId, + experimentExecutionId: payload.experimentExecutionId, + expectedExecutionVersion: payload.executionVersion, + handlerId: payload.handlerId, + handlerVersion: payload.handlerVersion, + claimToken: payload.claimToken, }) ); if (!claimed) { await log("info", "Discarding stale or duplicate readout message", { - submissionId, - submissionRunId, + experimentId: payload.experimentId, + experimentExecutionId: payload.experimentExecutionId, }); await this.safeDeleteMessage(message.messageId, heartbeat.stop()); return; } try { - if (!(await this.isCurrent(submissionId, submissionRunId))) return; - const evidence = await this.loadEvidence(claimed); + let completionVersion = await this.currentClaimVersion(payload); + if (completionVersion === null) return; + const evidence = await this.loadEvidence(payload); - if (!(await this.isCurrent(submissionId, submissionRunId))) return; + completionVersion = await this.currentClaimVersion(payload); + if (completionVersion === null) return; const githubToken = await this.tokenClient.acquireToken("copilot-sdk"); - if (!(await this.isCurrent(submissionId, submissionRunId))) return; - const readout = await generateSubmissionReadout({ + completionVersion = await this.currentClaimVersion(payload); + if (completionVersion === null) return; + const readout = await generateExperimentReadout({ evidence, model: this.readoutConfig.model, githubToken, timeoutMs: this.readoutConfig.sessionTimeoutMs, log, }); - const markdown = renderSubmissionReadoutMarkdown(readout); + const markdown = renderExperimentReadoutMarkdown(readout); - if (!(await this.isCurrent(submissionId, submissionRunId))) return; + completionVersion = await this.currentClaimVersion(payload); + if (completionVersion === null) return; const reportId = randomUUID(); const structuredUrl = await this.blobStorage.uploadJson( `reports/${reportId}/structured.json`, readout, ); - if (!(await this.isCurrent(submissionId, submissionRunId))) return; + completionVersion = await this.currentClaimVersion(payload); + if (completionVersion === null) return; const now = new Date(); const report: ReportDocument = { _id: reportId, - projectId: claimed.projectId, - submissionId, - submissionRunId, + projectId: payload.projectId, + experimentId: payload.experimentId, + experimentExecutionId: payload.experimentExecutionId, reporter: { id: HANDLER_ID, - name: "Submission Readout", + name: "Experiment Readout", gitHash: process.env.GIT_COMMIT || "unknown", model: this.readoutConfig.model, agentId: "copilot-sdk", @@ -142,8 +166,8 @@ export class ReadoutQueueProcessor extends BaseQueueProcessor("reports").insertOne(report) ); - const completed = await withRetry(() => - completeSubmissionHandler({ - submissions: this.collection, - submissionId, - submissionRunId, - handlerId: HANDLER_ID, - version: HANDLER_VERSION, - status: "done", - report: { - reportId, - url: `/api/v1/reports/${reportId}`, - updatedAt: now, - }, - }) - ); + completionVersion = await this.currentClaimVersion(payload); + const completed = completionVersion === null + ? null + : await withRetry(() => + store.completeHandler({ + projectId: payload.projectId, + experimentId: payload.experimentId, + experimentExecutionId: payload.experimentExecutionId, + expectedExecutionVersion: completionVersion, + handlerId: payload.handlerId, + handlerVersion: payload.handlerVersion, + claimToken: payload.claimToken, + status: "done", + report: { + reportId, + url: `/api/v1/reports/${reportId}`, + updatedAt: now, + }, + }) + ); if (!completed) { await log("warn", "Readout report remained historical after supersession", { - submissionId, - submissionRunId, + experimentId: payload.experimentId, + experimentExecutionId: payload.experimentExecutionId, reportId, }); } else { - await log("info", "Submission readout completed", { - submissionId, - submissionRunId, + await log("info", "Experiment readout completed", { + experimentId: payload.experimentId, + experimentExecutionId: payload.experimentExecutionId, reportId, }); } } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error); - if (await this.isCurrent(submissionId, submissionRunId)) { + const completionVersion = await this.currentClaimVersion(payload); + if (completionVersion !== null) { await withRetry(() => - completeSubmissionHandler({ - submissions: this.collection, - submissionId, - submissionRunId, - handlerId: HANDLER_ID, - version: HANDLER_VERSION, + store.completeHandler({ + projectId: payload.projectId, + experimentId: payload.experimentId, + experimentExecutionId: payload.experimentExecutionId, + expectedExecutionVersion: completionVersion, + handlerId: payload.handlerId, + handlerVersion: payload.handlerVersion, + claimToken: payload.claimToken, status: "failed", error: errorMessage, }) ); } - await log("error", `Submission readout failed: ${errorMessage}`, { - submissionId, - submissionRunId, + await log("error", `Experiment readout failed: ${errorMessage}`, { + experimentId: payload.experimentId, + experimentExecutionId: payload.experimentExecutionId, }); } finally { await this.safeDeleteMessage(message.messageId, heartbeat.stop()); } } - private isCurrent( - submissionId: string, - submissionRunId: string, - ): Promise { - return withRetry(() => - isCurrentSubmissionRun({ - submissions: this.collection, - submissionId, - submissionRunId, - }) + private experimentStore(): ExperimentStore { + return new ExperimentStore({ + experiments: this.collection, + executions: this.db.collection( + "experiment-executions", + ), + chunks: this.db.collection("experiment-chunks"), + }); + } + + private async currentClaimVersion( + payload: ReadoutQueueMessage, + ): Promise { + const experiment = await withRetry(() => + this.experimentStore().get(payload.projectId, payload.experimentId) ); + const execution = experiment?.execution; + const handler = execution?.handlerSummary[payload.handlerId]; + return execution?.experimentExecutionId === payload.experimentExecutionId && + !execution.frozenAt && + !execution.rollover && + handler?.status === "processing" && + handler.version === payload.handlerVersion && + handler.claimToken === payload.claimToken + ? execution.version + : null; } private async loadEvidence( - submission: SubmissionDocument, - ): Promise { + payload: ReadoutQueueMessage, + ): Promise { + const refs = await withRetry(() => + this.experimentStore().readExecutionRequests({ + projectId: payload.projectId, + experimentId: payload.experimentId, + experimentExecutionId: payload.experimentExecutionId, + }) + ); + if (!refs) { + throw new Error( + `Experiment execution ${payload.experimentId}/${payload.experimentExecutionId} was not found`, + ); + } const requests = await withRetry(() => this.db.collection("requests").find({ - _id: { $in: submission.run.requests.map((ref) => ref.requestId) }, - projectId: submission.projectId, + _id: { $in: refs.map((ref) => ref.requestId) }, + projectId: payload.projectId, }).toArray() ); const requestById = new Map(requests.map((request) => [request._id, request])); const history = this.db.collection("runs"); - const exactRuns: Array<{ request: RequestDocument; run: RunHistoryDocument | RequestDocument["run"] }> = []; - for (const ref of submission.run.requests) { + const exactRuns: Array<{ request: RequestDocument; run: RunState }> = []; + for (const ref of refs) { const request = requestById.get(ref.requestId); - if (!request) throw new Error(`Exact-vector request ${ref.requestId} was not found`); + if (!request) { + throw new Error(`Exact-vector request ${ref.requestId} was not found`); + } const run = request.run?._id === ref.runId ? request.run - : await withRetry(() => history.findOne({ - _id: ref.runId, - requestId: ref.requestId, - projectId: submission.projectId, - })); + : await withRetry(() => + history.findOne({ + _id: ref.runId, + requestId: ref.requestId, + projectId: payload.projectId, + }) + ); if (!run) { - throw new Error(`Exact-vector run ${ref.requestId}/${ref.runId} was not found`); + throw new Error( + `Exact-vector run ${ref.requestId}/${ref.runId} was not found`, + ); } exactRuns.push({ request, run }); } @@ -253,8 +318,8 @@ export class ReadoutQueueProcessor extends BaseQueueProcessor [ ...request.scenario.criteria, ...(request.observations ?? []), - ...(run?.observationResults ?? []).map((result) => result.criterionId), - ...(run?.turns ?? []).flatMap((turn) => + ...(run.observationResults ?? []).map((result) => result.criterionId), + ...(run.turns ?? []).flatMap((turn) => (turn.observationResults ?? []).map((result) => result.criterionId) ), ]), @@ -264,15 +329,16 @@ export class ReadoutQueueProcessor extends BaseQueueProcessor this.db.collection("criteria").find({ - projectId: submission.projectId, + projectId: payload.projectId, id: { $in: criterionIds }, }).toArray() ); - const criteriaById = new Map(criteria.map((criterion) => [criterion.id, criterion])); + const criteriaById = new Map( + criteria.map((criterion) => [criterion.id, criterion]), + ); const runs: ExactRunEvidence[] = []; for (const { request, run } of exactRuns) { - if (!run) continue; const runCriterionIds = new Set([ ...request.scenario.criteria, ...(request.observations ?? []), @@ -284,13 +350,14 @@ export class ReadoutQueueProcessor extends BaseQueueProcessor turn.atifUrl - ? [withRetry(() => - this.blobStorage.downloadJson(turn.atifUrl!) - ).then((trajectory) => ({ - iteration: turn.iteration, - trajectory, - }))] - : [], + ? [ + withRetry(() => this.blobStorage.downloadJson(turn.atifUrl!)) + .then((trajectory) => ({ + iteration: turn.iteration, + trajectory, + })), + ] + : [] ), ); runs.push({ @@ -304,8 +371,8 @@ export class ReadoutQueueProcessor extends BaseQueueProcessor **Status:** PR A foundation and client contract, current as of October 2026. +> **Status:** PR A foundation plus the PR B built-in readout, current as of +> October 2026. Experiments are project-scoped, named benchmark compositions. An Experiment stores durable metadata, one immutable revision, and one current processing execution. Previous executions are immutable history records. PR A provides the canonical storage model, project-scoped public API contract, -CLI commands, Portal creation/list/detail flows, and run correlation. It does -not add mutable membership or cascading lifecycle behavior. +CLI commands, Portal creation/list/detail flows, and run correlation. PR B adds +one built-in `pp-readout` Experiment handler and generic Reports presentation. +Neither contribution adds mutable membership or cascading lifecycle behavior. ## Model @@ -221,6 +223,52 @@ through shared `ExperimentStore` compare-and-swap primitives. Storage layout, routing, fingerprinting, and state transitions remain reusable typed modules under `packages/shared/src/experiments`; they do not own workflow policy. +## Built-in readout + +The built-in `pp-readout` Experiment handler synthesizes one report for one +exact immutable Experiment execution. It depends on `pp-observations` in +`all-runs` mode. Every `{requestId, runId}` pair in the execution vector must +have that dependency at `done` before dispatch: + +- a terminal coding outcome of `failed` is still valid evidence; +- an observation result with `passed: false` is still valid evidence; +- a failed `pp-observations` handler or one blocked by a failed dependency + blocks readout dispatch; +- failures in unrelated handlers do not block the readout. + +Dispatch messages carry `projectId`, `experimentId`, +`experimentExecutionId`, `handlerId`, `handlerVersion`, `claimToken`, and the +expected execution version. The worker claims the handler through +`ExperimentStore`, reconstructs the exact inline or chunked vector with +`readExecutionRequests()`, and verifies the current claim before and after each +external step. A frozen, rolled-over, superseded, or otherwise stale execution +cannot publish itself as current. + +The worker validates a structured `scope.experiment-readout@1.0` object, derives +Markdown from that object, uploads the JSON to blob storage, and inserts an +append-only `reports` document. Final handler completion and publication of the +execution's report pointer occur in one compare-and-swap. If the execution is +superseded after report insertion, the document remains available as historical +evidence but the execution pointer is not published. + +```mermaid +sequenceDiagram + participant Coordinator + participant Queue + participant Worker as pp-readout + participant Store as ExperimentStore + participant Blob + participant Reports + + Coordinator->>Queue: exact execution message and claim token + Worker->>Store: claim handler by execution/version/token + Worker->>Store: read and verify exact request/run vector + Worker->>Worker: compose and validate structured readout + Worker->>Blob: upload structured JSON + Worker->>Reports: insert append-only report document + Worker->>Store: atomic CAS complete + publish report pointer +``` + ## Public API All routes require `?projectId=`: @@ -232,6 +280,9 @@ All routes require `?projectId=`: | `GET` | `/api/v1/experiments/:id` | Read the public Experiment and current execution | | `GET` | `/api/v1/experiments/:id/executions` | List current and archived execution summaries | | `GET` | `/api/v1/experiments/:id/executions/:experimentExecutionId` | Read one exact current or archived execution | +| `GET` | `/api/v1/experiments/:id/executions/:experimentExecutionId/reports` | List reports for one exact execution, newest first | +| `GET` | `/api/v1/experiments/:id/executions/:experimentExecutionId/reports/:reportId/structured` | Download that exact report's structured JSON artifact | +| `GET` | `/api/v1/experiments/:id/executions/:experimentExecutionId/logs` | Stream attempt-scoped handler logs as SSE | | `POST` | `/api/v1/experiments/:id/retry` | Start a new processing attempt for the current exact vector | Creation metadata requires `name` and accepts optional `description` and @@ -239,11 +290,16 @@ Creation metadata requires `name` and accepts optional `description` and empty mode requires an empty request array. The public view exposes identity, project, metadata, revision, current -execution, readiness, blockers, handler progress, supersession, current report, +execution, readiness, blockers, handler progress, supersession, report, and linked request summaries when available. Creation additionally returns `requestIds`. List responses and execution-list responses use `{ data, nextCursor? }`. +The global generic Reports API accepts `experimentId` and +`experimentExecutionId` filters. Every nested and global report lookup retains +project scope; an Experiment ID or report ID is never used as a global business +key. + ## CLI and Portal The CLI provides: @@ -253,6 +309,8 @@ scope experiment create scope experiment list scope experiment get scope experiment executions [experiment-execution-id] +scope experiment reports +scope experiment logs scope experiment retry --execution ``` @@ -264,10 +322,16 @@ The Portal provides `/experiments`, `/experiments/new`, and `/experiments/:id`. The shared creation form retains the established run configuration and expansion count, adds Experiment metadata and an empty option, and navigates to Experiment Detail after creation. Detail shows current -state, linked runs, blockers, exact execution history, and retry. Run list and -detail views link `experimentId` back to the Experiment. A historical -correlation that has no Experiment document is view-only and cannot trigger -retry. +state, linked runs, blockers, exact execution history, retry, and the selected +attempt's built-in report. The selected execution is encoded as +`?execution=` so direct links and browser back/forward +navigation preserve the report scope. Report Markdown, the global report detail +link, structured JSON download, loading/error/retry states, and attempt-scoped +logs all use the exact `experimentId + experimentExecutionId` identity. Public +UI terminology remains generic **Reports**; `pp-readout` is only the internal +handler identifier. Run list and detail views link `experimentId` back to the +Experiment. A historical correlation that has no Experiment document is +view-only and cannot trigger retry. ## Permanent request creation and composition deprecation @@ -297,7 +361,6 @@ The following are explicitly outside PR A: | Follow-up | Deferred capability | | --- | --- | -| PR B | Additional derived readout orchestration beyond PR A's stored/public current and historical execution views | | PR C | Membership mutation and new revision creation after initial publication | | PR D | Report-template attachment or template-driven report orchestration | | PR E | Cascading lifecycle operations across an Experiment and its member requests/runs | @@ -318,7 +381,8 @@ through storage methods, client actions, or API contracts. | `packages/db-migrations/src/migrations/034-create-experiment-storage-indexes.ts` | Collection index ownership | | `apps/api/src/experiment-coordinator.ts` | API-owned workflow policy and repair | | `apps/api/src/experiment-dispatcher.ts` | Durable polling and queue delivery | +| `apps/workers/readout/` | Exact-execution readout synthesis, append-only report storage, and final CAS publication | | `apps/cli/src/commands/experiment.ts` | Project-scoped CLI surface | | `apps/portal/src/lib/experiments.ts` | Public response parsing | | `apps/portal/src/pages/ExperimentsList.tsx` | Experiment list | -| `apps/portal/src/pages/ExperimentDetail.tsx` | Current state, history, and retry | +| `apps/portal/src/pages/ExperimentDetail.tsx` | Current state, history, retry, and selected-attempt report | diff --git a/docs/architecture/post-processing.md b/docs/architecture/post-processing.md index ff90331dc..5a73d2960 100644 --- a/docs/architecture/post-processing.md +++ b/docs/architecture/post-processing.md @@ -147,7 +147,7 @@ Gate criteria are always `subject: "iteration"` (the API rejects `"run"` for gat Dimension grouping is **derived at read time** by joining each result's `criterionId` → criterion `taxonomyElementId` (the three R&A Readout dimensions; unset = unclassified). It never feeds back to the agent and never -affects pass/fail. Submission report synthesis consumes these classifications +affects pass/fail. Experiment readout synthesis consumes these classifications after every exact-vector dependency is ready. @@ -266,27 +266,33 @@ flowchart LR Experiment handlers use the same `services` registration endpoint but declare `type: experiment-handler` and `dependencyMode: all-runs`. Their dependencies -name run-level DAG handlers. The coordinator projects those dependencies for -the current Experiment's exact request/run vector and dispatches only its -embedded current execution. Built-in readout processing is not part of this -foundation; see [Experiments](experiments.md) for the queue and rollover -contracts. - -The public submission resource is generic: -`GET /api/v1/submissions/:id/reports` returns current and historical reports in -newest-first order, matching the run-level reports collection convention. The -embedded current attempt's `reportId` is resolved through -`GET /api/v1/reports/:reportId`; Structured JSON is downloaded through the -report artifact proxy. Internal handler, queue, and schema identifiers retain -`readout` because they identify this specific synthesis implementation rather -than the public resource. - -Submission-handler logs are persisted under the exact submission attempt and -streamed through -`GET /api/v1/submissions/:id/runs/:submissionRunId/logs`. Queue processors use -the message's `submissionRunId` as the log attempt key, so superseded attempts -remain independently replayable and cannot inherit logs from a newer current -attempt. +name run-level DAG handlers. The coordinator projects those dependencies for the current Experiment's exact +request/run vector and dispatches only its embedded current execution. + +The built-in `pp-readout` handler has `dependencyMode: all-runs` and depends on +`pp-observations`. A `done` observation handler is evidence even when the +coding outcome is `failed` or an observation has `passed: false`; failed or +dependency-blocked observations prevent dispatch, while unrelated handler +failures do not. Its queue message is attempt-scoped and includes the exact +`experimentId`, `experimentExecutionId`, execution version, handler version, +and claim token. + +The public surface remains generic: +`GET /api/v1/experiments/:id/executions/:experimentExecutionId/reports` returns +the exact execution's reports newest first. The global report endpoint resolves +the same append-only document, and the nested `/structured` route proxies its +schema-validated JSON artifact. The worker derives Markdown from that JSON and +atomically publishes the execution report pointer with final handler completion. +Internal handler, queue, and schema identifiers retain `readout` because they +identify this synthesis implementation rather than a public report type. + +Experiment-handler logs are persisted under the exact execution and streamed +through +`GET /api/v1/experiments/:id/executions/:experimentExecutionId/logs`. Queue +processors use `experimentExecutionId` as the log attempt key, so superseded +executions remain independently replayable and cannot inherit logs from a newer +current attempt. See [Experiments](experiments.md#built-in-readout) for the +full stale-work, storage, and final compare-and-swap contract. To **add or change a handler**, edit its `handler.yaml` (bump `version`, adjust `dependsOn`, etc.) — no code or migration changes are needed for registration. diff --git a/docs/architecture/prompt-evaluations.md b/docs/architecture/prompt-evaluations.md index ba40805f4..4c932d956 100644 --- a/docs/architecture/prompt-evaluations.md +++ b/docs/architecture/prompt-evaluations.md @@ -30,7 +30,7 @@ families: | Prompt-feature extraction | Positive, negative, ambiguous, ancestor/descendant, and empty-feature cases | | Judge instructions | Bundled and independent strategies; file, tool-history, response, conflicting, missing, and prior-pass evidence | | Developer feedback | Default persona, custom persona, descendant guard on/off, and multiple root failures | -| Run reports and submission readouts | Default and appended run-report system prompts; override as a control where the default prompt is intentionally absent; exact-vector submission readout synthesis | +| Run reports and Experiment readouts | Default and appended run-report system prompts; override as a control where the default prompt is intentionally absent; exact-execution Experiment readout synthesis | Judge and report tool descriptions are static AI-facing instructions, but they are evaluated through their owning end-to-end judge and report families rather diff --git a/evaluations/static-prompts/adapter/adapter.test.ts b/evaluations/static-prompts/adapter/adapter.test.ts index a5380d9f7..37e09aa77 100644 --- a/evaluations/static-prompts/adapter/adapter.test.ts +++ b/evaluations/static-prompts/adapter/adapter.test.ts @@ -71,7 +71,7 @@ describe("static prompt adapter registry", () => { "default", "append", "override-control", - "submission-readout", + "experiment-readout", ], }, ]); @@ -102,9 +102,9 @@ describe("static prompt adapter registry", () => { variant: "override-control", }); expect(targets).toContainEqual({ - adapterId: "run-report/submission-readout", + adapterId: "run-report/experiment-readout", family: "run-report", - variant: "submission-readout", + variant: "experiment-readout", }); expect(JSON.parse(JSON.stringify(targets))).toEqual(targets); }); @@ -137,6 +137,7 @@ describe("API prompt production contracts", () => { expect(result.output).toEqual({ prompt: "Use captured build output.", suggestedId: "builds_cleanly", + suggestedSubject: "run", }); }); @@ -434,18 +435,18 @@ describe("session prompt adapters", () => { ); }); - it("composes the exact submission-readout prompt without tools", async () => { + it("composes the exact experiment-readout prompt without tools", async () => { const evidence = { - submissionId: "submission-1", - submissionRunId: "submission-run-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-1", runs: [], }; const result = await getPromptTarget( "run-report", - "submission-readout", + "experiment-readout", ).run( { evidence }, - "submission-readout", + "experiment-readout", fakeContext(() => "{}"), ); @@ -454,7 +455,7 @@ describe("session prompt adapters", () => { { role: "user", content: [ - "Generate the submission readout synthesis for this exact immutable evidence vector.", + "Generate the Experiment readout synthesis for this exact immutable execution evidence vector.", JSON.stringify(evidence), ].join("\n\n"), }, diff --git a/evaluations/static-prompts/adapter/targets/reports.ts b/evaluations/static-prompts/adapter/targets/reports.ts index bef13c8e5..b8d25e28d 100644 --- a/evaluations/static-prompts/adapter/targets/reports.ts +++ b/evaluations/static-prompts/adapter/targets/reports.ts @@ -217,7 +217,7 @@ export function composeReportRequest( variant: string, defaultModel: string, ): ComposedPromptRequest { - if (variant === "submission-readout") { + if (variant === "experiment-readout") { const value = record(input); const evidence = record(value.evidence, "input.evidence"); return { @@ -234,7 +234,7 @@ export function composeReportRequest( optionalString(value.model, "input.model") ?? defaultModel, tools: [], metadata: { - adapterId: "run-report/submission-readout", + adapterId: "run-report/experiment-readout", productionSources: [ "apps/workers/readout/src/prompt.ts#READOUT_SYSTEM_PROMPT", "apps/workers/readout/src/readout-generator.ts#composeReadoutUserPrompt", @@ -252,9 +252,9 @@ export function composeReportRequest( export const reportAdapter: PromptTargetAdapter = { family: "run-report", - variants: ["default", "append", "override-control", "submission-readout"], + variants: ["default", "append", "override-control", "experiment-readout"], async run(input, variant, context) { - if (variant === "submission-readout") { + if (variant === "experiment-readout") { const request = composeReportRequest(input, variant, context.model); const completion = await context.complete(request); return { diff --git a/evaluations/static-prompts/datasets/manifest.json b/evaluations/static-prompts/datasets/manifest.json index 21a8b112b..0908d6b9b 100644 --- a/evaluations/static-prompts/datasets/manifest.json +++ b/evaluations/static-prompts/datasets/manifest.json @@ -92,7 +92,7 @@ }, { "path": "v1/readouts.jsonl", - "sha256": "a23f372bf8d4c4cb6668a2094facfe284bf8030685e6405b0c76a03a1f19f1e4", + "sha256": "1099b5cf2e076dfbba9e5284ec0e9e2dd326a4f49c6a7475cc655a2efb738886", "rows": 1, "families": [ "run-report" diff --git a/evaluations/static-prompts/datasets/v1/readouts.jsonl b/evaluations/static-prompts/datasets/v1/readouts.jsonl index 9ba053ab2..36d3dd633 100644 --- a/evaluations/static-prompts/datasets/v1/readouts.jsonl +++ b/evaluations/static-prompts/datasets/v1/readouts.jsonl @@ -1 +1 @@ -{"schemaVersion":1,"id":"run-report-0123456789abcdef","family":"run-report","variant":"submission-readout","smoke":true,"input":{"evidence":{"submissionId":"submission-1","submissionRunId":"submission-run-1","runs":[{"request":{"_id":"request-1","projectId":"project-1","workerType":"coder-acp-copilot","profileId":"profile-1","observations":["uses-current-api"],"scenario":{"task":"Update the API","criteria":[]},"createdAt":"2026-10-13T12:00:00.000Z","priority":0},"run":{"_id":"run-1","attemptNumber":1,"status":"done","outcome":"failed","observationResults":[{"criterionId":"uses-current-api","passed":false,"feedback":"The deprecated API remained.","evaluated":true}],"turns":[]},"criteria":[{"id":"uses-current-api","projectId":"project-1","prompt":"Use the current API","kind":"observation","subject":"run","taxonomyElementId":"dimension:dependency-currency","createdAt":"2026-10-13T12:00:00.000Z"}],"trajectories":[]}]}},"expected":{"referenceReport":"{\"title\":\"Submission readout\",\"executiveSummary\":\"The exact run failed and did not satisfy the selected observation.\",\"profiles\":[{\"profileId\":\"profile-1\",\"requestIds\":[\"request-1\"],\"summary\":\"One profiled run.\"}],\"runs\":[{\"requestId\":\"request-1\",\"runId\":\"run-1\",\"trajectoryHighlights\":[]}],\"findings\":[{\"id\":\"finding-1\",\"title\":\"Deprecated API remained\",\"description\":\"The final run evidence failed the selected observation.\",\"severity\":\"high\",\"evidence\":[{\"requestId\":\"request-1\",\"runId\":\"run-1\",\"criterionId\":\"uses-current-api\",\"excerpt\":\"The deprecated API remained.\"}]}],\"actions\":[{\"id\":\"action-1\",\"title\":\"Update the API usage\",\"priority\":\"high\",\"description\":\"Replace the deprecated API and verify the final state.\",\"relatedFindingIds\":[\"finding-1\"]}]}"},"evaluators":["generation_success","output_schema","non_empty_report","no_fabricated_references","relevance","coherence","groundedness","task_adherence","report_quality"],"tags":["submission-readout","failed"],"provenance":{"kind":"synthetic-reviewed","sourceEndpoints":[],"projectId":"synthetic-project","sourceEntityIds":["submission-1","submission-run-1","request-1","run-1"],"harvestedAt":"2026-10-13T12:00:00.000Z","selectionSeed":"scope-static-prompts-v1","sourceHash":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","note":"Reviewed synthetic exact-vector case covering failed coding outcome and passed:false observation evidence."},"review":{"status":"approved","method":"manual-contract-review-v1"}} +{"schemaVersion":1,"id":"run-report-0123456789abcdef","family":"run-report","variant":"experiment-readout","smoke":true,"input":{"evidence":{"experimentId":"experiment-1","experimentExecutionId":"execution-1","runs":[{"request":{"_id":"request-1","projectId":"project-1","workerType":"coder-acp-copilot","profileId":"profile-1","observations":["uses-current-api"],"scenario":{"task":"Update the API","criteria":[]},"createdAt":"2026-10-13T12:00:00.000Z","priority":0},"run":{"_id":"run-1","attemptNumber":1,"status":"done","outcome":"failed","observationResults":[{"criterionId":"uses-current-api","passed":false,"feedback":"The deprecated API remained.","evaluated":true}],"turns":[]},"criteria":[{"id":"uses-current-api","projectId":"project-1","prompt":"Use the current API","kind":"observation","subject":"run","taxonomyElementId":"dimension:dependency-currency","createdAt":"2026-10-13T12:00:00.000Z"}],"trajectories":[]}]}},"expected":{"referenceReport":"{\"title\":\"Experiment readout\",\"executiveSummary\":\"The exact run failed and did not satisfy the selected observation.\",\"profiles\":[{\"profileId\":\"profile-1\",\"requestIds\":[\"request-1\"],\"summary\":\"One profiled run.\"}],\"runs\":[{\"requestId\":\"request-1\",\"runId\":\"run-1\",\"trajectoryHighlights\":[]}],\"findings\":[{\"id\":\"finding-1\",\"title\":\"Deprecated API remained\",\"description\":\"The final run evidence failed the selected observation.\",\"severity\":\"high\",\"evidence\":[{\"requestId\":\"request-1\",\"runId\":\"run-1\",\"criterionId\":\"uses-current-api\",\"excerpt\":\"The deprecated API remained.\"}]}],\"actions\":[{\"id\":\"action-1\",\"title\":\"Update the API usage\",\"priority\":\"high\",\"description\":\"Replace the deprecated API and verify the final state.\",\"relatedFindingIds\":[\"finding-1\"]}]}"},"evaluators":["generation_success","output_schema","non_empty_report","no_fabricated_references","relevance","coherence","groundedness","task_adherence","report_quality"],"tags":["experiment-readout","failed"],"provenance":{"kind":"synthetic-reviewed","sourceEndpoints":[],"projectId":"synthetic-project","sourceEntityIds":["experiment-1","execution-1","request-1","run-1"],"harvestedAt":"2026-10-13T12:00:00.000Z","selectionSeed":"scope-static-prompts-v1","sourceHash":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","note":"Reviewed synthetic exact-vector case covering failed coding outcome and passed:false observation evidence."},"review":{"status":"approved","method":"manual-contract-review-v1"}} diff --git a/evaluations/static-prompts/evaluation-manifest.yaml b/evaluations/static-prompts/evaluation-manifest.yaml index 975574108..73f5a7f0f 100644 --- a/evaluations/static-prompts/evaluation-manifest.yaml +++ b/evaluations/static-prompts/evaluation-manifest.yaml @@ -19,4 +19,4 @@ qualityFamilies: - id: developer-feedback variants: [default, persona, descendant-guard] - id: run-report - variants: [default, append, override-control, submission-readout] + variants: [default, append, override-control, experiment-readout] diff --git a/evaluations/static-prompts/scripts/dataset-adapter-contract.test.ts b/evaluations/static-prompts/scripts/dataset-adapter-contract.test.ts index 943edc312..78f9f8278 100644 --- a/evaluations/static-prompts/scripts/dataset-adapter-contract.test.ts +++ b/evaluations/static-prompts/scripts/dataset-adapter-contract.test.ts @@ -57,10 +57,10 @@ function fakeResponse(row: QualityCaseRow, request: ComposedPromptRequest): stri case "developer-feedback": return "Implement the missing requirement and rerun the relevant check."; case "run-report": - if (row.variant === "submission-readout") { + if (row.variant === "experiment-readout") { expect(request.tools).toEqual([]); return JSON.stringify({ - title: "Submission readout", + title: "Experiment readout", executiveSummary: "The exact vector was reviewed.", profiles: [], runs: [], diff --git a/evaluations/static-prompts/scripts/validate-dataset.ts b/evaluations/static-prompts/scripts/validate-dataset.ts index 608d5cdf9..45d15a335 100644 --- a/evaluations/static-prompts/scripts/validate-dataset.ts +++ b/evaluations/static-prompts/scripts/validate-dataset.ts @@ -181,7 +181,7 @@ function validateFamilyContract(row: DatasetCase, prefix: string, errors: string assert(typeof input.includeDescendantGuard === "boolean", `${prefix}: input.includeDescendantGuard must be boolean`, errors); break; case "run-report": - if (row.variant === "submission-readout") { + if (row.variant === "experiment-readout") { assert(isRecord(input.evidence), `${prefix}: input.evidence is required`, errors); } else { assert(stringField(input, "requestId"), `${prefix}: input.requestId is required`, errors); @@ -352,10 +352,10 @@ export async function validateDataset( || (row.family === "run-report" && ( row.variant === "default" || row.variant === "override-control" - || row.variant === "submission-readout" + || row.variant === "experiment-readout" )) || (row.family === "developer-feedback" && row.variant === "persona")), - "synthetic cases are only permitted for unavailable judge evidence, feedback persona, report override controls, and submission readouts", + "synthetic cases are only permitted for unavailable judge evidence, feedback persona, report override controls, and experiment readouts", errors, ); const hasTag = (family: QualityFamily, tag: string): boolean => diff --git a/packages/shared/src/experiments/experiment-store.test.ts b/packages/shared/src/experiments/experiment-store.test.ts index fe823a92a..4561cfd01 100644 --- a/packages/shared/src/experiments/experiment-store.test.ts +++ b/packages/shared/src/experiments/experiment-store.test.ts @@ -326,12 +326,35 @@ describe("ExperimentStore CAS contracts", () => { handlerVersion: 1, claimToken: "claim-1", status: "done", + report: { + reportId: "report-1", + url: "/api/v1/reports/report-1", + updatedAt: new Date("2026-01-01T00:01:00.000Z"), + }, }); expect(completed?.execution.handlerSummary.report).toMatchObject({ status: "done", version: 1, claimToken: "claim-1", }); + expect(completed?.execution.report).toEqual({ + reportId: "report-1", + url: "/api/v1/reports/report-1", + updatedAt: new Date("2026-01-01T00:01:00.000Z"), + }); + expect(experiments.current().execution).toMatchObject({ + handlerSummary: { + report: { + status: "done", + version: 1, + claimToken: "claim-1", + }, + }, + report: { + reportId: "report-1", + url: "/api/v1/reports/report-1", + }, + }); }); it("serializes request-state writers against the freezer", async () => { diff --git a/packages/shared/src/experiments/experiment-store.ts b/packages/shared/src/experiments/experiment-store.ts index da21e733d..d209f6dc9 100644 --- a/packages/shared/src/experiments/experiment-store.ts +++ b/packages/shared/src/experiments/experiment-store.ts @@ -23,6 +23,7 @@ import { type ExperimentHandlerStatus, type ExperimentMetadata, type ExperimentPage, + type ExperimentReportReference, type ExperimentRequestState, type ExperimentRequestStateUpdateResult, type ExperimentRevision, @@ -285,6 +286,29 @@ export class ExperimentStore { return raw ? parseHistory(raw) : null; } + async readExecutionRequests(input: { + projectId: string; + experimentId: string; + experimentExecutionId: string; + }): Promise { + const experiment = await this.get(input.projectId, input.experimentId); + const execution = + experiment?.execution.experimentExecutionId === + input.experimentExecutionId + ? experiment.execution + : await this.getExecution(input); + if (!execution) return null; + return this.payloads.readRequests( + { + projectId: input.projectId, + experimentId: input.experimentId, + ownerId: execution.experimentExecutionId, + generation: execution.attemptNumber, + }, + execution.requests, + ); + } + async listExecutionHistory(input: { projectId: string; experimentId: string; @@ -358,6 +382,7 @@ export class ExperimentStore { claimToken: string; status: Extract; error?: string; + report?: ExperimentReportReference; now?: Date; }): Promise { assertHandlerId(input.handlerId); @@ -387,6 +412,7 @@ export class ExperimentStore { }, "execution.updatedAt": now, "execution.finishedAt": now, + ...(input.report ? { "execution.report": input.report } : {}), updatedAt: now, }, $inc: { "execution.version": 1, stateVersion: 1 }, diff --git a/packages/shared/src/experiments/schemas.ts b/packages/shared/src/experiments/schemas.ts index d3a96e088..0512e6f0e 100644 --- a/packages/shared/src/experiments/schemas.ts +++ b/packages/shared/src/experiments/schemas.ts @@ -122,6 +122,14 @@ const ExperimentExecutionBaseSchema = z message: "handlerSummary is limited to 64 handlers", }), blocking: z.array(ExperimentBlockerSchema).max(32), + report: z + .object({ + reportId: IdSchema, + url: z.string().min(1).max(2_048), + updatedAt: z.coerce.date(), + }) + .strict() + .optional(), createdAt: z.coerce.date(), updatedAt: z.coerce.date(), startedAt: z.coerce.date().optional(), diff --git a/packages/shared/src/experiments/types.ts b/packages/shared/src/experiments/types.ts index a3b64d2fa..8f780525d 100644 --- a/packages/shared/src/experiments/types.ts +++ b/packages/shared/src/experiments/types.ts @@ -110,6 +110,12 @@ export interface ExperimentRequestStateUpdateClaim { claimedAt: Date; } +export interface ExperimentReportReference { + reportId: string; + url: string; + updatedAt: Date; +} + export interface ExperimentExecution { experimentExecutionId: string; attemptNumber: number; @@ -118,6 +124,7 @@ export interface ExperimentExecution { vectorFingerprint: string; handlerSummary: Record; blocking: ExperimentBlocker[]; + report?: ExperimentReportReference; createdAt: Date; updatedAt: Date; startedAt?: Date; diff --git a/packages/shared/src/queue/base-queue-processor.ts b/packages/shared/src/queue/base-queue-processor.ts index 5a0d3772b..b5f881360 100644 --- a/packages/shared/src/queue/base-queue-processor.ts +++ b/packages/shared/src/queue/base-queue-processor.ts @@ -242,13 +242,13 @@ export abstract class BaseQueueProcessor { +describe("experimentReadoutSchema", () => { it("accepts a one-request readout with empty selected observations", () => { - expect(submissionReadoutSchema.parse({ + expect(experimentReadoutSchema.parse({ meta: { - schemaId: SUBMISSION_READOUT_SCHEMA_ID, - schemaVersion: SUBMISSION_READOUT_SCHEMA_VERSION, + schemaId: EXPERIMENT_READOUT_SCHEMA_ID, + schemaVersion: EXPERIMENT_READOUT_SCHEMA_VERSION, generatedAt: "2026-10-13T12:00:00.000Z", - submissionId: "submission-1", - submissionRunId: "submission-run-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-1", }, summary: { - title: "Submission readout", + title: "Experiment readout", executiveSummary: "The single run completed without selected observations.", totalRuns: 1, succeededRuns: 1, diff --git a/packages/shared/src/schemas/readout.ts b/packages/shared/src/schemas/readout.ts index f8e85ca7e..1ddae6780 100644 --- a/packages/shared/src/schemas/readout.ts +++ b/packages/shared/src/schemas/readout.ts @@ -3,8 +3,8 @@ import { z } from "zod"; -export const SUBMISSION_READOUT_SCHEMA_ID = "scope.submission-readout"; -export const SUBMISSION_READOUT_SCHEMA_VERSION = "1.0"; +export const EXPERIMENT_READOUT_SCHEMA_ID = "scope.experiment-readout"; +export const EXPERIMENT_READOUT_SCHEMA_VERSION = "1.0"; export const readoutObservationSchema = z.object({ criterionId: z.string().min(1), @@ -15,7 +15,7 @@ export const readoutObservationSchema = z.object({ }); export const readoutEvidenceSchema = z.object({ - submissionRunId: z.string().min(1), + experimentExecutionId: z.string().min(1), requestId: z.string().min(1), runId: z.string().min(1), iteration: z.number().int().min(0).optional(), @@ -53,13 +53,13 @@ export const readoutActionSchema = z.object({ relatedFindingIds: z.array(z.string().min(1)), }); -export const submissionReadoutSchema = z.object({ +export const experimentReadoutSchema = z.object({ meta: z.object({ - schemaId: z.literal(SUBMISSION_READOUT_SCHEMA_ID), - schemaVersion: z.literal(SUBMISSION_READOUT_SCHEMA_VERSION), + schemaId: z.literal(EXPERIMENT_READOUT_SCHEMA_ID), + schemaVersion: z.literal(EXPERIMENT_READOUT_SCHEMA_VERSION), generatedAt: z.string().datetime({ offset: true }), - submissionId: z.string().min(1), - submissionRunId: z.string().min(1), + experimentId: z.string().min(1), + experimentExecutionId: z.string().min(1), }), summary: z.object({ title: z.string().min(1), @@ -80,5 +80,5 @@ export const submissionReadoutSchema = z.object({ actions: z.array(readoutActionSchema), }); -export type SubmissionReadout = z.infer; +export type ExperimentReadout = z.infer; export type ReadoutRun = z.infer; diff --git a/packages/shared/src/schemas/report.ts b/packages/shared/src/schemas/report.ts index afeb0ac5d..e70cba99f 100644 --- a/packages/shared/src/schemas/report.ts +++ b/packages/shared/src/schemas/report.ts @@ -46,8 +46,8 @@ export const ReportResponseSchema = z .object({ _id: z.string(), requestId: z.string().optional(), - submissionId: z.string().optional(), - submissionRunId: z.string().optional(), + experimentId: z.string().optional(), + experimentExecutionId: z.string().optional(), templateId: z.string().optional(), reporter: ReporterSchema.optional(), content: z.string().optional(), diff --git a/packages/shared/src/types/types.ts b/packages/shared/src/types/types.ts index f98a33a61..28f5b5696 100644 --- a/packages/shared/src/types/types.ts +++ b/packages/shared/src/types/types.ts @@ -869,8 +869,8 @@ export interface StructuredReportArtifact { export interface ReportDocument { _id: string; // UUID requestId?: string; // FK → RequestDocument._id for run reports - submissionId?: string; // FK → SubmissionDocument._id for submission reports - submissionRunId?: string; // Exact immutable submission attempt + experimentId?: string; // FK → ExperimentDocument._id for Experiment reports + experimentExecutionId?: string; // Exact immutable Experiment execution projectId: string; // FK → ProjectDocument._id (denormalized from request) templateId?: string; // FK → ReportTemplateDocument.id (slug of template that generated this report) reporter?: Reporter; // Set by the worker when it picks up the job From af998077ed54aed0deb4e213783bdf8a81139027 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Fri, 16 Oct 2026 06:06:43 -0700 Subject: [PATCH 15/18] fix(readout): align experiment validation contracts Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4b4a63b3-29db-4d12-888f-051592682640 --- .../openapi-snapshot.test.ts.snap | 1167 +++++------------ apps/api/src/endpoints.test.ts | 10 +- apps/api/src/experiment-coordinator.test.ts | 2 + apps/api/src/prompt-request-transport.test.ts | 1 + .../__snapshots__/cli-commands.test.ts.snap | 19 + .../post-processor/src/post-processor.test.ts | 3 + .../src/queue/base-queue-processor.test.ts | 16 +- .../shared/src/queue/base-queue-processor.ts | 2 +- packages/shared/src/schemas/report.ts | 10 + .../src/scripts/register-handler.test.ts | 4 +- website/src/openapi/scope-openapi.json | 2 +- 11 files changed, 369 insertions(+), 867 deletions(-) diff --git a/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap b/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap index 65e96b19d..f8eaa21a9 100644 --- a/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap +++ b/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap @@ -1622,6 +1622,29 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` ], "type": "string", }, + "report": { + "properties": { + "reportId": { + "type": "string", + }, + "updatedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "url": { + "type": "string", + }, + }, + "required": [ + "reportId", + "url", + "updatedAt", + ], + "type": "object", + }, "startedAt": { "format": "date-time", "type": [ @@ -1809,6 +1832,11 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "type": "string", }, }, + "required": [ + "reportId", + "url", + "updatedAt", + ], "type": "object", }, "startedAt": { @@ -2998,6 +3026,12 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "error": { "type": "string", }, + "experimentExecutionId": { + "type": "string", + }, + "experimentId": { + "type": "string", + }, "insightReferences": { "items": { "$ref": "#/components/schemas/InsightReference", @@ -3025,12 +3059,6 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "structuredArtifact": { "$ref": "#/components/schemas/StructuredReportArtifact", }, - "submissionId": { - "type": "string", - }, - "submissionRunId": { - "type": "string", - }, "templateId": { "type": "string", }, @@ -3418,27 +3446,9 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, "type": "array", }, - "submissionExpectedRequestCount": { - "minimum": 1, - "type": "integer", - }, - "submissionFinalizedAt": { - "format": "date-time", - "type": [ - "string", - "null", - ], - }, "submissionId": { "type": "string", }, - "submissionInitialRunId": { - "type": "string", - }, - "submissionRequestIndex": { - "minimum": 0, - "type": "integer", - }, "taskPromptId": { "type": "string", }, @@ -4360,632 +4370,200 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` ], "type": "object", }, - "SubmissionBlockingDetail": { + "SuggestedPromptFeature": { "properties": { - "error": { - "type": "string", - }, - "handlerId": { - "type": "string", - }, - "reason": { - "enum": [ - "request-missing", - "run-changed", - "dependency-not-ready", - "dependency-failed", - "dependency-blocked", - ], + "behavior": { "type": "string", }, - "requestId": { + "prompt": { "type": "string", }, - "runId": { + "suggestedId": { "type": "string", }, }, "required": [ - "requestId", - "runId", - "handlerId", - "reason", + "suggestedId", + "behavior", + "prompt", ], "type": "object", }, - "SubmissionDependencyProjection": { + "TaskPromptResponse": { "properties": { - "error": { - "type": "string", - }, - "handlerId": { - "type": "string", - }, - "requestId": { + "_id": { "type": "string", }, - "runId": { + "contentBlobUrl": { "type": "string", }, - "state": { - "enum": [ - "waiting", - "ready", - "failed", - "blocked", + "createdAt": { + "format": "date-time", + "type": [ + "string", + "null", ], - "type": "string", }, - "updatedAt": { + "deletedAt": { "format": "date-time", "type": [ "string", "null", ], }, - "version": { - "type": "number", - }, - }, - "required": [ - "requestId", - "runId", - "handlerId", - "state", - ], - "type": "object", - }, - "SubmissionHandlerStatus": { - "properties": { - "error": { - "type": "string", - }, - "status": { - "enum": [ - "queued", - "processing", - "done", - "failed", - ], - "type": "string", + "features": { + "items": { + "$ref": "#/components/schemas/PromptFeatureResult", + }, + "type": "array", }, - "updatedAt": { + "featuresExtractedAt": { "format": "date-time", "type": [ "string", "null", ], }, - "version": { - "type": "number", + "keyId": { + "type": "string", }, - }, - "required": [ - "status", - ], - "type": "object", - }, - "SubmissionListResponse": { - "properties": { - "submissions": { - "items": { - "$ref": "#/components/schemas/SubmissionResponse", - }, - "type": "array", + "projectId": { + "type": "string", + }, + "text": { + "type": "string", + }, + "type": { + "$ref": "#/components/schemas/PromptType", }, }, "required": [ - "submissions", + "_id", + "keyId", + "createdAt", + "projectId", ], "type": "object", }, - "SubmissionReportPointer": { + "TokenUsage": { "properties": { - "reportId": { - "type": "string", + "completionTokens": { + "type": "number", }, - "updatedAt": { - "format": "date-time", - "type": [ - "string", - "null", - ], + "promptTokens": { + "type": "number", }, - "url": { - "type": "string", + "totalTokens": { + "type": "number", }, }, + "required": [ + "promptTokens", + "completionTokens", + "totalTokens", + ], "type": "object", }, - "SubmissionRequestRef": { + "TriggerReportsInput": { "properties": { "requestId": { "type": "string", }, - "runId": { - "type": "string", - }, }, "required": [ "requestId", - "runId", ], "type": "object", }, - "SubmissionRequestSummary": { + "UpdateAgentInput": { "properties": { - "model": { - "type": "string", + "available": { + "type": "boolean", }, - "outcome": { + "capabilities": { + "$ref": "#/components/schemas/AgentCapabilities", + }, + "defaultModel": { "type": "string", }, - "profileId": { + "description": { "type": "string", }, - "requestId": { + "modelProvider": { "type": "string", }, - "runId": { + "name": { "type": "string", }, - "status": { + "supportedModels": { + "items": { + "type": "string", + }, + "type": "array", + }, + }, + "type": "object", + }, + "UpdateCodebaseInput": { + "properties": { + "defaultBranch": { "type": "string", }, - "task": { + "description": { "type": "string", }, - "workerType": { + "name": { + "minLength": 1, "type": "string", }, }, - "required": [ - "requestId", - "runId", - "task", - "status", - "workerType", - ], "type": "object", }, - "SubmissionResponse": { + "UpdateCriteriaInput": { "properties": { - "attemptNumber": { - "type": "integer", - }, - "blocking": { + "dependsOn": { "items": { - "$ref": "#/components/schemas/SubmissionBlockingDetail", + "type": "string", }, "type": "array", }, - "createdAt": { - "format": "date-time", - "type": [ - "string", - "null", - ], - }, - "dependencyProjection": { + "gates": { "items": { - "$ref": "#/components/schemas/SubmissionDependencyProjection", + "enum": [ + "select", + "build", + "test", + "run", + "deploy", + ], + "type": "string", }, "type": "array", }, - "handlerProgress": { - "additionalProperties": { - "$ref": "#/components/schemas/SubmissionHandlerStatus", - }, - "type": "object", + "kind": { + "enum": [ + "gate", + "observation", + ], + "type": "string", }, - "projectId": { + "prompt": { "type": "string", }, - "readiness": { + "subject": { "enum": [ - "waiting", - "ready", - "blocked", - "processing", - "done", - "failed", + "run", + "iteration", ], "type": "string", }, - "report": { - "$ref": "#/components/schemas/SubmissionReportPointer", - }, - "requestIds": { - "items": { - "type": "string", - }, - "type": "array", - }, - "requests": { - "items": { - "$ref": "#/components/schemas/SubmissionRequestSummary", - }, - "type": "array", - }, - "submissionId": { - "type": "string", - }, - "submissionRunId": { - "type": "string", - }, - "supersededAt": { - "format": "date-time", - "type": [ - "string", - "null", - ], - }, - "supersededBySubmissionRunId": { - "type": "string", - }, - "supersededReason": { - "enum": [ - "request-vector-changed", - "manual-retry", - ], - "type": "string", - }, - "updatedAt": { - "format": "date-time", - "type": [ - "string", - "null", - ], - }, - }, - "required": [ - "submissionId", - "projectId", - "requestIds", - "createdAt", - "submissionRunId", - "attemptNumber", - "readiness", - "requests", - "dependencyProjection", - "blocking", - ], - "type": "object", - }, - "SubmissionRetryResponse": { - "properties": { - "attemptNumber": { - "type": "integer", - }, - "blocking": { - "items": { - "$ref": "#/components/schemas/SubmissionBlockingDetail", - }, - "type": "array", - }, - "readiness": { - "enum": [ - "waiting", - "ready", - "blocked", - "processing", - "done", - "failed", - ], - "type": "string", - }, - "submissionId": { - "type": "string", - }, - "submissionRunId": { - "type": "string", - }, - }, - "required": [ - "submissionId", - "submissionRunId", - "attemptNumber", - "readiness", - "blocking", - ], - "type": "object", - }, - "SubmissionRunResponse": { - "properties": { - "_id": { - "type": "string", - }, - "attemptNumber": { - "minimum": 1, - "type": "integer", - }, - "blocking": { - "default": [], - "items": { - "$ref": "#/components/schemas/SubmissionBlockingDetail", - }, - "type": "array", - }, - "createdAt": { - "format": "date-time", - "type": [ - "string", - "null", - ], - }, - "dependencyProjection": { - "default": [], - "items": { - "$ref": "#/components/schemas/SubmissionDependencyProjection", - }, - "type": "array", - }, - "finishedAt": { - "format": "date-time", - "type": [ - "string", - "null", - ], - }, - "handlerStatus": { - "additionalProperties": { - "$ref": "#/components/schemas/SubmissionHandlerStatus", - }, - "type": "object", - }, - "projectId": { - "type": "string", - }, - "report": { - "$ref": "#/components/schemas/SubmissionReportPointer", - }, - "requests": { - "items": { - "$ref": "#/components/schemas/SubmissionRequestRef", - }, - "type": "array", - }, - "startedAt": { - "format": "date-time", - "type": [ - "string", - "null", - ], - }, - "submissionId": { - "type": "string", - }, - "supersededAt": { - "format": "date-time", - "type": [ - "string", - "null", - ], - }, - "supersededBySubmissionRunId": { - "type": "string", - }, - "supersededReason": { - "enum": [ - "request-vector-changed", - "manual-retry", - ], - "type": "string", - }, - "updatedAt": { - "format": "date-time", - "type": [ - "string", - "null", - ], - }, - "vectorFingerprint": { - "type": "string", - }, - }, - "required": [ - "_id", - "attemptNumber", - "vectorFingerprint", - "requests", - "createdAt", - ], - "type": "object", - }, - "SuggestedPromptFeature": { - "properties": { - "behavior": { - "type": "string", - }, - "prompt": { - "type": "string", - }, - "suggestedId": { - "type": "string", - }, - }, - "required": [ - "suggestedId", - "behavior", - "prompt", - ], - "type": "object", - }, - "TaskPromptResponse": { - "properties": { - "_id": { - "type": "string", - }, - "contentBlobUrl": { - "type": "string", - }, - "createdAt": { - "format": "date-time", - "type": [ - "string", - "null", - ], - }, - "deletedAt": { - "format": "date-time", - "type": [ - "string", - "null", - ], - }, - "features": { - "items": { - "$ref": "#/components/schemas/PromptFeatureResult", - }, - "type": "array", - }, - "featuresExtractedAt": { - "format": "date-time", - "type": [ - "string", - "null", - ], - }, - "keyId": { - "type": "string", - }, - "projectId": { - "type": "string", - }, - "text": { - "type": "string", - }, - "type": { - "$ref": "#/components/schemas/PromptType", - }, - }, - "required": [ - "_id", - "keyId", - "createdAt", - "projectId", - ], - "type": "object", - }, - "TokenUsage": { - "properties": { - "completionTokens": { - "type": "number", - }, - "promptTokens": { - "type": "number", - }, - "totalTokens": { - "type": "number", - }, - }, - "required": [ - "promptTokens", - "completionTokens", - "totalTokens", - ], - "type": "object", - }, - "TriggerReportsInput": { - "properties": { - "requestId": { - "type": "string", - }, - }, - "required": [ - "requestId", - ], - "type": "object", - }, - "UpdateAgentInput": { - "properties": { - "available": { - "type": "boolean", - }, - "capabilities": { - "$ref": "#/components/schemas/AgentCapabilities", - }, - "defaultModel": { - "type": "string", - }, - "description": { - "type": "string", - }, - "modelProvider": { - "type": "string", - }, - "name": { - "type": "string", - }, - "supportedModels": { - "items": { - "type": "string", - }, - "type": "array", - }, - }, - "type": "object", - }, - "UpdateCodebaseInput": { - "properties": { - "defaultBranch": { - "type": "string", - }, - "description": { - "type": "string", - }, - "name": { - "minLength": 1, - "type": "string", - }, - }, - "type": "object", - }, - "UpdateCriteriaInput": { - "properties": { - "dependsOn": { - "items": { - "type": "string", - }, - "type": "array", - }, - "gates": { - "items": { - "enum": [ - "select", - "build", - "test", - "run", - "deploy", - ], - "type": "string", - }, - "type": "array", - }, - "kind": { - "enum": [ - "gate", - "observation", - ], - "type": "string", - }, - "prompt": { - "type": "string", - }, - "subject": { - "enum": [ - "run", - "iteration", - ], - "type": "string", - }, - "taxonomyElementId": { - "enum": [ - "dimension:idiomatic-use", - "dimension:dependency-currency", - "dimension:configuration-correctness", - ], - "type": "string", + "taxonomyElementId": { + "enum": [ + "dimension:idiomatic-use", + "dimension:dependency-currency", + "dimension:configuration-correctness", + ], + "type": "string", }, }, "type": "object", @@ -6964,12 +6542,187 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "description": "Success", }, "404": { - "description": "Experiment execution not found", + "description": "Experiment execution not found", + }, + }, + "summary": "Get one exact experiment execution", + "tags": [ + "Experiments", + ], + }, + }, + "/api/v1/experiments/{id}/executions/{experimentExecutionId}/logs": { + "get": { + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "minLength": 1, + "type": "string", + }, + }, + { + "in": "path", + "name": "experimentExecutionId", + "required": true, + "schema": { + "minLength": 1, + "type": "string", + }, + }, + { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "in": "query", + "name": "projectId", + "required": true, + "schema": { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, + "type": "string", + }, + }, + { + "in": "query", + "name": "fromStart", + "required": false, + "schema": { + "enum": [ + "true", + "false", + ], + "type": "string", + }, + }, + ], + "responses": { + "200": { + "description": "Server-sent event stream of log entries", + }, + "404": { + "description": "Experiment execution not found", + }, + }, + "summary": "Stream logs for one exact Experiment execution", + "tags": [ + "Experiments", + ], + }, + }, + "/api/v1/experiments/{id}/executions/{experimentExecutionId}/reports": { + "get": { + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "minLength": 1, + "type": "string", + }, + }, + { + "in": "path", + "name": "experimentExecutionId", + "required": true, + "schema": { + "minLength": 1, + "type": "string", + }, + }, + { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "in": "query", + "name": "projectId", + "required": true, + "schema": { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, + "type": "string", + }, + }, + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/ReportResponse", + }, + "type": "array", + }, + }, + }, + "description": "Success", + }, + "404": { + "description": "Experiment execution not found", + }, + }, + "summary": "List reports for one exact Experiment execution", + "tags": [ + "Reports", + ], + }, + }, + "/api/v1/experiments/{id}/executions/{experimentExecutionId}/reports/{reportId}/structured": { + "get": { + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "minLength": 1, + "type": "string", + }, + }, + { + "in": "path", + "name": "experimentExecutionId", + "required": true, + "schema": { + "minLength": 1, + "type": "string", + }, + }, + { + "in": "path", + "name": "reportId", + "required": true, + "schema": { + "minLength": 1, + "type": "string", + }, + }, + { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "in": "query", + "name": "projectId", + "required": true, + "schema": { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, + "type": "string", + }, + }, + ], + "responses": { + "200": { + "description": "Success", + }, + "404": { + "description": "Experiment report or structured artifact not found", }, }, - "summary": "Get one exact experiment execution", + "summary": "Download structured JSON for an exact Experiment report", "tags": [ - "Experiments", + "Reports", ], }, }, @@ -9827,7 +9580,7 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, { "in": "query", - "name": "submissionId", + "name": "experimentId", "required": false, "schema": { "type": "string", @@ -9835,7 +9588,7 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, { "in": "query", - "name": "submissionRunId", + "name": "experimentExecutionId", "required": false, "schema": { "type": "string", @@ -13124,292 +12877,6 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` ], }, }, - "/api/v1/submissions": { - "get": { - "parameters": [ - { - "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", - "in": "query", - "name": "projectId", - "required": true, - "schema": { - "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", - "example": "00000000-0000-0000-0000-000000000000", - "minLength": 1, - "type": "string", - }, - }, - { - "in": "query", - "name": "limit", - "required": false, - "schema": { - "default": 50, - "maximum": 100, - "minimum": 1, - "type": "integer", - }, - }, - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/SubmissionListResponse", - }, - }, - }, - "description": "Success", - }, - }, - "summary": "List submissions", - "tags": [ - "Submissions", - ], - }, - }, - "/api/v1/submissions/{id}": { - "get": { - "parameters": [ - { - "in": "path", - "name": "id", - "required": true, - "schema": { - "type": "string", - }, - }, - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/SubmissionResponse", - }, - }, - }, - "description": "Success", - }, - "404": { - "description": "Submission not found", - }, - }, - "summary": "Get a submission", - "tags": [ - "Submissions", - ], - }, - }, - "/api/v1/submissions/{id}/reports": { - "get": { - "parameters": [ - { - "in": "path", - "name": "id", - "required": true, - "schema": { - "type": "string", - }, - }, - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "items": { - "$ref": "#/components/schemas/ReportResponse", - }, - "type": "array", - }, - }, - }, - "description": "Success", - }, - "404": { - "description": "Submission not found", - }, - }, - "summary": "Get reports for submission", - "tags": [ - "Reports", - ], - }, - }, - "/api/v1/submissions/{id}/retry": { - "post": { - "parameters": [ - { - "in": "path", - "name": "id", - "required": true, - "schema": { - "type": "string", - }, - }, - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "submissionRunId": { - "type": "string", - }, - }, - "required": [ - "submissionRunId", - ], - "type": "object", - }, - }, - }, - }, - "responses": { - "201": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/SubmissionRetryResponse", - }, - }, - }, - "description": "Success", - }, - "404": { - "description": "Submission not found", - }, - "409": { - "description": "Submission attempt is stale or dependencies are blocked", - }, - }, - "summary": "Retry current submission processing", - "tags": [ - "Submissions", - ], - }, - }, - "/api/v1/submissions/{id}/runs": { - "get": { - "parameters": [ - { - "in": "path", - "name": "id", - "required": true, - "schema": { - "type": "string", - }, - }, - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "properties": { - "runs": { - "items": { - "$ref": "#/components/schemas/SubmissionRunResponse", - }, - "type": "array", - }, - }, - "required": [ - "runs", - ], - "type": "object", - }, - }, - }, - "description": "Success", - }, - "404": { - "description": "Submission not found", - }, - }, - "summary": "List current and historical submission attempts", - "tags": [ - "Submissions", - ], - }, - }, - "/api/v1/submissions/{id}/runs/{submissionRunId}": { - "get": { - "parameters": [ - { - "in": "path", - "name": "id", - "required": true, - "schema": { - "type": "string", - }, - }, - { - "in": "path", - "name": "submissionRunId", - "required": true, - "schema": { - "type": "string", - }, - }, - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/SubmissionRunResponse", - }, - }, - }, - "description": "Success", - }, - "404": { - "description": "Submission attempt not found", - }, - }, - "summary": "Get one submission attempt", - "tags": [ - "Submissions", - ], - }, - }, - "/api/v1/submissions/{id}/runs/{submissionRunId}/logs": { - "get": { - "parameters": [ - { - "in": "path", - "name": "id", - "required": true, - "schema": { - "type": "string", - }, - }, - { - "in": "path", - "name": "submissionRunId", - "required": true, - "schema": { - "type": "string", - }, - }, - ], - "responses": { - "200": { - "description": "Server-sent event stream of log entries", - }, - "404": { - "description": "Submission or submission attempt not found", - }, - }, - "summary": "Stream logs for a submission attempt (SSE)", - "tags": [ - "Submissions", - ], - }, - }, "/api/v1/task-prompts": { "get": { "parameters": [ diff --git a/apps/api/src/endpoints.test.ts b/apps/api/src/endpoints.test.ts index 3e1f50e5a..f3c93c4f5 100644 --- a/apps/api/src/endpoints.test.ts +++ b/apps/api/src/endpoints.test.ts @@ -274,7 +274,7 @@ describe("API Endpoints", () => { (mocks.criteriaCollection.findOne as any).mockResolvedValue(null); const res = await request(app) - .post("/api/v1/criteria") + .post(`/api/v1/criteria?projectId=${TEST_PROJECT_ID}`) .send({ id: "obs_crit", prompt: "Did the agent use current deps?", @@ -292,7 +292,7 @@ describe("API Endpoints", () => { (mocks.criteriaCollection.findOne as any).mockResolvedValue(null); const res = await request(app) - .post("/api/v1/criteria") + .post(`/api/v1/criteria?projectId=${TEST_PROJECT_ID}`) .send({ id: "bad_tax", prompt: "x", dependsOn: [], kind: "observation", taxonomyElementId: "dimension:nope" }); expect(res.status).toBe(400); @@ -302,7 +302,7 @@ describe("API Endpoints", () => { (mocks.criteriaCollection.findOne as any).mockResolvedValue(null); const res = await request(app) - .post("/api/v1/criteria") + .post(`/api/v1/criteria?projectId=${TEST_PROJECT_ID}`) .send({ id: "obs_subj", prompt: "Did the agent add then remove a dep?", dependsOn: [], kind: "observation" }); expect(res.status).toBe(201); @@ -313,7 +313,7 @@ describe("API Endpoints", () => { (mocks.criteriaCollection.findOne as any).mockResolvedValue(null); const res = await request(app) - .post("/api/v1/criteria") + .post(`/api/v1/criteria?projectId=${TEST_PROJECT_ID}`) .send({ id: "obs_iter", prompt: "x", dependsOn: [], kind: "observation", subject: "iteration" }); expect(res.status).toBe(201); @@ -324,7 +324,7 @@ describe("API Endpoints", () => { (mocks.criteriaCollection.findOne as any).mockResolvedValue(null); const res = await request(app) - .post("/api/v1/criteria") + .post(`/api/v1/criteria?projectId=${TEST_PROJECT_ID}`) .send({ id: "gate_run", prompt: "x", dependsOn: [], kind: "gate", subject: "run" }); expect(res.status).toBe(400); diff --git a/apps/api/src/experiment-coordinator.test.ts b/apps/api/src/experiment-coordinator.test.ts index 5da0c8a9e..3fe8a18a4 100644 --- a/apps/api/src/experiment-coordinator.test.ts +++ b/apps/api/src/experiment-coordinator.test.ts @@ -331,6 +331,8 @@ describe("ExperimentCoordinator", () => { vi.mocked(ExperimentStore.prototype.casUpdateRequestStates).mock .calls[0]?.[0].requests, ).toHaveLength(40); + }); + it.each([ { name: "failed observations", diff --git a/apps/api/src/prompt-request-transport.test.ts b/apps/api/src/prompt-request-transport.test.ts index f4f261411..5aab849dc 100644 --- a/apps/api/src/prompt-request-transport.test.ts +++ b/apps/api/src/prompt-request-transport.test.ts @@ -66,6 +66,7 @@ const cases = [ payload: { prompt: "Inspect captured build output.", suggestedId: "build_succeeds", + suggestedSubject: "run", suggestedParents: [], suggestedChildren: [], }, diff --git a/apps/cli/src/__snapshots__/cli-commands.test.ts.snap b/apps/cli/src/__snapshots__/cli-commands.test.ts.snap index 907676b78..be76f0a32 100644 --- a/apps/cli/src/__snapshots__/cli-commands.test.ts.snap +++ b/apps/cli/src/__snapshots__/cli-commands.test.ts.snap @@ -261,6 +261,22 @@ exports[`CLI command registration > full command tree matches snapshot 1`] = ` "--url", ], }, + "logs": { + "options": [ + "--from-start", + "--project", + "--url", + ], + }, + "reports": { + "options": [ + "--output", + "--project", + "--report", + "--structured", + "--url", + ], + }, "retry": { "options": [ "--execution", @@ -645,11 +661,14 @@ exports[`CLI command registration > full command tree matches snapshot 1`] = ` "options": [ "--id", "--output", + "--structured", "--url", ], }, "list": { "options": [ + "--experiment", + "--experiment-execution", "--output", "--project", "--run", diff --git a/apps/workers/post-processor/src/post-processor.test.ts b/apps/workers/post-processor/src/post-processor.test.ts index a0d0d3460..4d06a771a 100644 --- a/apps/workers/post-processor/src/post-processor.test.ts +++ b/apps/workers/post-processor/src/post-processor.test.ts @@ -48,6 +48,9 @@ function makeProcessor() { (processor as any).collection = collection; (processor as any).blobStorage = blobStorage; + (processor as any).db = { + collection: vi.fn().mockReturnValue({}), + }; const safeDeleteMessage = vi .spyOn(processor as any, "safeDeleteMessage") diff --git a/packages/shared/src/queue/base-queue-processor.test.ts b/packages/shared/src/queue/base-queue-processor.test.ts index a8ebf9d71..b4567994a 100644 --- a/packages/shared/src/queue/base-queue-processor.test.ts +++ b/packages/shared/src/queue/base-queue-processor.test.ts @@ -207,8 +207,8 @@ describe("BaseQueueProcessor visibility heartbeat lifecycle", () => { protected override extractDocumentId( payload: Record, ): string { - return typeof payload.submissionId === "string" - ? payload.submissionId + return typeof payload.experimentId === "string" + ? payload.experimentId : super.extractDocumentId(payload); } @@ -248,22 +248,22 @@ describe("BaseQueueProcessor visibility heartbeat lifecycle", () => { expect(updateMessage).not.toHaveBeenCalled(); }); - it("persists submission worker logs under the exact submission attempt", async () => { + it("persists Experiment worker logs under the exact execution", async () => { const { proc, logPublisher } = setupProcessor(async (_heartbeat, log) => { - await log("info", "Submission processing started"); + await log("info", "Experiment processing started"); }); await (proc as any).processMessage( makeMessage({ - submissionId: "doc-1", - submissionRunId: "submission-run-1", + experimentId: "doc-1", + experimentExecutionId: "experiment-execution-1", }), ); expect(logPublisher.publish).toHaveBeenCalledWith( "doc-1", - "submission-run-1", + "experiment-execution-1", "info", - "Submission processing started", + "Experiment processing started", undefined, ); }); diff --git a/packages/shared/src/queue/base-queue-processor.ts b/packages/shared/src/queue/base-queue-processor.ts index b5f881360..1ec8107cc 100644 --- a/packages/shared/src/queue/base-queue-processor.ts +++ b/packages/shared/src/queue/base-queue-processor.ts @@ -248,7 +248,7 @@ export abstract class BaseQueueProcessor + report.requestId !== undefined || + (report.experimentId !== undefined && + report.experimentExecutionId !== undefined), + { + message: + "Report must identify either a request or an exact Experiment execution", + }, + ) .openapi("ReportResponse"); export const CreateReportInputSchema = z diff --git a/packages/shared/src/scripts/register-handler.test.ts b/packages/shared/src/scripts/register-handler.test.ts index 7d873e452..f1723f5ba 100644 --- a/packages/shared/src/scripts/register-handler.test.ts +++ b/packages/shared/src/scripts/register-handler.test.ts @@ -52,10 +52,10 @@ describe("parseHandlerDocument", () => { expect(() => parseHandlerDocument("nope")).toThrow(/mapping/); }); - it("accepts the pp-readout submission handler contract", () => { + it("accepts the pp-readout Experiment handler contract", () => { const readout = { _id: "pp-readout", - type: "submission-handler", + type: "experiment-handler", version: 1, queue: "pp-readout-queue", selector: "readout", diff --git a/website/src/openapi/scope-openapi.json b/website/src/openapi/scope-openapi.json index 063624b4a..86faca9e5 100644 --- a/website/src/openapi/scope-openapi.json +++ b/website/src/openapi/scope-openapi.json @@ -1 +1 @@ -{"openapi":"3.1.0","info":{"title":"Scope API","version":"1.0.0","description":"REST API for the Scope platform — benchmarking AI coding agents"},"servers":[{"url":"/","description":"Current server"}],"components":{"securitySchemes":{"bearerAuth":{"type":"http","scheme":"bearer","bearerFormat":"JWT","description":"Use the unchanged IdP access token, without the Bearer prefix."}},"schemas":{"KeyResponse":{"type":"object","properties":{},"additionalProperties":{}},"KeyInput":{"type":"object","properties":{},"additionalProperties":{}},"ValidateKeyInput":{"type":"object","properties":{"token":{"type":"string"}},"required":["token"]},"AccountResponse":{"type":"object","properties":{},"additionalProperties":{}},"AccountInput":{"type":"object","properties":{},"additionalProperties":{}},"UserMeResponse":{"type":"object","properties":{"id":{"type":"string"},"role":{"type":"string"},"email":{"type":"string"},"displayName":{"type":"string"},"idp":{"type":"string"},"idpTenant":{"type":"string"}},"required":["id"]},"ProjectResponse":{"type":"object","properties":{"_id":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"creator":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"}},"required":["_id","name","createdAt"]},"CreateProjectInput":{"type":"object","properties":{"name":{"type":"string","minLength":1},"description":{"type":"string"},"creator":{"type":"string"}},"required":["name"]},"UpdateProjectInput":{"type":"object","properties":{"name":{"type":"string","minLength":1},"description":{"type":"string"}}},"CreateExperimentResponse":{"type":"object","properties":{"experiment":{"$ref":"#/components/schemas/ExperimentResponse"},"requestIds":{"type":"array","items":{"type":"string"}}},"required":["experiment","requestIds"]},"ExperimentResponse":{"type":"object","properties":{"experimentId":{"type":"string"},"projectId":{"type":"string"},"metadata":{"type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":256},"description":{"type":"string","maxLength":4096},"tags":{"type":"array","items":{"type":"string","minLength":1,"maxLength":64},"maxItems":32},"createdBy":{"type":"string"}},"required":["name"]},"revision":{"type":"object","properties":{"experimentRevisionId":{"type":"string"},"revisionNumber":{"type":"integer"},"memberCount":{"type":"integer"},"createdAt":{"type":["string","null"],"format":"date-time"}},"required":["experimentRevisionId","revisionNumber","memberCount","createdAt"]},"currentExecution":{"type":"object","properties":{"experimentExecutionId":{"type":"string"},"attemptNumber":{"type":"integer"},"readiness":{"type":"string","enum":["empty","waiting","blocked","ready","processing","done","failed"]},"blockers":{"type":"array","items":{"type":"object","properties":{"requestId":{"type":"string"},"runId":{"type":"string"},"handlerId":{"type":"string"},"reason":{"type":"string"},"error":{"type":"string"}},"required":["handlerId","reason"]}},"progress":{"type":"object","additionalProperties":{"type":"object","properties":{"status":{"type":"string","enum":["queued","processing","done","failed"]},"updatedAt":{"type":["string","null"],"format":"date-time"},"error":{"type":"string"}},"required":["status","updatedAt"]}},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"startedAt":{"type":["string","null"],"format":"date-time"},"finishedAt":{"type":["string","null"],"format":"date-time"},"discardedAt":{"type":["string","null"],"format":"date-time"},"discardReason":{"type":"string"},"report":{"type":"object","properties":{"reportId":{"type":"string"},"url":{"type":"string"},"updatedAt":{"type":["string","null"],"format":"date-time"}}}},"required":["experimentExecutionId","attemptNumber","readiness","blockers","progress","createdAt","updatedAt"]},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"}},"required":["experimentId","projectId","metadata","revision","currentExecution","createdAt","updatedAt"]},"CreateExperimentInput":{"type":"object","properties":{"mode":{"type":"string","enum":["compose","empty"]},"metadata":{"type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":256},"description":{"type":"string","maxLength":4096},"tags":{"type":"array","items":{"type":"string","minLength":1,"maxLength":64},"maxItems":32}},"required":["name"]},"requests":{"type":"array","items":{"type":"object","properties":{"scenario":{"$ref":"#/components/schemas/Scenario"},"model":{"type":"string"},"reasoningEffort":{"type":"string"},"maxIterations":{"type":"integer","minimum":1,"maximum":50},"personaInstructions":{"type":"string"},"persona":{"$ref":"#/components/schemas/Persona"},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"codebaseRevisionId":{"type":"string"},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceBindingSpec"}},"extensions":{"type":"array","items":{"type":"string"}},"profileId":{"type":"string"},"priority":{"type":"integer"},"agentsMd":{"type":"string"},"agentsMdParentIds":{"type":"array","items":{"type":"string"}},"gates":{"type":"array","items":{"$ref":"#/components/schemas/GateConfig"}},"observations":{"type":"array","items":{"type":"string"}},"workerType":{"type":"string","minLength":1}},"required":["scenario","workerType"]},"maxItems":250000,"default":[]}},"required":["mode","metadata"]},"Scenario":{"type":"object","properties":{"version":{"type":"string","enum":["v1","v2"]},"task":{"type":"string"},"criteria":{"type":"array","items":{"type":"string"}}},"required":["task","criteria"]},"Persona":{"type":"object","properties":{"personality":{"type":"string","enum":["demanding","friendly"]},"experience":{"type":"string","enum":["junior","senior"]},"verbosity":{"type":"string","enum":["brief","moderate"]},"type":{"type":"string","enum":["traditional","ai_assisted","vibe"]}},"required":["personality","experience","verbosity","type"]},"ResourceBindingSpec":{"anyOf":[{"type":"string"},{"type":"object","properties":{"ref":{"type":"string"},"params":{"type":"object","additionalProperties":{"type":"string"}}},"required":["ref"]}]},"GateConfig":{"type":"object","properties":{"gate":{"type":"string","enum":["select","build","test","run","deploy"]},"promptId":{"type":"string"},"promptText":{"type":"string"},"criteria":{"type":"array","items":{"type":"string"}},"maxIterations":{"type":"integer","minimum":1,"maximum":50}},"required":["gate","criteria"]},"ExperimentListResponse":{"type":"object","properties":{"data":{"type":"array","items":{"$ref":"#/components/schemas/ExperimentResponse"}},"nextCursor":{"type":"string"}},"required":["data"]},"ExperimentExecutionListResponse":{"type":"object","properties":{"data":{"type":"array","items":{"$ref":"#/components/schemas/ExperimentExecutionResponse"}},"nextCursor":{"type":"string"}},"required":["data"]},"ExperimentExecutionResponse":{"type":"object","properties":{"experimentExecutionId":{"type":"string"},"attemptNumber":{"type":"integer"},"readiness":{"type":"string","enum":["empty","waiting","blocked","ready","processing","done","failed"]},"blockers":{"type":"array","items":{"type":"object","properties":{"requestId":{"type":"string"},"runId":{"type":"string"},"handlerId":{"type":"string"},"reason":{"type":"string"},"error":{"type":"string"}},"required":["handlerId","reason"]}},"progress":{"type":"object","additionalProperties":{"type":"object","properties":{"status":{"type":"string","enum":["queued","processing","done","failed"]},"updatedAt":{"type":["string","null"],"format":"date-time"},"error":{"type":"string"}},"required":["status","updatedAt"]}},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"startedAt":{"type":["string","null"],"format":"date-time"},"finishedAt":{"type":["string","null"],"format":"date-time"},"discardedAt":{"type":["string","null"],"format":"date-time"},"discardReason":{"type":"string"},"supersededAt":{"type":["string","null"],"format":"date-time"},"supersededByExperimentExecutionId":{"type":"string"},"supersededReason":{"type":"string","enum":["rollover","retry"]}},"required":["experimentExecutionId","attemptNumber","readiness","blockers","progress","createdAt","updatedAt"]},"RetryExperimentResponse":{"type":"object","properties":{"experimentId":{"type":"string"},"experimentExecutionId":{"type":"string"},"attemptNumber":{"type":"integer"},"readiness":{"type":"string","enum":["empty","waiting","blocked","ready","processing","done","failed"]},"blockers":{"type":"array","items":{"type":"object","properties":{"requestId":{"type":"string"},"runId":{"type":"string"},"handlerId":{"type":"string"},"reason":{"type":"string"},"error":{"type":"string"}},"required":["handlerId","reason"]}}},"required":["experimentId","experimentExecutionId","attemptNumber","readiness","blockers"]},"RequestResponse":{"type":"object","properties":{"_id":{"type":"string"},"scenario":{"$ref":"#/components/schemas/Scenario"},"workerType":{"type":"string"},"model":{"type":"string"},"reasoningEffort":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"maxIterations":{"type":"number"},"personaInstructions":{"type":"string"},"persona":{"$ref":"#/components/schemas/Persona"},"deletedAt":{"type":["string","null"],"format":"date-time"},"taskPromptId":{"type":"string"},"agentsMdPromptId":{"type":"string"},"agentsMdParentIds":{"type":"array","items":{"type":"string"}},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"codebaseRevisionId":{"type":"string"},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceBinding"}},"extensions":{"type":"array","items":{"type":"string"}},"agentVersion":{"type":"string"},"profileId":{"type":"string"},"profileVersionId":{"type":"string"},"submissionId":{"type":"string"},"experimentId":{"type":"string"},"experimentExpectedRequestCount":{"type":"integer","minimum":0},"experimentRequestIndex":{"type":"integer","minimum":0},"experimentInitialRunId":{"type":"string"},"experimentFinalizedAt":{"type":["string","null"],"format":"date-time"},"priority":{"type":"integer","default":0},"gates":{"type":"array","items":{"$ref":"#/components/schemas/GateConfig"}},"observations":{"type":"array","items":{"type":"string"}},"gateSummaries":{"type":"array","items":{"$ref":"#/components/schemas/GateRunSummary"}},"run":{"$ref":"#/components/schemas/RunState"},"projectId":{"type":"string"}},"required":["_id","scenario","workerType","createdAt","projectId"]},"ResourceBinding":{"type":"object","properties":{"ref":{"type":"string"},"revisionId":{"type":"string"},"params":{"type":"object","additionalProperties":{"type":"string"}}},"required":["ref","revisionId","params"]},"GateRunSummary":{"type":"object","properties":{"gate":{"type":"string","enum":["select","build","test","run","deploy"]},"status":{"type":"string","enum":["passed","failed","skipped"]},"iterations":{"type":"integer","minimum":0}},"required":["gate","status","iterations"]},"RunState":{"type":"object","properties":{"_id":{"type":"string"},"attemptNumber":{"type":"integer","minimum":1},"status":{"type":"string","enum":["pending","queued","processing","paused","done"]},"queuedQueueName":{"type":"string"},"outcome":{"type":"string","enum":["succeeded","failed","finished"]},"result":{"type":"string"},"error":{"type":"string"},"logsUrl":{"type":"string"},"updatedAt":{"type":["string","null"],"format":"date-time"},"startedAt":{"type":["string","null"],"format":"date-time"},"finishedAt":{"type":["string","null"],"format":"date-time"},"durationMs":{"type":"number"},"turns":{"type":"array","items":{"$ref":"#/components/schemas/ConversationTurn"}},"workerVersion":{"type":"string"},"os":{"type":"object","properties":{"platform":{"type":"string"},"release":{"type":"string"},"arch":{"type":"string"}},"required":["platform","release","arch"]},"lastHeartbeatAt":{"type":["string","null"],"format":"date-time"},"worker":{"type":"object","properties":{"instanceId":{"type":"string"},"podName":{"type":"string"}},"required":["instanceId"]},"harUrl":{"type":"string"},"videoUrls":{"type":"array","items":{"type":"string"}},"setupVideoUrls":{"type":"array","items":{"type":"string"}},"tokenUsage":{"$ref":"#/components/schemas/TokenUsage"},"aiCallCount":{"type":"number"},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceRunOutcome"}},"mcpRegistered":{"type":"boolean"},"rawChatUrl":{"type":"string"},"rawChatFormat":{"type":"string"},"pausedAt":{"type":["string","null"],"format":"date-time"},"resumedAt":{"type":["string","null"],"format":"date-time"},"observationResults":{"type":"array","items":{"$ref":"#/components/schemas/CriterionResult"}}},"required":["_id","attemptNumber","status"]},"ConversationTurn":{"type":"object","properties":{"iteration":{"type":"number"},"gate":{"type":"string","enum":["select","build","test","run","deploy"]},"codingAgentResponse":{"type":"string"},"judgeFeedback":{"type":"string"},"snapshotUrl":{"type":"string"},"passed":{"type":"boolean"},"timestamp":{"type":["string","null"],"format":"date-time"},"criteriaResults":{"type":"array","items":{"$ref":"#/components/schemas/CriterionResult"}},"observationResults":{"type":"array","items":{"$ref":"#/components/schemas/CriterionResult"}},"harUrl":{"type":"string"},"videoUrls":{"type":"array","items":{"type":"string"}},"tokenUsage":{"$ref":"#/components/schemas/TokenUsage"},"startedAt":{"type":["string","null"],"format":"date-time"},"durationMs":{"type":"number"},"toolCalls":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"arguments":{"type":"object","additionalProperties":{}},"response":{"type":"string"},"timestamp":{"type":"string"}},"required":["id","name","arguments"]}},"toolCallsUrl":{"type":"string"},"toolCallCount":{"type":"number"},"aiCallCount":{"type":"number"},"rawChatUrl":{"type":"string"},"rawChatFormat":{"type":"string"},"chatResultUrl":{"type":"string"},"chatResultFormat":{"type":"string"}},"required":["iteration","judgeFeedback","snapshotUrl","passed","timestamp"]},"CriterionResult":{"type":"object","properties":{"criterionId":{"type":"string"},"passed":{"type":"boolean"},"feedback":{"type":"string"},"evaluated":{"type":"boolean"}},"required":["criterionId","passed","feedback","evaluated"]},"TokenUsage":{"type":"object","properties":{"promptTokens":{"type":"number"},"completionTokens":{"type":"number"},"totalTokens":{"type":"number"}},"required":["promptTokens","completionTokens","totalTokens"]},"ResourceRunOutcome":{"type":"object","properties":{"ref":{"type":"string"},"slug":{"type":"string"},"revisionId":{"type":"string"},"setupSucceeded":{"type":"boolean"},"published":{"type":"array","items":{"type":"string"}},"params":{"type":"object","additionalProperties":{"type":"string"}},"setupDurationMs":{"type":"number"},"error":{"type":"string"},"teardownRan":{"type":"boolean"}},"required":["ref","slug","revisionId","setupSucceeded","published"]},"CreateRequestInput":{"type":"object","properties":{"scenario":{"$ref":"#/components/schemas/Scenario"},"model":{"type":"string"},"reasoningEffort":{"type":"string"},"maxIterations":{"type":"integer","minimum":1,"maximum":50},"personaInstructions":{"type":"string"},"persona":{"$ref":"#/components/schemas/Persona"},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"codebaseRevisionId":{"type":"string"},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceBindingSpec"}},"extensions":{"type":"array","items":{"type":"string"}},"profileId":{"type":"string"},"profileVariations":{"type":"array","items":{"type":"string"},"deprecated":true,"description":"Deprecated for composition. Use POST /api/v1/experiments and its requests array."},"priority":{"type":"integer"},"agentsMd":{"type":"string"},"agentsMdParentIds":{"type":"array","items":{"type":"string"}},"gates":{"type":"array","items":{"$ref":"#/components/schemas/GateConfig"}},"observations":{"type":"array","items":{"type":"string"}}},"required":["scenario"]},"RunFacetsResponse":{"type":"object","properties":{"total":{"type":"number"},"facets":{"type":"object","properties":{"workerType":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"status":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"outcome":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"model":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"os":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"priority":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"agentVersion":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"profileId":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}}},"required":["workerType","status","outcome","model","os","priority","agentVersion","profileId"]}},"required":["total","facets"]},"RunFacetBucket":{"type":"object","properties":{"value":{"type":"string"},"count":{"type":"number"}},"required":["value","count"]},"PaginatedRunsResponse":{"type":"object","properties":{"data":{"type":"array","items":{"$ref":"#/components/schemas/RequestResponse"}},"limit":{"type":"number"},"estimatedTotal":{"type":"number"},"cursors":{"$ref":"#/components/schemas/Cursors"}},"required":["data","limit","estimatedTotal","cursors"]},"Cursors":{"type":"object","properties":{"next":{"type":["string","null"]},"prev":{"type":["string","null"]}},"required":["next","prev"]},"PaginatedRunGroupsResponse":{"type":"object","properties":{"data":{"type":"array","items":{"$ref":"#/components/schemas/RunGroup"}},"limit":{"type":"number"},"estimatedTotal":{"type":"number"},"cursors":{"$ref":"#/components/schemas/Cursors"}},"required":["data","limit","estimatedTotal","cursors"]},"RunGroup":{"type":"object","properties":{"key":{"type":"string"},"label":{"type":"string"},"runIds":{"type":"array","items":{"type":"string"}},"aggregates":{"$ref":"#/components/schemas/GroupAggregates"},"uniform":{"$ref":"#/components/schemas/GroupUniformValues"}},"required":["key","label","runIds","aggregates","uniform"]},"GroupAggregates":{"type":"object","properties":{"count":{"type":"number"},"turns":{"$ref":"#/components/schemas/AggregateStats"},"duration":{"$ref":"#/components/schemas/AggregateStats"},"promptTokens":{"$ref":"#/components/schemas/AggregateStats"},"completionTokens":{"$ref":"#/components/schemas/AggregateStats"},"statusCounts":{"type":"object","additionalProperties":{"type":"number"}},"outcomeCounts":{"type":"object","additionalProperties":{"type":"number"}}},"required":["count","turns","duration","promptTokens","completionTokens","statusCounts","outcomeCounts"]},"AggregateStats":{"type":["object","null"],"properties":{"min":{"type":"number"},"max":{"type":"number"},"mean":{"type":"number"},"stdDev":{"type":"number"}},"required":["min","max","mean","stdDev"]},"GroupUniformValues":{"type":"object","properties":{"workerType":{"type":"string"},"agentVersion":{"type":"string"},"model":{"type":"string"},"platform":{"type":"string"},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"codebaseRevisionId":{"type":"string"},"extensions":{"type":"array","items":{"type":"string"}},"status":{"type":"string","enum":["pending","queued","processing","paused","done"]},"submissionId":{"type":"string"},"experimentId":{"type":"string"},"task":{"type":"string"}}},"BulkResubmitInput":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1},"count":{"type":"integer","minimum":1,"maximum":10,"default":1},"overrides":{"type":"object","properties":{"profileId":{"type":["string","null"]},"workerType":{"type":"string"},"agentVersion":{"type":"string"},"model":{"type":["string","null"]},"reasoningEffort":{"type":["string","null"]},"maxIterations":{"type":["number","null"]},"mcpServers":{"type":["array","null"],"items":{"type":"string"}},"skillRevisions":{"type":["array","null"],"items":{"type":"string"}},"extensions":{"type":["array","null"],"items":{"type":"string"}}}}},"required":["ids"]},"ReportResponse":{"type":"object","properties":{"_id":{"type":"string"},"requestId":{"type":"string"},"templateId":{"type":"string"},"reporter":{"$ref":"#/components/schemas/Reporter"},"content":{"type":"string"},"status":{"type":"string","enum":["pending","generating","completed","failed"]},"error":{"type":"string"},"insightReferences":{"type":"array","items":{"$ref":"#/components/schemas/InsightReference"}},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","requestId","status","createdAt","projectId"]},"Reporter":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"gitHash":{"type":"string"},"model":{"type":"string"},"agentId":{"type":"string"},"agentVersion":{"type":"string"}},"required":["id","name","gitHash","model","agentId","agentVersion"]},"InsightReference":{"type":"object","properties":{"insightId":{"type":"string"},"referencedAt":{"type":["string","null"],"format":"date-time"},"isNew":{"type":"boolean"}},"required":["insightId","referencedAt","isNew"]},"CreateCriteriaInput":{"type":"object","properties":{"id":{"type":"string","pattern":"^[a-z][a-z0-9_]*$"},"prompt":{"type":"string"},"dependsOn":{"type":"array","items":{"type":"string"}},"gates":{"type":"array","items":{"type":"string","enum":["select","build","test","run","deploy"]}},"kind":{"type":"string","enum":["gate","observation"],"default":"gate"},"taxonomyElementId":{"type":"string","enum":["dimension:idiomatic-use","dimension:dependency-currency","dimension:configuration-correctness"]},"subject":{"type":"string","enum":["run","iteration"]}},"required":["id","prompt"]},"CriteriaResponse":{"type":"object","properties":{"id":{"type":"string"},"prompt":{"type":"string"},"dependsOn":{"type":"array","items":{"type":"string"}},"gates":{"type":"array","items":{"type":"string","enum":["select","build","test","run","deploy"]}},"kind":{"type":"string","enum":["gate","observation"]},"taxonomyElementId":{"type":"string","enum":["dimension:idiomatic-use","dimension:dependency-currency","dimension:configuration-correctness"]},"subject":{"type":"string","enum":["run","iteration"]},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["id","prompt","createdAt","projectId"]},"CriteriaGraph":{"type":"object","properties":{"nodes":{"type":"array","items":{"$ref":"#/components/schemas/CriteriaGraphNode"}},"edges":{"type":"array","items":{"$ref":"#/components/schemas/CriteriaGraphEdge"}}},"required":["nodes","edges"]},"CriteriaGraphNode":{"type":"object","properties":{"id":{"type":"string"},"prompt":{"type":"string"},"dependsOn":{"type":"array","items":{"type":"string"}},"gates":{"type":"array","items":{"type":"string","enum":["select","build","test","run","deploy"]}},"kind":{"type":"string","enum":["gate","observation"]},"taxonomyElementId":{"type":"string","enum":["dimension:idiomatic-use","dimension:dependency-currency","dimension:configuration-correctness"]},"subject":{"type":"string","enum":["run","iteration"]}},"required":["id","prompt"]},"CriteriaGraphEdge":{"type":"object","properties":{"from":{"type":"string"},"to":{"type":"string"}},"required":["from","to"]},"UpdateCriteriaInput":{"type":"object","properties":{"prompt":{"type":"string"},"dependsOn":{"type":"array","items":{"type":"string"}},"gates":{"type":"array","items":{"type":"string","enum":["select","build","test","run","deploy"]}},"kind":{"type":"string","enum":["gate","observation"]},"taxonomyElementId":{"type":"string","enum":["dimension:idiomatic-use","dimension:dependency-currency","dimension:configuration-correctness"]},"subject":{"type":"string","enum":["run","iteration"]}}},"CreatePromptFeatureInput":{"type":"object","properties":{"id":{"type":"string","pattern":"^[a-z][a-z0-9_]*$"},"prompt":{"type":"string"},"type":{"$ref":"#/components/schemas/PromptType"}},"required":["id","prompt"]},"PromptType":{"type":"string","enum":["select","build","test","run","deploy","agents.md"]},"PromptFeatureResult":{"type":"object","properties":{"featureId":{"type":"string"},"detected":{"type":"boolean"},"evaluated":{"type":"boolean"}},"required":["featureId","detected","evaluated"]},"SuggestedPromptFeature":{"type":"object","properties":{"suggestedId":{"type":"string"},"behavior":{"type":"string"},"prompt":{"type":"string"}},"required":["suggestedId","behavior","prompt"]},"PromptFeatureResponse":{"type":"object","properties":{"id":{"type":"string"},"prompt":{"type":"string"},"type":{"$ref":"#/components/schemas/PromptType"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["id","prompt","createdAt","projectId"]},"UpdatePromptFeatureInput":{"type":"object","properties":{"prompt":{"type":"string"}}},"TaskPromptResponse":{"type":"object","properties":{"_id":{"type":"string"},"keyId":{"type":"string"},"type":{"$ref":"#/components/schemas/PromptType"},"text":{"type":"string"},"contentBlobUrl":{"type":"string"},"features":{"type":"array","items":{"$ref":"#/components/schemas/PromptFeatureResult"}},"featuresExtractedAt":{"type":["string","null"],"format":"date-time"},"createdAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","keyId","createdAt","projectId"]},"CreateTaskPromptInput":{"type":"object","properties":{"text":{"type":"string"},"type":{"$ref":"#/components/schemas/PromptType"}},"required":["text"]},"PatchTaskPromptFeatureInput":{"type":"object","properties":{"detected":{"type":"boolean"}},"required":["detected"]},"CreateReportInput":{"type":"object","properties":{"requestId":{"type":"string"},"templateId":{"type":"string"}},"required":["requestId"]},"BulkCreateReportsInput":{"type":"object","properties":{"requestIds":{"type":"array","items":{"type":"string"}},"templateId":{"type":"string"}},"required":["requestIds"]},"BulkReportStatusInput":{"type":"object","properties":{"reportIds":{"type":"array","items":{"type":"string"}}},"required":["reportIds"]},"BulkReportSummaryResponse":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/ReportSummary"}},"ReportSummary":{"type":"object","properties":{"total":{"type":"number"},"pending":{"type":"number"},"generating":{"type":"number"},"completed":{"type":"number"},"failed":{"type":"number"}},"required":["total","pending","generating","completed","failed"]},"BulkReportSummaryInput":{"type":"object","properties":{"requestIds":{"type":"array","items":{"type":"string"}}},"required":["requestIds"]},"TriggerReportsInput":{"type":"object","properties":{"requestId":{"type":"string"}},"required":["requestId"]},"BulkTriggerReportsInput":{"type":"object","properties":{"requestIds":{"type":"array","items":{"type":"string"}}},"required":["requestIds"]},"InsightResponse":{"type":"object","properties":{"_id":{"type":"string"},"title":{"type":"string"},"description":{"type":"string"},"category":{"type":"string"},"tags":{"type":"array","items":{"type":"string"}},"upvotes":{"type":"number"},"downvotes":{"type":"number"},"blocked":{"type":"boolean"},"referenceCount":{"type":"number"},"createdBy":{"type":"string","enum":["agent","user"]},"sourceReportId":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","title","description","upvotes","downvotes","blocked","referenceCount","createdBy","createdAt","projectId"]},"ProfileWithVersionResponse":{"type":"object","properties":{"_id":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"latestVersion":{"type":"number"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"version":{"$ref":"#/components/schemas/ProfileVersionResponse"},"projectId":{"type":"string"}},"required":["_id","name","latestVersion","createdAt","version","projectId"]},"ProfileVersionResponse":{"type":"object","properties":{"_id":{"type":"string"},"profileId":{"type":"string"},"version":{"type":"number"},"workerType":{"type":"string"},"model":{"type":"string"},"reasoningEffort":{"type":"string"},"agentVersion":{"type":"string"},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceBindingSpec"}},"extensions":{"type":"array","items":{"type":"string"}},"createdAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","profileId","version","workerType","model","createdAt","projectId"]},"CreateProfileInput":{"type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":128},"description":{"type":"string","maxLength":512},"workerType":{"type":"string"},"model":{"type":"string"},"reasoningEffort":{"type":"string"},"agentVersion":{"type":"string"},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceBindingSpec"}},"extensions":{"type":"array","items":{"type":"string"}}},"required":["name","workerType","model"]},"ProfileResponse":{"type":"object","properties":{"_id":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"latestVersion":{"type":"number"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","name","latestVersion","createdAt","projectId"]},"UpdateProfileIdentity":{"type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":128},"description":{"type":"string","maxLength":512}}},"ReportTemplateResponse":{"type":"object","properties":{"_id":{"type":"string"},"id":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"userPrompt":{"type":"string"},"systemPrompt":{"$ref":"#/components/schemas/ReportTemplateSystemPrompt"},"trigger":{"$ref":"#/components/schemas/ReportTrigger"},"model":{"type":"string"},"timeoutMs":{"type":"integer","exclusiveMinimum":0},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","id","name","userPrompt","createdAt","projectId"]},"ReportTemplateSystemPrompt":{"type":"object","properties":{"mode":{"type":"string","enum":["append","override"]},"content":{"type":"string"}},"required":["mode","content"]},"ReportTrigger":{"oneOf":[{"type":"object","properties":{"type":{"type":"string","enum":["always"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["criteria"]},"criteriaIds":{"type":"array","items":{"type":"string"}},"match":{"type":"string","enum":["any","all"]}},"required":["type","criteriaIds"]},{"type":"object","properties":{"type":{"type":"string","enum":["taskPrompt"]},"taskPromptIds":{"type":"array","items":{"type":"string"}}},"required":["type","taskPromptIds"]},{"type":"object","properties":{"type":{"type":"string","enum":["promptFeature"]},"featureIds":{"type":"array","items":{"type":"string"}},"match":{"type":"string","enum":["any","all"]}},"required":["type","featureIds"]}]},"CreateReportTemplateInput":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"userPrompt":{"type":"string"},"systemPrompt":{"$ref":"#/components/schemas/ReportTemplateSystemPrompt"},"trigger":{"$ref":"#/components/schemas/ReportTrigger"},"model":{"type":"string"},"timeoutMs":{"type":"integer","exclusiveMinimum":0}},"required":["id","name","userPrompt"]},"UpdateReportTemplateInput":{"type":"object","properties":{"name":{"type":"string"},"description":{"type":"string"},"userPrompt":{"type":"string"},"systemPrompt":{"allOf":[{"$ref":"#/components/schemas/ReportTemplateSystemPrompt"},{"type":["object","null"]}]},"trigger":{"$ref":"#/components/schemas/ReportTrigger"},"model":{"type":["string","null"]},"timeoutMs":{"type":["integer","null"],"exclusiveMinimum":0}}},"AgentResponse":{"type":"object","properties":{"_id":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"modelProvider":{"type":"string"},"supportedModels":{"type":"array","items":{"type":"string"}},"defaultModel":{"type":"string"},"available":{"type":"boolean"},"capabilities":{"$ref":"#/components/schemas/AgentCapabilities"},"versions":{"type":"array","items":{"$ref":"#/components/schemas/AgentVersion"}},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"}},"required":["_id","name","supportedModels","createdAt"]},"AgentCapabilities":{"type":"object","properties":{"supportsReasoningEffort":{"type":"boolean"},"supportsMcpServers":{"type":"boolean"},"supportsSkills":{"type":"boolean"},"supportsExtensions":{"type":"boolean"},"supportsResources":{"type":"boolean"}}},"AgentVersion":{"type":"object","properties":{"agentVersion":{"type":"string","minLength":1},"workerVersion":{"type":"string","minLength":1},"components":{"type":"object","additionalProperties":{"type":"string"}},"gitCommit":{"type":"string","minLength":1},"buildTime":{"type":"string","minLength":1},"imageTag":{"type":"string","minLength":1},"queueName":{"type":"string"},"status":{"type":"string","enum":["active","retired"]},"createdAt":{"type":["string","null"],"format":"date-time"}},"required":["agentVersion","workerVersion","components","gitCommit","buildTime","imageTag","status","createdAt"]},"CreateAgentInput":{"type":"object","properties":{"_id":{"type":"string","minLength":1},"name":{"type":"string","minLength":1},"description":{"type":"string"},"modelProvider":{"type":"string"},"supportedModels":{"type":"array","items":{"type":"string"}},"defaultModel":{"type":"string"},"available":{"type":"boolean"},"capabilities":{"$ref":"#/components/schemas/AgentCapabilities"}},"required":["_id","name"]},"UpdateAgentInput":{"type":"object","properties":{"name":{"type":"string"},"description":{"type":"string"},"modelProvider":{"type":"string"},"supportedModels":{"type":"array","items":{"type":"string"}},"defaultModel":{"type":"string"},"available":{"type":"boolean"},"capabilities":{"$ref":"#/components/schemas/AgentCapabilities"}}},"RegisterAgentVersionInput":{"type":"object","properties":{"agentVersion":{"type":"string","minLength":1},"workerVersion":{"type":"string","minLength":1},"components":{"type":"object","additionalProperties":{"type":"string"}},"gitCommit":{"type":"string","minLength":1},"buildTime":{"type":"string","minLength":1},"imageTag":{"type":"string","minLength":1},"queueName":{"type":"string","minLength":1}},"required":["agentVersion","workerVersion","components","gitCommit","buildTime","imageTag","queueName"]},"PatchAgentVersionInput":{"type":"object","properties":{"status":{"type":"string","enum":["active","retired"]}},"required":["status"]},"ModelResponse":{"type":"object","properties":{"_id":{"type":"string"},"modelId":{"type":"string"},"provider":{"type":"string"},"agentId":{"type":"string"},"firstSeenAt":{"type":["string","null"],"format":"date-time"},"lastSeenAt":{"type":["string","null"],"format":"date-time"},"disappearedAt":{"type":["string","null"],"format":"date-time"},"providerAvailableFrom":{"type":["string","null"],"format":"date-time"},"providerEndOfLife":{"type":["string","null"],"format":"date-time"},"metadata":{"type":"object","additionalProperties":{}},"capabilities":{"$ref":"#/components/schemas/ModelCapabilities"}},"required":["_id","modelId","provider","agentId","firstSeenAt","lastSeenAt"]},"ModelCapabilities":{"type":"object","properties":{"reasoningEffort":{"type":"array","items":{"type":"string"}},"toolCalls":{"type":"boolean"},"vision":{"type":"boolean"},"streaming":{"type":"boolean"},"adaptiveThinking":{"type":"boolean"},"maxThinkingBudget":{"type":"number"}}},"McpServerResponse":{"type":"object","properties":{"_id":{"type":"string"},"slug":{"type":"string"},"name":{"type":"string"},"type":{"type":"string","enum":["sse","http","stdio"]},"url":{"type":"string"},"command":{"type":"string"},"args":{"type":"array","items":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"headers":{"type":"array","items":{"$ref":"#/components/schemas/McpServerHeader"}},"sessionMode":{"type":"string","enum":["stateful","stateless"]},"version":{"type":"string"},"description":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","slug","name","type","createdAt","projectId"]},"McpServerHeader":{"type":"object","properties":{"name":{"type":"string"},"value":{"type":"string"}},"required":["name","value"]},"UpdateMcpServerInput":{"type":"object","properties":{"name":{"type":"string"},"type":{"type":"string","enum":["sse","http","stdio"]},"url":{"type":"string"},"command":{"type":"string"},"args":{"type":"array","items":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"headers":{"type":"array","items":{"$ref":"#/components/schemas/McpServerHeader"}},"sessionMode":{"type":"string","enum":["stateful","stateless"]},"version":{"type":"string"},"description":{"type":"string"}}},"SkillResponse":{"type":"object","properties":{"_id":{"type":"string"},"slug":{"type":"string"},"source":{"type":"string"},"skillName":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"origin":{"type":"string","enum":["skills-sh","manual"]},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","slug","source","skillName","name","origin","createdAt","projectId"]},"SkillSearchResult":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"source":{"type":"string"},"description":{"type":"string"},"internal":{"type":"boolean"},"installs":{"type":"number"}},"required":["id","name","source","internal"]},"SkillDiscoveryResult":{"type":"object","properties":{"skillName":{"type":"string"},"skillPath":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"existsInLibrary":{"type":"boolean"},"currentRevisionCommitSha":{"type":"string"},"latestUpstreamCommitSha":{"type":"string"},"updateAvailable":{"type":"boolean"},"lastImportedAt":{"type":"string"}},"required":["skillName","skillPath"]},"SkillRevisionResponse":{"type":"object","properties":{"_id":{"type":"string"},"ref":{"type":"string"},"source":{"type":"string"},"skillName":{"type":"string"},"skillPath":{"type":"string"},"commitHash":{"type":"string"},"commitTimestamp":{"type":["string","null"],"format":"date-time"},"name":{"type":"string"},"description":{"type":"string"},"license":{"type":"string"},"compatibility":{"type":"string"},"allowedTools":{"type":"string"},"metadata":{"type":"object","additionalProperties":{"type":"string"}},"content":{"type":"string"},"archiveUrl":{"type":"string"},"validationWarnings":{"type":"array","items":{"type":"string"}},"resolvedAt":{"type":["string","null"],"format":"date-time"},"createdAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","ref","source","skillName","skillPath","commitHash","commitTimestamp","name","description","content","archiveUrl","resolvedAt","createdAt","projectId"]},"CreateSkillInput":{"type":"object","properties":{"source":{"type":"string"},"skillName":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"origin":{"type":"string","enum":["skills-sh","manual"]}},"required":["source","skillName","name","origin"]},"CodebaseResponse":{"type":"object","properties":{"_id":{"type":"string"},"slug":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"sourceType":{"type":"string","enum":["git","archive"]},"source":{"type":"string"},"defaultBranch":{"type":"string"},"revisionCounter":{"type":"number"},"latestRevisionId":{"type":"string"},"creator":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","slug","name","sourceType","revisionCounter","createdAt","projectId"]},"UpdateCodebaseInput":{"type":"object","properties":{"name":{"type":"string","minLength":1},"description":{"type":"string"},"defaultBranch":{"type":"string"}}},"CodebaseRevisionResponse":{"type":"object","properties":{"_id":{"type":"string"},"codebaseId":{"type":"string"},"slug":{"type":"string"},"revisionNumber":{"type":"number"},"ref":{"type":"string"},"sourceType":{"type":"string","enum":["git","archive"]},"source":{"type":"string"},"requestedRef":{"type":"string"},"resolvedCommitSha":{"type":"string"},"commitTimestamp":{"type":["string","null"],"format":"date-time"},"originalFilename":{"type":"string"},"contentSha256":{"type":"string"},"archiveUrl":{"type":"string"},"sizeBytes":{"type":"number"},"fileCount":{"type":"number"},"creator":{"type":"string"},"resolvedAt":{"type":["string","null"],"format":"date-time"},"createdAt":{"type":["string","null"],"format":"date-time"},"deduplicated":{"type":"boolean"},"projectId":{"type":"string"}},"required":["_id","codebaseId","slug","revisionNumber","ref","sourceType","archiveUrl","resolvedAt","createdAt","projectId"]},"ResolveCodebaseRevisionInput":{"type":"object","properties":{"requestedRef":{"type":"string"},"creator":{"type":"string"}}},"ResourceResponse":{"type":"object","properties":{"_id":{"type":"string"},"slug":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"revisionCounter":{"type":"number"},"latestRevisionId":{"type":"string"},"latestRevisionNumber":{"type":"number"},"creator":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","slug","name","revisionCounter","createdAt","projectId"]},"CreateResourceInput":{"type":"object","properties":{"name":{"type":"string","minLength":1},"slug":{"type":"string","minLength":1},"description":{"type":"string"},"setup":{"$ref":"#/components/schemas/ResourceScript"},"teardown":{"$ref":"#/components/schemas/ResourceScript"},"exports":{"type":"array","items":{"type":"string","pattern":"^[A-Za-z_][A-Za-z0-9_]*$"}},"parameters":{"type":"array","items":{"$ref":"#/components/schemas/ResourceParameter"}},"creator":{"type":"string"}},"required":["name","setup"]},"ResourceScript":{"type":"object","properties":{"sh":{"type":"string","minLength":1}}},"ResourceParameter":{"type":"object","properties":{"name":{"type":"string","pattern":"^[A-Za-z_][A-Za-z0-9_]*$"},"description":{"type":"string"},"required":{"type":"boolean","default":false},"default":{"type":"string"},"example":{"type":"string"}},"required":["name"]},"UpdateResourceInput":{"type":"object","properties":{"name":{"type":"string","minLength":1},"description":{"type":"string"}}},"ResourceRevisionResponse":{"type":"object","properties":{"_id":{"type":"string"},"resourceId":{"type":"string"},"slug":{"type":"string"},"revisionNumber":{"type":"number"},"ref":{"type":"string"},"setup":{"$ref":"#/components/schemas/ResourceScript"},"teardown":{"$ref":"#/components/schemas/ResourceScript"},"exports":{"type":"array","items":{"type":"string"}},"parameters":{"type":"array","items":{"$ref":"#/components/schemas/ResourceParameter"}},"contentSha256":{"type":"string"},"creator":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"deduplicated":{"type":"boolean"},"projectId":{"type":"string"}},"required":["_id","resourceId","slug","revisionNumber","ref","setup","exports","contentSha256","createdAt","projectId"]},"CreateResourceRevisionInput":{"type":"object","properties":{"setup":{"$ref":"#/components/schemas/ResourceScript"},"teardown":{"$ref":"#/components/schemas/ResourceScript"},"exports":{"type":"array","items":{"type":"string","pattern":"^[A-Za-z_][A-Za-z0-9_]*$"}},"parameters":{"type":"array","items":{"$ref":"#/components/schemas/ResourceParameter"}},"creator":{"type":"string"}},"required":["setup"]},"ExtensionResponse":{"type":"object","properties":{"_id":{"type":"string"},"slug":{"type":"string"},"publisher":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"origin":{"type":"string","enum":["marketplace","manual"]},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","slug","publisher","name","origin","createdAt","projectId"]},"ExtensionSearchResult":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"publisher":{"type":"string"},"description":{"type":"string"},"internal":{"type":"boolean"},"version":{"type":"string"}},"required":["id","name","publisher","internal"]},"CreateExtensionInput":{"type":"object","properties":{"_id":{"type":"string","pattern":"^[a-zA-Z0-9-]+\\.[a-zA-Z0-9-]+$"},"publisher":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"origin":{"type":"string","enum":["marketplace","manual"]}},"required":["_id","publisher","name","origin"]},"UpdateExtensionInput":{"type":"object","properties":{"name":{"type":"string"},"description":{"type":"string"}}},"ExtensionVersionInfo":{"type":"object","properties":{"version":{"type":"string"},"preRelease":{"type":"boolean"},"lastUpdated":{"type":"string"}},"required":["version","preRelease","lastUpdated"]},"CreateInsightInput":{"type":"object","properties":{"title":{"type":"string"},"description":{"type":"string"},"category":{"type":"string"},"tags":{"type":"array","items":{"type":"string"}},"createdBy":{"type":"string","enum":["agent","user"]},"sourceReportId":{"type":"string"}},"required":["title","description"]},"UpdateInsightInput":{"type":"object","properties":{"title":{"type":"string"},"description":{"type":"string"},"category":{"type":"string"},"tags":{"type":"array","items":{"type":"string"}}}},"FeatureFlagResponse":{"type":"object","properties":{"key":{"type":"string"},"label":{"type":"string"},"enabled":{"type":"boolean"},"updatedAt":{"type":["string","null"],"format":"date-time"}},"required":["key","label","enabled","updatedAt"]},"UpdateFeatureFlagInput":{"type":"object","properties":{"enabled":{"type":"boolean"}},"required":["enabled"]}},"parameters":{}},"paths":{"/api/v1/keys/preview":{"post":{"tags":["Keys"],"summary":"Preview key","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyInput"}}}},"responses":{"200":{"description":"Key preview","content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyResponse"}}}}}}},"/api/v1/keys":{"post":{"tags":["Keys"],"summary":"Create key","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyInput"}}}},"responses":{"201":{"description":"Key created","content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyResponse"}}}}}},"get":{"tags":["Keys"],"summary":"List keys","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/KeyResponse"}}}}}}}},"/api/v1/keys/{id}":{"get":{"tags":["Keys"],"summary":"Get key","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyResponse"}}}}}},"put":{"tags":["Keys"],"summary":"Update key","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyInput"}}}},"responses":{"200":{"description":"Key updated","content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyResponse"}}}}}},"delete":{"tags":["Keys"],"summary":"Delete key","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Key deleted","content":{"application/json":{"schema":{"type":"object","properties":{"success":{"type":"boolean"}},"required":["success"]}}}}}}},"/api/v1/keys/{id}/validate":{"post":{"tags":["Keys"],"summary":"Validate key","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ValidateKeyInput"}}}},"responses":{"200":{"description":"Validation result","content":{"application/json":{"schema":{"type":"object","properties":{},"additionalProperties":{},"description":"Key validation result"}}}}}}},"/api/v1/accounts":{"post":{"tags":["Accounts"],"summary":"Create account","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/AccountInput"}}}},"responses":{"201":{"description":"Account created","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AccountResponse"}}}}}},"get":{"tags":["Accounts"],"summary":"List accounts","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/AccountResponse"}}}}}}}},"/api/v1/accounts/{id}":{"get":{"tags":["Accounts"],"summary":"Get account","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AccountResponse"}}}}}},"put":{"tags":["Accounts"],"summary":"Update account","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/AccountInput"}}}},"responses":{"200":{"description":"Account updated","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AccountResponse"}}}}}},"delete":{"tags":["Accounts"],"summary":"Delete account","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Account deleted","content":{"application/json":{"schema":{"type":"object","properties":{"success":{"type":"boolean"}},"required":["success"]}}}}}}},"/api/v1/users/me":{"get":{"tags":["Users"],"summary":"Get the authenticated user's identity","description":"Read the existing authenticated Scope identity without enrollment or profile writes. Responses must not be HTTP-cached.","security":[{"bearerAuth":[]}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/UserMeResponse"}}}},"401":{"description":"Not authenticated"},"403":{"description":"User is not enrolled or is disabled"},"503":{"description":"Authentication service unavailable"}}},"post":{"tags":["Users"],"summary":"Enroll the authenticated user","description":"JIT-enroll the authenticated user, refresh their profile and lastLoginAt, apply bootstrap-admin rules, and warm the access cache. Use this only after an explicit IdP login; do not prefetch, poll, or automatically retry transient failures.","security":[{"bearerAuth":[]}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/UserMeResponse"}}}},"401":{"description":"Not authenticated"},"403":{"description":"User is disabled"},"503":{"description":"Authentication service unavailable"}}}},"/health":{"get":{"tags":["Health"],"summary":"Liveness probe","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"status":{"type":"string"},"version":{"type":"string"}},"required":["status","version"]}}}}}}},"/ready":{"get":{"tags":["Health"],"summary":"Readiness probe","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"status":{"type":"string"},"migrations":{}},"required":["status"]}}}},"503":{"description":"Service is not ready"}}}},"/about":{"get":{"tags":["System"],"summary":"API metadata","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"name":{"type":"string"},"version":{"type":"string"},"buildTime":{"type":"string"},"environment":{"type":"string"},"description":{"type":"string"},"workers":{"type":"array","items":{"type":"string"}}},"required":["name","version","buildTime","environment","description","workers"]}}}}}}},"/api/v1/version":{"get":{"tags":["System"],"summary":"Version info","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"commit":{"type":"string"},"buildTime":{"type":"string"},"environment":{"type":"string"},"strictAgentCapabilities":{"type":"boolean"}},"required":["commit","buildTime","environment","strictAgentCapabilities"]}}}}}}},"/api/v1/projects":{"get":{"tags":["Projects"],"summary":"List all projects","parameters":[{"schema":{"type":"string","enum":["true","false"],"description":"When true, include soft-deleted projects in the result"},"required":false,"description":"When true, include soft-deleted projects in the result","name":"includeDeleted","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ProjectResponse"}}}}}}},"post":{"tags":["Projects"],"summary":"Create a project","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateProjectInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProjectResponse"}}}},"400":{"description":"Invalid input"}}}},"/api/v1/projects/{id}":{"get":{"tags":["Projects"],"summary":"Get a project","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProjectResponse"}}}},"404":{"description":"Project not found"}}},"patch":{"tags":["Projects"],"summary":"Update a project","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateProjectInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProjectResponse"}}}},"404":{"description":"Project not found"}}},"delete":{"tags":["Projects"],"summary":"Soft-delete a project","responses":{"204":{"description":"Success"},"404":{"description":"Project not found"}}}},"/api/v1/projects/{id}/restore":{"post":{"tags":["Projects"],"summary":"Restore a soft-deleted project","responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProjectResponse"}}}},"404":{"description":"Project not found"}}}},"/api/v1/experiments":{"post":{"tags":["Experiments"],"summary":"Create an experiment and zero or more new requests","description":"Canonical composition endpoint. Request and run IDs are minted before insertion; the exact revision/execution is then finalized idempotently.","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateExperimentInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateExperimentResponse"}}}},"409":{"description":"Experiment identity conflict"}}},"get":{"tags":["Experiments"],"summary":"List experiments","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"},{"schema":{"type":"integer","minimum":1,"maximum":100,"default":50},"required":false,"name":"limit","in":"query"},{"schema":{"type":"string","minLength":1},"required":false,"name":"after","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ExperimentListResponse"}}}}}}},"/api/v1/experiments/{id}":{"get":{"tags":["Experiments"],"summary":"Get an experiment","parameters":[{"schema":{"type":"string","minLength":1},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ExperimentResponse"}}}},"404":{"description":"Experiment not found"}}}},"/api/v1/experiments/{id}/executions":{"get":{"tags":["Experiments"],"summary":"List current and historical experiment executions","parameters":[{"schema":{"type":"string","minLength":1},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"},{"schema":{"type":"integer","minimum":1,"maximum":100,"default":50},"required":false,"name":"limit","in":"query"},{"schema":{"type":"string","minLength":1},"required":false,"name":"after","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ExperimentExecutionListResponse"}}}},"404":{"description":"Experiment not found"}}}},"/api/v1/experiments/{id}/executions/{experimentExecutionId}":{"get":{"tags":["Experiments"],"summary":"Get one exact experiment execution","parameters":[{"schema":{"type":"string","minLength":1},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1},"required":true,"name":"experimentExecutionId","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ExperimentExecutionResponse"}}}},"404":{"description":"Experiment execution not found"}}}},"/api/v1/experiments/{id}/retry":{"post":{"tags":["Experiments"],"summary":"Retry current experiment processing","parameters":[{"schema":{"type":"string","minLength":1},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"experimentExecutionId":{"type":"string","minLength":1}},"required":["experimentExecutionId"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/RetryExperimentResponse"}}}},"404":{"description":"Experiment not found"},"409":{"description":"Execution is stale or cannot be retried"}}}},"/api/v1/requests":{"post":{"tags":["Requests"],"summary":"Submit request(s)","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"allOf":[{"$ref":"#/components/schemas/CreateRequestInput"},{"type":"object","properties":{"count":{"type":"number","minimum":1,"maximum":10,"default":1,"deprecated":true,"description":"Deprecated for composition. Use POST /api/v1/experiments and its requests array."},"skills":{"type":"array","items":{"type":"string"}},"agentVersion":{"type":"string"},"codebase":{"type":"string"}}}]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"anyOf":[{"$ref":"#/components/schemas/RequestResponse"},{"type":"array","items":{"$ref":"#/components/schemas/RequestResponse"}}]}}}}}},"get":{"tags":["Requests"],"summary":"List requests","parameters":[{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"worker","in":"query"},{"schema":{"type":"string"},"required":false,"name":"taskPromptId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"criteria","in":"query"},{"schema":{"type":"string"},"required":false,"name":"submissionId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"experimentId","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"profileId","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"status","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"outcome","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"model","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"os","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"priority","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"agentVersion","in":"query"},{"schema":{"type":"string"},"required":false,"name":"search","in":"query"},{"schema":{"type":"string","enum":["task","submissionId","experimentId","profile"]},"required":false,"name":"groupBy","in":"query"},{"schema":{"type":"integer","minimum":1,"maximum":100},"required":false,"name":"limit","in":"query"},{"schema":{"type":"string"},"required":false,"name":"after","in":"query"},{"schema":{"type":"string"},"required":false,"name":"before","in":"query"},{"schema":{"type":"string","enum":["true","false"]},"required":false,"name":"last","in":"query"},{"schema":{"type":"string","enum":["created","updated","priority","worker","status","id","duration","createdAt"]},"required":false,"name":"sortBy","in":"query"},{"schema":{"type":"string","enum":["asc","desc"]},"required":false,"name":"sortDir","in":"query"},{"schema":{"type":["string","null"],"format":"date-time"},"required":false,"name":"createdAfter","in":"query"},{"schema":{"type":["string","null"],"format":"date-time"},"required":false,"name":"createdBefore","in":"query"},{"schema":{"type":["integer","null"],"minimum":0},"required":false,"name":"turns","in":"query"},{"schema":{"type":"string","enum":["eq","gte","lte"]},"required":false,"name":"turnsOp","in":"query"},{"schema":{"type":["integer","null"],"minimum":0},"required":false,"name":"maxIterations","in":"query"},{"schema":{"type":"string","enum":["eq","gte","lte"]},"required":false,"name":"maxIterationsOp","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"anyOf":[{"$ref":"#/components/schemas/PaginatedRunsResponse"},{"$ref":"#/components/schemas/PaginatedRunGroupsResponse"}]}}}}}}},"/api/v1/requests/facets":{"get":{"tags":["Requests"],"summary":"List run filter facets","description":"Returns every distinct value and its full-dataset count per categorical filter dimension for the Runs list rail. Counts are absolute over all non-deleted runs **in the given project**: they intentionally ignore the active search, date range, iteration, and categorical selections so every selectable value stays visible with a stable count. Requires ?projectId=. Computed with parallel $group aggregations (Cosmos has no $facet) and cached per-project for a short TTL.","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/RunFacetsResponse"}}}}}}},"/api/v1/requests/{id}":{"get":{"tags":["Requests"],"summary":"Get request","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/RequestResponse"}}}},"404":{"description":"Not found"}}},"delete":{"tags":["Requests"],"summary":"Soft-delete request","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}},"404":{"description":"Not found"}}}},"/api/v1/analysis":{"get":{"tags":["Requests"],"summary":"Compute pass@k / success@T metrics","description":"Aggregates pass@k / success@T metrics over a single project's done runs. Requires ?projectId=.","parameters":[{"schema":{"type":"string"},"required":false,"name":"worker","in":"query"},{"schema":{"type":"string"},"required":false,"name":"taskPromptId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"criteria","in":"query"},{"schema":{"type":"string"},"required":false,"name":"features","in":"query"},{"schema":{"type":"string"},"required":false,"name":"submissionId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"experimentId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"k","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{},"additionalProperties":{},"description":"Analysis metrics"}}}}}}},"/api/v1/requests/bulk-resubmit":{"post":{"tags":["Requests"],"summary":"Bulk resubmit requests","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/BulkResubmitInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/RequestResponse"}}}}}}}},"/api/v1/requests/bulk":{"delete":{"tags":["Requests"],"summary":"Bulk soft-delete requests","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"}}},"required":["ids"]}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"deleted":{"type":"number"}},"required":["deleted"]}}}}}}},"/api/v1/requests/archive":{"post":{"tags":["Requests"],"summary":"Download batch archive of multiple runs","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1,"maxItems":100}},"required":["ids"]}}}},"responses":{"201":{"description":"Gzipped batch archive containing individual run archives"},"400":{"description":"Invalid input"},"404":{"description":"One or more runs not found"}}}},"/api/v1/runs/upload":{"post":{"tags":["Requests"],"summary":"Import run archive","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"201":{"description":"Success"}}}},"/api/v1/runs/upload-batch":{"post":{"tags":["Requests"],"summary":"Import batch run archive","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"207":{"description":"Multi-status: 201 if all runs imported, 400 if none imported, 207 if partial"},"400":{"description":"No archive uploaded, empty archive, or all runs failed"}}}},"/api/v1/requests/{id}/reports":{"get":{"tags":["Reports"],"summary":"Get reports for request","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ReportResponse"}}}}},"404":{"description":"Run not found"}}}},"/api/v1/requests/{id}/runs":{"get":{"tags":["Requests"],"summary":"List attempts for a request","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/RunState"}}}}},"404":{"description":"Request not found"}}}},"/api/v1/requests/{id}/runs/{runId}":{"get":{"tags":["Requests"],"summary":"Get a single attempt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/RunState"}}}},"404":{"description":"Request or run not found"}}}},"/api/v1/requests/bulk-retry":{"post":{"tags":["Requests"],"summary":"Bulk retry requests (start new attempts)","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1},"force":{"type":"boolean"}},"required":["ids"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"retried":{"type":"integer"},"skipped":{"type":"integer"},"results":{"type":"array","items":{"type":"object","properties":{"requestId":{"type":"string"},"runId":{"type":"string"},"attemptNumber":{"type":"integer"},"error":{"type":"string"}},"required":["requestId"]}}},"required":["retried","skipped","results"]}}}}}}},"/api/v1/requests/{id}/retry":{"post":{"tags":["Requests"],"summary":"Retry a request (start a new attempt)","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"force":{"type":"boolean"}}}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"requestId":{"type":"string"},"runId":{"type":"string"},"attemptNumber":{"type":"integer"}},"required":["requestId","runId","attemptNumber"]}}}},"404":{"description":"Request not found"},"409":{"description":"Conflict — request is not in a retryable state, or a concurrent retry won the race"},"422":{"description":"Cannot retry — current run not yet terminal"}}}},"/api/v1/requests/{id}/priority":{"post":{"tags":["Requests"],"summary":"Set priority on a single request","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"priority":{"type":"integer"}},"required":["priority"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"priority":{"type":"number"}},"required":["id","priority"]}}}}}}},"/api/v1/requests/bulk-priority":{"post":{"tags":["Requests"],"summary":"Set priority on multiple requests","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1,"maxItems":100},"priority":{"type":"integer"}},"required":["ids","priority"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"updated":{"type":"number"},"skipped":{"type":"number"}},"required":["updated","skipped"]}}}}}}},"/api/v1/requests/{id}/pause":{"post":{"tags":["Requests"],"summary":"Pause a single request","responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"status":{"type":"string"}},"required":["id","status"]}}}}}}},"/api/v1/requests/{id}/resume":{"post":{"tags":["Requests"],"summary":"Resume a paused request","responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"status":{"type":"string"}},"required":["id","status"]}}}}}}},"/api/v1/requests/bulk-pause":{"post":{"tags":["Requests"],"summary":"Pause multiple requests","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1,"maxItems":100}},"required":["ids"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"updated":{"type":"number"},"skipped":{"type":"number"}},"required":["updated","skipped"]}}}}}}},"/api/v1/requests/bulk-resume":{"post":{"tags":["Requests"],"summary":"Resume multiple paused requests","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1,"maxItems":100}},"required":["ids"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"updated":{"type":"number"},"skipped":{"type":"number"}},"required":["updated","skipped"]}}}}}}},"/api/v1/requests/{id}/cancel":{"post":{"tags":["Requests"],"summary":"Cancel a request (marks as done/failed and signals worker to exit)","responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"previousStatus":{"type":"string"},"status":{"type":"string"},"outcome":{"type":"string"}},"required":["id","previousStatus","status","outcome"]}}}}}}},"/api/v1/requests/bulk-cancel":{"post":{"tags":["Requests"],"summary":"Cancel multiple requests","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1,"maxItems":100}},"required":["ids"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"cancelled":{"type":"number"},"skipped":{"type":"number"},"results":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string"},"cancelled":{"type":"boolean"},"previousStatus":{"type":"string"},"error":{"type":"string"}},"required":["id","cancelled"]}}},"required":["cancelled","skipped","results"]}}}}}}},"/api/v1/requests/{id}/logs":{"get":{"tags":["Requests"],"summary":"Stream request logs (SSE)","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Server-sent event stream of log entries"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/logs":{"get":{"tags":["Requests"],"summary":"Stream logs for a specific attempt (SSE)","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"}],"responses":{"200":{"description":"Server-sent event stream of log entries"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/har":{"get":{"tags":["Requests"],"summary":"Download HAR file","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"HAR-format JSON file"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/har":{"get":{"tags":["Requests"],"summary":"Download HAR file for a specific attempt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"}],"responses":{"200":{"description":"HAR-format JSON file"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/atif":{"get":{"tags":["Requests"],"summary":"Download ATIF trajectory file for a specific iteration","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","description":"The iteration number (1-based)"},"required":true,"description":"The iteration number (1-based)","name":"iteration","in":"query"}],"responses":{"200":{"description":"ATIF v1.7 trajectory JSON file"},"400":{"description":"Missing or invalid iteration"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/atif":{"get":{"tags":["Requests"],"summary":"Download ATIF trajectory file for a specific attempt and iteration","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"},{"schema":{"type":"string","description":"The iteration number (1-based)"},"required":true,"description":"The iteration number (1-based)","name":"iteration","in":"query"}],"responses":{"200":{"description":"ATIF v1.7 trajectory JSON file"},"400":{"description":"Missing or invalid iteration"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/video":{"get":{"tags":["Requests"],"summary":"Download session recording","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"WebM video recording (supports Range requests)"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/video":{"get":{"tags":["Requests"],"summary":"Download session recording for a specific attempt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"}],"responses":{"200":{"description":"WebM video recording (supports Range requests)"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/tool-calls":{"get":{"tags":["Requests"],"summary":"Download per-iteration tool-calls JSONL","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"JSONL stream — one ToolCall per line"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/tool-calls":{"get":{"tags":["Requests"],"summary":"Download per-iteration tool-calls JSONL for a specific attempt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"}],"responses":{"200":{"description":"JSONL stream — one ToolCall per line"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/snapshots/{iteration}":{"get":{"tags":["Requests"],"summary":"Download iteration snapshot","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"iteration","in":"path"}],"responses":{"200":{"description":"Gzipped snapshot archive"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/snapshots/{iteration}":{"get":{"tags":["Requests"],"summary":"Download iteration snapshot for a specific attempt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"},{"schema":{"type":"string"},"required":true,"name":"iteration","in":"path"}],"responses":{"200":{"description":"Gzipped snapshot archive"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/archive":{"get":{"tags":["Requests"],"summary":"Download full run archive","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Gzipped run archive"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/archive":{"get":{"tags":["Requests"],"summary":"Download full run archive for a specific attempt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"}],"responses":{"200":{"description":"Gzipped run archive"},"404":{"description":"Not found"}}}},"/api/v1/criteria/generate-prompt":{"post":{"tags":["Criteria"],"summary":"Generate criterion prompt from behavior","parameters":[{"schema":{"type":"string","minLength":1,"description":"Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.","example":"00000000-0000-0000-0000-000000000000"},"required":false,"description":"Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"behavior":{"type":"string"},"currentId":{"type":"string"},"gates":{"type":"array","items":{"type":"string"}}},"required":["behavior"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"prompt":{"type":"string"},"suggestedId":{"type":"string"},"suggestedParents":{"type":"array","items":{"type":"string"}},"suggestedChildren":{"type":"array","items":{"type":"string"}}},"required":["prompt","suggestedId","suggestedParents","suggestedChildren"]}}}},"400":{"description":"Empty behavior string"},"503":{"description":"LLM not configured"}}}},"/api/v1/criteria/seed":{"post":{"tags":["Criteria"],"summary":"Seed criteria in bulk","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"criteria":{"type":"array","items":{"$ref":"#/components/schemas/CreateCriteriaInput"}}},"required":["criteria"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"seeded":{"type":"number"},"errors":{"type":"array","items":{"type":"string"}}},"required":["seeded","errors"]}}}},"400":{"description":"Seed batch would introduce a dependency cycle"}}}},"/api/v1/criteria":{"get":{"tags":["Criteria"],"summary":"List criteria","parameters":[{"schema":{"type":"string"},"required":false,"name":"q","in":"query"},{"schema":{"type":"string","description":"Comma-separated criterion IDs to include"},"required":false,"description":"Comma-separated criterion IDs to include","name":"ids","in":"query"},{"schema":{"type":"string","enum":["true","false"],"description":"When true and ids is set, also include dependency ancestors"},"required":false,"description":"When true and ids is set, also include dependency ancestors","name":"ancestors","in":"query"},{"schema":{"type":"string","enum":["gate","observation"],"description":"Filter by criterion kind"},"required":false,"description":"Filter by criterion kind","name":"kind","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/CriteriaResponse"}}}}}}},"post":{"tags":["Criteria"],"summary":"Create criterion","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateCriteriaInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CriteriaResponse"}}}},"409":{"description":"Criterion already exists"}}}},"/api/v1/criteria/mdp":{"get":{"tags":["Criteria"],"summary":"Compute MDP transitions","parameters":[{"schema":{"type":"string"},"required":false,"name":"criteria","in":"query"},{"schema":{"type":"string"},"required":false,"name":"features","in":"query"},{"schema":{"type":"string"},"required":false,"name":"since","in":"query"},{"schema":{"type":"string"},"required":false,"name":"worker","in":"query"},{"schema":{"type":"string"},"required":false,"name":"taskPromptId","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{},"additionalProperties":{}}}}}}}},"/api/v1/criteria/graph":{"get":{"tags":["Criteria"],"summary":"Get criteria DAG","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CriteriaGraph"}}}}}}},"/api/v1/criteria/{id}":{"get":{"tags":["Criteria"],"summary":"Get criterion","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CriteriaResponse"}}}},"404":{"description":"Criterion not found"}}},"put":{"tags":["Criteria"],"summary":"Update criterion","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateCriteriaInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CriteriaResponse"}}}},"400":{"description":"Invalid dependency reference or self-reference"},"404":{"description":"Criterion not found"}}},"delete":{"tags":["Criteria"],"summary":"Soft-delete criterion","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"deleted":{"type":"boolean"}},"required":["id","deleted"]}}}},"404":{"description":"Criterion not found"},"409":{"description":"Criterion has dependents"}}}},"/api/v1/prompt-features/generate-prompt":{"post":{"tags":["Prompt Features"],"summary":"Generate prompt feature from behavior","parameters":[{"schema":{"type":"string","minLength":1,"description":"Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.","example":"00000000-0000-0000-0000-000000000000"},"required":false,"description":"Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"behavior":{"type":"string"},"currentId":{"type":"string"}},"required":["behavior"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"prompt":{"type":"string"}},"required":["prompt"]}}}},"400":{"description":"Empty behavior string"},"503":{"description":"LLM not configured"}}}},"/api/v1/prompt-features/seed":{"post":{"tags":["Prompt Features"],"summary":"Seed prompt features in bulk","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"features":{"type":"array","items":{"$ref":"#/components/schemas/CreatePromptFeatureInput"}}},"required":["features"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"seeded":{"type":"number"},"errors":{"type":"array","items":{"type":"string"}}},"required":["seeded","errors"]}}}}}}},"/api/v1/prompt-features/extract-from-text":{"post":{"tags":["Prompt Features"],"summary":"Extract features from text","parameters":[{"schema":{"type":"string","minLength":1,"description":"Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.","example":"00000000-0000-0000-0000-000000000000"},"required":false,"description":"Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"text":{"type":"string"},"model":{"type":"string"},"type":{"type":"string","enum":["select","agents.md"]}},"required":["text"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"features":{"type":"array","items":{"$ref":"#/components/schemas/PromptFeatureResult"}},"suggestedFeatures":{"type":"array","items":{"$ref":"#/components/schemas/SuggestedPromptFeature"}},"cached":{"type":"boolean"}},"required":["features","cached"]}}}},"400":{"description":"Empty text string"},"503":{"description":"LLM not configured"}}}},"/api/v1/prompt-features":{"get":{"tags":["Prompt Features"],"summary":"List features","parameters":[{"schema":{"type":"string"},"required":false,"name":"q","in":"query"},{"schema":{"type":"string","enum":["select","agents.md"]},"required":false,"name":"type","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/PromptFeatureResponse"}}}}}}},"post":{"tags":["Prompt Features"],"summary":"Create feature","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreatePromptFeatureInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/PromptFeatureResponse"}}}},"400":{"description":"Invalid input"},"409":{"description":"Feature already exists"}}}},"/api/v1/prompt-features/{id}":{"get":{"tags":["Prompt Features"],"summary":"Get feature","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/PromptFeatureResponse"}}}},"404":{"description":"Feature not found"}}},"put":{"tags":["Prompt Features"],"summary":"Update feature","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdatePromptFeatureInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/PromptFeatureResponse"}}}},"400":{"description":"Invalid input"},"404":{"description":"Feature not found"}}},"delete":{"tags":["Prompt Features"],"summary":"Soft-delete feature","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"deleted":{"type":"boolean"}},"required":["id","deleted"]}}}},"404":{"description":"Feature not found"}}}},"/api/v1/task-prompts/generate":{"post":{"tags":["Task Prompts"],"summary":"Generate task prompts","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"description":{"type":"string"},"existingPrompt":{"type":"string"}}}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"tasks":{"type":"array","items":{"type":"string"}}},"required":["tasks"]}}}},"503":{"description":"LLM not configured"}}}},"/api/v1/task-prompts":{"get":{"tags":["Task Prompts"],"summary":"List task prompts","parameters":[{"schema":{"type":["number","null"]},"required":false,"name":"limit","in":"query"},{"schema":{"type":["number","null"]},"required":false,"name":"offset","in":"query"},{"schema":{"type":"string"},"required":false,"name":"search","in":"query"},{"schema":{"$ref":"#/components/schemas/PromptType"},"required":false,"name":"type","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"items":{"type":"array","items":{"$ref":"#/components/schemas/TaskPromptResponse"}},"total":{"type":"number"},"limit":{"type":"number"},"offset":{"type":"number"}},"required":["items","total","limit","offset"]}}}}}},"post":{"tags":["Task Prompts"],"summary":"Create or find task prompt","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateTaskPromptInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/TaskPromptResponse"}}}},"400":{"description":"Empty text string"}}}},"/api/v1/task-prompts/{id}":{"get":{"tags":["Task Prompts"],"summary":"Get task prompt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/TaskPromptResponse"}}}},"404":{"description":"Task prompt not found"}}},"delete":{"tags":["Task Prompts"],"summary":"Soft-delete task prompt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"deleted":{"type":"boolean"}},"required":["deleted"]}}}},"404":{"description":"Task prompt not found"}}}},"/api/v1/task-prompts/{id}/content":{"get":{"tags":["Task Prompts"],"summary":"Get resolved task prompt content","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"text":{"type":"string"}},"required":["id","text"]}}}},"404":{"description":"Task prompt not found"}}}},"/api/v1/task-prompts/{id}/extract-features":{"post":{"tags":["Task Prompts"],"summary":"Extract features from task prompt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":false,"name":"force","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"model":{"type":"string"}}}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"taskPromptId":{"type":"string"},"features":{"type":"array","items":{"$ref":"#/components/schemas/PromptFeatureResult"}},"featuresExtractedAt":{"type":["string","null"],"format":"date-time"},"suggestedFeatures":{"type":"array","items":{"$ref":"#/components/schemas/SuggestedPromptFeature"}},"cached":{"type":"boolean"}},"required":["taskPromptId","features","cached"]}}}},"404":{"description":"Task prompt not found"},"503":{"description":"LLM not configured"}}}},"/api/v1/task-prompts/{id}/features/{featureId}":{"patch":{"tags":["Task Prompts"],"summary":"Toggle feature flag on task prompt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"featureId","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/PatchTaskPromptFeatureInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/TaskPromptResponse"}}}},"400":{"description":"Invalid detected value"},"404":{"description":"Task prompt or feature not found"}}}},"/api/v1/reports":{"post":{"tags":["Reports"],"summary":"Create report","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateReportInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ReportResponse"}}}},"400":{"description":"Invalid input"},"404":{"description":"Run or template not found"}}},"get":{"tags":["Reports"],"summary":"List reports","parameters":[{"schema":{"type":"string"},"required":false,"name":"requestId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ReportResponse"}}}}}}}},"/api/v1/reports/bulk-create":{"post":{"tags":["Reports"],"summary":"Bulk create reports","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/BulkCreateReportsInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ReportResponse"}}}}},"400":{"description":"Invalid input"}}}},"/api/v1/reports/bulk-status":{"post":{"tags":["Reports"],"summary":"Bulk get report statuses","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/BulkReportStatusInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ReportResponse"}}}}},"400":{"description":"Invalid input"}}}},"/api/v1/reports/bulk-summary":{"post":{"tags":["Reports"],"summary":"Bulk get report summary per run","description":"Returns aggregated report status counts per run. Only the latest report per template is counted (re-triggers are deduplicated).","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/BulkReportSummaryInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/BulkReportSummaryResponse"}}}},"400":{"description":"Invalid input"}}}},"/api/v1/reports/{id}":{"get":{"tags":["Reports"],"summary":"Get report","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ReportResponse"}}}},"404":{"description":"Report not found"}}}},"/api/v1/reports/{id}/logs":{"get":{"tags":["Reports"],"summary":"Stream report logs (SSE)","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Server-sent event stream of log entries"},"404":{"description":"Report not found"}}}},"/api/v1/reports/trigger":{"post":{"tags":["Reports"],"summary":"Trigger reports","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/TriggerReportsInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"triggered":{"type":"number"},"reports":{"type":"array","items":{"$ref":"#/components/schemas/ReportResponse"}}},"required":["triggered","reports"]}}}},"400":{"description":"Invalid input"},"404":{"description":"Run not found"}}}},"/api/v1/reports/bulk-trigger":{"post":{"tags":["Reports"],"summary":"Bulk trigger reports","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/BulkTriggerReportsInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"type":"object","properties":{},"additionalProperties":{}}}}}},"400":{"description":"Invalid input"}}}},"/api/v1/reports/{id}/insights":{"get":{"tags":["Reports"],"summary":"Get insights for report","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"allOf":[{"$ref":"#/components/schemas/InsightResponse"},{"type":"object","properties":{"referencedAt":{"type":["string","null"],"format":"date-time"},"isNew":{"type":"boolean"}}}]}}}}},"404":{"description":"Report not found"}}},"post":{"tags":["Reports"],"summary":"Link insight to report","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"insightId":{"type":"string"},"isNew":{"type":"boolean"}},"required":["insightId"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"insightId":{"type":"string"},"referencedAt":{"type":["string","null"],"format":"date-time"},"isNew":{"type":"boolean"}},"required":["insightId","referencedAt","isNew"]}}}},"404":{"description":"Report or insight not found"},"409":{"description":"Insight already referenced by this report"}}}},"/api/v1/profiles":{"post":{"tags":["Profiles"],"summary":"Create a new profile","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateProfileInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProfileWithVersionResponse"}}}}}},"get":{"tags":["Profiles"],"summary":"List profiles","parameters":[{"schema":{"type":"string"},"required":false,"name":"workerType","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ProfileWithVersionResponse"}}}}}}}},"/api/v1/profiles/{profileId}":{"get":{"tags":["Profiles"],"summary":"Get profile with latest version","parameters":[{"schema":{"type":"string"},"required":true,"name":"profileId","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProfileWithVersionResponse"}}}}}},"post":{"tags":["Profiles"],"summary":"Create new profile version","parameters":[{"schema":{"type":"string"},"required":true,"name":"profileId","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"workerType":{"type":"string"},"model":{"type":"string"},"reasoningEffort":{"type":"string"},"agentVersion":{"type":"string"},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceBindingSpec"}},"extensions":{"type":"array","items":{"type":"string"}}},"required":["workerType","model"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProfileVersionResponse"}}}}}},"put":{"tags":["Profiles"],"summary":"Update profile identity","parameters":[{"schema":{"type":"string"},"required":true,"name":"profileId","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateProfileIdentity"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProfileResponse"}}}}}},"delete":{"tags":["Profiles"],"summary":"Delete profile","parameters":[{"schema":{"type":"string"},"required":true,"name":"profileId","in":"path"}],"responses":{"204":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}}}}},"/api/v1/profiles/{profileId}/versions":{"get":{"tags":["Profiles"],"summary":"List profile versions","parameters":[{"schema":{"type":"string"},"required":true,"name":"profileId","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ProfileVersionResponse"}}}}}}}},"/api/v1/profiles/{profileId}/versions/{version}":{"get":{"tags":["Profiles"],"summary":"Get profile version","parameters":[{"schema":{"type":"string"},"required":true,"name":"profileId","in":"path"},{"schema":{"type":["number","null"]},"required":false,"name":"version","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProfileVersionResponse"}}}}}}},"/api/v1/report-templates/default-system-prompt":{"get":{"tags":["Report Templates"],"summary":"Get default system prompt","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"content":{"type":"string"}},"required":["content"]}}}}}}},"/api/v1/report-templates/available-models":{"get":{"tags":["Report Templates"],"summary":"List models available for report generation","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"type":"object","properties":{"modelId":{"type":"string"}},"required":["modelId"]}}}}}}}},"/api/v1/report-templates":{"get":{"tags":["Report Templates"],"summary":"List report templates","parameters":[{"schema":{"type":"string"},"required":false,"name":"q","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ReportTemplateResponse"}}}}}}},"post":{"tags":["Report Templates"],"summary":"Create report template","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateReportTemplateInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ReportTemplateResponse"}}}},"409":{"description":"Report template already exists"}}}},"/api/v1/report-templates/{id}":{"get":{"tags":["Report Templates"],"summary":"Get report template","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ReportTemplateResponse"}}}},"404":{"description":"Report template not found"}}},"put":{"tags":["Report Templates"],"summary":"Update report template","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateReportTemplateInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ReportTemplateResponse"}}}},"404":{"description":"Report template not found"}}},"delete":{"tags":["Report Templates"],"summary":"Delete report template","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"204":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}},"404":{"description":"Report template not found"}}}},"/api/v1/agents":{"get":{"tags":["Agents"],"summary":"List agents","parameters":[{"schema":{"type":"string"},"required":false,"name":"modelProvider","in":"query"},{"schema":{"type":"string","enum":["true","false"]},"required":false,"name":"includeDeleted","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/AgentResponse"}}}}}}},"post":{"tags":["Agents"],"summary":"Create or update agent (upsert)","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateAgentInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AgentResponse"}}}},"400":{"description":"Validation error"},"409":{"description":"Restored agent versions conflict with an active queue owner"}}}},"/api/v1/agents/{id}":{"get":{"tags":["Agents"],"summary":"Get agent","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","enum":["true","false"]},"required":false,"name":"includeDeleted","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AgentResponse"}}}},"404":{"description":"Agent not found"}}},"put":{"tags":["Agents"],"summary":"Update agent","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateAgentInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AgentResponse"}}}},"400":{"description":"Validation error"},"404":{"description":"Agent not found"}}},"delete":{"tags":["Agents"],"summary":"Delete agent","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"204":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}},"404":{"description":"Agent not found"}}}},"/api/v1/agents/{id}/versions":{"get":{"tags":["Agents"],"summary":"List agent versions","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":false,"name":"status","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/AgentVersion"}}}}},"404":{"description":"Agent not found"}}},"post":{"tags":["Agents"],"summary":"Register agent version (upsert)","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/RegisterAgentVersionInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AgentVersion"}}}},"400":{"description":"Validation error"},"404":{"description":"Agent not found"},"409":{"description":"Queue is assigned to another agent"},"503":{"description":"Concurrent registration did not stabilize"}}}},"/api/v1/agents/{id}/versions/{agentVersion}":{"patch":{"tags":["Agents"],"summary":"Patch agent version","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"agentVersion","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/PatchAgentVersionInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AgentVersion"}}}},"400":{"description":"Invalid status value"},"404":{"description":"Agent or version not found"},"409":{"description":"Queue is assigned to another agent"},"503":{"description":"Concurrent activation did not stabilize"}}}},"/api/v1/models":{"get":{"tags":["Models"],"summary":"List models","parameters":[{"schema":{"type":"string"},"required":false,"name":"agentId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"provider","in":"query"},{"schema":{"type":"string","enum":["active","disappeared"]},"required":false,"name":"status","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ModelResponse"}}}}}}}},"/api/v1/models/{id}":{"get":{"tags":["Models"],"summary":"Get model","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ModelResponse"}}}}}}},"/api/v1/models/sync":{"post":{"tags":["Models"],"summary":"Sync models from provider","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"agentId":{"type":"string"},"provider":{"type":"string"},"models":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string"},"providerAvailableFrom":{"type":"string"},"providerEndOfLife":{"type":"string"},"metadata":{"type":"object","additionalProperties":{}},"capabilities":{"type":"object","properties":{"reasoningEffort":{"type":"array","items":{"type":"string"}},"toolCalls":{"type":"boolean"},"vision":{"type":"boolean"},"streaming":{"type":"boolean"},"adaptiveThinking":{"type":"boolean"},"maxThinkingBudget":{"type":"number"}}}},"required":["id"]}},"scannedAt":{"type":"string"}},"required":["agentId","provider","models","scannedAt"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"added":{"type":"array","items":{"type":"string"}},"removed":{"type":"array","items":{"type":"string"}},"unchanged":{"type":"array","items":{"type":"string"}}},"required":["added","removed","unchanged"]}}}}}}},"/api/v1/mcp/servers":{"get":{"tags":["MCP Servers"],"summary":"List MCP servers","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/McpServerResponse"}}}}}}},"post":{"tags":["MCP Servers"],"summary":"Create MCP server","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"_id":{"type":"string","pattern":"^[a-z0-9]([a-z0-9-]*[a-z0-9])?$"},"name":{"type":"string"},"type":{"type":"string","enum":["sse","http","stdio"]},"url":{"type":"string"},"command":{"type":"string"},"args":{"type":"array","items":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"headers":{"type":"array","items":{"$ref":"#/components/schemas/McpServerHeader"}},"sessionMode":{"type":"string","enum":["stateful","stateless"]},"version":{"type":"string"},"description":{"type":"string"}},"required":["_id","name","type"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/McpServerResponse"}}}}}}},"/api/v1/mcp/servers/{id}":{"get":{"tags":["MCP Servers"],"summary":"Get MCP server","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/McpServerResponse"}}}}}},"put":{"tags":["MCP Servers"],"summary":"Update MCP server","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateMcpServerInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/McpServerResponse"}}}}}},"delete":{"tags":["MCP Servers"],"summary":"Delete MCP server","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"204":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}}}}},"/api/v1/skills":{"get":{"tags":["Skills"],"summary":"List all skills","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/SkillResponse"}}}}}}},"post":{"tags":["Skills"],"summary":"Create or import a skill","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateSkillInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SkillResponse"}}}}}}},"/api/v1/skills/search":{"get":{"tags":["Skills"],"summary":"Search skills (internal + external)","parameters":[{"schema":{"type":"string"},"required":true,"name":"q","in":"query"},{"schema":{"type":"string"},"required":false,"name":"limit","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/SkillSearchResult"}}}}},"400":{"description":"Missing query parameter"}}}},"/api/v1/skills/search/external":{"get":{"tags":["Skills"],"summary":"Search external skills registry","parameters":[{"schema":{"type":"string"},"required":true,"name":"q","in":"query"},{"schema":{"type":"string"},"required":false,"name":"limit","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Optional project scope. When provided, external results are de-duplicated only against skills already installed in that project."},"required":false,"description":"Optional project scope. When provided, external results are de-duplicated only against skills already installed in that project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/SkillSearchResult"}}}}},"400":{"description":"Missing query parameter"}}}},"/api/v1/skills/discover":{"get":{"tags":["Skills"],"summary":"Discover skills in a GitHub repository","parameters":[{"schema":{"type":"string"},"required":true,"name":"source","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/SkillDiscoveryResult"}}}}},"400":{"description":"Missing or malformed source parameter"},"404":{"description":"Repository not found"},"502":{"description":"GitHub API error"}}}},"/api/v1/skills/{id}(*)/revisions":{"get":{"tags":["Skills"],"summary":"List skill revisions","parameters":[{"schema":{"type":"string"},"required":false,"name":"limit","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/SkillRevisionResponse"}}}}},"404":{"description":"Skill not found"}}}},"/api/v1/skills/{id}(*)":{"get":{"tags":["Skills"],"summary":"Get skill by slug","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SkillResponse"}}}},"404":{"description":"Skill not found"}}},"delete":{"tags":["Skills"],"summary":"Soft-delete a skill","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"204":{"description":"Success"},"404":{"description":"Skill not found"}}}},"/api/v1/skill-revisions/by-ref/{ref}(*)/archive":{"get":{"tags":["Skill Revisions"],"summary":"Download skill revision archive","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Binary tar.gz archive"},"404":{"description":"Skill revision or archive not found"},"500":{"description":"Blob storage error"}}}},"/api/v1/skill-revisions/by-ref/{ref}(*)":{"get":{"tags":["Skill Revisions"],"summary":"Get skill revision by ref","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SkillRevisionResponse"}}}},"404":{"description":"Skill revision not found"}}}},"/api/v1/skill-revisions/{id}":{"get":{"tags":["Skill Revisions"],"summary":"Get skill revision by ID","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SkillRevisionResponse"}}}},"404":{"description":"Skill revision not found"}}}},"/api/v1/skills/{id}(*)/resolve":{"post":{"tags":["Skills"],"summary":"Trigger skill resolution","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SkillRevisionResponse"}}}},"404":{"description":"Skill not found"},"500":{"description":"Blob storage error"}}}},"/api/v1/codebases":{"get":{"tags":["Codebases"],"summary":"List all codebases","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/CodebaseResponse"}}}}}}},"post":{"tags":["Codebases"],"summary":"Create a codebase (archive codebases require the archive file)","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"201":{"description":"Success"},"400":{"description":"Invalid input or missing archive"}}}},"/api/v1/codebases/{id}":{"get":{"tags":["Codebases"],"summary":"Get a codebase","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CodebaseResponse"}}}},"404":{"description":"Codebase not found"}}},"patch":{"tags":["Codebases"],"summary":"Update a codebase","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateCodebaseInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CodebaseResponse"}}}},"404":{"description":"Codebase not found"}}},"delete":{"tags":["Codebases"],"summary":"Delete a codebase","responses":{"204":{"description":"Success"},"404":{"description":"Codebase not found"}}}},"/api/v1/codebases/{id}/revisions":{"get":{"tags":["Codebases"],"summary":"List codebase revisions","parameters":[{"schema":{"type":"string"},"required":false,"name":"limit","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/CodebaseRevisionResponse"}}}}},"404":{"description":"Codebase not found"}}},"post":{"tags":["Codebases"],"summary":"Resolve a new git codebase revision","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ResolveCodebaseRevisionInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CodebaseRevisionResponse"}}}},"400":{"description":"Not a git codebase"},"404":{"description":"Codebase not found"},"502":{"description":"GitHub resolution failed"}}}},"/api/v1/codebases/{id}/upload":{"post":{"tags":["Codebases"],"summary":"Upload a codebase archive as a new revision","responses":{"201":{"description":"Success"},"400":{"description":"Missing archive or not an archive codebase"},"404":{"description":"Codebase not found"}}}},"/api/v1/codebase-revisions/{id}/archive":{"get":{"tags":["Codebase Revisions"],"summary":"Download codebase revision archive","responses":{"200":{"description":"Binary tar.gz archive"},"404":{"description":"Revision or archive not found"},"500":{"description":"Blob storage error"}}}},"/api/v1/codebase-revisions/{id}":{"get":{"tags":["Codebase Revisions"],"summary":"Get codebase revision by id","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CodebaseRevisionResponse"}}}},"404":{"description":"Revision not found"}}}},"/api/v1/resources":{"get":{"tags":["Resources"],"summary":"List all resources","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ResourceResponse"}}}}}}},"post":{"tags":["Resources"],"summary":"Create a resource and its first revision","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateResourceInput"}}}},"responses":{"201":{"description":"Success"},"400":{"description":"Invalid input"},"409":{"description":"Resource slug or revision ref already exists in this project"}}}},"/api/v1/resources/{id}":{"get":{"tags":["Resources"],"summary":"Get a resource","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ResourceResponse"}}}},"404":{"description":"Resource not found"}}},"patch":{"tags":["Resources"],"summary":"Update a resource","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateResourceInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ResourceResponse"}}}},"404":{"description":"Resource not found"}}},"delete":{"tags":["Resources"],"summary":"Delete a resource","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"204":{"description":"Success"},"404":{"description":"Resource not found"}}}},"/api/v1/resources/{id}/revisions":{"get":{"tags":["Resource Revisions"],"summary":"List resource revisions","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"limit","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ResourceRevisionResponse"}}}}},"404":{"description":"Resource not found"}}},"post":{"tags":["Resource Revisions"],"summary":"Create a resource revision","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateResourceRevisionInput"}}}},"responses":{"201":{"description":"Success"},"404":{"description":"Resource not found"},"409":{"description":"Resource revision ref already exists in this project"}}}},"/api/v1/resources/{id}/revisions/latest":{"get":{"tags":["Resource Revisions"],"summary":"Get the latest resource revision","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ResourceRevisionResponse"}}}},"404":{"description":"Revision not found"}}}},"/api/v1/resources/{id}/revisions/{revisionNumber}":{"get":{"tags":["Resource Revisions"],"summary":"Get a resource revision by resource and revision number","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"revisionNumber","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ResourceRevisionResponse"}}}},"404":{"description":"Revision not found"}}}},"/api/v1/resources/revisions/{id}":{"get":{"tags":["Resource Revisions"],"summary":"Get resource revision by id","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ResourceRevisionResponse"}}}},"404":{"description":"Revision not found"}}}},"/api/v1/extensions":{"get":{"tags":["Extensions"],"summary":"List all extensions","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ExtensionResponse"}}}}}}},"post":{"tags":["Extensions"],"summary":"Create or import an extension","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateExtensionInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ExtensionResponse"}}}}}}},"/api/v1/extensions/search":{"get":{"tags":["Extensions"],"summary":"Search extensions (internal + marketplace)","parameters":[{"schema":{"type":"string"},"required":true,"name":"q","in":"query"},{"schema":{"type":"string"},"required":false,"name":"limit","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ExtensionSearchResult"}}}}},"400":{"description":"Missing query parameter"}}}},"/api/v1/extensions/{id}":{"get":{"tags":["Extensions"],"summary":"Get extension by ID","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ExtensionResponse"}}}}}},"put":{"tags":["Extensions"],"summary":"Update extension","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateExtensionInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ExtensionResponse"}}}}}},"delete":{"tags":["Extensions"],"summary":"Delete extension","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"204":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}}}}},"/api/v1/extensions/{id}/versions":{"get":{"tags":["Extensions"],"summary":"List available versions for an extension","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":false,"name":"preRelease","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ExtensionVersionInfo"}}}}}}}},"/api/v1/insights":{"get":{"tags":["Insights"],"summary":"List insights","parameters":[{"schema":{"type":"string"},"required":false,"name":"q","in":"query"},{"schema":{"type":"string"},"required":false,"name":"blocked","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/InsightResponse"}}}}}}},"post":{"tags":["Insights"],"summary":"Create insight","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateInsightInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}}}}},"/api/v1/insights/search":{"get":{"tags":["Insights"],"summary":"Search insights","parameters":[{"schema":{"type":"string"},"required":true,"name":"q","in":"query"},{"schema":{"type":"string"},"required":false,"name":"blocked","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/InsightResponse"}}}}}}}},"/api/v1/insights/{id}":{"get":{"tags":["Insights"],"summary":"Get insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}},"404":{"description":"Insight not found"}}},"put":{"tags":["Insights"],"summary":"Update insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateInsightInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}},"404":{"description":"Insight not found"}}},"delete":{"tags":["Insights"],"summary":"Delete insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"204":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}},"404":{"description":"Insight not found"}}}},"/api/v1/insights/{id}/upvote":{"post":{"tags":["Insights"],"summary":"Upvote insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}},"404":{"description":"Insight not found"}}}},"/api/v1/insights/{id}/downvote":{"post":{"tags":["Insights"],"summary":"Downvote insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}},"404":{"description":"Insight not found"}}}},"/api/v1/insights/{id}/block":{"post":{"tags":["Insights"],"summary":"Block insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}},"404":{"description":"Insight not found"}}}},"/api/v1/insights/{id}/unblock":{"post":{"tags":["Insights"],"summary":"Unblock insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}},"404":{"description":"Insight not found"}}}},"/api/v1/insights/{id}/reports":{"get":{"tags":["Insights"],"summary":"Get reports referencing insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ReportResponse"}}}}},"404":{"description":"Insight not found"}}}},"/api/v1/feature-flags":{"get":{"tags":["Feature Flags"],"summary":"List feature flags","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/FeatureFlagResponse"}}}}}}}},"/api/v1/feature-flags/{key}":{"put":{"tags":["Feature Flags"],"summary":"Update feature flag","parameters":[{"schema":{"type":"string"},"required":true,"name":"key","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateFeatureFlagInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/FeatureFlagResponse"}}}}}}}},"webhooks":{}} \ No newline at end of file +{"openapi":"3.1.0","info":{"title":"Scope API","version":"1.0.0","description":"REST API for the Scope platform — benchmarking AI coding agents"},"servers":[{"url":"/","description":"Current server"}],"components":{"securitySchemes":{"bearerAuth":{"type":"http","scheme":"bearer","bearerFormat":"JWT","description":"Use the unchanged IdP access token, without the Bearer prefix."}},"schemas":{"KeyResponse":{"type":"object","properties":{},"additionalProperties":{}},"KeyInput":{"type":"object","properties":{},"additionalProperties":{}},"ValidateKeyInput":{"type":"object","properties":{"token":{"type":"string"}},"required":["token"]},"AccountResponse":{"type":"object","properties":{},"additionalProperties":{}},"AccountInput":{"type":"object","properties":{},"additionalProperties":{}},"UserMeResponse":{"type":"object","properties":{"id":{"type":"string"},"role":{"type":"string"},"email":{"type":"string"},"displayName":{"type":"string"},"idp":{"type":"string"},"idpTenant":{"type":"string"}},"required":["id"]},"ProjectResponse":{"type":"object","properties":{"_id":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"creator":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"}},"required":["_id","name","createdAt"]},"CreateProjectInput":{"type":"object","properties":{"name":{"type":"string","minLength":1},"description":{"type":"string"},"creator":{"type":"string"}},"required":["name"]},"UpdateProjectInput":{"type":"object","properties":{"name":{"type":"string","minLength":1},"description":{"type":"string"}}},"CreateExperimentResponse":{"type":"object","properties":{"experiment":{"$ref":"#/components/schemas/ExperimentResponse"},"requestIds":{"type":"array","items":{"type":"string"}}},"required":["experiment","requestIds"]},"ExperimentResponse":{"type":"object","properties":{"experimentId":{"type":"string"},"projectId":{"type":"string"},"metadata":{"type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":256},"description":{"type":"string","maxLength":4096},"tags":{"type":"array","items":{"type":"string","minLength":1,"maxLength":64},"maxItems":32},"createdBy":{"type":"string"}},"required":["name"]},"revision":{"type":"object","properties":{"experimentRevisionId":{"type":"string"},"revisionNumber":{"type":"integer"},"memberCount":{"type":"integer"},"createdAt":{"type":["string","null"],"format":"date-time"}},"required":["experimentRevisionId","revisionNumber","memberCount","createdAt"]},"currentExecution":{"type":"object","properties":{"experimentExecutionId":{"type":"string"},"attemptNumber":{"type":"integer"},"readiness":{"type":"string","enum":["empty","waiting","blocked","ready","processing","done","failed"]},"blockers":{"type":"array","items":{"type":"object","properties":{"requestId":{"type":"string"},"runId":{"type":"string"},"handlerId":{"type":"string"},"reason":{"type":"string"},"error":{"type":"string"}},"required":["handlerId","reason"]}},"progress":{"type":"object","additionalProperties":{"type":"object","properties":{"status":{"type":"string","enum":["queued","processing","done","failed"]},"updatedAt":{"type":["string","null"],"format":"date-time"},"error":{"type":"string"}},"required":["status","updatedAt"]}},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"startedAt":{"type":["string","null"],"format":"date-time"},"finishedAt":{"type":["string","null"],"format":"date-time"},"discardedAt":{"type":["string","null"],"format":"date-time"},"discardReason":{"type":"string"},"report":{"type":"object","properties":{"reportId":{"type":"string"},"url":{"type":"string"},"updatedAt":{"type":["string","null"],"format":"date-time"}},"required":["reportId","url","updatedAt"]}},"required":["experimentExecutionId","attemptNumber","readiness","blockers","progress","createdAt","updatedAt"]},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"}},"required":["experimentId","projectId","metadata","revision","currentExecution","createdAt","updatedAt"]},"CreateExperimentInput":{"type":"object","properties":{"mode":{"type":"string","enum":["compose","empty"]},"metadata":{"type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":256},"description":{"type":"string","maxLength":4096},"tags":{"type":"array","items":{"type":"string","minLength":1,"maxLength":64},"maxItems":32}},"required":["name"]},"requests":{"type":"array","items":{"type":"object","properties":{"scenario":{"$ref":"#/components/schemas/Scenario"},"model":{"type":"string"},"reasoningEffort":{"type":"string"},"maxIterations":{"type":"integer","minimum":1,"maximum":50},"personaInstructions":{"type":"string"},"persona":{"$ref":"#/components/schemas/Persona"},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"codebaseRevisionId":{"type":"string"},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceBindingSpec"}},"extensions":{"type":"array","items":{"type":"string"}},"profileId":{"type":"string"},"priority":{"type":"integer"},"agentsMd":{"type":"string"},"agentsMdParentIds":{"type":"array","items":{"type":"string"}},"gates":{"type":"array","items":{"$ref":"#/components/schemas/GateConfig"}},"observations":{"type":"array","items":{"type":"string"}},"workerType":{"type":"string","minLength":1}},"required":["scenario","workerType"]},"maxItems":250000,"default":[]}},"required":["mode","metadata"]},"Scenario":{"type":"object","properties":{"version":{"type":"string","enum":["v1","v2"]},"task":{"type":"string"},"criteria":{"type":"array","items":{"type":"string"}}},"required":["task","criteria"]},"Persona":{"type":"object","properties":{"personality":{"type":"string","enum":["demanding","friendly"]},"experience":{"type":"string","enum":["junior","senior"]},"verbosity":{"type":"string","enum":["brief","moderate"]},"type":{"type":"string","enum":["traditional","ai_assisted","vibe"]}},"required":["personality","experience","verbosity","type"]},"ResourceBindingSpec":{"anyOf":[{"type":"string"},{"type":"object","properties":{"ref":{"type":"string"},"params":{"type":"object","additionalProperties":{"type":"string"}}},"required":["ref"]}]},"GateConfig":{"type":"object","properties":{"gate":{"type":"string","enum":["select","build","test","run","deploy"]},"promptId":{"type":"string"},"promptText":{"type":"string"},"criteria":{"type":"array","items":{"type":"string"}},"maxIterations":{"type":"integer","minimum":1,"maximum":50}},"required":["gate","criteria"]},"ExperimentListResponse":{"type":"object","properties":{"data":{"type":"array","items":{"$ref":"#/components/schemas/ExperimentResponse"}},"nextCursor":{"type":"string"}},"required":["data"]},"ExperimentExecutionListResponse":{"type":"object","properties":{"data":{"type":"array","items":{"$ref":"#/components/schemas/ExperimentExecutionResponse"}},"nextCursor":{"type":"string"}},"required":["data"]},"ExperimentExecutionResponse":{"type":"object","properties":{"experimentExecutionId":{"type":"string"},"attemptNumber":{"type":"integer"},"readiness":{"type":"string","enum":["empty","waiting","blocked","ready","processing","done","failed"]},"blockers":{"type":"array","items":{"type":"object","properties":{"requestId":{"type":"string"},"runId":{"type":"string"},"handlerId":{"type":"string"},"reason":{"type":"string"},"error":{"type":"string"}},"required":["handlerId","reason"]}},"progress":{"type":"object","additionalProperties":{"type":"object","properties":{"status":{"type":"string","enum":["queued","processing","done","failed"]},"updatedAt":{"type":["string","null"],"format":"date-time"},"error":{"type":"string"}},"required":["status","updatedAt"]}},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"startedAt":{"type":["string","null"],"format":"date-time"},"finishedAt":{"type":["string","null"],"format":"date-time"},"discardedAt":{"type":["string","null"],"format":"date-time"},"discardReason":{"type":"string"},"supersededAt":{"type":["string","null"],"format":"date-time"},"supersededByExperimentExecutionId":{"type":"string"},"supersededReason":{"type":"string","enum":["rollover","retry"]},"report":{"type":"object","properties":{"reportId":{"type":"string"},"url":{"type":"string"},"updatedAt":{"type":["string","null"],"format":"date-time"}},"required":["reportId","url","updatedAt"]}},"required":["experimentExecutionId","attemptNumber","readiness","blockers","progress","createdAt","updatedAt"]},"ReportResponse":{"type":"object","properties":{"_id":{"type":"string"},"requestId":{"type":"string"},"experimentId":{"type":"string"},"experimentExecutionId":{"type":"string"},"templateId":{"type":"string"},"reporter":{"$ref":"#/components/schemas/Reporter"},"content":{"type":"string"},"structuredArtifact":{"$ref":"#/components/schemas/StructuredReportArtifact"},"status":{"type":"string","enum":["pending","generating","completed","failed"]},"error":{"type":"string"},"insightReferences":{"type":"array","items":{"$ref":"#/components/schemas/InsightReference"}},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","status","createdAt","projectId"]},"Reporter":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"gitHash":{"type":"string"},"model":{"type":"string"},"agentId":{"type":"string"},"agentVersion":{"type":"string"}},"required":["id","name","gitHash","model","agentId","agentVersion"]},"StructuredReportArtifact":{"type":"object","properties":{"schemaId":{"type":"string"},"schemaVersion":{"type":"string"},"contentType":{"type":"string","enum":["application/json"]},"url":{"type":"string"}},"required":["schemaId","schemaVersion","contentType","url"]},"InsightReference":{"type":"object","properties":{"insightId":{"type":"string"},"referencedAt":{"type":["string","null"],"format":"date-time"},"isNew":{"type":"boolean"}},"required":["insightId","referencedAt","isNew"]},"RetryExperimentResponse":{"type":"object","properties":{"experimentId":{"type":"string"},"experimentExecutionId":{"type":"string"},"attemptNumber":{"type":"integer"},"readiness":{"type":"string","enum":["empty","waiting","blocked","ready","processing","done","failed"]},"blockers":{"type":"array","items":{"type":"object","properties":{"requestId":{"type":"string"},"runId":{"type":"string"},"handlerId":{"type":"string"},"reason":{"type":"string"},"error":{"type":"string"}},"required":["handlerId","reason"]}}},"required":["experimentId","experimentExecutionId","attemptNumber","readiness","blockers"]},"RequestResponse":{"type":"object","properties":{"_id":{"type":"string"},"scenario":{"$ref":"#/components/schemas/Scenario"},"workerType":{"type":"string"},"model":{"type":"string"},"reasoningEffort":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"maxIterations":{"type":"number"},"personaInstructions":{"type":"string"},"persona":{"$ref":"#/components/schemas/Persona"},"deletedAt":{"type":["string","null"],"format":"date-time"},"taskPromptId":{"type":"string"},"agentsMdPromptId":{"type":"string"},"agentsMdParentIds":{"type":"array","items":{"type":"string"}},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"codebaseRevisionId":{"type":"string"},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceBinding"}},"extensions":{"type":"array","items":{"type":"string"}},"agentVersion":{"type":"string"},"profileId":{"type":"string"},"profileVersionId":{"type":"string"},"submissionId":{"type":"string"},"experimentId":{"type":"string"},"experimentExpectedRequestCount":{"type":"integer","minimum":0},"experimentRequestIndex":{"type":"integer","minimum":0},"experimentInitialRunId":{"type":"string"},"experimentFinalizedAt":{"type":["string","null"],"format":"date-time"},"priority":{"type":"integer","default":0},"gates":{"type":"array","items":{"$ref":"#/components/schemas/GateConfig"}},"observations":{"type":"array","items":{"type":"string"}},"gateSummaries":{"type":"array","items":{"$ref":"#/components/schemas/GateRunSummary"}},"run":{"$ref":"#/components/schemas/RunState"},"projectId":{"type":"string"}},"required":["_id","scenario","workerType","createdAt","projectId"]},"ResourceBinding":{"type":"object","properties":{"ref":{"type":"string"},"revisionId":{"type":"string"},"params":{"type":"object","additionalProperties":{"type":"string"}}},"required":["ref","revisionId","params"]},"GateRunSummary":{"type":"object","properties":{"gate":{"type":"string","enum":["select","build","test","run","deploy"]},"status":{"type":"string","enum":["passed","failed","skipped"]},"iterations":{"type":"integer","minimum":0}},"required":["gate","status","iterations"]},"RunState":{"type":"object","properties":{"_id":{"type":"string"},"attemptNumber":{"type":"integer","minimum":1},"status":{"type":"string","enum":["pending","queued","processing","paused","done"]},"queuedQueueName":{"type":"string"},"outcome":{"type":"string","enum":["succeeded","failed","finished"]},"result":{"type":"string"},"error":{"type":"string"},"logsUrl":{"type":"string"},"updatedAt":{"type":["string","null"],"format":"date-time"},"startedAt":{"type":["string","null"],"format":"date-time"},"finishedAt":{"type":["string","null"],"format":"date-time"},"durationMs":{"type":"number"},"turns":{"type":"array","items":{"$ref":"#/components/schemas/ConversationTurn"}},"workerVersion":{"type":"string"},"os":{"type":"object","properties":{"platform":{"type":"string"},"release":{"type":"string"},"arch":{"type":"string"}},"required":["platform","release","arch"]},"lastHeartbeatAt":{"type":["string","null"],"format":"date-time"},"worker":{"type":"object","properties":{"instanceId":{"type":"string"},"podName":{"type":"string"}},"required":["instanceId"]},"harUrl":{"type":"string"},"videoUrls":{"type":"array","items":{"type":"string"}},"setupVideoUrls":{"type":"array","items":{"type":"string"}},"tokenUsage":{"$ref":"#/components/schemas/TokenUsage"},"aiCallCount":{"type":"number"},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceRunOutcome"}},"mcpRegistered":{"type":"boolean"},"rawChatUrl":{"type":"string"},"rawChatFormat":{"type":"string"},"pausedAt":{"type":["string","null"],"format":"date-time"},"resumedAt":{"type":["string","null"],"format":"date-time"},"observationResults":{"type":"array","items":{"$ref":"#/components/schemas/CriterionResult"}}},"required":["_id","attemptNumber","status"]},"ConversationTurn":{"type":"object","properties":{"iteration":{"type":"number"},"gate":{"type":"string","enum":["select","build","test","run","deploy"]},"codingAgentResponse":{"type":"string"},"judgeFeedback":{"type":"string"},"snapshotUrl":{"type":"string"},"passed":{"type":"boolean"},"timestamp":{"type":["string","null"],"format":"date-time"},"criteriaResults":{"type":"array","items":{"$ref":"#/components/schemas/CriterionResult"}},"observationResults":{"type":"array","items":{"$ref":"#/components/schemas/CriterionResult"}},"harUrl":{"type":"string"},"videoUrls":{"type":"array","items":{"type":"string"}},"tokenUsage":{"$ref":"#/components/schemas/TokenUsage"},"startedAt":{"type":["string","null"],"format":"date-time"},"durationMs":{"type":"number"},"toolCalls":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"arguments":{"type":"object","additionalProperties":{}},"response":{"type":"string"},"timestamp":{"type":"string"}},"required":["id","name","arguments"]}},"toolCallsUrl":{"type":"string"},"toolCallCount":{"type":"number"},"aiCallCount":{"type":"number"},"rawChatUrl":{"type":"string"},"rawChatFormat":{"type":"string"},"chatResultUrl":{"type":"string"},"chatResultFormat":{"type":"string"}},"required":["iteration","judgeFeedback","snapshotUrl","passed","timestamp"]},"CriterionResult":{"type":"object","properties":{"criterionId":{"type":"string"},"passed":{"type":"boolean"},"feedback":{"type":"string"},"evaluated":{"type":"boolean"}},"required":["criterionId","passed","feedback","evaluated"]},"TokenUsage":{"type":"object","properties":{"promptTokens":{"type":"number"},"completionTokens":{"type":"number"},"totalTokens":{"type":"number"}},"required":["promptTokens","completionTokens","totalTokens"]},"ResourceRunOutcome":{"type":"object","properties":{"ref":{"type":"string"},"slug":{"type":"string"},"revisionId":{"type":"string"},"setupSucceeded":{"type":"boolean"},"published":{"type":"array","items":{"type":"string"}},"params":{"type":"object","additionalProperties":{"type":"string"}},"setupDurationMs":{"type":"number"},"error":{"type":"string"},"teardownRan":{"type":"boolean"}},"required":["ref","slug","revisionId","setupSucceeded","published"]},"CreateRequestInput":{"type":"object","properties":{"scenario":{"$ref":"#/components/schemas/Scenario"},"model":{"type":"string"},"reasoningEffort":{"type":"string"},"maxIterations":{"type":"integer","minimum":1,"maximum":50},"personaInstructions":{"type":"string"},"persona":{"$ref":"#/components/schemas/Persona"},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"codebaseRevisionId":{"type":"string"},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceBindingSpec"}},"extensions":{"type":"array","items":{"type":"string"}},"profileId":{"type":"string"},"profileVariations":{"type":"array","items":{"type":"string"},"deprecated":true,"description":"Deprecated for composition. Use POST /api/v1/experiments and its requests array."},"priority":{"type":"integer"},"agentsMd":{"type":"string"},"agentsMdParentIds":{"type":"array","items":{"type":"string"}},"gates":{"type":"array","items":{"$ref":"#/components/schemas/GateConfig"}},"observations":{"type":"array","items":{"type":"string"}}},"required":["scenario"]},"RunFacetsResponse":{"type":"object","properties":{"total":{"type":"number"},"facets":{"type":"object","properties":{"workerType":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"status":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"outcome":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"model":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"os":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"priority":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"agentVersion":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"profileId":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}}},"required":["workerType","status","outcome","model","os","priority","agentVersion","profileId"]}},"required":["total","facets"]},"RunFacetBucket":{"type":"object","properties":{"value":{"type":"string"},"count":{"type":"number"}},"required":["value","count"]},"PaginatedRunsResponse":{"type":"object","properties":{"data":{"type":"array","items":{"$ref":"#/components/schemas/RequestResponse"}},"limit":{"type":"number"},"estimatedTotal":{"type":"number"},"cursors":{"$ref":"#/components/schemas/Cursors"}},"required":["data","limit","estimatedTotal","cursors"]},"Cursors":{"type":"object","properties":{"next":{"type":["string","null"]},"prev":{"type":["string","null"]}},"required":["next","prev"]},"PaginatedRunGroupsResponse":{"type":"object","properties":{"data":{"type":"array","items":{"$ref":"#/components/schemas/RunGroup"}},"limit":{"type":"number"},"estimatedTotal":{"type":"number"},"cursors":{"$ref":"#/components/schemas/Cursors"}},"required":["data","limit","estimatedTotal","cursors"]},"RunGroup":{"type":"object","properties":{"key":{"type":"string"},"label":{"type":"string"},"runIds":{"type":"array","items":{"type":"string"}},"aggregates":{"$ref":"#/components/schemas/GroupAggregates"},"uniform":{"$ref":"#/components/schemas/GroupUniformValues"}},"required":["key","label","runIds","aggregates","uniform"]},"GroupAggregates":{"type":"object","properties":{"count":{"type":"number"},"turns":{"$ref":"#/components/schemas/AggregateStats"},"duration":{"$ref":"#/components/schemas/AggregateStats"},"promptTokens":{"$ref":"#/components/schemas/AggregateStats"},"completionTokens":{"$ref":"#/components/schemas/AggregateStats"},"statusCounts":{"type":"object","additionalProperties":{"type":"number"}},"outcomeCounts":{"type":"object","additionalProperties":{"type":"number"}}},"required":["count","turns","duration","promptTokens","completionTokens","statusCounts","outcomeCounts"]},"AggregateStats":{"type":["object","null"],"properties":{"min":{"type":"number"},"max":{"type":"number"},"mean":{"type":"number"},"stdDev":{"type":"number"}},"required":["min","max","mean","stdDev"]},"GroupUniformValues":{"type":"object","properties":{"workerType":{"type":"string"},"agentVersion":{"type":"string"},"model":{"type":"string"},"platform":{"type":"string"},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"codebaseRevisionId":{"type":"string"},"extensions":{"type":"array","items":{"type":"string"}},"status":{"type":"string","enum":["pending","queued","processing","paused","done"]},"submissionId":{"type":"string"},"experimentId":{"type":"string"},"task":{"type":"string"}}},"BulkResubmitInput":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1},"count":{"type":"integer","minimum":1,"maximum":10,"default":1},"overrides":{"type":"object","properties":{"profileId":{"type":["string","null"]},"workerType":{"type":"string"},"agentVersion":{"type":"string"},"model":{"type":["string","null"]},"reasoningEffort":{"type":["string","null"]},"maxIterations":{"type":["number","null"]},"mcpServers":{"type":["array","null"],"items":{"type":"string"}},"skillRevisions":{"type":["array","null"],"items":{"type":"string"}},"extensions":{"type":["array","null"],"items":{"type":"string"}}}}},"required":["ids"]},"CreateCriteriaInput":{"type":"object","properties":{"id":{"type":"string","pattern":"^[a-z][a-z0-9_]*$"},"prompt":{"type":"string"},"dependsOn":{"type":"array","items":{"type":"string"}},"gates":{"type":"array","items":{"type":"string","enum":["select","build","test","run","deploy"]}},"kind":{"type":"string","enum":["gate","observation"],"default":"gate"},"taxonomyElementId":{"type":"string","enum":["dimension:idiomatic-use","dimension:dependency-currency","dimension:configuration-correctness"]},"subject":{"type":"string","enum":["run","iteration"]}},"required":["id","prompt"]},"CriteriaResponse":{"type":"object","properties":{"id":{"type":"string"},"prompt":{"type":"string"},"dependsOn":{"type":"array","items":{"type":"string"}},"gates":{"type":"array","items":{"type":"string","enum":["select","build","test","run","deploy"]}},"kind":{"type":"string","enum":["gate","observation"]},"taxonomyElementId":{"type":"string","enum":["dimension:idiomatic-use","dimension:dependency-currency","dimension:configuration-correctness"]},"subject":{"type":"string","enum":["run","iteration"]},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["id","prompt","createdAt","projectId"]},"CriteriaGraph":{"type":"object","properties":{"nodes":{"type":"array","items":{"$ref":"#/components/schemas/CriteriaGraphNode"}},"edges":{"type":"array","items":{"$ref":"#/components/schemas/CriteriaGraphEdge"}}},"required":["nodes","edges"]},"CriteriaGraphNode":{"type":"object","properties":{"id":{"type":"string"},"prompt":{"type":"string"},"dependsOn":{"type":"array","items":{"type":"string"}},"gates":{"type":"array","items":{"type":"string","enum":["select","build","test","run","deploy"]}},"kind":{"type":"string","enum":["gate","observation"]},"taxonomyElementId":{"type":"string","enum":["dimension:idiomatic-use","dimension:dependency-currency","dimension:configuration-correctness"]},"subject":{"type":"string","enum":["run","iteration"]}},"required":["id","prompt"]},"CriteriaGraphEdge":{"type":"object","properties":{"from":{"type":"string"},"to":{"type":"string"}},"required":["from","to"]},"UpdateCriteriaInput":{"type":"object","properties":{"prompt":{"type":"string"},"dependsOn":{"type":"array","items":{"type":"string"}},"gates":{"type":"array","items":{"type":"string","enum":["select","build","test","run","deploy"]}},"kind":{"type":"string","enum":["gate","observation"]},"taxonomyElementId":{"type":"string","enum":["dimension:idiomatic-use","dimension:dependency-currency","dimension:configuration-correctness"]},"subject":{"type":"string","enum":["run","iteration"]}}},"CreatePromptFeatureInput":{"type":"object","properties":{"id":{"type":"string","pattern":"^[a-z][a-z0-9_]*$"},"prompt":{"type":"string"},"type":{"$ref":"#/components/schemas/PromptType"}},"required":["id","prompt"]},"PromptType":{"type":"string","enum":["select","build","test","run","deploy","agents.md"]},"PromptFeatureResult":{"type":"object","properties":{"featureId":{"type":"string"},"detected":{"type":"boolean"},"evaluated":{"type":"boolean"}},"required":["featureId","detected","evaluated"]},"SuggestedPromptFeature":{"type":"object","properties":{"suggestedId":{"type":"string"},"behavior":{"type":"string"},"prompt":{"type":"string"}},"required":["suggestedId","behavior","prompt"]},"PromptFeatureResponse":{"type":"object","properties":{"id":{"type":"string"},"prompt":{"type":"string"},"type":{"$ref":"#/components/schemas/PromptType"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["id","prompt","createdAt","projectId"]},"UpdatePromptFeatureInput":{"type":"object","properties":{"prompt":{"type":"string"}}},"TaskPromptResponse":{"type":"object","properties":{"_id":{"type":"string"},"keyId":{"type":"string"},"type":{"$ref":"#/components/schemas/PromptType"},"text":{"type":"string"},"contentBlobUrl":{"type":"string"},"features":{"type":"array","items":{"$ref":"#/components/schemas/PromptFeatureResult"}},"featuresExtractedAt":{"type":["string","null"],"format":"date-time"},"createdAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","keyId","createdAt","projectId"]},"CreateTaskPromptInput":{"type":"object","properties":{"text":{"type":"string"},"type":{"$ref":"#/components/schemas/PromptType"}},"required":["text"]},"PatchTaskPromptFeatureInput":{"type":"object","properties":{"detected":{"type":"boolean"}},"required":["detected"]},"CreateReportInput":{"type":"object","properties":{"requestId":{"type":"string"},"templateId":{"type":"string"}},"required":["requestId"]},"BulkCreateReportsInput":{"type":"object","properties":{"requestIds":{"type":"array","items":{"type":"string"}},"templateId":{"type":"string"}},"required":["requestIds"]},"BulkReportStatusInput":{"type":"object","properties":{"reportIds":{"type":"array","items":{"type":"string"}}},"required":["reportIds"]},"BulkReportSummaryResponse":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/ReportSummary"}},"ReportSummary":{"type":"object","properties":{"total":{"type":"number"},"pending":{"type":"number"},"generating":{"type":"number"},"completed":{"type":"number"},"failed":{"type":"number"}},"required":["total","pending","generating","completed","failed"]},"BulkReportSummaryInput":{"type":"object","properties":{"requestIds":{"type":"array","items":{"type":"string"}}},"required":["requestIds"]},"TriggerReportsInput":{"type":"object","properties":{"requestId":{"type":"string"}},"required":["requestId"]},"BulkTriggerReportsInput":{"type":"object","properties":{"requestIds":{"type":"array","items":{"type":"string"}}},"required":["requestIds"]},"InsightResponse":{"type":"object","properties":{"_id":{"type":"string"},"title":{"type":"string"},"description":{"type":"string"},"category":{"type":"string"},"tags":{"type":"array","items":{"type":"string"}},"upvotes":{"type":"number"},"downvotes":{"type":"number"},"blocked":{"type":"boolean"},"referenceCount":{"type":"number"},"createdBy":{"type":"string","enum":["agent","user"]},"sourceReportId":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","title","description","upvotes","downvotes","blocked","referenceCount","createdBy","createdAt","projectId"]},"ProfileWithVersionResponse":{"type":"object","properties":{"_id":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"latestVersion":{"type":"number"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"version":{"$ref":"#/components/schemas/ProfileVersionResponse"},"projectId":{"type":"string"}},"required":["_id","name","latestVersion","createdAt","version","projectId"]},"ProfileVersionResponse":{"type":"object","properties":{"_id":{"type":"string"},"profileId":{"type":"string"},"version":{"type":"number"},"workerType":{"type":"string"},"model":{"type":"string"},"reasoningEffort":{"type":"string"},"agentVersion":{"type":"string"},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceBindingSpec"}},"extensions":{"type":"array","items":{"type":"string"}},"createdAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","profileId","version","workerType","model","createdAt","projectId"]},"CreateProfileInput":{"type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":128},"description":{"type":"string","maxLength":512},"workerType":{"type":"string"},"model":{"type":"string"},"reasoningEffort":{"type":"string"},"agentVersion":{"type":"string"},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceBindingSpec"}},"extensions":{"type":"array","items":{"type":"string"}}},"required":["name","workerType","model"]},"ProfileResponse":{"type":"object","properties":{"_id":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"latestVersion":{"type":"number"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","name","latestVersion","createdAt","projectId"]},"UpdateProfileIdentity":{"type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":128},"description":{"type":"string","maxLength":512}}},"ReportTemplateResponse":{"type":"object","properties":{"_id":{"type":"string"},"id":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"userPrompt":{"type":"string"},"systemPrompt":{"$ref":"#/components/schemas/ReportTemplateSystemPrompt"},"trigger":{"$ref":"#/components/schemas/ReportTrigger"},"model":{"type":"string"},"timeoutMs":{"type":"integer","exclusiveMinimum":0},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","id","name","userPrompt","createdAt","projectId"]},"ReportTemplateSystemPrompt":{"type":"object","properties":{"mode":{"type":"string","enum":["append","override"]},"content":{"type":"string"}},"required":["mode","content"]},"ReportTrigger":{"oneOf":[{"type":"object","properties":{"type":{"type":"string","enum":["always"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["criteria"]},"criteriaIds":{"type":"array","items":{"type":"string"}},"match":{"type":"string","enum":["any","all"]}},"required":["type","criteriaIds"]},{"type":"object","properties":{"type":{"type":"string","enum":["taskPrompt"]},"taskPromptIds":{"type":"array","items":{"type":"string"}}},"required":["type","taskPromptIds"]},{"type":"object","properties":{"type":{"type":"string","enum":["promptFeature"]},"featureIds":{"type":"array","items":{"type":"string"}},"match":{"type":"string","enum":["any","all"]}},"required":["type","featureIds"]}]},"CreateReportTemplateInput":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"userPrompt":{"type":"string"},"systemPrompt":{"$ref":"#/components/schemas/ReportTemplateSystemPrompt"},"trigger":{"$ref":"#/components/schemas/ReportTrigger"},"model":{"type":"string"},"timeoutMs":{"type":"integer","exclusiveMinimum":0}},"required":["id","name","userPrompt"]},"UpdateReportTemplateInput":{"type":"object","properties":{"name":{"type":"string"},"description":{"type":"string"},"userPrompt":{"type":"string"},"systemPrompt":{"allOf":[{"$ref":"#/components/schemas/ReportTemplateSystemPrompt"},{"type":["object","null"]}]},"trigger":{"$ref":"#/components/schemas/ReportTrigger"},"model":{"type":["string","null"]},"timeoutMs":{"type":["integer","null"],"exclusiveMinimum":0}}},"AgentResponse":{"type":"object","properties":{"_id":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"modelProvider":{"type":"string"},"supportedModels":{"type":"array","items":{"type":"string"}},"defaultModel":{"type":"string"},"available":{"type":"boolean"},"capabilities":{"$ref":"#/components/schemas/AgentCapabilities"},"versions":{"type":"array","items":{"$ref":"#/components/schemas/AgentVersion"}},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"}},"required":["_id","name","supportedModels","createdAt"]},"AgentCapabilities":{"type":"object","properties":{"supportsReasoningEffort":{"type":"boolean"},"supportsMcpServers":{"type":"boolean"},"supportsSkills":{"type":"boolean"},"supportsExtensions":{"type":"boolean"},"supportsResources":{"type":"boolean"}}},"AgentVersion":{"type":"object","properties":{"agentVersion":{"type":"string","minLength":1},"workerVersion":{"type":"string","minLength":1},"components":{"type":"object","additionalProperties":{"type":"string"}},"gitCommit":{"type":"string","minLength":1},"buildTime":{"type":"string","minLength":1},"imageTag":{"type":"string","minLength":1},"queueName":{"type":"string"},"status":{"type":"string","enum":["active","retired"]},"createdAt":{"type":["string","null"],"format":"date-time"}},"required":["agentVersion","workerVersion","components","gitCommit","buildTime","imageTag","status","createdAt"]},"CreateAgentInput":{"type":"object","properties":{"_id":{"type":"string","minLength":1},"name":{"type":"string","minLength":1},"description":{"type":"string"},"modelProvider":{"type":"string"},"supportedModels":{"type":"array","items":{"type":"string"}},"defaultModel":{"type":"string"},"available":{"type":"boolean"},"capabilities":{"$ref":"#/components/schemas/AgentCapabilities"}},"required":["_id","name"]},"UpdateAgentInput":{"type":"object","properties":{"name":{"type":"string"},"description":{"type":"string"},"modelProvider":{"type":"string"},"supportedModels":{"type":"array","items":{"type":"string"}},"defaultModel":{"type":"string"},"available":{"type":"boolean"},"capabilities":{"$ref":"#/components/schemas/AgentCapabilities"}}},"RegisterAgentVersionInput":{"type":"object","properties":{"agentVersion":{"type":"string","minLength":1},"workerVersion":{"type":"string","minLength":1},"components":{"type":"object","additionalProperties":{"type":"string"}},"gitCommit":{"type":"string","minLength":1},"buildTime":{"type":"string","minLength":1},"imageTag":{"type":"string","minLength":1},"queueName":{"type":"string","minLength":1}},"required":["agentVersion","workerVersion","components","gitCommit","buildTime","imageTag","queueName"]},"PatchAgentVersionInput":{"type":"object","properties":{"status":{"type":"string","enum":["active","retired"]}},"required":["status"]},"ModelResponse":{"type":"object","properties":{"_id":{"type":"string"},"modelId":{"type":"string"},"provider":{"type":"string"},"agentId":{"type":"string"},"firstSeenAt":{"type":["string","null"],"format":"date-time"},"lastSeenAt":{"type":["string","null"],"format":"date-time"},"disappearedAt":{"type":["string","null"],"format":"date-time"},"providerAvailableFrom":{"type":["string","null"],"format":"date-time"},"providerEndOfLife":{"type":["string","null"],"format":"date-time"},"metadata":{"type":"object","additionalProperties":{}},"capabilities":{"$ref":"#/components/schemas/ModelCapabilities"}},"required":["_id","modelId","provider","agentId","firstSeenAt","lastSeenAt"]},"ModelCapabilities":{"type":"object","properties":{"reasoningEffort":{"type":"array","items":{"type":"string"}},"toolCalls":{"type":"boolean"},"vision":{"type":"boolean"},"streaming":{"type":"boolean"},"adaptiveThinking":{"type":"boolean"},"maxThinkingBudget":{"type":"number"}}},"McpServerResponse":{"type":"object","properties":{"_id":{"type":"string"},"slug":{"type":"string"},"name":{"type":"string"},"type":{"type":"string","enum":["sse","http","stdio"]},"url":{"type":"string"},"command":{"type":"string"},"args":{"type":"array","items":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"headers":{"type":"array","items":{"$ref":"#/components/schemas/McpServerHeader"}},"sessionMode":{"type":"string","enum":["stateful","stateless"]},"version":{"type":"string"},"description":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","slug","name","type","createdAt","projectId"]},"McpServerHeader":{"type":"object","properties":{"name":{"type":"string"},"value":{"type":"string"}},"required":["name","value"]},"UpdateMcpServerInput":{"type":"object","properties":{"name":{"type":"string"},"type":{"type":"string","enum":["sse","http","stdio"]},"url":{"type":"string"},"command":{"type":"string"},"args":{"type":"array","items":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"headers":{"type":"array","items":{"$ref":"#/components/schemas/McpServerHeader"}},"sessionMode":{"type":"string","enum":["stateful","stateless"]},"version":{"type":"string"},"description":{"type":"string"}}},"SkillResponse":{"type":"object","properties":{"_id":{"type":"string"},"slug":{"type":"string"},"source":{"type":"string"},"skillName":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"origin":{"type":"string","enum":["skills-sh","manual"]},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","slug","source","skillName","name","origin","createdAt","projectId"]},"SkillSearchResult":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"source":{"type":"string"},"description":{"type":"string"},"internal":{"type":"boolean"},"installs":{"type":"number"}},"required":["id","name","source","internal"]},"SkillDiscoveryResult":{"type":"object","properties":{"skillName":{"type":"string"},"skillPath":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"existsInLibrary":{"type":"boolean"},"currentRevisionCommitSha":{"type":"string"},"latestUpstreamCommitSha":{"type":"string"},"updateAvailable":{"type":"boolean"},"lastImportedAt":{"type":"string"}},"required":["skillName","skillPath"]},"SkillRevisionResponse":{"type":"object","properties":{"_id":{"type":"string"},"ref":{"type":"string"},"source":{"type":"string"},"skillName":{"type":"string"},"skillPath":{"type":"string"},"commitHash":{"type":"string"},"commitTimestamp":{"type":["string","null"],"format":"date-time"},"name":{"type":"string"},"description":{"type":"string"},"license":{"type":"string"},"compatibility":{"type":"string"},"allowedTools":{"type":"string"},"metadata":{"type":"object","additionalProperties":{"type":"string"}},"content":{"type":"string"},"archiveUrl":{"type":"string"},"validationWarnings":{"type":"array","items":{"type":"string"}},"resolvedAt":{"type":["string","null"],"format":"date-time"},"createdAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","ref","source","skillName","skillPath","commitHash","commitTimestamp","name","description","content","archiveUrl","resolvedAt","createdAt","projectId"]},"CreateSkillInput":{"type":"object","properties":{"source":{"type":"string"},"skillName":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"origin":{"type":"string","enum":["skills-sh","manual"]}},"required":["source","skillName","name","origin"]},"CodebaseResponse":{"type":"object","properties":{"_id":{"type":"string"},"slug":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"sourceType":{"type":"string","enum":["git","archive"]},"source":{"type":"string"},"defaultBranch":{"type":"string"},"revisionCounter":{"type":"number"},"latestRevisionId":{"type":"string"},"creator":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","slug","name","sourceType","revisionCounter","createdAt","projectId"]},"UpdateCodebaseInput":{"type":"object","properties":{"name":{"type":"string","minLength":1},"description":{"type":"string"},"defaultBranch":{"type":"string"}}},"CodebaseRevisionResponse":{"type":"object","properties":{"_id":{"type":"string"},"codebaseId":{"type":"string"},"slug":{"type":"string"},"revisionNumber":{"type":"number"},"ref":{"type":"string"},"sourceType":{"type":"string","enum":["git","archive"]},"source":{"type":"string"},"requestedRef":{"type":"string"},"resolvedCommitSha":{"type":"string"},"commitTimestamp":{"type":["string","null"],"format":"date-time"},"originalFilename":{"type":"string"},"contentSha256":{"type":"string"},"archiveUrl":{"type":"string"},"sizeBytes":{"type":"number"},"fileCount":{"type":"number"},"creator":{"type":"string"},"resolvedAt":{"type":["string","null"],"format":"date-time"},"createdAt":{"type":["string","null"],"format":"date-time"},"deduplicated":{"type":"boolean"},"projectId":{"type":"string"}},"required":["_id","codebaseId","slug","revisionNumber","ref","sourceType","archiveUrl","resolvedAt","createdAt","projectId"]},"ResolveCodebaseRevisionInput":{"type":"object","properties":{"requestedRef":{"type":"string"},"creator":{"type":"string"}}},"ResourceResponse":{"type":"object","properties":{"_id":{"type":"string"},"slug":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"revisionCounter":{"type":"number"},"latestRevisionId":{"type":"string"},"latestRevisionNumber":{"type":"number"},"creator":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","slug","name","revisionCounter","createdAt","projectId"]},"CreateResourceInput":{"type":"object","properties":{"name":{"type":"string","minLength":1},"slug":{"type":"string","minLength":1},"description":{"type":"string"},"setup":{"$ref":"#/components/schemas/ResourceScript"},"teardown":{"$ref":"#/components/schemas/ResourceScript"},"exports":{"type":"array","items":{"type":"string","pattern":"^[A-Za-z_][A-Za-z0-9_]*$"}},"parameters":{"type":"array","items":{"$ref":"#/components/schemas/ResourceParameter"}},"creator":{"type":"string"}},"required":["name","setup"]},"ResourceScript":{"type":"object","properties":{"sh":{"type":"string","minLength":1}}},"ResourceParameter":{"type":"object","properties":{"name":{"type":"string","pattern":"^[A-Za-z_][A-Za-z0-9_]*$"},"description":{"type":"string"},"required":{"type":"boolean","default":false},"default":{"type":"string"},"example":{"type":"string"}},"required":["name"]},"UpdateResourceInput":{"type":"object","properties":{"name":{"type":"string","minLength":1},"description":{"type":"string"}}},"ResourceRevisionResponse":{"type":"object","properties":{"_id":{"type":"string"},"resourceId":{"type":"string"},"slug":{"type":"string"},"revisionNumber":{"type":"number"},"ref":{"type":"string"},"setup":{"$ref":"#/components/schemas/ResourceScript"},"teardown":{"$ref":"#/components/schemas/ResourceScript"},"exports":{"type":"array","items":{"type":"string"}},"parameters":{"type":"array","items":{"$ref":"#/components/schemas/ResourceParameter"}},"contentSha256":{"type":"string"},"creator":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"deduplicated":{"type":"boolean"},"projectId":{"type":"string"}},"required":["_id","resourceId","slug","revisionNumber","ref","setup","exports","contentSha256","createdAt","projectId"]},"CreateResourceRevisionInput":{"type":"object","properties":{"setup":{"$ref":"#/components/schemas/ResourceScript"},"teardown":{"$ref":"#/components/schemas/ResourceScript"},"exports":{"type":"array","items":{"type":"string","pattern":"^[A-Za-z_][A-Za-z0-9_]*$"}},"parameters":{"type":"array","items":{"$ref":"#/components/schemas/ResourceParameter"}},"creator":{"type":"string"}},"required":["setup"]},"ExtensionResponse":{"type":"object","properties":{"_id":{"type":"string"},"slug":{"type":"string"},"publisher":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"origin":{"type":"string","enum":["marketplace","manual"]},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","slug","publisher","name","origin","createdAt","projectId"]},"ExtensionSearchResult":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"publisher":{"type":"string"},"description":{"type":"string"},"internal":{"type":"boolean"},"version":{"type":"string"}},"required":["id","name","publisher","internal"]},"CreateExtensionInput":{"type":"object","properties":{"_id":{"type":"string","pattern":"^[a-zA-Z0-9-]+\\.[a-zA-Z0-9-]+$"},"publisher":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"origin":{"type":"string","enum":["marketplace","manual"]}},"required":["_id","publisher","name","origin"]},"UpdateExtensionInput":{"type":"object","properties":{"name":{"type":"string"},"description":{"type":"string"}}},"ExtensionVersionInfo":{"type":"object","properties":{"version":{"type":"string"},"preRelease":{"type":"boolean"},"lastUpdated":{"type":"string"}},"required":["version","preRelease","lastUpdated"]},"CreateInsightInput":{"type":"object","properties":{"title":{"type":"string"},"description":{"type":"string"},"category":{"type":"string"},"tags":{"type":"array","items":{"type":"string"}},"createdBy":{"type":"string","enum":["agent","user"]},"sourceReportId":{"type":"string"}},"required":["title","description"]},"UpdateInsightInput":{"type":"object","properties":{"title":{"type":"string"},"description":{"type":"string"},"category":{"type":"string"},"tags":{"type":"array","items":{"type":"string"}}}},"FeatureFlagResponse":{"type":"object","properties":{"key":{"type":"string"},"label":{"type":"string"},"enabled":{"type":"boolean"},"updatedAt":{"type":["string","null"],"format":"date-time"}},"required":["key","label","enabled","updatedAt"]},"UpdateFeatureFlagInput":{"type":"object","properties":{"enabled":{"type":"boolean"}},"required":["enabled"]}},"parameters":{}},"paths":{"/api/v1/keys/preview":{"post":{"tags":["Keys"],"summary":"Preview key","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyInput"}}}},"responses":{"200":{"description":"Key preview","content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyResponse"}}}}}}},"/api/v1/keys":{"post":{"tags":["Keys"],"summary":"Create key","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyInput"}}}},"responses":{"201":{"description":"Key created","content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyResponse"}}}}}},"get":{"tags":["Keys"],"summary":"List keys","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/KeyResponse"}}}}}}}},"/api/v1/keys/{id}":{"get":{"tags":["Keys"],"summary":"Get key","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyResponse"}}}}}},"put":{"tags":["Keys"],"summary":"Update key","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyInput"}}}},"responses":{"200":{"description":"Key updated","content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyResponse"}}}}}},"delete":{"tags":["Keys"],"summary":"Delete key","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Key deleted","content":{"application/json":{"schema":{"type":"object","properties":{"success":{"type":"boolean"}},"required":["success"]}}}}}}},"/api/v1/keys/{id}/validate":{"post":{"tags":["Keys"],"summary":"Validate key","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ValidateKeyInput"}}}},"responses":{"200":{"description":"Validation result","content":{"application/json":{"schema":{"type":"object","properties":{},"additionalProperties":{},"description":"Key validation result"}}}}}}},"/api/v1/accounts":{"post":{"tags":["Accounts"],"summary":"Create account","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/AccountInput"}}}},"responses":{"201":{"description":"Account created","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AccountResponse"}}}}}},"get":{"tags":["Accounts"],"summary":"List accounts","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/AccountResponse"}}}}}}}},"/api/v1/accounts/{id}":{"get":{"tags":["Accounts"],"summary":"Get account","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AccountResponse"}}}}}},"put":{"tags":["Accounts"],"summary":"Update account","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/AccountInput"}}}},"responses":{"200":{"description":"Account updated","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AccountResponse"}}}}}},"delete":{"tags":["Accounts"],"summary":"Delete account","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Account deleted","content":{"application/json":{"schema":{"type":"object","properties":{"success":{"type":"boolean"}},"required":["success"]}}}}}}},"/api/v1/users/me":{"get":{"tags":["Users"],"summary":"Get the authenticated user's identity","description":"Read the existing authenticated Scope identity without enrollment or profile writes. Responses must not be HTTP-cached.","security":[{"bearerAuth":[]}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/UserMeResponse"}}}},"401":{"description":"Not authenticated"},"403":{"description":"User is not enrolled or is disabled"},"503":{"description":"Authentication service unavailable"}}},"post":{"tags":["Users"],"summary":"Enroll the authenticated user","description":"JIT-enroll the authenticated user, refresh their profile and lastLoginAt, apply bootstrap-admin rules, and warm the access cache. Use this only after an explicit IdP login; do not prefetch, poll, or automatically retry transient failures.","security":[{"bearerAuth":[]}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/UserMeResponse"}}}},"401":{"description":"Not authenticated"},"403":{"description":"User is disabled"},"503":{"description":"Authentication service unavailable"}}}},"/health":{"get":{"tags":["Health"],"summary":"Liveness probe","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"status":{"type":"string"},"version":{"type":"string"}},"required":["status","version"]}}}}}}},"/ready":{"get":{"tags":["Health"],"summary":"Readiness probe","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"status":{"type":"string"},"migrations":{}},"required":["status"]}}}},"503":{"description":"Service is not ready"}}}},"/about":{"get":{"tags":["System"],"summary":"API metadata","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"name":{"type":"string"},"version":{"type":"string"},"buildTime":{"type":"string"},"environment":{"type":"string"},"description":{"type":"string"},"workers":{"type":"array","items":{"type":"string"}}},"required":["name","version","buildTime","environment","description","workers"]}}}}}}},"/api/v1/version":{"get":{"tags":["System"],"summary":"Version info","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"commit":{"type":"string"},"buildTime":{"type":"string"},"environment":{"type":"string"},"strictAgentCapabilities":{"type":"boolean"}},"required":["commit","buildTime","environment","strictAgentCapabilities"]}}}}}}},"/api/v1/projects":{"get":{"tags":["Projects"],"summary":"List all projects","parameters":[{"schema":{"type":"string","enum":["true","false"],"description":"When true, include soft-deleted projects in the result"},"required":false,"description":"When true, include soft-deleted projects in the result","name":"includeDeleted","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ProjectResponse"}}}}}}},"post":{"tags":["Projects"],"summary":"Create a project","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateProjectInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProjectResponse"}}}},"400":{"description":"Invalid input"}}}},"/api/v1/projects/{id}":{"get":{"tags":["Projects"],"summary":"Get a project","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProjectResponse"}}}},"404":{"description":"Project not found"}}},"patch":{"tags":["Projects"],"summary":"Update a project","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateProjectInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProjectResponse"}}}},"404":{"description":"Project not found"}}},"delete":{"tags":["Projects"],"summary":"Soft-delete a project","responses":{"204":{"description":"Success"},"404":{"description":"Project not found"}}}},"/api/v1/projects/{id}/restore":{"post":{"tags":["Projects"],"summary":"Restore a soft-deleted project","responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProjectResponse"}}}},"404":{"description":"Project not found"}}}},"/api/v1/experiments":{"post":{"tags":["Experiments"],"summary":"Create an experiment and zero or more new requests","description":"Canonical composition endpoint. Request and run IDs are minted before insertion; the exact revision/execution is then finalized idempotently.","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateExperimentInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateExperimentResponse"}}}},"409":{"description":"Experiment identity conflict"}}},"get":{"tags":["Experiments"],"summary":"List experiments","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"},{"schema":{"type":"integer","minimum":1,"maximum":100,"default":50},"required":false,"name":"limit","in":"query"},{"schema":{"type":"string","minLength":1},"required":false,"name":"after","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ExperimentListResponse"}}}}}}},"/api/v1/experiments/{id}":{"get":{"tags":["Experiments"],"summary":"Get an experiment","parameters":[{"schema":{"type":"string","minLength":1},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ExperimentResponse"}}}},"404":{"description":"Experiment not found"}}}},"/api/v1/experiments/{id}/executions":{"get":{"tags":["Experiments"],"summary":"List current and historical experiment executions","parameters":[{"schema":{"type":"string","minLength":1},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"},{"schema":{"type":"integer","minimum":1,"maximum":100,"default":50},"required":false,"name":"limit","in":"query"},{"schema":{"type":"string","minLength":1},"required":false,"name":"after","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ExperimentExecutionListResponse"}}}},"404":{"description":"Experiment not found"}}}},"/api/v1/experiments/{id}/executions/{experimentExecutionId}":{"get":{"tags":["Experiments"],"summary":"Get one exact experiment execution","parameters":[{"schema":{"type":"string","minLength":1},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1},"required":true,"name":"experimentExecutionId","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ExperimentExecutionResponse"}}}},"404":{"description":"Experiment execution not found"}}}},"/api/v1/experiments/{id}/executions/{experimentExecutionId}/reports":{"get":{"tags":["Reports"],"summary":"List reports for one exact Experiment execution","parameters":[{"schema":{"type":"string","minLength":1},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1},"required":true,"name":"experimentExecutionId","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ReportResponse"}}}}},"404":{"description":"Experiment execution not found"}}}},"/api/v1/experiments/{id}/executions/{experimentExecutionId}/reports/{reportId}/structured":{"get":{"tags":["Reports"],"summary":"Download structured JSON for an exact Experiment report","parameters":[{"schema":{"type":"string","minLength":1},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1},"required":true,"name":"experimentExecutionId","in":"path"},{"schema":{"type":"string","minLength":1},"required":true,"name":"reportId","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success"},"404":{"description":"Experiment report or structured artifact not found"}}}},"/api/v1/experiments/{id}/executions/{experimentExecutionId}/logs":{"get":{"tags":["Experiments"],"summary":"Stream logs for one exact Experiment execution","parameters":[{"schema":{"type":"string","minLength":1},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1},"required":true,"name":"experimentExecutionId","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"},{"schema":{"type":"string","enum":["true","false"]},"required":false,"name":"fromStart","in":"query"}],"responses":{"200":{"description":"Server-sent event stream of log entries"},"404":{"description":"Experiment execution not found"}}}},"/api/v1/experiments/{id}/retry":{"post":{"tags":["Experiments"],"summary":"Retry current experiment processing","parameters":[{"schema":{"type":"string","minLength":1},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"experimentExecutionId":{"type":"string","minLength":1}},"required":["experimentExecutionId"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/RetryExperimentResponse"}}}},"404":{"description":"Experiment not found"},"409":{"description":"Execution is stale or cannot be retried"}}}},"/api/v1/requests":{"post":{"tags":["Requests"],"summary":"Submit request(s)","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"allOf":[{"$ref":"#/components/schemas/CreateRequestInput"},{"type":"object","properties":{"count":{"type":"number","minimum":1,"maximum":10,"default":1,"deprecated":true,"description":"Deprecated for composition. Use POST /api/v1/experiments and its requests array."},"skills":{"type":"array","items":{"type":"string"}},"agentVersion":{"type":"string"},"codebase":{"type":"string"}}}]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"anyOf":[{"$ref":"#/components/schemas/RequestResponse"},{"type":"array","items":{"$ref":"#/components/schemas/RequestResponse"}}]}}}}}},"get":{"tags":["Requests"],"summary":"List requests","parameters":[{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"worker","in":"query"},{"schema":{"type":"string"},"required":false,"name":"taskPromptId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"criteria","in":"query"},{"schema":{"type":"string"},"required":false,"name":"submissionId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"experimentId","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"profileId","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"status","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"outcome","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"model","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"os","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"priority","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"agentVersion","in":"query"},{"schema":{"type":"string"},"required":false,"name":"search","in":"query"},{"schema":{"type":"string","enum":["task","submissionId","experimentId","profile"]},"required":false,"name":"groupBy","in":"query"},{"schema":{"type":"integer","minimum":1,"maximum":100},"required":false,"name":"limit","in":"query"},{"schema":{"type":"string"},"required":false,"name":"after","in":"query"},{"schema":{"type":"string"},"required":false,"name":"before","in":"query"},{"schema":{"type":"string","enum":["true","false"]},"required":false,"name":"last","in":"query"},{"schema":{"type":"string","enum":["created","updated","priority","worker","status","id","duration","createdAt"]},"required":false,"name":"sortBy","in":"query"},{"schema":{"type":"string","enum":["asc","desc"]},"required":false,"name":"sortDir","in":"query"},{"schema":{"type":["string","null"],"format":"date-time"},"required":false,"name":"createdAfter","in":"query"},{"schema":{"type":["string","null"],"format":"date-time"},"required":false,"name":"createdBefore","in":"query"},{"schema":{"type":["integer","null"],"minimum":0},"required":false,"name":"turns","in":"query"},{"schema":{"type":"string","enum":["eq","gte","lte"]},"required":false,"name":"turnsOp","in":"query"},{"schema":{"type":["integer","null"],"minimum":0},"required":false,"name":"maxIterations","in":"query"},{"schema":{"type":"string","enum":["eq","gte","lte"]},"required":false,"name":"maxIterationsOp","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"anyOf":[{"$ref":"#/components/schemas/PaginatedRunsResponse"},{"$ref":"#/components/schemas/PaginatedRunGroupsResponse"}]}}}}}}},"/api/v1/requests/facets":{"get":{"tags":["Requests"],"summary":"List run filter facets","description":"Returns every distinct value and its full-dataset count per categorical filter dimension for the Runs list rail. Counts are absolute over all non-deleted runs **in the given project**: they intentionally ignore the active search, date range, iteration, and categorical selections so every selectable value stays visible with a stable count. Requires ?projectId=. Computed with parallel $group aggregations (Cosmos has no $facet) and cached per-project for a short TTL.","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/RunFacetsResponse"}}}}}}},"/api/v1/requests/{id}":{"get":{"tags":["Requests"],"summary":"Get request","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/RequestResponse"}}}},"404":{"description":"Not found"}}},"delete":{"tags":["Requests"],"summary":"Soft-delete request","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}},"404":{"description":"Not found"}}}},"/api/v1/analysis":{"get":{"tags":["Requests"],"summary":"Compute pass@k / success@T metrics","description":"Aggregates pass@k / success@T metrics over a single project's done runs. Requires ?projectId=.","parameters":[{"schema":{"type":"string"},"required":false,"name":"worker","in":"query"},{"schema":{"type":"string"},"required":false,"name":"taskPromptId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"criteria","in":"query"},{"schema":{"type":"string"},"required":false,"name":"features","in":"query"},{"schema":{"type":"string"},"required":false,"name":"submissionId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"experimentId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"k","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{},"additionalProperties":{},"description":"Analysis metrics"}}}}}}},"/api/v1/requests/bulk-resubmit":{"post":{"tags":["Requests"],"summary":"Bulk resubmit requests","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/BulkResubmitInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/RequestResponse"}}}}}}}},"/api/v1/requests/bulk":{"delete":{"tags":["Requests"],"summary":"Bulk soft-delete requests","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"}}},"required":["ids"]}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"deleted":{"type":"number"}},"required":["deleted"]}}}}}}},"/api/v1/requests/archive":{"post":{"tags":["Requests"],"summary":"Download batch archive of multiple runs","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1,"maxItems":100}},"required":["ids"]}}}},"responses":{"201":{"description":"Gzipped batch archive containing individual run archives"},"400":{"description":"Invalid input"},"404":{"description":"One or more runs not found"}}}},"/api/v1/runs/upload":{"post":{"tags":["Requests"],"summary":"Import run archive","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"201":{"description":"Success"}}}},"/api/v1/runs/upload-batch":{"post":{"tags":["Requests"],"summary":"Import batch run archive","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"207":{"description":"Multi-status: 201 if all runs imported, 400 if none imported, 207 if partial"},"400":{"description":"No archive uploaded, empty archive, or all runs failed"}}}},"/api/v1/requests/{id}/reports":{"get":{"tags":["Reports"],"summary":"Get reports for request","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ReportResponse"}}}}},"404":{"description":"Run not found"}}}},"/api/v1/requests/{id}/runs":{"get":{"tags":["Requests"],"summary":"List attempts for a request","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/RunState"}}}}},"404":{"description":"Request not found"}}}},"/api/v1/requests/{id}/runs/{runId}":{"get":{"tags":["Requests"],"summary":"Get a single attempt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/RunState"}}}},"404":{"description":"Request or run not found"}}}},"/api/v1/requests/bulk-retry":{"post":{"tags":["Requests"],"summary":"Bulk retry requests (start new attempts)","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1},"force":{"type":"boolean"}},"required":["ids"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"retried":{"type":"integer"},"skipped":{"type":"integer"},"results":{"type":"array","items":{"type":"object","properties":{"requestId":{"type":"string"},"runId":{"type":"string"},"attemptNumber":{"type":"integer"},"error":{"type":"string"}},"required":["requestId"]}}},"required":["retried","skipped","results"]}}}}}}},"/api/v1/requests/{id}/retry":{"post":{"tags":["Requests"],"summary":"Retry a request (start a new attempt)","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"force":{"type":"boolean"}}}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"requestId":{"type":"string"},"runId":{"type":"string"},"attemptNumber":{"type":"integer"}},"required":["requestId","runId","attemptNumber"]}}}},"404":{"description":"Request not found"},"409":{"description":"Conflict — request is not in a retryable state, or a concurrent retry won the race"},"422":{"description":"Cannot retry — current run not yet terminal"}}}},"/api/v1/requests/{id}/priority":{"post":{"tags":["Requests"],"summary":"Set priority on a single request","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"priority":{"type":"integer"}},"required":["priority"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"priority":{"type":"number"}},"required":["id","priority"]}}}}}}},"/api/v1/requests/bulk-priority":{"post":{"tags":["Requests"],"summary":"Set priority on multiple requests","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1,"maxItems":100},"priority":{"type":"integer"}},"required":["ids","priority"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"updated":{"type":"number"},"skipped":{"type":"number"}},"required":["updated","skipped"]}}}}}}},"/api/v1/requests/{id}/pause":{"post":{"tags":["Requests"],"summary":"Pause a single request","responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"status":{"type":"string"}},"required":["id","status"]}}}}}}},"/api/v1/requests/{id}/resume":{"post":{"tags":["Requests"],"summary":"Resume a paused request","responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"status":{"type":"string"}},"required":["id","status"]}}}}}}},"/api/v1/requests/bulk-pause":{"post":{"tags":["Requests"],"summary":"Pause multiple requests","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1,"maxItems":100}},"required":["ids"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"updated":{"type":"number"},"skipped":{"type":"number"}},"required":["updated","skipped"]}}}}}}},"/api/v1/requests/bulk-resume":{"post":{"tags":["Requests"],"summary":"Resume multiple paused requests","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1,"maxItems":100}},"required":["ids"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"updated":{"type":"number"},"skipped":{"type":"number"}},"required":["updated","skipped"]}}}}}}},"/api/v1/requests/{id}/cancel":{"post":{"tags":["Requests"],"summary":"Cancel a request (marks as done/failed and signals worker to exit)","responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"previousStatus":{"type":"string"},"status":{"type":"string"},"outcome":{"type":"string"}},"required":["id","previousStatus","status","outcome"]}}}}}}},"/api/v1/requests/bulk-cancel":{"post":{"tags":["Requests"],"summary":"Cancel multiple requests","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1,"maxItems":100}},"required":["ids"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"cancelled":{"type":"number"},"skipped":{"type":"number"},"results":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string"},"cancelled":{"type":"boolean"},"previousStatus":{"type":"string"},"error":{"type":"string"}},"required":["id","cancelled"]}}},"required":["cancelled","skipped","results"]}}}}}}},"/api/v1/requests/{id}/logs":{"get":{"tags":["Requests"],"summary":"Stream request logs (SSE)","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Server-sent event stream of log entries"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/logs":{"get":{"tags":["Requests"],"summary":"Stream logs for a specific attempt (SSE)","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"}],"responses":{"200":{"description":"Server-sent event stream of log entries"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/har":{"get":{"tags":["Requests"],"summary":"Download HAR file","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"HAR-format JSON file"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/har":{"get":{"tags":["Requests"],"summary":"Download HAR file for a specific attempt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"}],"responses":{"200":{"description":"HAR-format JSON file"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/atif":{"get":{"tags":["Requests"],"summary":"Download ATIF trajectory file for a specific iteration","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","description":"The iteration number (1-based)"},"required":true,"description":"The iteration number (1-based)","name":"iteration","in":"query"}],"responses":{"200":{"description":"ATIF v1.7 trajectory JSON file"},"400":{"description":"Missing or invalid iteration"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/atif":{"get":{"tags":["Requests"],"summary":"Download ATIF trajectory file for a specific attempt and iteration","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"},{"schema":{"type":"string","description":"The iteration number (1-based)"},"required":true,"description":"The iteration number (1-based)","name":"iteration","in":"query"}],"responses":{"200":{"description":"ATIF v1.7 trajectory JSON file"},"400":{"description":"Missing or invalid iteration"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/video":{"get":{"tags":["Requests"],"summary":"Download session recording","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"WebM video recording (supports Range requests)"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/video":{"get":{"tags":["Requests"],"summary":"Download session recording for a specific attempt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"}],"responses":{"200":{"description":"WebM video recording (supports Range requests)"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/tool-calls":{"get":{"tags":["Requests"],"summary":"Download per-iteration tool-calls JSONL","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"JSONL stream — one ToolCall per line"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/tool-calls":{"get":{"tags":["Requests"],"summary":"Download per-iteration tool-calls JSONL for a specific attempt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"}],"responses":{"200":{"description":"JSONL stream — one ToolCall per line"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/snapshots/{iteration}":{"get":{"tags":["Requests"],"summary":"Download iteration snapshot","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"iteration","in":"path"}],"responses":{"200":{"description":"Gzipped snapshot archive"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/snapshots/{iteration}":{"get":{"tags":["Requests"],"summary":"Download iteration snapshot for a specific attempt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"},{"schema":{"type":"string"},"required":true,"name":"iteration","in":"path"}],"responses":{"200":{"description":"Gzipped snapshot archive"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/archive":{"get":{"tags":["Requests"],"summary":"Download full run archive","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Gzipped run archive"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/archive":{"get":{"tags":["Requests"],"summary":"Download full run archive for a specific attempt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"}],"responses":{"200":{"description":"Gzipped run archive"},"404":{"description":"Not found"}}}},"/api/v1/criteria/generate-prompt":{"post":{"tags":["Criteria"],"summary":"Generate criterion prompt from behavior","parameters":[{"schema":{"type":"string","minLength":1,"description":"Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.","example":"00000000-0000-0000-0000-000000000000"},"required":false,"description":"Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"behavior":{"type":"string"},"currentId":{"type":"string"},"gates":{"type":"array","items":{"type":"string"}}},"required":["behavior"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"prompt":{"type":"string"},"suggestedId":{"type":"string"},"suggestedParents":{"type":"array","items":{"type":"string"}},"suggestedChildren":{"type":"array","items":{"type":"string"}}},"required":["prompt","suggestedId","suggestedParents","suggestedChildren"]}}}},"400":{"description":"Empty behavior string"},"503":{"description":"LLM not configured"}}}},"/api/v1/criteria/seed":{"post":{"tags":["Criteria"],"summary":"Seed criteria in bulk","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"criteria":{"type":"array","items":{"$ref":"#/components/schemas/CreateCriteriaInput"}}},"required":["criteria"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"seeded":{"type":"number"},"errors":{"type":"array","items":{"type":"string"}}},"required":["seeded","errors"]}}}},"400":{"description":"Seed batch would introduce a dependency cycle"}}}},"/api/v1/criteria":{"get":{"tags":["Criteria"],"summary":"List criteria","parameters":[{"schema":{"type":"string"},"required":false,"name":"q","in":"query"},{"schema":{"type":"string","description":"Comma-separated criterion IDs to include"},"required":false,"description":"Comma-separated criterion IDs to include","name":"ids","in":"query"},{"schema":{"type":"string","enum":["true","false"],"description":"When true and ids is set, also include dependency ancestors"},"required":false,"description":"When true and ids is set, also include dependency ancestors","name":"ancestors","in":"query"},{"schema":{"type":"string","enum":["gate","observation"],"description":"Filter by criterion kind"},"required":false,"description":"Filter by criterion kind","name":"kind","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/CriteriaResponse"}}}}}}},"post":{"tags":["Criteria"],"summary":"Create criterion","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateCriteriaInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CriteriaResponse"}}}},"409":{"description":"Criterion already exists"}}}},"/api/v1/criteria/mdp":{"get":{"tags":["Criteria"],"summary":"Compute MDP transitions","parameters":[{"schema":{"type":"string"},"required":false,"name":"criteria","in":"query"},{"schema":{"type":"string"},"required":false,"name":"features","in":"query"},{"schema":{"type":"string"},"required":false,"name":"since","in":"query"},{"schema":{"type":"string"},"required":false,"name":"worker","in":"query"},{"schema":{"type":"string"},"required":false,"name":"taskPromptId","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{},"additionalProperties":{}}}}}}}},"/api/v1/criteria/graph":{"get":{"tags":["Criteria"],"summary":"Get criteria DAG","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CriteriaGraph"}}}}}}},"/api/v1/criteria/{id}":{"get":{"tags":["Criteria"],"summary":"Get criterion","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CriteriaResponse"}}}},"404":{"description":"Criterion not found"}}},"put":{"tags":["Criteria"],"summary":"Update criterion","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateCriteriaInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CriteriaResponse"}}}},"400":{"description":"Invalid dependency reference or self-reference"},"404":{"description":"Criterion not found"}}},"delete":{"tags":["Criteria"],"summary":"Soft-delete criterion","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"deleted":{"type":"boolean"}},"required":["id","deleted"]}}}},"404":{"description":"Criterion not found"},"409":{"description":"Criterion has dependents"}}}},"/api/v1/prompt-features/generate-prompt":{"post":{"tags":["Prompt Features"],"summary":"Generate prompt feature from behavior","parameters":[{"schema":{"type":"string","minLength":1,"description":"Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.","example":"00000000-0000-0000-0000-000000000000"},"required":false,"description":"Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"behavior":{"type":"string"},"currentId":{"type":"string"}},"required":["behavior"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"prompt":{"type":"string"}},"required":["prompt"]}}}},"400":{"description":"Empty behavior string"},"503":{"description":"LLM not configured"}}}},"/api/v1/prompt-features/seed":{"post":{"tags":["Prompt Features"],"summary":"Seed prompt features in bulk","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"features":{"type":"array","items":{"$ref":"#/components/schemas/CreatePromptFeatureInput"}}},"required":["features"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"seeded":{"type":"number"},"errors":{"type":"array","items":{"type":"string"}}},"required":["seeded","errors"]}}}}}}},"/api/v1/prompt-features/extract-from-text":{"post":{"tags":["Prompt Features"],"summary":"Extract features from text","parameters":[{"schema":{"type":"string","minLength":1,"description":"Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.","example":"00000000-0000-0000-0000-000000000000"},"required":false,"description":"Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"text":{"type":"string"},"model":{"type":"string"},"type":{"type":"string","enum":["select","agents.md"]}},"required":["text"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"features":{"type":"array","items":{"$ref":"#/components/schemas/PromptFeatureResult"}},"suggestedFeatures":{"type":"array","items":{"$ref":"#/components/schemas/SuggestedPromptFeature"}},"cached":{"type":"boolean"}},"required":["features","cached"]}}}},"400":{"description":"Empty text string"},"503":{"description":"LLM not configured"}}}},"/api/v1/prompt-features":{"get":{"tags":["Prompt Features"],"summary":"List features","parameters":[{"schema":{"type":"string"},"required":false,"name":"q","in":"query"},{"schema":{"type":"string","enum":["select","agents.md"]},"required":false,"name":"type","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/PromptFeatureResponse"}}}}}}},"post":{"tags":["Prompt Features"],"summary":"Create feature","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreatePromptFeatureInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/PromptFeatureResponse"}}}},"400":{"description":"Invalid input"},"409":{"description":"Feature already exists"}}}},"/api/v1/prompt-features/{id}":{"get":{"tags":["Prompt Features"],"summary":"Get feature","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/PromptFeatureResponse"}}}},"404":{"description":"Feature not found"}}},"put":{"tags":["Prompt Features"],"summary":"Update feature","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdatePromptFeatureInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/PromptFeatureResponse"}}}},"400":{"description":"Invalid input"},"404":{"description":"Feature not found"}}},"delete":{"tags":["Prompt Features"],"summary":"Soft-delete feature","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"deleted":{"type":"boolean"}},"required":["id","deleted"]}}}},"404":{"description":"Feature not found"}}}},"/api/v1/task-prompts/generate":{"post":{"tags":["Task Prompts"],"summary":"Generate task prompts","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"description":{"type":"string"},"existingPrompt":{"type":"string"}}}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"tasks":{"type":"array","items":{"type":"string"}}},"required":["tasks"]}}}},"503":{"description":"LLM not configured"}}}},"/api/v1/task-prompts":{"get":{"tags":["Task Prompts"],"summary":"List task prompts","parameters":[{"schema":{"type":["number","null"]},"required":false,"name":"limit","in":"query"},{"schema":{"type":["number","null"]},"required":false,"name":"offset","in":"query"},{"schema":{"type":"string"},"required":false,"name":"search","in":"query"},{"schema":{"$ref":"#/components/schemas/PromptType"},"required":false,"name":"type","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"items":{"type":"array","items":{"$ref":"#/components/schemas/TaskPromptResponse"}},"total":{"type":"number"},"limit":{"type":"number"},"offset":{"type":"number"}},"required":["items","total","limit","offset"]}}}}}},"post":{"tags":["Task Prompts"],"summary":"Create or find task prompt","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateTaskPromptInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/TaskPromptResponse"}}}},"400":{"description":"Empty text string"}}}},"/api/v1/task-prompts/{id}":{"get":{"tags":["Task Prompts"],"summary":"Get task prompt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/TaskPromptResponse"}}}},"404":{"description":"Task prompt not found"}}},"delete":{"tags":["Task Prompts"],"summary":"Soft-delete task prompt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"deleted":{"type":"boolean"}},"required":["deleted"]}}}},"404":{"description":"Task prompt not found"}}}},"/api/v1/task-prompts/{id}/content":{"get":{"tags":["Task Prompts"],"summary":"Get resolved task prompt content","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"text":{"type":"string"}},"required":["id","text"]}}}},"404":{"description":"Task prompt not found"}}}},"/api/v1/task-prompts/{id}/extract-features":{"post":{"tags":["Task Prompts"],"summary":"Extract features from task prompt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":false,"name":"force","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"model":{"type":"string"}}}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"taskPromptId":{"type":"string"},"features":{"type":"array","items":{"$ref":"#/components/schemas/PromptFeatureResult"}},"featuresExtractedAt":{"type":["string","null"],"format":"date-time"},"suggestedFeatures":{"type":"array","items":{"$ref":"#/components/schemas/SuggestedPromptFeature"}},"cached":{"type":"boolean"}},"required":["taskPromptId","features","cached"]}}}},"404":{"description":"Task prompt not found"},"503":{"description":"LLM not configured"}}}},"/api/v1/task-prompts/{id}/features/{featureId}":{"patch":{"tags":["Task Prompts"],"summary":"Toggle feature flag on task prompt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"featureId","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/PatchTaskPromptFeatureInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/TaskPromptResponse"}}}},"400":{"description":"Invalid detected value"},"404":{"description":"Task prompt or feature not found"}}}},"/api/v1/reports":{"post":{"tags":["Reports"],"summary":"Create report","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateReportInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ReportResponse"}}}},"400":{"description":"Invalid input"},"404":{"description":"Run or template not found"}}},"get":{"tags":["Reports"],"summary":"List reports","parameters":[{"schema":{"type":"string"},"required":false,"name":"requestId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"experimentId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"experimentExecutionId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ReportResponse"}}}}}}}},"/api/v1/reports/bulk-create":{"post":{"tags":["Reports"],"summary":"Bulk create reports","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/BulkCreateReportsInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ReportResponse"}}}}},"400":{"description":"Invalid input"}}}},"/api/v1/reports/{id}/structured":{"get":{"tags":["Reports"],"summary":"Download a report's structured JSON artifact","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success"},"404":{"description":"Report or structured artifact not found"}}}},"/api/v1/reports/bulk-status":{"post":{"tags":["Reports"],"summary":"Bulk get report statuses","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/BulkReportStatusInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ReportResponse"}}}}},"400":{"description":"Invalid input"}}}},"/api/v1/reports/bulk-summary":{"post":{"tags":["Reports"],"summary":"Bulk get report summary per run","description":"Returns aggregated report status counts per run. Only the latest report per template is counted (re-triggers are deduplicated).","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/BulkReportSummaryInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/BulkReportSummaryResponse"}}}},"400":{"description":"Invalid input"}}}},"/api/v1/reports/{id}":{"get":{"tags":["Reports"],"summary":"Get report","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ReportResponse"}}}},"404":{"description":"Report not found"}}}},"/api/v1/reports/{id}/logs":{"get":{"tags":["Reports"],"summary":"Stream report logs (SSE)","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Server-sent event stream of log entries"},"404":{"description":"Report not found"}}}},"/api/v1/reports/trigger":{"post":{"tags":["Reports"],"summary":"Trigger reports","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/TriggerReportsInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"triggered":{"type":"number"},"reports":{"type":"array","items":{"$ref":"#/components/schemas/ReportResponse"}}},"required":["triggered","reports"]}}}},"400":{"description":"Invalid input"},"404":{"description":"Run not found"}}}},"/api/v1/reports/bulk-trigger":{"post":{"tags":["Reports"],"summary":"Bulk trigger reports","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/BulkTriggerReportsInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"type":"object","properties":{},"additionalProperties":{}}}}}},"400":{"description":"Invalid input"}}}},"/api/v1/reports/{id}/insights":{"get":{"tags":["Reports"],"summary":"Get insights for report","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"allOf":[{"$ref":"#/components/schemas/InsightResponse"},{"type":"object","properties":{"referencedAt":{"type":["string","null"],"format":"date-time"},"isNew":{"type":"boolean"}}}]}}}}},"404":{"description":"Report not found"}}},"post":{"tags":["Reports"],"summary":"Link insight to report","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"insightId":{"type":"string"},"isNew":{"type":"boolean"}},"required":["insightId"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"insightId":{"type":"string"},"referencedAt":{"type":["string","null"],"format":"date-time"},"isNew":{"type":"boolean"}},"required":["insightId","referencedAt","isNew"]}}}},"404":{"description":"Report or insight not found"},"409":{"description":"Insight already referenced by this report"}}}},"/api/v1/profiles":{"post":{"tags":["Profiles"],"summary":"Create a new profile","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateProfileInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProfileWithVersionResponse"}}}}}},"get":{"tags":["Profiles"],"summary":"List profiles","parameters":[{"schema":{"type":"string"},"required":false,"name":"workerType","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ProfileWithVersionResponse"}}}}}}}},"/api/v1/profiles/{profileId}":{"get":{"tags":["Profiles"],"summary":"Get profile with latest version","parameters":[{"schema":{"type":"string"},"required":true,"name":"profileId","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProfileWithVersionResponse"}}}}}},"post":{"tags":["Profiles"],"summary":"Create new profile version","parameters":[{"schema":{"type":"string"},"required":true,"name":"profileId","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"workerType":{"type":"string"},"model":{"type":"string"},"reasoningEffort":{"type":"string"},"agentVersion":{"type":"string"},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceBindingSpec"}},"extensions":{"type":"array","items":{"type":"string"}}},"required":["workerType","model"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProfileVersionResponse"}}}}}},"put":{"tags":["Profiles"],"summary":"Update profile identity","parameters":[{"schema":{"type":"string"},"required":true,"name":"profileId","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateProfileIdentity"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProfileResponse"}}}}}},"delete":{"tags":["Profiles"],"summary":"Delete profile","parameters":[{"schema":{"type":"string"},"required":true,"name":"profileId","in":"path"}],"responses":{"204":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}}}}},"/api/v1/profiles/{profileId}/versions":{"get":{"tags":["Profiles"],"summary":"List profile versions","parameters":[{"schema":{"type":"string"},"required":true,"name":"profileId","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ProfileVersionResponse"}}}}}}}},"/api/v1/profiles/{profileId}/versions/{version}":{"get":{"tags":["Profiles"],"summary":"Get profile version","parameters":[{"schema":{"type":"string"},"required":true,"name":"profileId","in":"path"},{"schema":{"type":["number","null"]},"required":false,"name":"version","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProfileVersionResponse"}}}}}}},"/api/v1/report-templates/default-system-prompt":{"get":{"tags":["Report Templates"],"summary":"Get default system prompt","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"content":{"type":"string"}},"required":["content"]}}}}}}},"/api/v1/report-templates/available-models":{"get":{"tags":["Report Templates"],"summary":"List models available for report generation","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"type":"object","properties":{"modelId":{"type":"string"}},"required":["modelId"]}}}}}}}},"/api/v1/report-templates":{"get":{"tags":["Report Templates"],"summary":"List report templates","parameters":[{"schema":{"type":"string"},"required":false,"name":"q","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ReportTemplateResponse"}}}}}}},"post":{"tags":["Report Templates"],"summary":"Create report template","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateReportTemplateInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ReportTemplateResponse"}}}},"409":{"description":"Report template already exists"}}}},"/api/v1/report-templates/{id}":{"get":{"tags":["Report Templates"],"summary":"Get report template","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ReportTemplateResponse"}}}},"404":{"description":"Report template not found"}}},"put":{"tags":["Report Templates"],"summary":"Update report template","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateReportTemplateInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ReportTemplateResponse"}}}},"404":{"description":"Report template not found"}}},"delete":{"tags":["Report Templates"],"summary":"Delete report template","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"204":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}},"404":{"description":"Report template not found"}}}},"/api/v1/agents":{"get":{"tags":["Agents"],"summary":"List agents","parameters":[{"schema":{"type":"string"},"required":false,"name":"modelProvider","in":"query"},{"schema":{"type":"string","enum":["true","false"]},"required":false,"name":"includeDeleted","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/AgentResponse"}}}}}}},"post":{"tags":["Agents"],"summary":"Create or update agent (upsert)","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateAgentInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AgentResponse"}}}},"400":{"description":"Validation error"},"409":{"description":"Restored agent versions conflict with an active queue owner"}}}},"/api/v1/agents/{id}":{"get":{"tags":["Agents"],"summary":"Get agent","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","enum":["true","false"]},"required":false,"name":"includeDeleted","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AgentResponse"}}}},"404":{"description":"Agent not found"}}},"put":{"tags":["Agents"],"summary":"Update agent","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateAgentInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AgentResponse"}}}},"400":{"description":"Validation error"},"404":{"description":"Agent not found"}}},"delete":{"tags":["Agents"],"summary":"Delete agent","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"204":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}},"404":{"description":"Agent not found"}}}},"/api/v1/agents/{id}/versions":{"get":{"tags":["Agents"],"summary":"List agent versions","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":false,"name":"status","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/AgentVersion"}}}}},"404":{"description":"Agent not found"}}},"post":{"tags":["Agents"],"summary":"Register agent version (upsert)","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/RegisterAgentVersionInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AgentVersion"}}}},"400":{"description":"Validation error"},"404":{"description":"Agent not found"},"409":{"description":"Queue is assigned to another agent"},"503":{"description":"Concurrent registration did not stabilize"}}}},"/api/v1/agents/{id}/versions/{agentVersion}":{"patch":{"tags":["Agents"],"summary":"Patch agent version","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"agentVersion","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/PatchAgentVersionInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AgentVersion"}}}},"400":{"description":"Invalid status value"},"404":{"description":"Agent or version not found"},"409":{"description":"Queue is assigned to another agent"},"503":{"description":"Concurrent activation did not stabilize"}}}},"/api/v1/models":{"get":{"tags":["Models"],"summary":"List models","parameters":[{"schema":{"type":"string"},"required":false,"name":"agentId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"provider","in":"query"},{"schema":{"type":"string","enum":["active","disappeared"]},"required":false,"name":"status","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ModelResponse"}}}}}}}},"/api/v1/models/{id}":{"get":{"tags":["Models"],"summary":"Get model","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ModelResponse"}}}}}}},"/api/v1/models/sync":{"post":{"tags":["Models"],"summary":"Sync models from provider","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"agentId":{"type":"string"},"provider":{"type":"string"},"models":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string"},"providerAvailableFrom":{"type":"string"},"providerEndOfLife":{"type":"string"},"metadata":{"type":"object","additionalProperties":{}},"capabilities":{"type":"object","properties":{"reasoningEffort":{"type":"array","items":{"type":"string"}},"toolCalls":{"type":"boolean"},"vision":{"type":"boolean"},"streaming":{"type":"boolean"},"adaptiveThinking":{"type":"boolean"},"maxThinkingBudget":{"type":"number"}}}},"required":["id"]}},"scannedAt":{"type":"string"}},"required":["agentId","provider","models","scannedAt"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"added":{"type":"array","items":{"type":"string"}},"removed":{"type":"array","items":{"type":"string"}},"unchanged":{"type":"array","items":{"type":"string"}}},"required":["added","removed","unchanged"]}}}}}}},"/api/v1/mcp/servers":{"get":{"tags":["MCP Servers"],"summary":"List MCP servers","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/McpServerResponse"}}}}}}},"post":{"tags":["MCP Servers"],"summary":"Create MCP server","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"_id":{"type":"string","pattern":"^[a-z0-9]([a-z0-9-]*[a-z0-9])?$"},"name":{"type":"string"},"type":{"type":"string","enum":["sse","http","stdio"]},"url":{"type":"string"},"command":{"type":"string"},"args":{"type":"array","items":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"headers":{"type":"array","items":{"$ref":"#/components/schemas/McpServerHeader"}},"sessionMode":{"type":"string","enum":["stateful","stateless"]},"version":{"type":"string"},"description":{"type":"string"}},"required":["_id","name","type"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/McpServerResponse"}}}}}}},"/api/v1/mcp/servers/{id}":{"get":{"tags":["MCP Servers"],"summary":"Get MCP server","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/McpServerResponse"}}}}}},"put":{"tags":["MCP Servers"],"summary":"Update MCP server","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateMcpServerInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/McpServerResponse"}}}}}},"delete":{"tags":["MCP Servers"],"summary":"Delete MCP server","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"204":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}}}}},"/api/v1/skills":{"get":{"tags":["Skills"],"summary":"List all skills","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/SkillResponse"}}}}}}},"post":{"tags":["Skills"],"summary":"Create or import a skill","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateSkillInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SkillResponse"}}}}}}},"/api/v1/skills/search":{"get":{"tags":["Skills"],"summary":"Search skills (internal + external)","parameters":[{"schema":{"type":"string"},"required":true,"name":"q","in":"query"},{"schema":{"type":"string"},"required":false,"name":"limit","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/SkillSearchResult"}}}}},"400":{"description":"Missing query parameter"}}}},"/api/v1/skills/search/external":{"get":{"tags":["Skills"],"summary":"Search external skills registry","parameters":[{"schema":{"type":"string"},"required":true,"name":"q","in":"query"},{"schema":{"type":"string"},"required":false,"name":"limit","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Optional project scope. When provided, external results are de-duplicated only against skills already installed in that project."},"required":false,"description":"Optional project scope. When provided, external results are de-duplicated only against skills already installed in that project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/SkillSearchResult"}}}}},"400":{"description":"Missing query parameter"}}}},"/api/v1/skills/discover":{"get":{"tags":["Skills"],"summary":"Discover skills in a GitHub repository","parameters":[{"schema":{"type":"string"},"required":true,"name":"source","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/SkillDiscoveryResult"}}}}},"400":{"description":"Missing or malformed source parameter"},"404":{"description":"Repository not found"},"502":{"description":"GitHub API error"}}}},"/api/v1/skills/{id}(*)/revisions":{"get":{"tags":["Skills"],"summary":"List skill revisions","parameters":[{"schema":{"type":"string"},"required":false,"name":"limit","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/SkillRevisionResponse"}}}}},"404":{"description":"Skill not found"}}}},"/api/v1/skills/{id}(*)":{"get":{"tags":["Skills"],"summary":"Get skill by slug","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SkillResponse"}}}},"404":{"description":"Skill not found"}}},"delete":{"tags":["Skills"],"summary":"Soft-delete a skill","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"204":{"description":"Success"},"404":{"description":"Skill not found"}}}},"/api/v1/skill-revisions/by-ref/{ref}(*)/archive":{"get":{"tags":["Skill Revisions"],"summary":"Download skill revision archive","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Binary tar.gz archive"},"404":{"description":"Skill revision or archive not found"},"500":{"description":"Blob storage error"}}}},"/api/v1/skill-revisions/by-ref/{ref}(*)":{"get":{"tags":["Skill Revisions"],"summary":"Get skill revision by ref","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SkillRevisionResponse"}}}},"404":{"description":"Skill revision not found"}}}},"/api/v1/skill-revisions/{id}":{"get":{"tags":["Skill Revisions"],"summary":"Get skill revision by ID","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SkillRevisionResponse"}}}},"404":{"description":"Skill revision not found"}}}},"/api/v1/skills/{id}(*)/resolve":{"post":{"tags":["Skills"],"summary":"Trigger skill resolution","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SkillRevisionResponse"}}}},"404":{"description":"Skill not found"},"500":{"description":"Blob storage error"}}}},"/api/v1/codebases":{"get":{"tags":["Codebases"],"summary":"List all codebases","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/CodebaseResponse"}}}}}}},"post":{"tags":["Codebases"],"summary":"Create a codebase (archive codebases require the archive file)","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"201":{"description":"Success"},"400":{"description":"Invalid input or missing archive"}}}},"/api/v1/codebases/{id}":{"get":{"tags":["Codebases"],"summary":"Get a codebase","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CodebaseResponse"}}}},"404":{"description":"Codebase not found"}}},"patch":{"tags":["Codebases"],"summary":"Update a codebase","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateCodebaseInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CodebaseResponse"}}}},"404":{"description":"Codebase not found"}}},"delete":{"tags":["Codebases"],"summary":"Delete a codebase","responses":{"204":{"description":"Success"},"404":{"description":"Codebase not found"}}}},"/api/v1/codebases/{id}/revisions":{"get":{"tags":["Codebases"],"summary":"List codebase revisions","parameters":[{"schema":{"type":"string"},"required":false,"name":"limit","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/CodebaseRevisionResponse"}}}}},"404":{"description":"Codebase not found"}}},"post":{"tags":["Codebases"],"summary":"Resolve a new git codebase revision","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ResolveCodebaseRevisionInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CodebaseRevisionResponse"}}}},"400":{"description":"Not a git codebase"},"404":{"description":"Codebase not found"},"502":{"description":"GitHub resolution failed"}}}},"/api/v1/codebases/{id}/upload":{"post":{"tags":["Codebases"],"summary":"Upload a codebase archive as a new revision","responses":{"201":{"description":"Success"},"400":{"description":"Missing archive or not an archive codebase"},"404":{"description":"Codebase not found"}}}},"/api/v1/codebase-revisions/{id}/archive":{"get":{"tags":["Codebase Revisions"],"summary":"Download codebase revision archive","responses":{"200":{"description":"Binary tar.gz archive"},"404":{"description":"Revision or archive not found"},"500":{"description":"Blob storage error"}}}},"/api/v1/codebase-revisions/{id}":{"get":{"tags":["Codebase Revisions"],"summary":"Get codebase revision by id","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CodebaseRevisionResponse"}}}},"404":{"description":"Revision not found"}}}},"/api/v1/resources":{"get":{"tags":["Resources"],"summary":"List all resources","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ResourceResponse"}}}}}}},"post":{"tags":["Resources"],"summary":"Create a resource and its first revision","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateResourceInput"}}}},"responses":{"201":{"description":"Success"},"400":{"description":"Invalid input"},"409":{"description":"Resource slug or revision ref already exists in this project"}}}},"/api/v1/resources/{id}":{"get":{"tags":["Resources"],"summary":"Get a resource","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ResourceResponse"}}}},"404":{"description":"Resource not found"}}},"patch":{"tags":["Resources"],"summary":"Update a resource","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateResourceInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ResourceResponse"}}}},"404":{"description":"Resource not found"}}},"delete":{"tags":["Resources"],"summary":"Delete a resource","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"204":{"description":"Success"},"404":{"description":"Resource not found"}}}},"/api/v1/resources/{id}/revisions":{"get":{"tags":["Resource Revisions"],"summary":"List resource revisions","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"limit","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ResourceRevisionResponse"}}}}},"404":{"description":"Resource not found"}}},"post":{"tags":["Resource Revisions"],"summary":"Create a resource revision","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateResourceRevisionInput"}}}},"responses":{"201":{"description":"Success"},"404":{"description":"Resource not found"},"409":{"description":"Resource revision ref already exists in this project"}}}},"/api/v1/resources/{id}/revisions/latest":{"get":{"tags":["Resource Revisions"],"summary":"Get the latest resource revision","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ResourceRevisionResponse"}}}},"404":{"description":"Revision not found"}}}},"/api/v1/resources/{id}/revisions/{revisionNumber}":{"get":{"tags":["Resource Revisions"],"summary":"Get a resource revision by resource and revision number","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"revisionNumber","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ResourceRevisionResponse"}}}},"404":{"description":"Revision not found"}}}},"/api/v1/resources/revisions/{id}":{"get":{"tags":["Resource Revisions"],"summary":"Get resource revision by id","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ResourceRevisionResponse"}}}},"404":{"description":"Revision not found"}}}},"/api/v1/extensions":{"get":{"tags":["Extensions"],"summary":"List all extensions","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ExtensionResponse"}}}}}}},"post":{"tags":["Extensions"],"summary":"Create or import an extension","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateExtensionInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ExtensionResponse"}}}}}}},"/api/v1/extensions/search":{"get":{"tags":["Extensions"],"summary":"Search extensions (internal + marketplace)","parameters":[{"schema":{"type":"string"},"required":true,"name":"q","in":"query"},{"schema":{"type":"string"},"required":false,"name":"limit","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ExtensionSearchResult"}}}}},"400":{"description":"Missing query parameter"}}}},"/api/v1/extensions/{id}":{"get":{"tags":["Extensions"],"summary":"Get extension by ID","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ExtensionResponse"}}}}}},"put":{"tags":["Extensions"],"summary":"Update extension","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateExtensionInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ExtensionResponse"}}}}}},"delete":{"tags":["Extensions"],"summary":"Delete extension","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"204":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}}}}},"/api/v1/extensions/{id}/versions":{"get":{"tags":["Extensions"],"summary":"List available versions for an extension","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":false,"name":"preRelease","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ExtensionVersionInfo"}}}}}}}},"/api/v1/insights":{"get":{"tags":["Insights"],"summary":"List insights","parameters":[{"schema":{"type":"string"},"required":false,"name":"q","in":"query"},{"schema":{"type":"string"},"required":false,"name":"blocked","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/InsightResponse"}}}}}}},"post":{"tags":["Insights"],"summary":"Create insight","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateInsightInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}}}}},"/api/v1/insights/search":{"get":{"tags":["Insights"],"summary":"Search insights","parameters":[{"schema":{"type":"string"},"required":true,"name":"q","in":"query"},{"schema":{"type":"string"},"required":false,"name":"blocked","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/InsightResponse"}}}}}}}},"/api/v1/insights/{id}":{"get":{"tags":["Insights"],"summary":"Get insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}},"404":{"description":"Insight not found"}}},"put":{"tags":["Insights"],"summary":"Update insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateInsightInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}},"404":{"description":"Insight not found"}}},"delete":{"tags":["Insights"],"summary":"Delete insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"204":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}},"404":{"description":"Insight not found"}}}},"/api/v1/insights/{id}/upvote":{"post":{"tags":["Insights"],"summary":"Upvote insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}},"404":{"description":"Insight not found"}}}},"/api/v1/insights/{id}/downvote":{"post":{"tags":["Insights"],"summary":"Downvote insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}},"404":{"description":"Insight not found"}}}},"/api/v1/insights/{id}/block":{"post":{"tags":["Insights"],"summary":"Block insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}},"404":{"description":"Insight not found"}}}},"/api/v1/insights/{id}/unblock":{"post":{"tags":["Insights"],"summary":"Unblock insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}},"404":{"description":"Insight not found"}}}},"/api/v1/insights/{id}/reports":{"get":{"tags":["Insights"],"summary":"Get reports referencing insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ReportResponse"}}}}},"404":{"description":"Insight not found"}}}},"/api/v1/feature-flags":{"get":{"tags":["Feature Flags"],"summary":"List feature flags","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/FeatureFlagResponse"}}}}}}}},"/api/v1/feature-flags/{key}":{"put":{"tags":["Feature Flags"],"summary":"Update feature flag","parameters":[{"schema":{"type":"string"},"required":true,"name":"key","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateFeatureFlagInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/FeatureFlagResponse"}}}}}}}},"webhooks":{}} \ No newline at end of file From fb80f62df3a6cde5005190e2fd3f169b7e95a225 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Fri, 16 Oct 2026 08:49:46 -0700 Subject: [PATCH 16/18] test(cli): refresh tag-free experiment snapshot Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4b4a63b3-29db-4d12-888f-051592682640 --- apps/cli/src/__snapshots__/cli-commands.test.ts.snap | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/cli/src/__snapshots__/cli-commands.test.ts.snap b/apps/cli/src/__snapshots__/cli-commands.test.ts.snap index be76f0a32..6fe8b7040 100644 --- a/apps/cli/src/__snapshots__/cli-commands.test.ts.snap +++ b/apps/cli/src/__snapshots__/cli-commands.test.ts.snap @@ -230,7 +230,6 @@ exports[`CLI command registration > full command tree matches snapshot 1`] = ` "--resources", "--scenario", "--skills", - "--tags", "--traits", "--url", "--worker", From afe6e85a7e5d4566a953bad1d4d6e2ae87e6b39c Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Fri, 16 Oct 2026 11:45:16 -0700 Subject: [PATCH 17/18] feat(portal): organize experiment detail reports Add compact attempt-scoped Runs, Reports, and Processing tabs with URL-backed report type and version history selection. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4b4a63b3-29db-4d12-888f-051592682640 --- .../ExperimentReportCard.stories.tsx | 137 +++- .../src/components/ExperimentReportCard.tsx | 122 +++- apps/portal/src/lib/experiment-reports.ts | 94 +++ .../src/pages/ExperimentDetail.play.test.tsx | 26 + .../src/pages/ExperimentDetail.stories.tsx | 87 ++- .../src/pages/ExperimentDetail.test.tsx | 298 ++++++++- apps/portal/src/pages/ExperimentDetail.tsx | 594 ++++++++++++------ docs/architecture/experiments.md | 48 +- .../docs/guides/creating-experiments.mdx | 20 +- 9 files changed, 1177 insertions(+), 249 deletions(-) create mode 100644 apps/portal/src/lib/experiment-reports.ts diff --git a/apps/portal/src/components/ExperimentReportCard.stories.tsx b/apps/portal/src/components/ExperimentReportCard.stories.tsx index c47db044a..f1727dca9 100644 --- a/apps/portal/src/components/ExperimentReportCard.stories.tsx +++ b/apps/portal/src/components/ExperimentReportCard.stories.tsx @@ -2,7 +2,12 @@ // Licensed under the MIT License. import type { Meta, StoryObj } from "@storybook/react-vite"; -import { expect, within } from "storybook/test"; +import { useMemo, useState } from "react"; +import { expect, screen, userEvent, within } from "storybook/test"; +import { + groupExperimentReports, + resolveExperimentReportSelection, +} from "@/lib/experiment-reports"; import type { Report } from "@/types"; import { ExperimentReportCard } from "./ExperimentReportCard"; @@ -30,6 +35,31 @@ const report: Report = { createdAt: "2026-01-02T00:00:00.000Z", }; +const historicalReport: Report = { + ...report, + _id: "report-0", + id: "report-0", + content: "# Previous readout\n\nThe earlier retry output.", + createdAt: "2026-01-01T00:00:00.000Z", +}; + +const qualityReport: Report = { + ...report, + _id: "report-quality", + id: "report-quality", + templateId: "quality-summary", + content: "# Quality summary\n\nA second report type.", + reporter: { + ...report.reporter!, + id: "quality-summary", + name: "Quality Summary", + }, + createdAt: "2026-01-03T00:00:00.000Z", +}; + +const completedGroups = groupExperimentReports([report]); +const completedSelection = resolveExperimentReportSelection(completedGroups); + const meta = { title: "Experiments/Reports", component: ExperimentReportCard, @@ -37,7 +67,8 @@ const meta = { experimentId: "experiment-1", experimentExecutionId: "execution-2", attemptNumber: 2, - report, + groups: completedGroups, + selection: completedSelection, structuredUrl: "/structured.json", }, parameters: { layout: "padded" }, @@ -55,13 +86,14 @@ export const Completed: Story = { await expect( canvas.getByRole("link", { name: "Structured JSON" }), ).toBeVisible(); - await expect(canvas.queryByRole("tab")).toBeNull(); + await expect(canvas.queryByRole("combobox")).toBeNull(); }, }; export const Processing: Story = { args: { - report: undefined, + groups: [], + selection: undefined, structuredUrl: undefined, isProcessing: true, }, @@ -69,9 +101,104 @@ export const Processing: Story = { export const ErrorWithRetry: Story = { args: { - report: undefined, + groups: [], + selection: undefined, structuredUrl: undefined, error: "Report service unavailable", onRetry: () => undefined, }, }; + +function InteractiveReports({ + reports, + initialType, + initialReportId, +}: { + reports: Report[]; + initialType?: string; + initialReportId?: string; +}) { + const groups = useMemo(() => groupExperimentReports(reports), [reports]); + const initialSelection = resolveExperimentReportSelection( + groups, + initialType, + initialReportId, + ); + const [typeKey, setTypeKey] = useState(initialSelection?.group.key); + const [reportId, setReportId] = useState(initialSelection?.report._id); + const selection = resolveExperimentReportSelection( + groups, + typeKey, + reportId, + ); + + return ( + { + const group = groups.find((candidate) => candidate.key === nextType); + setTypeKey(nextType); + setReportId(group?.reports[0]._id); + }} + onReportVersionChange={setReportId} + /> + ); +} + +export const RetryHistory: Story = { + render: () => ( + + ), + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await expect( + canvas.getByRole("heading", { name: "Experiment readout" }), + ).toBeVisible(); + await expect(canvas.getByText("Latest")).toBeVisible(); + + await userEvent.click( + canvas.getByRole("combobox", { name: "Report version" }), + ); + await userEvent.click( + await screen.findByRole("option", { + name: /Version 1 · Historical/, + }), + ); + + await expect( + canvas.getByRole("heading", { name: "Previous readout" }), + ).toBeVisible(); + await expect(canvas.getByText("Historical")).toBeVisible(); + }, +}; + +export const MultipleReportTypes: Story = { + render: () => ( + + ), + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await expect( + canvas.getByRole("heading", { name: "Quality summary" }), + ).toBeVisible(); + + await userEvent.click( + canvas.getByRole("combobox", { name: "Report type" }), + ); + await userEvent.click( + await screen.findByRole("option", { name: "Experiment Readout" }), + ); + + await expect( + canvas.getByRole("heading", { name: "Experiment readout" }), + ).toBeVisible(); + await expect( + canvas.getByRole("combobox", { name: "Report version" }), + ).toBeVisible(); + }, +}; diff --git a/apps/portal/src/components/ExperimentReportCard.tsx b/apps/portal/src/components/ExperimentReportCard.tsx index 5d7ee6726..359bc6600 100644 --- a/apps/portal/src/components/ExperimentReportCard.tsx +++ b/apps/portal/src/components/ExperimentReportCard.tsx @@ -5,19 +5,34 @@ import { Download, ExternalLink } from "lucide-react"; import { Link } from "react-router-dom"; import { MarkdownRenderer } from "@/components/MarkdownRenderer"; import { ReportStatusBadge } from "@/components/ReportStatusBadge"; +import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; -import type { Report } from "@/types"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import type { + ExperimentReportGroup, + ExperimentReportSelection, +} from "@/lib/experiment-reports"; +import { formatDate } from "@/lib/utils"; interface ExperimentReportCardProps { experimentId: string; experimentExecutionId: string; attemptNumber?: number; - report?: Report; + groups: ExperimentReportGroup[]; + selection?: ExperimentReportSelection; structuredUrl?: string; isLoading?: boolean; error?: string; isProcessing?: boolean; + onReportTypeChange?: (typeKey: string) => void; + onReportVersionChange?: (reportId: string) => void; onRetry?: () => void; } @@ -25,16 +40,20 @@ export function ExperimentReportCard({ experimentId, experimentExecutionId, attemptNumber, - report, + groups, + selection, structuredUrl, isLoading, error, isProcessing, + onReportTypeChange, + onReportVersionChange, onRetry, }: ExperimentReportCardProps) { + const report = selection?.report; return ( - +
Reports

@@ -42,27 +61,84 @@ export function ExperimentReportCard({ {experimentExecutionId}

- {report && ( -
- - {structuredUrl && ( +
+ {groups.length > 1 && selection && ( +
+ + +
+ )} + {selection && selection.group.reports.length > 1 && ( +
+ + +
+ )} + {report && ( + <> + + {selection?.isLatest ? "Latest" : "Historical"} + + + {structuredUrl && ( + + )} - )} - -
- )} + + )} +
{isLoading ? ( diff --git a/apps/portal/src/lib/experiment-reports.ts b/apps/portal/src/lib/experiment-reports.ts new file mode 100644 index 000000000..e072c7e19 --- /dev/null +++ b/apps/portal/src/lib/experiment-reports.ts @@ -0,0 +1,94 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { Report } from "@/types"; + +export interface ExperimentReportGroup { + key: string; + label: string; + reports: Report[]; +} + +export interface ExperimentReportSelection { + group: ExperimentReportGroup; + report: Report; + isLatest: boolean; +} + +function reportTimestamp(report: Report): number { + const parsed = Date.parse(report.createdAt); + return Number.isNaN(parsed) ? 0 : parsed; +} + +export function experimentReportTypeKey(report: Report): string { + if (report.templateId) return `template:${report.templateId}`; + if (report.reporter?.id) return `reporter:${report.reporter.id}`; + return "report:untyped"; +} + +function reportTypeLabel(report: Report): string { + if (report.templateId && report.reporter?.name) { + return `${report.reporter.name} · ${report.templateId}`; + } + return report.reporter?.name ?? report.templateId ?? "Report"; +} + +export function groupExperimentReports( + reports: Report[], +): ExperimentReportGroup[] { + const byType = new Map(); + for (const report of reports) { + const key = experimentReportTypeKey(report); + const group = byType.get(key); + if (group) { + group.reports.push(report); + } else { + byType.set(key, { + key, + label: reportTypeLabel(report), + reports: [report], + }); + } + } + + const groups = [...byType.values()]; + for (const group of groups) { + group.reports.sort( + (left, right) => + reportTimestamp(right) - reportTimestamp(left) || + right._id.localeCompare(left._id), + ); + } + return groups.sort( + (left, right) => + reportTimestamp(right.reports[0]) - reportTimestamp(left.reports[0]) || + left.label.localeCompare(right.label), + ); +} + +export function resolveExperimentReportSelection( + groups: ExperimentReportGroup[], + requestedType?: string, + requestedReportId?: string, +): ExperimentReportSelection | undefined { + if (groups.length === 0) return undefined; + + const reportGroup = requestedReportId + ? groups.find((group) => + group.reports.some((report) => report._id === requestedReportId), + ) + : undefined; + const group = + groups.find((candidate) => candidate.key === requestedType) ?? + reportGroup ?? + groups[0]; + const report = + group.reports.find((candidate) => candidate._id === requestedReportId) ?? + group.reports[0]; + + return { + group, + report, + isLatest: report._id === group.reports[0]._id, + }; +} diff --git a/apps/portal/src/pages/ExperimentDetail.play.test.tsx b/apps/portal/src/pages/ExperimentDetail.play.test.tsx index ebda77fc6..21468bd48 100644 --- a/apps/portal/src/pages/ExperimentDetail.play.test.tsx +++ b/apps/portal/src/pages/ExperimentDetail.play.test.tsx @@ -18,6 +18,8 @@ vi.mock("@/lib/api", async (importOriginal) => { getExperiment: vi.fn(), listExperimentExecutions: vi.fn(), getExperimentExecution: vi.fn(), + listExperimentReports: vi.fn(), + experimentReportStructuredUrl: vi.fn(() => "/structured.json"), retryExperiment: vi.fn(), }, }; @@ -109,6 +111,30 @@ beforeAll(() => { vi.mocked(api.getExperimentExecution).mockResolvedValue( historicalExecution, ); + vi.mocked(api.listExperimentReports).mockImplementation( + async (_experimentId, executionId) => + executionId === "execution-current" + ? [ + { + _id: "report-current", + id: "report-current", + experimentId: "experiment-demo", + experimentExecutionId: "execution-current", + status: "completed", + content: "# Current readout\n\nThe latest exact-vector analysis.", + reporter: { + id: "pp-readout", + name: "Experiment Readout", + gitHash: "abc", + model: "gpt-5.4", + agentId: "copilot-sdk", + agentVersion: "1.0.0", + }, + createdAt: "2026-10-16T10:30:00.000Z", + }, + ] + : [], + ); }); describe("ExperimentDetail stories", () => { diff --git a/apps/portal/src/pages/ExperimentDetail.stories.tsx b/apps/portal/src/pages/ExperimentDetail.stories.tsx index 35330ccff..679d4f019 100644 --- a/apps/portal/src/pages/ExperimentDetail.stories.tsx +++ b/apps/portal/src/pages/ExperimentDetail.stories.tsx @@ -70,6 +70,67 @@ const experiment = { createdAt: "2026-10-16T09:00:00.000Z", }; +const currentReports = [ + { + _id: "report-current", + id: "report-current", + experimentId: "experiment-demo", + experimentExecutionId: "execution-current", + status: "completed", + content: "# Current readout\n\nThe latest exact-vector analysis.", + reporter: { + id: "pp-readout", + name: "Experiment Readout", + gitHash: "abc", + model: "gpt-5.4", + agentId: "copilot-sdk", + agentVersion: "1.0.0", + }, + structuredArtifact: { + schemaId: "scope.experiment-readout", + schemaVersion: "1.0", + contentType: "application/json", + url: "reports/report-current/structured.json", + }, + createdAt: "2026-10-16T10:30:00.000Z", + }, + { + _id: "report-previous", + id: "report-previous", + experimentId: "experiment-demo", + experimentExecutionId: "execution-current", + status: "completed", + content: "# Previous readout\n\nThe first report version.", + reporter: { + id: "pp-readout", + name: "Experiment Readout", + gitHash: "abc", + model: "gpt-5.4", + agentId: "copilot-sdk", + agentVersion: "1.0.0", + }, + createdAt: "2026-10-16T10:15:00.000Z", + }, + { + _id: "report-quality", + id: "report-quality", + experimentId: "experiment-demo", + experimentExecutionId: "execution-current", + templateId: "quality-summary", + status: "completed", + content: "# Quality summary\n\nAn additional attached report.", + reporter: { + id: "quality-summary", + name: "Quality Summary", + gitHash: "def", + model: "gpt-5.4", + agentId: "copilot-sdk", + agentVersion: "1.0.0", + }, + createdAt: "2026-10-16T10:20:00.000Z", + }, +]; + function ExperimentDetailStory() { const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } }, @@ -109,6 +170,15 @@ const meta = { "/api/v1/experiments/experiment-demo/executions/execution-history", () => HttpResponse.json(historicalExecution), ), + http.get( + "/api/v1/experiments/experiment-demo/executions/:executionId/reports", + ({ params }) => + HttpResponse.json( + params.executionId === "execution-current" + ? currentReports + : [], + ), + ), ], }, }, @@ -120,6 +190,19 @@ type Story = StoryObj; export const AttemptSelectionAndRunsTable: Story = { play: async ({ canvas, canvasElement }) => { + await expect( + await screen.findByRole("heading", { name: "Current readout" }), + ).toBeVisible(); + const tabList = canvas.getByRole("tablist", { + name: "Experiment detail sections", + }); + const header = canvasElement.querySelector( + '[data-slot="experiment-detail-header"]', + ); + if (!header) throw new Error("Compact Experiment header is missing"); + await expect(header.nextElementSibling).toBe(tabList.parentElement); + + await userEvent.click(canvas.getByRole("tab", { name: "Runs" })); await expect( await screen.findByText("Runs in attempt 2"), ).toBeVisible(); @@ -143,10 +226,10 @@ export const AttemptSelectionAndRunsTable: Story = { ); await expect( - await screen.findByText(/Viewing processing attempt #1 \(historical\)/), + await screen.findByText("Historical attempt"), ).toBeVisible(); await expect( - screen.getAllByText("Inspect the historical benchmark task")[0], + (await screen.findAllByText("Inspect the historical benchmark task"))[0], ).toBeVisible(); await expect(screen.queryAllByText("Implement the current benchmark task")) .toHaveLength(0); diff --git a/apps/portal/src/pages/ExperimentDetail.test.tsx b/apps/portal/src/pages/ExperimentDetail.test.tsx index 3b23cf023..e84adb84c 100644 --- a/apps/portal/src/pages/ExperimentDetail.test.tsx +++ b/apps/portal/src/pages/ExperimentDetail.test.tsx @@ -3,6 +3,7 @@ // @vitest-environment happy-dom import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { afterEach, @@ -37,14 +38,14 @@ vi.mock("@/lib/api", async (importOriginal) => { listExperimentReports: vi.fn(), retryExperiment: vi.fn(), experimentReportStructuredUrl: vi.fn( - () => - "/api/v1/experiments/experiment-1/executions/execution-history/reports/report-1/structured?projectId=project-1", + (_experimentId: string, executionId: string, reportId: string) => + `/api/v1/experiments/experiment-1/executions/${executionId}/reports/${reportId}/structured?projectId=project-1`, ), }, }; }); -import { api } from "@/lib/api"; +import { ApiError, api } from "@/lib/api"; import { ExperimentDetail } from "./ExperimentDetail"; class ResizeObserverMock { @@ -142,13 +143,47 @@ const historicalReport: Report = { createdAt: "2026-10-16T10:30:00.000Z", }; +const currentReport: Report = { + ...historicalReport, + _id: "report-current", + id: "report-current", + experimentExecutionId: "execution-current", + content: "# Current readout\n\nLatest exact-attempt evidence.", + createdAt: "2026-10-16T11:30:00.000Z", +}; + +const previousCurrentReport: Report = { + ...currentReport, + _id: "report-previous", + id: "report-previous", + content: "# Previous readout\n\nEarlier retry output.", + createdAt: "2026-10-16T11:00:00.000Z", +}; + +const qualityReport: Report = { + ...currentReport, + _id: "report-quality", + id: "report-quality", + templateId: "quality-summary", + content: "# Quality summary\n\nAn additional report type.", + reporter: { + ...currentReport.reporter!, + id: "quality-summary", + name: "Quality Summary", + }, + createdAt: "2026-10-16T11:15:00.000Z", +}; + function LocationProbe() { const location = useLocation(); return
{location.pathname}{location.search}
; } -function renderDetail(initialEntry = "/experiments/experiment-1") { - vi.mocked(api.getExperiment).mockResolvedValue(experiment); +function renderDetail( + initialEntry = "/experiments/experiment-1", + experimentValue = experiment, +) { + vi.mocked(api.getExperiment).mockResolvedValue(experimentValue); vi.mocked(api.listExperimentExecutions).mockResolvedValue({ items: [currentExecution, historicalExecution], }); @@ -168,7 +203,15 @@ function renderDetail(initialEntry = "/experiments/experiment-1") { - } /> + + + + + } + /> } /> @@ -212,6 +255,14 @@ describe("ExperimentDetail", () => { if (!embeddedTable) throw new Error("Embedded runs table block is missing"); expect(embeddedTable.className).toContain("space-y-3"); expect(embeddedTable.parentElement?.className).toContain("p-6"); + const header = document.querySelector( + '[data-slot="experiment-detail-header"]', + ); + const tabList = screen.getByRole("tablist", { + name: "Experiment detail sections", + }); + expect(header?.nextElementSibling).toBe(tabList.parentElement); + expect(screen.queryByText("Execution attempt 2")).toBeNull(); fireEvent.click(screen.getByRole("button", { name: "Next page" })); expect((await screen.findAllByText("Current task 11")).length).toBeGreaterThan(0); @@ -231,31 +282,45 @@ describe("ExperimentDetail", () => { ); expect( - await screen.findByText(/Viewing processing attempt #1 \(historical\)/), + await screen.findByText("Historical attempt"), ).toBeTruthy(); expect(screen.getByText("Runs in attempt 1")).toBeTruthy(); expect(screen.getAllByText("Historical task").length).toBeGreaterThan(0); expect(screen.queryAllByText("Current task 1")).toHaveLength(0); expect( - screen.getByRole("button", { name: "Retry processing" }).hasAttribute( - "disabled", - ), - ).toBe(true); - + screen.queryByRole("button", { name: "Retry processing" }), + ).toBeNull(); fireEvent.click(screen.getAllByText("Historical task")[0]); expect((await screen.findByTestId("location")).textContent).toBe( "/runs/request-history?runId=run-history", ); + + }); + + it("keeps processing retry disabled for a historical attempt", async () => { + renderDetail( + "/experiments/experiment-1?executionId=execution-history&tab=processing", + ); + + expect( + ( + await screen.findByRole("button", { name: "Retry processing" }) + ).hasAttribute("disabled"), + ).toBe(true); }); it("returns from a historical attempt to the current attempt", async () => { + const user = userEvent.setup(); renderDetail( "/experiments/experiment-1?executionId=execution-history", ); - fireEvent.click(await screen.findByRole("button", { - name: "View current attempt", - })); + await user.click( + await screen.findByRole("combobox", { name: "Experiment attempt" }), + ); + await user.click( + screen.getByRole("option", { name: "Attempt 2 · current" }), + ); await waitFor(() => { expect(screen.getByText("Runs in attempt 2")).toBeTruthy(); @@ -265,6 +330,7 @@ describe("ExperimentDetail", () => { }); it("loads the report and structured artifact for the exact selected attempt", async () => { + const user = userEvent.setup(); vi.mocked(api.listExperimentReports).mockImplementation( async (_experimentId, executionId) => executionId === historicalExecution.experimentExecutionId @@ -287,8 +353,11 @@ describe("ExperimentDetail", () => { screen.getByRole("link", { name: "Structured JSON" }).getAttribute("href"), ).toContain("/executions/execution-history/reports/report-1/structured"); - fireEvent.click( - screen.getByRole("button", { name: "View current attempt" }), + await user.click( + screen.getByRole("combobox", { name: "Experiment attempt" }), + ); + await user.click( + screen.getByRole("option", { name: "Attempt 2 · current" }), ); await waitFor(() => { expect(api.listExperimentReports).toHaveBeenCalledWith( @@ -315,9 +384,200 @@ describe("ExperimentDetail", () => { "execution-current", ); expect( - await screen.findByText( - "No built-in report is available for this execution.", + await screen.findByText("Runs in attempt 2"), + ).toBeTruthy(); + }); + + it("defaults to Reports for one report without showing unnecessary selectors", async () => { + vi.mocked(api.listExperimentReports).mockResolvedValue([currentReport]); + + renderDetail(); + + expect( + await screen.findByRole("heading", { name: "Current readout" }), + ).toBeTruthy(); + expect( + screen.getByRole("tab", { name: "Reports" }).getAttribute( + "data-state", ), + ).toBe("active"); + expect(screen.queryByRole("combobox", { name: "Report type" })).toBeNull(); + expect( + screen.queryByRole("combobox", { name: "Report version" }), + ).toBeNull(); + }); + + it("defaults to the latest retry version and selects historical versions", async () => { + const user = userEvent.setup(); + vi.mocked(api.listExperimentReports).mockResolvedValue([ + previousCurrentReport, + currentReport, + ]); + + renderDetail(); + + expect( + await screen.findByRole("heading", { name: "Current readout" }), + ).toBeTruthy(); + await user.click( + screen.getByRole("combobox", { name: "Report version" }), + ); + await user.click( + screen.getByRole("option", { name: /Version 1 · Historical/ }), + ); + + expect( + await screen.findByRole("heading", { name: "Previous readout" }), + ).toBeTruthy(); + expect(screen.getByTestId("location").textContent).toContain( + "reportType=reporter%3App-readout", + ); + expect(screen.getByTestId("location").textContent).toContain( + "reportId=report-previous", + ); + expect( + screen.getByRole("link", { name: "Direct link" }).getAttribute("href"), + ).toContain("/reports/report-previous?"); + }); + + it("selects one report type at a time and honors direct query selection", async () => { + const user = userEvent.setup(); + vi.mocked(api.listExperimentReports).mockResolvedValue([ + previousCurrentReport, + currentReport, + qualityReport, + ]); + + renderDetail( + "/experiments/experiment-1?tab=reports&reportType=reporter%3App-readout&reportId=report-previous", + ); + + expect( + await screen.findByRole("heading", { name: "Previous readout" }), + ).toBeTruthy(); + await user.click(screen.getByRole("combobox", { name: "Report type" })); + await user.click( + screen.getByRole("option", { name: /Quality Summary/ }), + ); + + expect( + await screen.findByRole("heading", { name: "Quality summary" }), + ).toBeTruthy(); + expect(screen.queryByRole("heading", { name: "Current readout" })).toBeNull(); + expect(screen.getByTestId("location").textContent).toContain( + "reportType=template%3Aquality-summary", + ); + expect(screen.getByTestId("location").textContent).toContain( + "reportId=report-quality", + ); + }); + + it("falls back safely from stale tab, report type, and report version links", async () => { + vi.mocked(api.listExperimentReports).mockResolvedValue([ + currentReport, + qualityReport, + ]); + + renderDetail( + "/experiments/experiment-1?tab=missing&reportType=missing&reportId=missing", + ); + + expect( + await screen.findByRole("heading", { name: "Current readout" }), + ).toBeTruthy(); + await waitFor(() => { + expect(screen.getByTestId("location").textContent).toBe( + "/experiments/experiment-1", + ); + }); + }); + + it("falls back safely from a stale attempt while preserving a valid tab", async () => { + vi.mocked(api.getExperimentExecution).mockRejectedValueOnce( + new ApiError("Experiment execution not found", 404), + ); + + renderDetail( + "/experiments/experiment-1?executionId=execution-missing&tab=runs&reportId=report-missing", + ); + + expect(await screen.findByText("Runs in attempt 2")).toBeTruthy(); + await waitFor(() => { + expect(screen.getByTestId("location").textContent).toBe( + "/experiments/experiment-1?tab=runs", + ); + }); + }); + + it("defaults to Processing while blocked and keeps retry recovery scoped there", async () => { + vi.mocked(api.retryExperiment).mockResolvedValue({ + experimentId: "experiment-1", + experimentExecutionId: "execution-retry", + attemptNumber: 3, + }); + const processingExperiment: ExperimentView = { + ...experiment, + readiness: "blocked", + execution: { + ...currentExecution, + readiness: "blocked", + handlerProgress: { + "pp-observations": { status: "done" }, + "pp-readout": { + status: "failed", + error: "A dependency must be retried.", + }, + }, + }, + }; + + renderDetail("/experiments/experiment-1", processingExperiment); + + expect( + ( + await screen.findByRole("tab", { name: "Processing" }) + ).getAttribute("data-state"), + ).toBe("active"); + expect(screen.getByText("pp-observations")).toBeTruthy(); + expect(screen.getByText("pp-readout")).toBeTruthy(); + fireEvent.click(screen.getByRole("button", { name: "Retry processing" })); + await waitFor(() => { + expect(api.retryExperiment).toHaveBeenCalledWith( + "experiment-1", + "execution-current", + ); + }); + }); + + it("preserves the selected tab across attempts and clears report history state", async () => { + const user = userEvent.setup(); + vi.mocked(api.listExperimentReports).mockImplementation( + async (_experimentId, executionId) => + executionId === "execution-current" + ? [previousCurrentReport, currentReport] + : [historicalReport], + ); + + renderDetail( + "/experiments/experiment-1?tab=reports&reportType=reporter%3App-readout&reportId=report-previous", + ); + + expect( + await screen.findByRole("heading", { name: "Previous readout" }), + ).toBeTruthy(); + await user.click( + screen.getByRole("combobox", { name: "Experiment attempt" }), + ); + await user.click( + screen.getByRole("option", { name: "Attempt 1 · historical" }), + ); + + expect( + await screen.findByRole("heading", { name: "Historical readout" }), ).toBeTruthy(); + const location = screen.getByTestId("location").textContent ?? ""; + expect(location).toContain("executionId=execution-history"); + expect(location).toContain("tab=reports"); + expect(location).not.toContain("report-previous"); }); }); diff --git a/apps/portal/src/pages/ExperimentDetail.tsx b/apps/portal/src/pages/ExperimentDetail.tsx index dc2aece3d..68a1c83ec 100644 --- a/apps/portal/src/pages/ExperimentDetail.tsx +++ b/apps/portal/src/pages/ExperimentDetail.tsx @@ -35,7 +35,17 @@ import { SelectTrigger, SelectValue, } from "@/components/ui/select"; +import { + Tabs, + TabsContent, + TabsList, + TabsTrigger, +} from "@/components/ui/tabs"; import { ApiError, api } from "@/lib/api"; +import { + groupExperimentReports, + resolveExperimentReportSelection, +} from "@/lib/experiment-reports"; import type { ExperimentExecutionView, ExperimentRequestSummaryView, @@ -58,6 +68,13 @@ const DEFAULT_PAGE_SIZE = 10; type RunSortColumn = (typeof RUN_COLUMN_IDS)[number]; type SortDirection = "asc" | "desc"; +type ExperimentDetailTab = "runs" | "reports" | "processing"; + +const EXPERIMENT_DETAIL_TABS: ExperimentDetailTab[] = [ + "runs", + "reports", + "processing", +]; function handlerStatus(value: unknown): string | undefined { if ( @@ -71,6 +88,18 @@ function handlerStatus(value: unknown): string | undefined { return undefined; } +function handlerError(value: unknown): string | undefined { + if ( + value && + typeof value === "object" && + "error" in value && + typeof value.error === "string" + ) { + return value.error; + } + return undefined; +} + function requestSortValue( request: ExperimentRequestSummaryView, column: RunSortColumn, @@ -134,6 +163,30 @@ function ExecutionSummary({
Created
{formatDate(execution.createdAt)}
+ {execution.version !== undefined && ( +
+
Version
+
{execution.version}
+
+ )} + {execution.startedAt && ( +
+
Started
+
{formatDate(execution.startedAt)}
+
+ )} + {execution.finishedAt && ( +
+
Finished
+
{formatDate(execution.finishedAt)}
+
+ )} + {execution.supersededReason && ( +
+
Superseded by
+
{execution.supersededReason}
+
+ )} ); @@ -162,6 +215,10 @@ export function ExperimentDetail() { const queryClient = useQueryClient(); const [searchParams, setSearchParams] = useSearchParams(); const selectedExecutionId = searchParams.get("executionId") ?? undefined; + const selectedReportType = searchParams.get("reportType") ?? undefined; + const selectedReportId = + searchParams.get("reportId") ?? searchParams.get("report") ?? undefined; + const requestedTab = searchParams.get("tab") ?? undefined; const [customizingColumns, setCustomizingColumns] = useState(false); const [page, setPage] = useState(1); const [pageSize, setPageSize] = useState(DEFAULT_PAGE_SIZE); @@ -212,8 +269,15 @@ export function ExperimentDetail() { queryFn: () => api.listExperimentReports(id!, reportExecutionId!), enabled: Boolean(id && reportExecutionId), refetchInterval: (query) => { - const report = query.state.data?.[0]; - if (report?.status === "completed" || report?.status === "failed") { + const reports = query.state.data; + if ( + reports && + reports.length > 0 && + reports.every( + (report) => + report.status === "completed" || report.status === "failed", + ) + ) { return false; } return 5_000; @@ -273,10 +337,103 @@ export function ExperimentDetail() { experimentQuery.data && activeExecution ? selectedExperimentView(experimentQuery.data, activeExecution) : undefined; - const readout = reportsQuery.data?.find( - (report) => report.reporter?.id === "pp-readout", + const reportGroups = useMemo( + () => groupExperimentReports(reportsQuery.data ?? []), + [reportsQuery.data], + ); + const reportSelection = useMemo( + () => + resolveExperimentReportSelection( + reportGroups, + selectedReportType, + selectedReportId, + ), + [reportGroups, selectedReportId, selectedReportType], ); + useEffect(() => { + if ( + !selectedExecutionId || + !(selectedExecutionQuery.error instanceof ApiError) || + selectedExecutionQuery.error.status !== 404 + ) { + return; + } + const next = new URLSearchParams(searchParams); + next.delete("executionId"); + next.delete("reportType"); + next.delete("reportId"); + next.delete("report"); + setSearchParams(next, { replace: true }); + }, [ + searchParams, + selectedExecutionId, + selectedExecutionQuery.error, + setSearchParams, + ]); + + const defaultTab: ExperimentDetailTab = + reportSelection || reportsQuery.isError + ? "reports" + : activeExperiment && + ["waiting", "processing", "blocked"].includes( + activeExperiment.readiness ?? "", + ) + ? "processing" + : "runs"; + const activeTab: ExperimentDetailTab = EXPERIMENT_DETAIL_TABS.includes( + requestedTab as ExperimentDetailTab, + ) + ? (requestedTab as ExperimentDetailTab) + : defaultTab; + + useEffect(() => { + if ( + !reportsQuery.isSuccess || + (selectedExecutionQuery.error instanceof ApiError && + selectedExecutionQuery.error.status === 404) + ) { + return; + } + const next = new URLSearchParams(searchParams); + let changed = false; + + if ( + selectedReportType && + selectedReportType !== reportSelection?.group.key + ) { + next.delete("reportType"); + changed = true; + } + if (selectedReportId && selectedReportId !== reportSelection?.report._id) { + next.delete("reportId"); + next.delete("report"); + changed = true; + } else if (searchParams.has("report") && reportSelection) { + next.delete("report"); + next.set("reportId", reportSelection.report._id); + changed = true; + } + if ( + requestedTab && + !EXPERIMENT_DETAIL_TABS.includes(requestedTab as ExperimentDetailTab) + ) { + next.delete("tab"); + changed = true; + } + + if (changed) setSearchParams(next, { replace: true }); + }, [ + reportSelection, + reportsQuery.isSuccess, + searchParams, + selectedReportId, + selectedReportType, + selectedExecutionQuery.error, + requestedTab, + setSearchParams, + ]); + const sortedRequests = useMemo(() => { if (!activeExecution) return []; const direction = sortDirection === "asc" ? 1 : -1; @@ -410,6 +567,35 @@ export function ExperimentDetail() { } else { next.set("executionId", executionId); } + next.delete("reportType"); + next.delete("reportId"); + next.delete("report"); + setSearchParams(next, { replace: true }); + }; + + const selectReportType = (typeKey: string) => { + const group = reportGroups.find((candidate) => candidate.key === typeKey); + if (!group) return; + const next = new URLSearchParams(searchParams); + next.set("reportType", group.key); + next.set("reportId", group.reports[0]._id); + next.delete("report"); + setSearchParams(next, { replace: true }); + }; + + const selectReportVersion = (reportId: string) => { + if (!reportSelection) return; + const next = new URLSearchParams(searchParams); + next.set("reportType", reportSelection.group.key); + next.set("reportId", reportId); + next.delete("report"); + setSearchParams(next, { replace: true }); + }; + + const selectTab = (value: string) => { + if (!EXPERIMENT_DETAIL_TABS.includes(value as ExperimentDetailTab)) return; + const next = new URLSearchParams(searchParams); + next.set("tab", value); setSearchParams(next, { replace: true }); }; @@ -487,10 +673,13 @@ export function ExperimentDetail() { experiment.execution.experimentExecutionId; return ( -
-
+
+
-
-

- Created {formatDate(experiment.createdAt)} -

+ {experiment.metadata.description && ( +

+ {experiment.metadata.description} +

+ )}
@@ -540,44 +744,8 @@ export function ExperimentDetail() {
-
-
- - {experiment.metadata.description && ( -

- {experiment.metadata.description} -

- )} - - {isViewingHistorical && ( -
- - - Viewing processing attempt #{activeExecution?.attemptNumber ?? "…"}{" "} - (historical).{" "} - - -
- )} + {selectedExecutionQuery.isLoading && isViewingHistorical && (

@@ -595,149 +763,211 @@ export function ExperimentDetail() { )} {activeExperiment && activeExecution && ( - <> - - - void reportsQuery.refetch()} - /> + + + Runs + Reports + Processing + - {activeExperiment.blocking.length > 0 && ( - - - - Blocking dependencies - - - - {activeExperiment.blocking.map((blocking, index) => ( -

- {blocking.requestId && ( - - {blocking.requestId.slice(0, 8)} - + + + + setCustomizingColumns((open) => !open)} + /> + } + > +
+ {customizingColumns && ( +
- ))} + - )} + + + + void reportsQuery.refetch()} + /> + - - - setCustomizingColumns((open) => !open)} - /> + +
+ +
+ + + + + Handler progress + + + {Object.entries(activeExecution.handlerProgress).length === + 0 ? ( +

+ No handlers are registered for this attempt. +

+ ) : ( + Object.entries(activeExecution.handlerProgress).map( + ([handlerId, progress]) => ( +
+ {handlerId} + + {handlerStatus(progress) ?? "unknown"} + + {handlerError(progress) && ( + + {handlerError(progress)} + + )} +
+ ), + ) + )} + + + {activeExperiment.blocking.length > 0 && ( + + + + Blocking dependencies + + + + {activeExperiment.blocking.map((blocking, index) => ( +
+ {blocking.requestId && ( + + {blocking.requestId.slice(0, 8)} + + )} + {blocking.reason} + + {blocking.handlerId} + + {blocking.error && {blocking.error}} +
+ ))} + + + )} + + )}
); diff --git a/docs/architecture/experiments.md b/docs/architecture/experiments.md index 075b097e7..7be9475b0 100644 --- a/docs/architecture/experiments.md +++ b/docs/architecture/experiments.md @@ -268,21 +268,39 @@ composition controls and creates the expanded run count. The Portal navigates to Experiment Detail after creation. The Experiments list uses the shared `ListLayout`, `FilterRail`, `DataTable`, column customization, and cursor-aware `Pagination` interaction model while retaining Experiment-specific status, -revision, attempt, and run-count columns. Detail selects the current processing -attempt by default and supports direct historical links via `?executionId=`. -A header attempt selector renders exactly one current or historical execution -at a time, including its state, blockers, and a shared -sortable/customizable/paginated Runs `DataTable`. Run rows link to the exact -stored request/run pair. The same selected attempt owns the built-in report: -Markdown, structured JSON, the global report-detail link, loading/error/retry -states, and logs all use the exact -`experimentId + experimentExecutionId` identity. Browser navigation preserves -that attempt scope through `?executionId=`. Public UI -terminology remains generic **Reports**; `pp-readout` is only the internal -handler identifier. Processing retry remains current-attempt-only. Run list -and detail views link `experimentId` back to the Experiment. A historical -correlation that has no Experiment document is view-only and cannot trigger -retry. +revision, attempt, and run-count columns. + +Experiment Detail keeps the Experiment identity, concise status, and global +attempt selector in a compact header. The horizontal **Runs**, **Reports**, and +**Processing** tabs follow that header directly so their active content is +available without scrolling through overview cards. The selected attempt +scopes all three tabs: + +- **Runs** renders the exact stored request/run vector in an inset, + sortable, customizable, paginated shared `DataTable`. Rows link to the exact + request/run pair. +- **Reports** groups append-only report documents by persisted `templateId` or + reporter identity. It shows a type selector only when multiple report types + exist and a version selector only when a type has history. The newest type + and newest version are the defaults, and exactly one version is rendered. + Markdown, structured JSON, the global report-detail link, loading/error + retry, and status use the exact + `experimentId + experimentExecutionId + reportId` identity. +- **Processing** owns handler progress, blockers, errors, execution metadata, + and processing retry/recovery. Retry remains current-attempt-only. + +Browser history preserves the selected attempt, tab, report type, and report +version through `executionId`, `tab`, `reportType`, and `reportId` query +parameters. Missing parameters retain backward-compatible defaults. Stale +values fall back safely to the current data and are removed from the URL. +Detail defaults to **Reports** when a report document exists, **Processing** +while an artifact-free attempt is active or blocked, and **Runs** otherwise. +Public UI terminology remains generic **Reports**; `pp-readout` is only the +internal handler identifier. This projection does not add Experiment report +template attachment, selection, or triggering; those remain deferred to PR D. +Run list and detail views link `experimentId` back to the Experiment. A +historical correlation that has no Experiment document is view-only and cannot +trigger retry. ## Permanent request creation and composition deprecation diff --git a/website/src/content/docs/guides/creating-experiments.mdx b/website/src/content/docs/guides/creating-experiments.mdx index 871788dad..423e5e7fb 100644 --- a/website/src/content/docs/guides/creating-experiments.mdx +++ b/website/src/content/docs/guides/creating-experiments.mdx @@ -35,9 +35,23 @@ Experiment-specific status, revision, attempt, and run-count columns. After creation, the Portal opens Experiment Detail, where you can inspect one processing attempt at a time. The current attempt is selected by default; use the header selector to open a historical attempt, whose URL can be shared -directly. Each selected attempt shows its blockers, handler progress, and a -sortable, customizable, paginated table of exact linked runs. Empty Experiments -remain useful as durable placeholders. +directly. The compact header is followed by three tabs: + +- **Runs** shows the selected attempt's exact linked runs in a sortable, + customizable, paginated table. +- **Reports** shows one report at a time. The latest report is the default. + When retries have created previous versions, use the compact version selector + to open one historical version. A report-type selector appears only when the + Experiment has more than one report type. Structured JSON and the report + detail link remain scoped to the selected attempt and version. +- **Processing** shows handler progress, blockers, errors, execution metadata, + and retry controls. Processing retry is available only for the current + attempt. + +The selected attempt, tab, report type, and report version are preserved in +the URL. Old links without these values continue to use the latest defaults, +and stale selections fall back to available data. Empty Experiments remain +useful as durable placeholders. ## Create from the CLI From 0e1a4d77ecd036d960117899e80159cd241831b5 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Fri, 16 Oct 2026 12:10:32 -0700 Subject: [PATCH 18/18] fix(portal): stabilize experiment report states Keep report identity, controls, badges, and actions anchored while only the report body changes across completed, processing, blocked, failed, and empty states. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4b4a63b3-29db-4d12-888f-051592682640 --- .../ExperimentReportCard.stories.tsx | 106 ++++++++ .../components/ExperimentReportCard.test.tsx | 143 +++++++++++ .../src/components/ExperimentReportCard.tsx | 238 +++++++++++------- apps/portal/src/pages/ExperimentDetail.tsx | 1 + docs/architecture/experiments.md | 6 +- 5 files changed, 406 insertions(+), 88 deletions(-) create mode 100644 apps/portal/src/components/ExperimentReportCard.test.tsx diff --git a/apps/portal/src/components/ExperimentReportCard.stories.tsx b/apps/portal/src/components/ExperimentReportCard.stories.tsx index f1727dca9..cbc700163 100644 --- a/apps/portal/src/components/ExperimentReportCard.stories.tsx +++ b/apps/portal/src/components/ExperimentReportCard.stories.tsx @@ -43,6 +43,17 @@ const historicalReport: Report = { createdAt: "2026-01-01T00:00:00.000Z", }; +const failedReport: Report = { + ...report, + _id: "report-failed", + id: "report-failed", + status: "failed", + content: undefined, + structuredArtifact: undefined, + error: "The readout worker stopped before publishing an artifact.", + createdAt: "2026-01-04T00:00:00.000Z", +}; + const qualityReport: Report = { ...report, _id: "report-quality", @@ -87,6 +98,12 @@ export const Completed: Story = { canvas.getByRole("link", { name: "Structured JSON" }), ).toBeVisible(); await expect(canvas.queryByRole("combobox")).toBeNull(); + await expect( + canvasElement.querySelector('[data-slot="experiment-report-toolbar"]'), + ).not.toBeNull(); + await expect( + canvasElement.querySelector('[data-slot="experiment-report-body"]'), + ).not.toBeNull(); }, }; @@ -97,6 +114,17 @@ export const Processing: Story = { structuredUrl: undefined, isProcessing: true, }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await expect(canvas.getByText("Report processing")).toBeVisible(); + await expect(canvas.getByText("Processing")).toBeVisible(); + await expect( + canvasElement.querySelector('[data-slot="experiment-report-toolbar"]'), + ).not.toBeNull(); + await expect( + canvasElement.querySelector('[data-slot="experiment-report-body"]'), + ).not.toBeNull(); + }, }; export const ErrorWithRetry: Story = { @@ -107,6 +135,84 @@ export const ErrorWithRetry: Story = { error: "Report service unavailable", onRetry: () => undefined, }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await expect(canvas.getByText("Report unavailable")).toBeVisible(); + await expect(canvas.getByRole("button", { name: "Retry" })).toBeVisible(); + await expect( + canvasElement.querySelector('[data-slot="experiment-report-toolbar"]'), + ).not.toBeNull(); + await expect( + canvasElement.querySelector('[data-slot="experiment-report-body"]'), + ).not.toBeNull(); + }, +}; + +export const FailedVersion: Story = { + args: { + groups: groupExperimentReports([failedReport]), + selection: resolveExperimentReportSelection( + groupExperimentReports([failedReport]), + ), + structuredUrl: undefined, + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await expect(canvas.getByText("Report generation failed")).toBeVisible(); + await expect( + canvas.getByRole("button", { name: "Structured JSON" }).hasAttribute( + "disabled", + ), + ).toBe(true); + await expect( + canvas.getByRole("link", { name: "Direct link" }), + ).toBeVisible(); + await expect( + canvasElement.querySelector('[data-slot="experiment-report-toolbar"]'), + ).not.toBeNull(); + await expect( + canvasElement.querySelector('[data-slot="experiment-report-body"]'), + ).not.toBeNull(); + }, +}; + +export const NoArtifact: Story = { + args: { + groups: [], + selection: undefined, + structuredUrl: undefined, + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await expect(canvas.getByText("No report available")).toBeVisible(); + await expect(canvas.getByText("No report")).toBeVisible(); + await expect( + canvasElement.querySelector('[data-slot="experiment-report-toolbar"]'), + ).not.toBeNull(); + await expect( + canvasElement.querySelector('[data-slot="experiment-report-body"]'), + ).not.toBeNull(); + }, +}; + +export const Blocked: Story = { + args: { + groups: [], + selection: undefined, + structuredUrl: undefined, + isBlocked: true, + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await expect(canvas.getByText("Report blocked")).toBeVisible(); + await expect(canvas.getByText("Blocked")).toBeVisible(); + await expect( + canvasElement.querySelector('[data-slot="experiment-report-toolbar"]'), + ).not.toBeNull(); + await expect( + canvasElement.querySelector('[data-slot="experiment-report-body"]'), + ).not.toBeNull(); + }, }; function InteractiveReports({ diff --git a/apps/portal/src/components/ExperimentReportCard.test.tsx b/apps/portal/src/components/ExperimentReportCard.test.tsx new file mode 100644 index 000000000..8856de763 --- /dev/null +++ b/apps/portal/src/components/ExperimentReportCard.test.tsx @@ -0,0 +1,143 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// @vitest-environment happy-dom +import { cleanup, render, screen } from "@testing-library/react"; +import type { ComponentProps } from "react"; +import { afterEach, describe, expect, it } from "vitest"; +import { MemoryRouter } from "react-router-dom"; +import { + groupExperimentReports, + resolveExperimentReportSelection, +} from "@/lib/experiment-reports"; +import type { Report } from "@/types"; +import { ExperimentReportCard } from "./ExperimentReportCard"; + +const completedReport: Report = { + _id: "report-completed", + id: "report-completed", + experimentId: "experiment-1", + experimentExecutionId: "execution-2", + status: "completed", + content: "# Completed report\n\nExact evidence.", + reporter: { + id: "pp-readout", + name: "Experiment Readout", + gitHash: "abc", + model: "gpt-5.4", + agentId: "copilot-sdk", + agentVersion: "1.0.0", + }, + structuredArtifact: { + schemaId: "scope.experiment-readout", + schemaVersion: "1.0", + contentType: "application/json", + url: "reports/report-completed/structured.json", + }, + createdAt: "2026-10-16T10:00:00.000Z", +}; + +const failedReport: Report = { + ...completedReport, + _id: "report-failed", + id: "report-failed", + status: "failed", + content: undefined, + structuredArtifact: undefined, + error: "Generation failed.", +}; + +const baseProps: ComponentProps = { + experimentId: "experiment-1", + experimentExecutionId: "execution-2", + attemptNumber: 2, + groups: [], +}; + +function reportProps( + report: Report, +): ComponentProps { + const groups = groupExperimentReports([report]); + return { + ...baseProps, + groups, + selection: resolveExperimentReportSelection(groups), + structuredUrl: + report.status === "completed" ? "/structured.json" : undefined, + }; +} + +function renderCard(props: ComponentProps) { + return render( + + + , + ); +} + +function expectStableShell(container: HTMLElement) { + const panel = container.querySelector( + '[data-slot="experiment-report-panel"]', + ); + const header = container.querySelector( + '[data-slot="experiment-report-header"]', + ); + const toolbar = container.querySelector( + '[data-slot="experiment-report-toolbar"]', + ); + const selectors = container.querySelector( + '[data-slot="experiment-report-selectors"]', + ); + const actions = container.querySelector( + '[data-slot="experiment-report-actions"]', + ); + const body = container.querySelector('[data-slot="experiment-report-body"]'); + + expect(panel).not.toBeNull(); + expect(header).not.toBeNull(); + expect(toolbar).not.toBeNull(); + expect(selectors).not.toBeNull(); + expect(actions).not.toBeNull(); + expect(body).not.toBeNull(); + expect(header?.nextElementSibling).toBe(body); + expect(header?.contains(toolbar)).toBe(true); +} + +afterEach(() => cleanup()); + +describe("ExperimentReportCard", () => { + it.each([ + ["completed", reportProps(completedReport)], + ["failed", reportProps(failedReport)], + ["processing", { ...baseProps, isProcessing: true }], + ["blocked", { ...baseProps, isBlocked: true }], + ["no artifact", baseProps], + ])("keeps the stable report shell for %s", (_state, props) => { + const { container } = renderCard(props); + + expectStableShell(container); + expect(screen.getByRole("heading", { name: "Reports" })).toBeTruthy(); + expect(screen.getByText(/Attempt 2/)).toBeTruthy(); + }); + + it("keeps report actions aligned while disabling unavailable artifacts", () => { + const { rerender } = renderCard(reportProps(completedReport)); + + expect(screen.getByRole("link", { name: "Structured JSON" })).toBeTruthy(); + expect(screen.getByRole("link", { name: "Direct link" })).toBeTruthy(); + + rerender( + + + , + ); + + expect( + screen + .getByRole("button", { name: "Structured JSON" }) + .hasAttribute("disabled"), + ).toBe(true); + expect(screen.getByRole("link", { name: "Direct link" })).toBeTruthy(); + expect(screen.getByText("Report generation failed")).toBeTruthy(); + }); +}); diff --git a/apps/portal/src/components/ExperimentReportCard.tsx b/apps/portal/src/components/ExperimentReportCard.tsx index 359bc6600..833f425a3 100644 --- a/apps/portal/src/components/ExperimentReportCard.tsx +++ b/apps/portal/src/components/ExperimentReportCard.tsx @@ -31,6 +31,7 @@ interface ExperimentReportCardProps { isLoading?: boolean; error?: string; isProcessing?: boolean; + isBlocked?: boolean; onReportTypeChange?: (typeKey: string) => void; onReportVersionChange?: (reportId: string) => void; onRetry?: () => void; @@ -46,105 +47,154 @@ export function ExperimentReportCard({ isLoading, error, isProcessing, + isBlocked, onReportTypeChange, onReportVersionChange, onRetry, }: ExperimentReportCardProps) { const report = selection?.report; + const emptyStateClassName = + "flex min-h-44 flex-col items-start justify-center gap-3 rounded-lg border border-dashed bg-muted/20 p-6"; + return ( - - -
+ + +
Reports

Attempt {attemptNumber ?? "…"} ·{" "} {experimentExecutionId}

-
- {groups.length > 1 && selection && ( -
- - - - - - {groups.map((group) => ( - - {group.label} - - ))} - - -
- )} - {selection && selection.group.reports.length > 1 && ( -
- - +
+ )} + {selection && selection.group.reports.length > 1 && ( +
+ + -
- )} - {report && ( - <> - - {selection?.isLatest ? "Latest" : "Historical"} - - - {structuredUrl && ( -
+ )} +
+
+ {report ? ( + <> + + {selection?.isLatest ? "Latest" : "Historical"} + + + {structuredUrl ? ( + + ) : ( + + )} + - )} - - - )} + + ) : isLoading ? ( + Loading + ) : error ? ( + Unavailable + ) : isBlocked ? ( + Blocked + ) : isProcessing ? ( + Processing + ) : ( + No report + )} +
- + {isLoading ? ( -

Loading report…

+
+

Loading report

+

+ The selected report version is being retrieved. +

+
) : error ? ( -
+
+

Report unavailable

{error}

{onRetry && (
) : report?.status === "failed" ? ( -

- {report.error ?? "Readout generation failed."} -

+
+

Report generation failed

+

+ {report.error ?? "Readout generation failed."} +

+
) : report?.content ? ( -
+
{report.content}
) : ( -

- {isProcessing - ? "The report is being generated." - : "No built-in report is available for this execution."} -

+
+

+ {isBlocked + ? "Report blocked" + : isProcessing + ? "Report processing" + : "No report available"} +

+

+ {isBlocked + ? "Required processing must recover before a report can be published." + : isProcessing + ? "The report is being generated." + : "No built-in report is available for this execution."} +

+
)} diff --git a/apps/portal/src/pages/ExperimentDetail.tsx b/apps/portal/src/pages/ExperimentDetail.tsx index 68a1c83ec..51ff99ae4 100644 --- a/apps/portal/src/pages/ExperimentDetail.tsx +++ b/apps/portal/src/pages/ExperimentDetail.tsx @@ -872,6 +872,7 @@ export function ExperimentDetail() { activeExecution.readiness !== "failed" && activeExecution.readiness !== "blocked" } + isBlocked={activeExecution.readiness === "blocked"} onReportTypeChange={selectReportType} onReportVersionChange={selectReportVersion} onRetry={() => void reportsQuery.refetch()} diff --git a/docs/architecture/experiments.md b/docs/architecture/experiments.md index 7be9475b0..ed16b7f99 100644 --- a/docs/architecture/experiments.md +++ b/docs/architecture/experiments.md @@ -285,7 +285,11 @@ scopes all three tabs: and newest version are the defaults, and exactly one version is rendered. Markdown, structured JSON, the global report-detail link, loading/error retry, and status use the exact - `experimentId + experimentExecutionId + reportId` identity. + `experimentId + experimentExecutionId + reportId` identity. Completed, + processing, failed, blocked, and no-artifact states share one stable panel + shell: identity, selectors, badges, and actions keep their positions while + only the content region changes. Artifact-only actions are disabled when no + artifact exists, while addressable report versions retain their direct link. - **Processing** owns handler progress, blockers, errors, execution metadata, and processing retry/recovery. Retry remains current-attempt-only.