diff --git a/AGENTS.md b/AGENTS.md index 119d758fc..326cc3edd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,6 +19,7 @@ apps/ coder-acp-copilot/ # GitHub Copilot worker (ACP SDK) coder-acp-claude-code/ # Claude Code worker (ACP SDK) report-generator/ # Post-run report generation + readout/ # Submission-level generated readout model-scanners/ # Feature detection for Copilot and Anthropic models version-checkers/ # Poll for new agent/tool releases key-updaters/ # GitHub auth cookie management for VS Code Web @@ -69,6 +70,7 @@ Each worker implements the same queue-processor interface but adapts a different | `coder-acp-copilot` | GitHub Copilot | ACP SDK v0.14.1 | | `coder-acp-claude-code` | Claude Code | ACP SDK v0.13.1 | | `report-generator` | — | Copilot SDK | +| `readout` | — | Copilot SDK | Workers consume tasks from Azure Storage Queues (named `queue-`) and write results to MongoDB and Blob Storage. Each has its own Dockerfile and docker-compose profile. diff --git a/ENV_VARIABLES.md b/ENV_VARIABLES.md index 4f20cc04b..c79b005f5 100644 --- a/ENV_VARIABLES.md +++ b/ENV_VARIABLES.md @@ -889,6 +889,33 @@ Timeout for the Copilot SDK session used by the report-generator worker. If the Git commit hash embedded in reporter metadata. Automatically set during CI/CD builds. Used to track which version of the report-generator produced a given report. +## Experiment Readout Handler Configuration + +### AZURE_STORAGE_QUEUE_READOUT +**Default:** `pp-readout-queue` +**Type:** string + +Azure Storage Queue name for Experiment readout jobs. The coordinator enqueues +an exact execution message containing `projectId`, `experimentId`, +`experimentExecutionId`, handler identity/version, claim token, and execution +version only after `pp-observations` is done for every exact request/run pair. + +### READOUT_MODEL +**Default:** `gpt-5.4` +**Type:** string + +The Copilot SDK model used by the dedicated `pp-readout` worker to synthesize a +schema-validated structured Experiment readout. Markdown is derived from the +validated structured object. + +### READOUT_SESSION_TIMEOUT_MS +**Default:** `300000` (5 minutes) +**Type:** integer (milliseconds) + +Timeout for one readout-generation attempt. Exhausted generation or schema +validation marks the exact current Experiment execution's `pp-readout` handler +as failed; historical and superseded executions remain immutable. + ## Scheduler Configuration ### SCHEDULER_POLL_INTERVAL_MS @@ -936,13 +963,13 @@ TTL for the DAG dispatcher's in-memory cache of handler topology (the `services` **Default:** `post-processor-queue` **Type:** string -Azure Storage Queue name used by both the scheduler (to enqueue post-processing work) and the post-processor worker (to dequeue). Must match between the two services. The `pp-taxonomy` observation handler shares this queue. +Azure Storage Queue name used by both the scheduler (to enqueue post-processing work) and the post-processor worker (to dequeue). Must match between the two services. The `pp-observations` handler shares this queue. ### JUDGE_URL **Default:** falls back to `JUDGE_SERVICE_URL`, then `http://localhost:3002` **Type:** URL string -Base URL of the judge service used by the `pp-taxonomy` post-process handler to evaluate observation criteria per iteration (snapshot + ATIF trajectory → boolean + evidence). The handler reuses the shared `JudgeClient`, so `JUDGE_CLIENT_TIMEOUT` / `JUDGE_CLIENT_RETRIES` apply. In K8s the post-processor reads `JUDGE_SERVICE_URL` from the `worker-config` ConfigMap (`envFrom`); in docker-compose the `post-processor` service sets `JUDGE_SERVICE_URL: http://judge:80` explicitly (it uses the `common-env` anchor, which — unlike `worker-env` — does not carry it). See issue #1156 and [docs/architecture/post-processing.md](docs/architecture/post-processing.md). +Base URL of the judge service used by the `pp-observations` post-process handler to evaluate observation criteria per iteration (snapshot + ATIF trajectory → boolean + evidence). The handler reuses the shared `JudgeClient`, so `JUDGE_CLIENT_TIMEOUT` / `JUDGE_CLIENT_RETRIES` apply. In K8s the post-processor reads `JUDGE_SERVICE_URL` from the `worker-config` ConfigMap (`envFrom`); in docker-compose the `post-processor` service sets `JUDGE_SERVICE_URL: http://judge:80` explicitly (it uses the `common-env` anchor, which — unlike `worker-env` — does not carry it). See issue #1156 and [docs/architecture/post-processing.md](docs/architecture/post-processing.md). ### SCHEDULER_URL **Default:** (not set) diff --git a/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap b/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap index d602570fc..d2b8bdf02 100644 --- a/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap +++ b/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap @@ -1614,6 +1614,29 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` ], "type": "string", }, + "report": { + "properties": { + "reportId": { + "type": "string", + }, + "updatedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "url": { + "type": "string", + }, + }, + "required": [ + "reportId", + "url", + "updatedAt", + ], + "type": "object", + }, "startedAt": { "format": "date-time", "type": [ @@ -1801,6 +1824,11 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "type": "string", }, }, + "required": [ + "reportId", + "url", + "updatedAt", + ], "type": "object", }, "startedAt": { @@ -2982,6 +3010,12 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "error": { "type": "string", }, + "experimentExecutionId": { + "type": "string", + }, + "experimentId": { + "type": "string", + }, "insightReferences": { "items": { "$ref": "#/components/schemas/InsightReference", @@ -3006,6 +3040,9 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` ], "type": "string", }, + "structuredArtifact": { + "$ref": "#/components/schemas/StructuredReportArtifact", + }, "templateId": { "type": "string", }, @@ -3019,7 +3056,6 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, "required": [ "_id", - "requestId", "status", "createdAt", "projectId", @@ -4292,6 +4328,32 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` ], "type": "object", }, + "StructuredReportArtifact": { + "properties": { + "contentType": { + "enum": [ + "application/json", + ], + "type": "string", + }, + "schemaId": { + "type": "string", + }, + "schemaVersion": { + "type": "string", + }, + "url": { + "type": "string", + }, + }, + "required": [ + "schemaId", + "schemaVersion", + "contentType", + "url", + ], + "type": "object", + }, "SuggestedPromptFeature": { "properties": { "behavior": { @@ -6473,6 +6535,181 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` ], }, }, + "/api/v1/experiments/{id}/executions/{experimentExecutionId}/logs": { + "get": { + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "minLength": 1, + "type": "string", + }, + }, + { + "in": "path", + "name": "experimentExecutionId", + "required": true, + "schema": { + "minLength": 1, + "type": "string", + }, + }, + { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "in": "query", + "name": "projectId", + "required": true, + "schema": { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, + "type": "string", + }, + }, + { + "in": "query", + "name": "fromStart", + "required": false, + "schema": { + "enum": [ + "true", + "false", + ], + "type": "string", + }, + }, + ], + "responses": { + "200": { + "description": "Server-sent event stream of log entries", + }, + "404": { + "description": "Experiment execution not found", + }, + }, + "summary": "Stream logs for one exact Experiment execution", + "tags": [ + "Experiments", + ], + }, + }, + "/api/v1/experiments/{id}/executions/{experimentExecutionId}/reports": { + "get": { + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "minLength": 1, + "type": "string", + }, + }, + { + "in": "path", + "name": "experimentExecutionId", + "required": true, + "schema": { + "minLength": 1, + "type": "string", + }, + }, + { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "in": "query", + "name": "projectId", + "required": true, + "schema": { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, + "type": "string", + }, + }, + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/ReportResponse", + }, + "type": "array", + }, + }, + }, + "description": "Success", + }, + "404": { + "description": "Experiment execution not found", + }, + }, + "summary": "List reports for one exact Experiment execution", + "tags": [ + "Reports", + ], + }, + }, + "/api/v1/experiments/{id}/executions/{experimentExecutionId}/reports/{reportId}/structured": { + "get": { + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "minLength": 1, + "type": "string", + }, + }, + { + "in": "path", + "name": "experimentExecutionId", + "required": true, + "schema": { + "minLength": 1, + "type": "string", + }, + }, + { + "in": "path", + "name": "reportId", + "required": true, + "schema": { + "minLength": 1, + "type": "string", + }, + }, + { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "in": "query", + "name": "projectId", + "required": true, + "schema": { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, + "type": "string", + }, + }, + ], + "responses": { + "200": { + "description": "Success", + }, + "404": { + "description": "Experiment report or structured artifact not found", + }, + }, + "summary": "Download structured JSON for an exact Experiment report", + "tags": [ + "Reports", + ], + }, + }, "/api/v1/experiments/{id}/retry": { "post": { "parameters": [ @@ -9325,6 +9562,22 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "type": "string", }, }, + { + "in": "query", + "name": "experimentId", + "required": false, + "schema": { + "type": "string", + }, + }, + { + "in": "query", + "name": "experimentExecutionId", + "required": false, + "schema": { + "type": "string", + }, + }, { "in": "query", "name": "projectId", @@ -9767,6 +10020,32 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` ], }, }, + "/api/v1/reports/{id}/structured": { + "get": { + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "type": "string", + }, + }, + ], + "responses": { + "200": { + "description": "Success", + }, + "404": { + "description": "Report or structured artifact not found", + }, + }, + "summary": "Download a report's structured JSON artifact", + "tags": [ + "Reports", + ], + }, + }, "/api/v1/requests": { "get": { "parameters": [ diff --git a/apps/api/src/archive-har.test.ts b/apps/api/src/archive-har.test.ts index 9d01fc261..f123b4b8c 100644 --- a/apps/api/src/archive-har.test.ts +++ b/apps/api/src/archive-har.test.ts @@ -99,6 +99,7 @@ describe("rewriteHarUrlsForArchive", () => { expect(result.run!.turns![1].toolCallsUrl).toBe("iteration-2.tool-calls.jsonl"); expect(result.run!.turns![2].toolCallsUrl).toBeUndefined(); }); + }); // --- blobNameFromSnapshotsUrl --- @@ -902,7 +903,6 @@ describe("packRunIntoTar", () => { await packRunIntoTar(p, run, container, "run-011", () => true); p.finalize(); await entriesPromise; - expect(run.codebase!.archiveUrl).toBe(blobUrl); }); }); diff --git a/apps/api/src/endpoints.test.ts b/apps/api/src/endpoints.test.ts index 3e1f50e5a..f3c93c4f5 100644 --- a/apps/api/src/endpoints.test.ts +++ b/apps/api/src/endpoints.test.ts @@ -274,7 +274,7 @@ describe("API Endpoints", () => { (mocks.criteriaCollection.findOne as any).mockResolvedValue(null); const res = await request(app) - .post("/api/v1/criteria") + .post(`/api/v1/criteria?projectId=${TEST_PROJECT_ID}`) .send({ id: "obs_crit", prompt: "Did the agent use current deps?", @@ -292,7 +292,7 @@ describe("API Endpoints", () => { (mocks.criteriaCollection.findOne as any).mockResolvedValue(null); const res = await request(app) - .post("/api/v1/criteria") + .post(`/api/v1/criteria?projectId=${TEST_PROJECT_ID}`) .send({ id: "bad_tax", prompt: "x", dependsOn: [], kind: "observation", taxonomyElementId: "dimension:nope" }); expect(res.status).toBe(400); @@ -302,7 +302,7 @@ describe("API Endpoints", () => { (mocks.criteriaCollection.findOne as any).mockResolvedValue(null); const res = await request(app) - .post("/api/v1/criteria") + .post(`/api/v1/criteria?projectId=${TEST_PROJECT_ID}`) .send({ id: "obs_subj", prompt: "Did the agent add then remove a dep?", dependsOn: [], kind: "observation" }); expect(res.status).toBe(201); @@ -313,7 +313,7 @@ describe("API Endpoints", () => { (mocks.criteriaCollection.findOne as any).mockResolvedValue(null); const res = await request(app) - .post("/api/v1/criteria") + .post(`/api/v1/criteria?projectId=${TEST_PROJECT_ID}`) .send({ id: "obs_iter", prompt: "x", dependsOn: [], kind: "observation", subject: "iteration" }); expect(res.status).toBe(201); @@ -324,7 +324,7 @@ describe("API Endpoints", () => { (mocks.criteriaCollection.findOne as any).mockResolvedValue(null); const res = await request(app) - .post("/api/v1/criteria") + .post(`/api/v1/criteria?projectId=${TEST_PROJECT_ID}`) .send({ id: "gate_run", prompt: "x", dependsOn: [], kind: "gate", subject: "run" }); expect(res.status).toBe(400); diff --git a/apps/api/src/experiment-coordinator.test.ts b/apps/api/src/experiment-coordinator.test.ts index 30a0afd09..3fe8a18a4 100644 --- a/apps/api/src/experiment-coordinator.test.ts +++ b/apps/api/src/experiment-coordinator.test.ts @@ -165,13 +165,26 @@ describe("ExperimentCoordinator", () => { _id: "run-1", attemptNumber: 1, status: "done", + outcome: "failed", logsUrl: "https://logs", + observationResults: [{ + criterionId: "criterion-1", + evaluated: true, + passed: false, + feedback: "Valid negative evidence", + }], handlerStatus: { "pp-observations": { status: "done", version: 3, updatedAt: now, }, + "unrelated-handler": { + status: "failed", + version: 1, + updatedAt: now, + error: "Unrelated failure", + }, }, }, }, @@ -320,6 +333,151 @@ describe("ExperimentCoordinator", () => { ).toHaveLength(40); }); + it.each([ + { + name: "failed observations", + services: [ + { + _id: "pp-observations", + type: "post-process-handler" as const, + version: 1, + queue: "queue-pp-observations", + selector: "observations", + autoBackfill: false, + dependsOn: [], + }, + ], + handlerStatus: { + "pp-observations": { + status: "failed" as const, + version: 1, + updatedAt: now, + error: "Observation processing failed", + }, + }, + expectedState: "failed", + expectedReason: "dependency-failed", + }, + { + name: "dependency-blocked observations", + services: [ + { + _id: "observation-prerequisite", + type: "post-process-handler" as const, + version: 1, + queue: "queue-observation-prerequisite", + selector: "observation-prerequisite", + autoBackfill: false, + dependsOn: [], + }, + { + _id: "pp-observations", + type: "post-process-handler" as const, + version: 1, + queue: "queue-pp-observations", + selector: "observations", + autoBackfill: false, + dependsOn: ["observation-prerequisite"], + }, + ], + handlerStatus: { + "observation-prerequisite": { + status: "failed" as const, + version: 1, + updatedAt: now, + error: "Prerequisite failed", + }, + }, + expectedState: "blocked", + expectedReason: "dependency-blocked", + }, + ])("does not dispatch when $name", async ({ + services, + handlerStatus, + expectedState, + expectedReason, + }) => { + const initialRequest: ExperimentRequestState = { + requestId: "request-1", + runId: "run-1", + dependencies: [], + blockers: [], + }; + const projectedRequest: ExperimentRequestState = { + ...initialRequest, + dependencies: [{ + handlerId: "pp-observations", + state: expectedState as "failed" | "blocked", + }], + blockers: [{ + requestId: "request-1", + runId: "run-1", + handlerId: "pp-observations", + reason: expectedReason as "dependency-failed" | "dependency-blocked", + ...(expectedReason === "dependency-failed" + ? { error: "Observation processing failed" } + : {}), + }], + }; + const initial = experiment([initialRequest]); + const projected = experiment([projectedRequest], { + version: 2, + blocking: projectedRequest.blockers, + }); + vi.spyOn(ExperimentStore.prototype, "get").mockResolvedValue(initial); + vi.spyOn(FixedPayloadStore.prototype, "readRequests").mockResolvedValue([ + initialRequest, + ]); + vi.spyOn( + ExperimentStore.prototype, + "casUpdateRequestStates", + ).mockResolvedValue({ + status: "applied", + experiment: projected, + requests: [projectedRequest], + }); + const enqueue = vi.fn().mockResolvedValue(undefined); + const coordinator = new ExperimentCoordinator( + collections({ + requests: [{ + _id: "request-1", + projectId: "project-1", + scenario: { task: "task", criteria: [] }, + workerType: "worker", + createdAt: now, + taskPromptId: "prompt-1", + priority: 0, + run: { + _id: "run-1", + attemptNumber: 1, + status: "done", + logsUrl: "https://logs", + handlerStatus, + }, + }], + services: [ + ...services, + { + _id: "pp-readout", + type: "experiment-handler", + version: 1, + queue: "queue-pp-readout", + selector: "readout", + autoBackfill: false, + dependsOn: ["pp-observations"], + dependencyMode: "all-runs", + }, + ], + }), + enqueue, + ); + + const result = await coordinator.reconcile("project-1", "experiment-1"); + + expect(result?.dispatched).toBe(false); + expect(enqueue).not.toHaveBeenCalled(); + }); + it("re-enqueues a dropped notice without changing its exact identity", async () => { const requestState: ExperimentRequestState = { requestId: "request-1", diff --git a/apps/api/src/experiment-openapi.test.ts b/apps/api/src/experiment-openapi.test.ts index aeaddb2bd..926ed9e16 100644 --- a/apps/api/src/experiment-openapi.test.ts +++ b/apps/api/src/experiment-openapi.test.ts @@ -50,6 +50,15 @@ describe("Experiment OpenAPI contract", () => { expect(document.paths).toHaveProperty( "/api/v1/experiments/{id}/executions/{experimentExecutionId}", ); + expect(document.paths).toHaveProperty( + "/api/v1/experiments/{id}/executions/{experimentExecutionId}/reports", + ); + expect(document.paths).toHaveProperty( + "/api/v1/experiments/{id}/executions/{experimentExecutionId}/reports/{reportId}/structured", + ); + expect(document.paths).toHaveProperty( + "/api/v1/experiments/{id}/executions/{experimentExecutionId}/logs", + ); expect(document.paths).toHaveProperty("/api/v1/experiments/{id}/retry"); expect( Object.keys(document.paths ?? {}).filter((path) => diff --git a/apps/api/src/prompt-request-transport.test.ts b/apps/api/src/prompt-request-transport.test.ts index f4f261411..5aab849dc 100644 --- a/apps/api/src/prompt-request-transport.test.ts +++ b/apps/api/src/prompt-request-transport.test.ts @@ -66,6 +66,7 @@ const cases = [ payload: { prompt: "Inspect captured build output.", suggestedId: "build_succeeds", + suggestedSubject: "run", suggestedParents: [], suggestedChildren: [], }, diff --git a/apps/api/src/routes/experiments.test.ts b/apps/api/src/routes/experiments.test.ts index c3ff88c3d..a59d339f7 100644 --- a/apps/api/src/routes/experiments.test.ts +++ b/apps/api/src/routes/experiments.test.ts @@ -11,7 +11,7 @@ import { import { describe, expect, it, vi } from "vitest"; import { z } from "zod"; import type { RouteContext } from "../route-context.js"; -import type { ExperimentDocument } from "shared"; +import { ExperimentStore, type ExperimentDocument } from "shared"; import { registerExperimentsRoutes } from "./experiments.js"; extendZodWithOpenApi(z); @@ -271,6 +271,9 @@ describe("Experiment API contract", () => { "/api/v1/experiments/{id}", "/api/v1/experiments/{id}/executions", "/api/v1/experiments/{id}/executions/{experimentExecutionId}", + "/api/v1/experiments/{id}/executions/{experimentExecutionId}/logs", + "/api/v1/experiments/{id}/executions/{experimentExecutionId}/reports", + "/api/v1/experiments/{id}/executions/{experimentExecutionId}/reports/{reportId}/structured", "/api/v1/experiments/{id}/retry", ]); expect(document.paths).not.toHaveProperty("/api/v1/submissions"); @@ -290,6 +293,7 @@ describe("Experiment API contract", () => { readiness: expect.any(Object), blockers: expect.any(Object), progress: expect.any(Object), + report: expect.any(Object), supersededByExperimentExecutionId: { type: "string" }, }, }); @@ -297,4 +301,112 @@ describe("Experiment API contract", () => { /claimToken|stateVersion|vectorFingerprint|storageProfile/, ); }); + + it("lists reports for one exact current or archived execution", async () => { + const execution = { + experimentExecutionId: "execution-2", + attemptNumber: 2, + version: 4, + requests: { + storage: "inline" as const, + entryCount: 0, + payloadFingerprint: "payload", + payloadVersion: 1, + entries: [], + }, + vectorFingerprint: "vector", + handlerSummary: {}, + blocking: [], + createdAt: new Date(), + updatedAt: new Date(), + }; + const getExecution = vi + .spyOn(ExperimentStore.prototype, "getExecution") + .mockResolvedValue(execution); + const toArray = vi.fn().mockResolvedValue([{ + _id: "report-1", + projectId: "project-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-2", + status: "completed", + createdAt: new Date(), + }]); + const find = vi.fn(() => ({ + sort: vi.fn(() => ({ toArray })), + })); + const { app } = harness({ + experimentCollection: {}, + experimentExecutionsCollection: {}, + experimentChunksCollection: {}, + reportCollection: { find }, + } as unknown as Partial); + + const response = await request(app).get( + "/api/v1/experiments/experiment-1/executions/execution-2/reports?projectId=project-1", + ); + + expect(response.status).toBe(200); + expect(getExecution).toHaveBeenCalledWith({ + projectId: "project-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-2", + }); + expect(find).toHaveBeenCalledWith({ + projectId: "project-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-2", + }); + expect(response.body).toEqual([ + expect.objectContaining({ + id: "report-1", + experimentExecutionId: "execution-2", + }), + ]); + }); + + it("retrieves structured JSON only through the exact execution identity", async () => { + const findOne = vi.fn().mockResolvedValue({ + _id: "report-1", + projectId: "project-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-2", + status: "completed", + structuredArtifact: { + schemaId: "scope.experiment-readout", + schemaVersion: "1.0", + contentType: "application/json", + url: "https://storage.invalid/readout.json", + }, + createdAt: new Date(), + }); + const downloadJson = vi.fn().mockResolvedValue({ + meta: { + experimentId: "experiment-1", + experimentExecutionId: "execution-2", + }, + }); + const { app } = harness({ + reportCollection: { findOne }, + blobStorage: { downloadJson }, + } as unknown as Partial); + + const response = await request(app).get( + "/api/v1/experiments/experiment-1/executions/execution-2/reports/report-1/structured?projectId=project-1", + ); + + expect(response.status).toBe(200); + expect(findOne).toHaveBeenCalledWith({ + _id: "report-1", + projectId: "project-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-2", + }); + expect(downloadJson).toHaveBeenCalledWith( + "https://storage.invalid/readout.json", + ); + expect(response.body.meta).toEqual({ + experimentId: "experiment-1", + experimentExecutionId: "execution-2", + }); + }); }); diff --git a/apps/api/src/routes/experiments.ts b/apps/api/src/routes/experiments.ts index 1c43fd8bf..e126ab12a 100644 --- a/apps/api/src/routes/experiments.ts +++ b/apps/api/src/routes/experiments.ts @@ -4,10 +4,13 @@ import { v4 as uuidv4 } from "uuid"; import { z } from "zod"; import type { Document } from "mongodb"; +import type { Response } from "express"; import { CreateRequestInputSchema, ExperimentStore, fingerprintExperimentPayload, + ReportResponseSchema, + withRetry, type ExperimentDocument, type ExperimentExecution, type ExperimentExecutionHistoryDocument, @@ -20,6 +23,8 @@ import { ProjectIdQuerySchema, getQueryProjectId, } from "../utils/project-scope.js"; +import { subscribeClient, unsubscribeClient } from "../utils/sse.js"; +import type { SSEClient } from "../utils/sse.js"; const CursorSchema = z.string().min(1).optional(); const PageQuerySchema = ProjectIdQuerySchema.extend({ @@ -81,6 +86,12 @@ const ReadinessSchema = z.enum([ "failed", ]); +const ExperimentReportReferenceSchema = z.object({ + reportId: z.string(), + url: z.string(), + updatedAt: z.coerce.date(), +}); + const ExperimentResponseSchema = z .object({ experimentId: z.string(), @@ -114,13 +125,7 @@ const ExperimentResponseSchema = z finishedAt: z.coerce.date().optional(), discardedAt: z.coerce.date().optional(), discardReason: z.string().optional(), - report: z - .object({ - reportId: z.string().optional(), - url: z.string().optional(), - updatedAt: z.coerce.date().optional(), - }) - .optional(), + report: ExperimentReportReferenceSchema.optional(), }), createdAt: z.coerce.date(), updatedAt: z.coerce.date(), @@ -156,6 +161,7 @@ const ExecutionResponseSchema = z supersededAt: z.coerce.date().optional(), supersededByExperimentExecutionId: z.string().optional(), supersededReason: z.enum(["rollover", "retry"]).optional(), + report: ExperimentReportReferenceSchema.optional(), }) .openapi("ExperimentExecutionResponse"); @@ -257,7 +263,154 @@ function executionResponse(execution: ExperimentExecution) { ...(execution.supersededReason ? { supersededReason: execution.supersededReason } : {}), + ...(execution.report ? { report: execution.report } : {}), + }; +} + +async function streamExperimentExecutionLogs( + ctx: RouteContext, + req: import("express").Request, + res: Response, + experiment: ExperimentDocument, + execution: ExperimentExecution, + fromStart: boolean, +): Promise { + const isCurrent = + experiment.execution.experimentExecutionId === + execution.experimentExecutionId; + res.setHeader("Content-Type", "text/event-stream"); + res.setHeader("Cache-Control", "no-cache"); + res.setHeader("Connection", "keep-alive"); + res.setHeader("X-Accel-Buffering", "no"); + res.flushHeaders(); + + if (fromStart) { + try { + const pastLogs = await ctx.blobStorage.getLogEvents( + experiment.experimentId, + execution.experimentExecutionId, + ); + for (const log of pastLogs) { + res.write(`data: ${JSON.stringify(log)}\n\n`); + } + } catch (error) { + console.error( + `Failed to replay Experiment logs for ${experiment.experimentId}/${execution.experimentExecutionId}:`, + error, + ); + res.write( + `event: error\ndata: ${JSON.stringify({ message: "Cannot connect to log storage" })}\n\n`, + ); + res.end(); + return; + } + } + + const executionReadiness = readiness(execution); + if ( + !isCurrent || + executionReadiness === "done" || + executionReadiness === "failed" || + executionReadiness === "blocked" + ) { + res.write( + `event: done\ndata: ${JSON.stringify({ status: executionReadiness })}\n\n`, + ); + res.end(); + return; + } + + let cleaned = false; + let redisSubscribed = false; + let changeStream: ReturnType | null = + null; + const heartbeat = setInterval(() => { + if (!cleaned) res.write(`:\n\n`); + }, 30_000); + const client: SSEClient = { + res, + accepts: (event) => + event.data?.runId === execution.experimentExecutionId, + cleanup: () => { + if (cleaned) return; + cleaned = true; + clearInterval(heartbeat); + if (changeStream) { + changeStream.close().catch((error: unknown) => { + console.error("Error closing Experiment log change stream:", error); + }); + } + if (redisSubscribed) { + unsubscribeClient(experiment.experimentId, client); + } + res.end(); + }, }; + + if (process.env.REDIS_HOST) { + try { + await subscribeClient(experiment.experimentId, client); + redisSubscribed = true; + } catch (error) { + console.error( + `Redis subscription failed for Experiment ${experiment.experimentId}:`, + error, + ); + } + } + try { + changeStream = ctx.experimentCollection.watch( + [ + { + $match: { + "documentKey._id": experiment.experimentId, + operationType: "update", + }, + }, + ], + { fullDocument: "updateLookup" }, + ); + changeStream.on("change", (change: { + operationType?: string; + fullDocument?: ExperimentDocument; + }) => { + const updated = change.fullDocument; + if (change.operationType !== "update" || !updated) return; + if ( + updated.execution.experimentExecutionId !== + execution.experimentExecutionId + ) { + res.write( + `event: done\ndata: ${JSON.stringify({ status: "superseded" })}\n\n`, + ); + client.cleanup(); + return; + } + const updatedReadiness = readiness(updated.execution); + if ( + updatedReadiness === "done" || + updatedReadiness === "failed" || + updatedReadiness === "blocked" + ) { + res.write( + `event: done\ndata: ${JSON.stringify({ status: updatedReadiness })}\n\n`, + ); + client.cleanup(); + } + }); + changeStream.on("error", (error: unknown) => { + console.error( + `Experiment log change stream failed for ${experiment.experimentId}:`, + error, + ); + }); + } catch (error) { + console.error( + `Failed to create Experiment log change stream for ${experiment.experimentId}:`, + error, + ); + } + req.on("close", () => client.cleanup()); } function experimentResponse(experiment: ExperimentDocument) { @@ -621,6 +774,124 @@ export function registerExperimentsRoutes(ctx: RouteContext): void { }, }); + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/experiments/:id/executions/:experimentExecutionId/reports", + tags: ["Reports"], + summary: "List reports for one exact Experiment execution", + params: z.object({ + id: z.string().min(1), + experimentExecutionId: z.string().min(1), + }), + query: ProjectIdQuerySchema, + response: z.array(ReportResponseSchema), + errorResponses: { 404: { description: "Experiment execution not found" } }, + handler: async (req, res) => { + const projectId = getQueryProjectId(req); + const execution = await store(ctx).getExecution({ + projectId, + experimentId: req.params.id, + experimentExecutionId: req.params.experimentExecutionId, + }); + if (!execution) { + res.status(404).json({ error: "Experiment execution not found" }); + return; + } + const reports = await ctx.reportCollection + .find({ + projectId, + experimentId: req.params.id, + experimentExecutionId: req.params.experimentExecutionId, + }) + .sort({ createdAt: -1 }) + .toArray(); + res.json(reports.map((report) => ({ ...report, id: report._id }))); + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/experiments/:id/executions/:experimentExecutionId/reports/:reportId/structured", + tags: ["Reports"], + summary: "Download structured JSON for an exact Experiment report", + params: z.object({ + id: z.string().min(1), + experimentExecutionId: z.string().min(1), + reportId: z.string().min(1), + }), + query: ProjectIdQuerySchema, + response: z.any(), + rawResponse: true, + errorResponses: { + 404: { description: "Experiment report or structured artifact not found" }, + }, + handler: async (req, res) => { + const projectId = getQueryProjectId(req); + const report = await ctx.reportCollection.findOne({ + _id: req.params.reportId, + projectId, + experimentId: req.params.id, + experimentExecutionId: req.params.experimentExecutionId, + }); + if (!report?.structuredArtifact) { + res.status(404).json({ + error: "Experiment report structured artifact not found", + }); + return; + } + const structured = await withRetry(() => + ctx.blobStorage.downloadJson(report.structuredArtifact!.url) + ); + res.setHeader("Content-Type", "application/json"); + res.setHeader( + "Content-Disposition", + `attachment; filename="${req.params.reportId}.structured.json"`, + ); + res.json(structured); + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/experiments/:id/executions/:experimentExecutionId/logs", + tags: ["Experiments"], + summary: "Stream logs for one exact Experiment execution", + params: z.object({ + id: z.string().min(1), + experimentExecutionId: z.string().min(1), + }), + query: ProjectIdQuerySchema.extend({ + fromStart: z.enum(["true", "false"]).optional(), + }), + response: z.any(), + rawResponse: true, + responseDescription: "Server-sent event stream of log entries", + errorResponses: { 404: { description: "Experiment execution not found" } }, + handler: async (req, res) => { + const projectId = getQueryProjectId(req); + const experiment = await store(ctx).get(projectId, req.params.id); + const execution = experiment + ? await store(ctx).getExecution({ + projectId, + experimentId: req.params.id, + experimentExecutionId: req.params.experimentExecutionId, + }) + : null; + if (!experiment || !execution) { + res.status(404).json({ error: "Experiment execution not found" }); + return; + } + await streamExperimentExecutionLogs( + ctx, + req, + res, + experiment, + execution, + req.query.fromStart === "true", + ); + }, + }); + apiRoute(ctx.app, ctx.registry, { method: "post", path: "/api/v1/experiments/:id/retry", diff --git a/apps/api/src/routes/reports-structured.test.ts b/apps/api/src/routes/reports-structured.test.ts new file mode 100644 index 000000000..159613f8a --- /dev/null +++ b/apps/api/src/routes/reports-structured.test.ts @@ -0,0 +1,101 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { OpenAPIRegistry } from "@asteasolutions/zod-to-openapi"; +import express from "express"; +import request from "supertest"; +import { describe, expect, it, vi } from "vitest"; +import type { RouteContext } from "../route-context.js"; +import { registerReportsRoutes } from "./reports.js"; + +describe("structured report download", () => { + it("proxies the stored JSON artifact through the API", async () => { + const app = express(); + app.use(express.json()); + const downloadJson = vi.fn().mockResolvedValue({ + meta: { experimentExecutionId: "execution-1" }, + }); + const ctx = { + app, + registry: new OpenAPIRegistry(), + reportCollection: { + findOne: vi.fn().mockResolvedValue({ + _id: "report-1", + projectId: "project-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-1", + status: "completed", + structuredArtifact: { + schemaId: "scope.experiment-readout", + schemaVersion: "1.0", + contentType: "application/json", + url: "https://storage.example/structured.json", + }, + createdAt: new Date(), + }), + }, + blobStorage: { downloadJson }, + } as unknown as RouteContext; + registerReportsRoutes(ctx); + app.use(( + error: Error, + _req: express.Request, + res: express.Response, + _next: express.NextFunction, + ) => res.status(500).json({ error: error.message })); + + const response = await request(app).get("/api/v1/reports/report-1/structured"); + + expect(response.status).toBe(200); + expect(response.headers["content-disposition"]).toContain( + "report-1.structured.json", + ); + expect(response.body).toEqual({ + meta: { experimentExecutionId: "execution-1" }, + }); + expect(downloadJson).toHaveBeenCalledWith( + "https://storage.example/structured.json", + ); + }); + + it("filters experiment reports by the exact execution and project", async () => { + const app = express(); + app.use(express.json()); + const toArray = vi.fn().mockResolvedValue([{ + _id: "report-1", + projectId: "project-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-1", + status: "completed", + createdAt: new Date("2026-10-13T12:00:00.000Z"), + }]); + const find = vi.fn(() => ({ + sort: vi.fn(() => ({ toArray })), + })); + const ctx = { + app, + registry: new OpenAPIRegistry(), + reportCollection: { find }, + requestCollection: { + find: vi.fn(() => ({ toArray: vi.fn().mockResolvedValue([]) })), + }, + } as unknown as RouteContext; + registerReportsRoutes(ctx); + + const response = await request(app).get( + "/api/v1/reports?projectId=project-1&experimentId=experiment-1&experimentExecutionId=execution-1", + ); + + expect(response.status).toBe(200); + expect(find).toHaveBeenCalledWith({ + projectId: "project-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-1", + }); + expect(response.body[0]).toMatchObject({ + id: "report-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-1", + }); + }); +}); diff --git a/apps/api/src/routes/reports.ts b/apps/api/src/routes/reports.ts index d6dd7aa32..5a2655a95 100644 --- a/apps/api/src/routes/reports.ts +++ b/apps/api/src/routes/reports.ts @@ -15,6 +15,7 @@ import { ReportResponseSchema, TriggerReportsInputSchema, evaluateTrigger, + withRetry, } from "shared"; import type { TaskPromptDocument } from "shared"; import { apiRoute } from "../openapi/api-route.js"; @@ -108,15 +109,37 @@ apiRoute(ctx.app, ctx.registry, { path: "/api/v1/reports", tags: ["Reports"], summary: "List reports", - query: z.object({ requestId: z.string().optional(), projectId: z.string().optional() }), + query: z.object({ + requestId: z.string().optional(), + experimentId: z.string().optional(), + experimentExecutionId: z.string().optional(), + projectId: z.string().optional(), + }).superRefine((value, ctx) => { + if (value.experimentExecutionId && !value.experimentId) { + ctx.addIssue({ + code: "custom", + path: ["experimentId"], + message: "experimentId is required with experimentExecutionId", + }); + } + }), response: z.array(ReportResponseSchema), handler: async (req, res, next) => { try { const requestIdFilter = req.query.requestId as string; + const experimentIdFilter = req.query.experimentId as string; + const experimentExecutionIdFilter = + req.query.experimentExecutionId as string; const filter: Record = {}; if (requestIdFilter) { // Parent-scoped: the run id already pins the project. filter.requestId = requestIdFilter; + } else if (experimentIdFilter) { + filter.projectId = getQueryProjectId(req); + filter.experimentId = experimentIdFilter; + if (experimentExecutionIdFilter) { + filter.experimentExecutionId = experimentExecutionIdFilter; + } } else { // Top-level reports list requires an explicit project (400 if absent). filter.projectId = getQueryProjectId(req); @@ -128,13 +151,21 @@ apiRoute(ctx.app, ctx.registry, { .toArray(); // Enrich reports with the task from their associated run - const requestIds = [...new Set(reports.map(r => r.requestId))]; + const requestIds = [ + ...new Set(reports.flatMap((report) => report.requestId ? [report.requestId] : [])), + ]; const runs = requestIds.length > 0 ? await ctx.requestCollection.find({ _id: { $in: requestIds } as any }, { projection: { _id: 1, "scenario.task": 1 } }).toArray() : []; const taskByRequestId = new Map(runs.map(r => [r._id, r.scenario?.task])); - res.json(reports.map(r => ({ ...r, id: r._id, task: taskByRequestId.get(r.requestId) }))); + res.json(reports.map((report) => ({ + ...report, + id: report._id, + ...(report.requestId + ? { task: taskByRequestId.get(report.requestId) } + : {}), + }))); } catch (error) { next(error); } @@ -201,6 +232,39 @@ apiRoute(ctx.app, ctx.registry, { }, }); +apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/reports/:id/structured", + tags: ["Reports"], + summary: "Download a report's structured JSON artifact", + params: z.object({ id: z.string() }), + response: z.any(), + rawResponse: true, + errorResponses: { + 404: { description: "Report or structured artifact not found" }, + }, + handler: async (req, res, next) => { + try { + const report = await ctx.reportCollection.findOne({ _id: req.params.id }); + if (!report?.structuredArtifact) { + res.status(404).json({ error: "Structured report artifact not found" }); + return; + } + const structured = await withRetry(() => + ctx.blobStorage.downloadJson(report.structuredArtifact!.url) + ); + res.setHeader("Content-Type", "application/json"); + res.setHeader( + "Content-Disposition", + `attachment; filename="${req.params.id}.structured.json"`, + ); + res.json(structured); + } catch (error) { + next(error); + } + }, +}); + apiRoute(ctx.app, ctx.registry, { method: "post", path: "/api/v1/reports/bulk-status", @@ -229,7 +293,7 @@ apiRoute(ctx.app, ctx.registry, { // Build a map of requestId → latest report status const statusMap: Record = {}; for (const report of reports) { - if (!statusMap[report.requestId]) { + if (report.requestId && !statusMap[report.requestId]) { statusMap[report.requestId] = { reportId: report._id, status: report.status, diff --git a/apps/api/src/utils/sse.ts b/apps/api/src/utils/sse.ts index 2e073662d..093faa2ff 100644 --- a/apps/api/src/utils/sse.ts +++ b/apps/api/src/utils/sse.ts @@ -13,6 +13,7 @@ export type SSEClient = { res: Response; cleanup: () => void; onActivity?: () => void; + accepts?: (event: LogEvent) => boolean; }; const sseClients: Map> = new Map(); @@ -50,20 +51,24 @@ function getOrCreateSubscriber(): InstanceType { sharedSubscriber.on("message", (channel: string, message: string) => { const requestId = channel.replace("logs:", ""); const clients = sseClients.get(requestId); + let logEvent: LogEvent | undefined; + try { + logEvent = JSON.parse(message) as LogEvent; + } catch { + // Preserve legacy forwarding for clients that do not filter events. + } if (clients) { for (const client of clients) { + if (client.accepts && (!logEvent || !client.accepts(logEvent))) { + continue; + } client.res.write(`data: ${message}\n\n`); client.onActivity?.(); // Check for completion - try { - const logEvent = JSON.parse(message) as LogEvent; - if (logEvent.data?.final === true) { - client.res.write(`event: done\ndata: {"status":"done"}\n\n`); - client.cleanup(); - } - } catch { - // Ignore parse errors + if (logEvent?.data?.final === true) { + client.res.write(`event: done\ndata: {"status":"done"}\n\n`); + client.cleanup(); } } } diff --git a/apps/cli/src/__snapshots__/cli-commands.test.ts.snap b/apps/cli/src/__snapshots__/cli-commands.test.ts.snap index 907676b78..6fe8b7040 100644 --- a/apps/cli/src/__snapshots__/cli-commands.test.ts.snap +++ b/apps/cli/src/__snapshots__/cli-commands.test.ts.snap @@ -230,7 +230,6 @@ exports[`CLI command registration > full command tree matches snapshot 1`] = ` "--resources", "--scenario", "--skills", - "--tags", "--traits", "--url", "--worker", @@ -261,6 +260,22 @@ exports[`CLI command registration > full command tree matches snapshot 1`] = ` "--url", ], }, + "logs": { + "options": [ + "--from-start", + "--project", + "--url", + ], + }, + "reports": { + "options": [ + "--output", + "--project", + "--report", + "--structured", + "--url", + ], + }, "retry": { "options": [ "--execution", @@ -645,11 +660,14 @@ exports[`CLI command registration > full command tree matches snapshot 1`] = ` "options": [ "--id", "--output", + "--structured", "--url", ], }, "list": { "options": [ + "--experiment", + "--experiment-execution", "--output", "--project", "--run", diff --git a/apps/cli/src/commands/experiment.test.ts b/apps/cli/src/commands/experiment.test.ts index 60e55c8c1..204635bb7 100644 --- a/apps/cli/src/commands/experiment.test.ts +++ b/apps/cli/src/commands/experiment.test.ts @@ -212,4 +212,61 @@ describe("experiment commands", () => { "http://localhost:3100/api/v1/experiments/experiment-1/retry?projectId=project-1", ); }); + + it("retrieves structured JSON for one exact experiment execution", async () => { + const requests: CapturedRequest[] = []; + vi.stubGlobal("fetch", vi.fn(async (request: Request) => { + requests.push({ + url: request.url, + method: request.method, + body: await request.text(), + }); + const structured = request.url.endsWith( + "/structured?projectId=project-1", + ); + return new Response(JSON.stringify( + structured + ? { meta: { experimentExecutionId: "execution-1" } } + : [{ + id: "report-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-1", + status: "completed", + createdAt: "2026-01-01T00:00:00.000Z", + }], + ), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + })); + const output: string[] = []; + vi.spyOn(console, "log").mockImplementation((...parts: unknown[]) => { + output.push(parts.map(String).join(" ")); + }); + + await makeProgram().parseAsync( + [ + "experiment", + "reports", + "experiment-1", + "execution-1", + "--report", + "report-1", + "--structured", + "--project", + "project-1", + "--url", + "http://localhost:3100", + ], + { from: "user" }, + ); + + expect(requests.map((item) => item.url)).toEqual([ + "http://localhost:3100/api/v1/experiments/experiment-1/executions/execution-1/reports?projectId=project-1", + "http://localhost:3100/api/v1/experiments/experiment-1/executions/execution-1/reports/report-1/structured?projectId=project-1", + ]); + expect(output.join("\n")).toContain( + '"experimentExecutionId": "execution-1"', + ); + }); }); diff --git a/apps/cli/src/commands/experiment.ts b/apps/cli/src/commands/experiment.ts index a6c8041bc..19e57f871 100644 --- a/apps/cli/src/commands/experiment.ts +++ b/apps/cli/src/commands/experiment.ts @@ -4,9 +4,10 @@ import { readFileSync } from "node:fs"; import { resolve } from "node:path"; import { Command } from "commander"; +import EventSource from "eventsource"; import { stringify as stringifyYaml } from "yaml"; import { resolveScenarioAndPersona } from "../config-loader.js"; -import { apiFetch } from "../utils/api-client.js"; +import { apiFetch, getApiBasePath } from "../utils/api-client.js"; import { requireProjectId } from "../utils/config.js"; import { formatData, isMachineReadable } from "../utils/formatters.js"; import { parseGatesOption } from "../utils/gates.js"; @@ -14,6 +15,7 @@ import { configureHelp } from "../utils/helpFormatter.js"; import { buildResourceBindingSpecs, collectRepeatable } from "../utils/resources.js"; import { getDefaultApiUrl, + normalizeUrl, withOutputOption, withProjectOption, } from "../utils/shared.js"; @@ -34,10 +36,10 @@ interface ExperimentExecution { requests: ExperimentRequestSummary[]; handlerProgress: JsonRecord; blocking: ExperimentBlocker[]; - currentReport?: { - reportId?: string; - url?: string; - updatedAt?: string; + report?: { + reportId: string; + url: string; + updatedAt: string; }; createdAt: string; updatedAt?: string; @@ -188,9 +190,9 @@ function parseExecution(value: unknown): ExperimentExecution { {}, "experiment handler progress", ); - const report = execution.currentReport === undefined + const report = execution.report === undefined ? undefined - : asRecord(execution.currentReport, "experiment report"); + : asRecord(execution.report, "experiment report"); return { experimentExecutionId: stringField( execution, @@ -214,11 +216,11 @@ function parseExecution(value: unknown): ExperimentExecution { requests: requestVector.requests, handlerProgress: progress, blocking, - currentReport: report + report: report ? { - reportId: stringField(report, "reportId", "experiment report", true), - url: stringField(report, "url", "experiment report", true), - updatedAt: stringField(report, "updatedAt", "experiment report", true), + reportId: stringField(report, "reportId", "experiment report")!, + url: stringField(report, "url", "experiment report")!, + updatedAt: stringField(report, "updatedAt", "experiment report")!, } : undefined, createdAt: stringField( @@ -837,6 +839,129 @@ export function registerExperimentCommands(program: Command): void { } }); + withProjectOption( + withOutputOption( + experiment + .command("reports") + .description("List or retrieve reports for one exact experiment execution") + .argument("", "Experiment ID") + .argument("", "Experiment execution ID") + .option("--report ", "Select one report") + .option("--structured", "Print the selected report's structured JSON artifact") + .option("-u, --url ", "API base URL", getDefaultApiUrl()), + ["markdown"], + ), + ).action(async (id: string, executionId: string, options) => { + try { + const projectId = requireProjectId(options.project); + const path = + `/experiments/${encodeURIComponent(id)}/executions/${encodeURIComponent(executionId)}/reports`; + const response = await apiFetch(options.url, path, { projectId }); + if (!response.ok) await throwApiError(response); + const reportsValue = await json(response); + if (!Array.isArray(reportsValue)) { + throw new Error("Invalid experiment reports response from Scope"); + } + const reports = reportsValue.map((item) => + asRecord(item, "experiment report"), + ); + const selected = options.report + ? reports.find((item) => item.id === options.report) + : reports[0]; + if (options.report && !selected) { + throw new Error(`Report ${options.report} was not found for this execution`); + } + if (options.structured) { + if (!selected) { + throw new Error("No report is available for this execution"); + } + const reportId = stringField(selected, "id", "experiment report")!; + const structuredResponse = await apiFetch( + options.url, + `${path}/${encodeURIComponent(reportId)}/structured`, + { projectId }, + ); + if (!structuredResponse.ok) await throwApiError(structuredResponse); + console.log(JSON.stringify(await json(structuredResponse), null, 2)); + return; + } + const format = (options.output ?? "table") as OutputFormat; + if (format === "markdown") { + console.log(selected?.content ?? ""); + return; + } + console.log( + formatData( + options.report && selected ? [selected] : reports, + [ + { key: "id", label: "Report" }, + { key: "status", label: "Status" }, + { + key: "reporter", + label: "Reporter", + formatter: (row: JsonRecord) => + String((row.reporter as JsonRecord | undefined)?.name ?? ""), + }, + { key: "createdAt", label: "Created" }, + { key: "updatedAt", label: "Updated" }, + ], + format, + ), + ); + } catch (error) { + console.error( + errorText("Error:"), + error instanceof Error ? error.message : error, + ); + process.exitCode = 1; + } + }); + + withProjectOption( + experiment + .command("logs") + .description("Stream logs for one exact experiment execution") + .argument("", "Experiment ID") + .argument("", "Experiment execution ID") + .option("--from-start", "Include historical logs from the beginning", false) + .option("-u, --url ", "API base URL", getDefaultApiUrl()), + ).action(async (id: string, executionId: string, options) => { + const projectId = requireProjectId(options.project); + const params = new URLSearchParams({ projectId }); + if (options.fromStart) params.set("fromStart", "true"); + const eventSource = new EventSource( + `${normalizeUrl(options.url)}${getApiBasePath()}/experiments/${encodeURIComponent(id)}/executions/${encodeURIComponent(executionId)}/logs?${params}`, + ); + eventSource.onmessage = (event: MessageEvent) => { + try { + const log = JSON.parse(event.data) as { + timestamp: string; + level: string; + source?: string; + message: string; + }; + const source = log.source ? ` [${log.source}]` : ""; + console.log( + `${new Date(log.timestamp).toLocaleTimeString()} ${log.level}${source} ${log.message}`, + ); + } catch { + console.log(event.data); + } + }; + eventSource.addEventListener("done", () => { + eventSource.close(); + process.exit(0); + }); + eventSource.addEventListener("timeout", () => { + eventSource.close(); + process.exit(0); + }); + eventSource.onerror = () => { + eventSource.close(); + process.exit(1); + }; + }); + withProjectOption( experiment .command("retry") diff --git a/apps/cli/src/commands/report.test.ts b/apps/cli/src/commands/report.test.ts new file mode 100644 index 000000000..c4d223593 --- /dev/null +++ b/apps/cli/src/commands/report.test.ts @@ -0,0 +1,130 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { afterEach, describe, expect, it, vi } from "vitest"; +import { Command } from "commander"; +import { registerReportCommands } from "./report.js"; + +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + process.exitCode = undefined; +}); + +describe("report get", () => { + it("downloads Structured JSON through the standard report-by-ID command", async () => { + const urls: string[] = []; + vi.stubGlobal("fetch", vi.fn(async (request: Request) => { + urls.push(request.url); + return { + ok: true, + status: 200, + json: async () => ({ + meta: { schemaId: "scope.experiment-readout" }, + }), + }; + })); + const output: string[] = []; + vi.spyOn(console, "log").mockImplementation((...parts: unknown[]) => { + output.push(parts.map(String).join(" ")); + }); + const program = new Command(); + registerReportCommands(program); + + await program.parseAsync( + [ + "report", + "get", + "-i", + "report-1", + "--structured", + "-u", + "http://localhost:3100", + ], + { from: "user" }, + ); + + expect(urls).toEqual([ + "http://localhost:3100/api/v1/reports/report-1/structured", + ]); + expect(output.join("\n")).toContain('"schemaId": "scope.experiment-readout"'); + }); + + it("renders experiment subject metadata through standard report retrieval", async () => { + vi.stubGlobal("fetch", vi.fn(async () => ({ + ok: true, + status: 200, + json: async () => ({ + id: "report-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-1", + status: "completed", + createdAt: "2026-01-01T00:00:00.000Z", + }), + }))); + const output: string[] = []; + vi.spyOn(console, "log").mockImplementation((...parts: unknown[]) => { + output.push(parts.map(String).join(" ")); + }); + const program = new Command(); + registerReportCommands(program); + + await program.parseAsync( + [ + "report", + "get", + "-i", + "report-1", + "-o", + "json", + "-u", + "http://localhost:3100", + ], + { from: "user" }, + ); + + expect(output.join("\n")).toContain('"experimentId": "experiment-1"'); + expect(output.join("\n")).toContain( + '"experimentExecutionId": "execution-1"', + ); + }); +}); + +describe("report list", () => { + it("passes experiment filters to the global reports collection", async () => { + const urls: string[] = []; + vi.stubGlobal("fetch", vi.fn(async (request: Request) => { + urls.push(request.url); + return { + ok: true, + status: 200, + json: async () => [], + }; + })); + vi.spyOn(console, "log").mockImplementation(() => undefined); + const program = new Command(); + registerReportCommands(program); + + await program.parseAsync( + [ + "report", + "list", + "--experiment", + "experiment-1", + "--experiment-execution", + "execution-1", + "--project", + "project-1", + "-o", + "json", + "-u", + "http://localhost:3100", + ], + { from: "user" }, + ); + + expect(urls).toEqual([ + "http://localhost:3100/api/v1/reports?experimentId=experiment-1&experimentExecutionId=execution-1&projectId=project-1", + ]); + }); +}); diff --git a/apps/cli/src/commands/report.ts b/apps/cli/src/commands/report.ts index 1dce6f3b6..2a9807138 100644 --- a/apps/cli/src/commands/report.ts +++ b/apps/cli/src/commands/report.ts @@ -16,7 +16,7 @@ export function registerReportCommands(program: Command): void { const report = program .command("report") - .description("Generate, view, and monitor run reports") + .description("Generate, view, and monitor reports") .action(() => { report.help(); }); @@ -106,11 +106,26 @@ report .command("get") .description("Get a report by ID") .requiredOption("-i, --id ", "Report ID") + .option("--structured", "Get the report's Structured JSON artifact") .option("-u, --url ", "API base URL", getDefaultApiUrl()) , ['markdown']) .action(async (options) => { const format = (options.output || 'table') as OutputFormat; try { + if (options.structured) { + const structuredResponse = await apiFetch( + options.url, + `/reports/${encodeURIComponent(options.id)}/structured`, + ); + if (!structuredResponse.ok) { + const error = await structuredResponse.json(); + console.error(errorText("Error:"), error.error || JSON.stringify(error)); + process.exitCode = 1; + return; + } + console.log(JSON.stringify(await structuredResponse.json(), null, 2)); + return; + } const response = await apiFetch(options.url, `/reports/${options.id}`); if (!response.ok) { @@ -121,7 +136,9 @@ report const report = await response.json() as { id: string; - requestId: string; + requestId?: string; + experimentId?: string; + experimentExecutionId?: string; status: string; reporter?: { id: string; name: string; model: string; agentId: string; agentVersion: string }; content?: string; @@ -139,6 +156,8 @@ report const fields: DisplayField[] = [ { key: 'id', label: 'ID' }, { key: 'requestId', label: 'Run' }, + { key: 'experimentId', label: 'Experiment' }, + { key: 'experimentExecutionId', label: 'Experiment Execution' }, { key: 'status', label: 'Status' }, { key: 'reporter', label: 'Reporter', formatter: (r: any) => r.reporter ? `${r.reporter.name} (${r.reporter.agentId}@${r.reporter.agentVersion})` : '' }, { key: 'model', label: 'Model', formatter: (r: any) => r.reporter?.model || '' }, @@ -152,7 +171,15 @@ report } console.log(`${label('Report:')} ${value(report.id)}`); - console.log(`${label('Run:')} ${value(report.requestId)}`); + if (report.requestId) { + console.log(`${label('Run:')} ${value(report.requestId)}`); + } + if (report.experimentId) { + console.log(`${label('Experiment:')} ${value(report.experimentId)}`); + } + if (report.experimentExecutionId) { + console.log(`${label('Execution:')} ${value(report.experimentExecutionId)}`); + } console.log(`${label('Status:')} ${colorLevel(report.status === "completed" ? "info" : report.status === "failed" ? "error" : "warn")} ${report.status}`); console.log(`${label('Created:')} ${dimTimestamp(new Date(report.createdAt).toLocaleString())}`); if (report.updatedAt) { @@ -181,8 +208,10 @@ report withProjectOption(withOutputOption( report .command("list") - .description("List all reports (optionally filter by run)") + .description("List all reports") .option("-r, --run ", "Filter by run ID") + .option("-e, --experiment ", "Filter by experiment ID") + .option("--experiment-execution ", "Filter by experiment execution ID") .option("-u, --url ", "API base URL", getDefaultApiUrl()) )) .action(async (options) => { @@ -191,6 +220,10 @@ report try { const params = new URLSearchParams(); if (options.run) params.set("requestId", options.run); + if (options.experiment) params.set("experimentId", options.experiment); + if (options.experimentExecution) { + params.set("experimentExecutionId", options.experimentExecution); + } const qs = params.toString(); const response = await apiFetch(options.url, `/reports${qs ? `?${qs}` : ""}`, { projectId }); @@ -202,7 +235,9 @@ report const reports = await response.json() as Array<{ id: string; - requestId: string; + requestId?: string; + experimentId?: string; + experimentExecutionId?: string; status: string; reporter?: { model: string }; createdAt: string; @@ -222,7 +257,13 @@ report tableFormatter: (r: any) => value(r.id), }, { key: 'requestId', label: 'Run ID', - tableFormatter: (r: any) => dimTimestamp(r.requestId), + tableFormatter: (r: any) => r.requestId ? dimTimestamp(r.requestId) : 'N/A', + }, + { key: 'experimentId', label: 'Experiment', + tableFormatter: (r: any) => r.experimentId ? dimTimestamp(r.experimentId) : 'N/A', + }, + { key: 'experimentExecutionId', label: 'Experiment Execution', + tableFormatter: (r: any) => r.experimentExecutionId ? dimTimestamp(r.experimentExecutionId) : 'N/A', }, { key: 'status', label: 'Status', tableFormatter: (r: any) => { diff --git a/apps/portal/src/components/ExperimentReportCard.stories.tsx b/apps/portal/src/components/ExperimentReportCard.stories.tsx new file mode 100644 index 000000000..cbc700163 --- /dev/null +++ b/apps/portal/src/components/ExperimentReportCard.stories.tsx @@ -0,0 +1,310 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { Meta, StoryObj } from "@storybook/react-vite"; +import { useMemo, useState } from "react"; +import { expect, screen, userEvent, within } from "storybook/test"; +import { + groupExperimentReports, + resolveExperimentReportSelection, +} from "@/lib/experiment-reports"; +import type { Report } from "@/types"; +import { ExperimentReportCard } from "./ExperimentReportCard"; + +const report: Report = { + _id: "report-1", + id: "report-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-2", + status: "completed", + content: "# Experiment readout\n\nFive exact-vector runs were analyzed.", + reporter: { + id: "pp-readout", + name: "Experiment Readout", + gitHash: "abc", + model: "gpt-5.4", + agentId: "copilot-sdk", + agentVersion: "1.0.0", + }, + structuredArtifact: { + schemaId: "scope.experiment-readout", + schemaVersion: "1.0", + contentType: "application/json", + url: "reports/report-1/structured.json", + }, + createdAt: "2026-01-02T00:00:00.000Z", +}; + +const historicalReport: Report = { + ...report, + _id: "report-0", + id: "report-0", + content: "# Previous readout\n\nThe earlier retry output.", + createdAt: "2026-01-01T00:00:00.000Z", +}; + +const failedReport: Report = { + ...report, + _id: "report-failed", + id: "report-failed", + status: "failed", + content: undefined, + structuredArtifact: undefined, + error: "The readout worker stopped before publishing an artifact.", + createdAt: "2026-01-04T00:00:00.000Z", +}; + +const qualityReport: Report = { + ...report, + _id: "report-quality", + id: "report-quality", + templateId: "quality-summary", + content: "# Quality summary\n\nA second report type.", + reporter: { + ...report.reporter!, + id: "quality-summary", + name: "Quality Summary", + }, + createdAt: "2026-01-03T00:00:00.000Z", +}; + +const completedGroups = groupExperimentReports([report]); +const completedSelection = resolveExperimentReportSelection(completedGroups); + +const meta = { + title: "Experiments/Reports", + component: ExperimentReportCard, + args: { + experimentId: "experiment-1", + experimentExecutionId: "execution-2", + attemptNumber: 2, + groups: completedGroups, + selection: completedSelection, + structuredUrl: "/structured.json", + }, + parameters: { layout: "padded" }, +} satisfies Meta; + +export default meta; +type Story = StoryObj; + +export const Completed: Story = { + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await expect( + canvas.getByRole("heading", { name: "Experiment readout" }), + ).toBeVisible(); + await expect( + canvas.getByRole("link", { name: "Structured JSON" }), + ).toBeVisible(); + await expect(canvas.queryByRole("combobox")).toBeNull(); + await expect( + canvasElement.querySelector('[data-slot="experiment-report-toolbar"]'), + ).not.toBeNull(); + await expect( + canvasElement.querySelector('[data-slot="experiment-report-body"]'), + ).not.toBeNull(); + }, +}; + +export const Processing: Story = { + args: { + groups: [], + selection: undefined, + structuredUrl: undefined, + isProcessing: true, + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await expect(canvas.getByText("Report processing")).toBeVisible(); + await expect(canvas.getByText("Processing")).toBeVisible(); + await expect( + canvasElement.querySelector('[data-slot="experiment-report-toolbar"]'), + ).not.toBeNull(); + await expect( + canvasElement.querySelector('[data-slot="experiment-report-body"]'), + ).not.toBeNull(); + }, +}; + +export const ErrorWithRetry: Story = { + args: { + groups: [], + selection: undefined, + structuredUrl: undefined, + error: "Report service unavailable", + onRetry: () => undefined, + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await expect(canvas.getByText("Report unavailable")).toBeVisible(); + await expect(canvas.getByRole("button", { name: "Retry" })).toBeVisible(); + await expect( + canvasElement.querySelector('[data-slot="experiment-report-toolbar"]'), + ).not.toBeNull(); + await expect( + canvasElement.querySelector('[data-slot="experiment-report-body"]'), + ).not.toBeNull(); + }, +}; + +export const FailedVersion: Story = { + args: { + groups: groupExperimentReports([failedReport]), + selection: resolveExperimentReportSelection( + groupExperimentReports([failedReport]), + ), + structuredUrl: undefined, + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await expect(canvas.getByText("Report generation failed")).toBeVisible(); + await expect( + canvas.getByRole("button", { name: "Structured JSON" }).hasAttribute( + "disabled", + ), + ).toBe(true); + await expect( + canvas.getByRole("link", { name: "Direct link" }), + ).toBeVisible(); + await expect( + canvasElement.querySelector('[data-slot="experiment-report-toolbar"]'), + ).not.toBeNull(); + await expect( + canvasElement.querySelector('[data-slot="experiment-report-body"]'), + ).not.toBeNull(); + }, +}; + +export const NoArtifact: Story = { + args: { + groups: [], + selection: undefined, + structuredUrl: undefined, + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await expect(canvas.getByText("No report available")).toBeVisible(); + await expect(canvas.getByText("No report")).toBeVisible(); + await expect( + canvasElement.querySelector('[data-slot="experiment-report-toolbar"]'), + ).not.toBeNull(); + await expect( + canvasElement.querySelector('[data-slot="experiment-report-body"]'), + ).not.toBeNull(); + }, +}; + +export const Blocked: Story = { + args: { + groups: [], + selection: undefined, + structuredUrl: undefined, + isBlocked: true, + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await expect(canvas.getByText("Report blocked")).toBeVisible(); + await expect(canvas.getByText("Blocked")).toBeVisible(); + await expect( + canvasElement.querySelector('[data-slot="experiment-report-toolbar"]'), + ).not.toBeNull(); + await expect( + canvasElement.querySelector('[data-slot="experiment-report-body"]'), + ).not.toBeNull(); + }, +}; + +function InteractiveReports({ + reports, + initialType, + initialReportId, +}: { + reports: Report[]; + initialType?: string; + initialReportId?: string; +}) { + const groups = useMemo(() => groupExperimentReports(reports), [reports]); + const initialSelection = resolveExperimentReportSelection( + groups, + initialType, + initialReportId, + ); + const [typeKey, setTypeKey] = useState(initialSelection?.group.key); + const [reportId, setReportId] = useState(initialSelection?.report._id); + const selection = resolveExperimentReportSelection( + groups, + typeKey, + reportId, + ); + + return ( + { + const group = groups.find((candidate) => candidate.key === nextType); + setTypeKey(nextType); + setReportId(group?.reports[0]._id); + }} + onReportVersionChange={setReportId} + /> + ); +} + +export const RetryHistory: Story = { + render: () => ( + + ), + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await expect( + canvas.getByRole("heading", { name: "Experiment readout" }), + ).toBeVisible(); + await expect(canvas.getByText("Latest")).toBeVisible(); + + await userEvent.click( + canvas.getByRole("combobox", { name: "Report version" }), + ); + await userEvent.click( + await screen.findByRole("option", { + name: /Version 1 · Historical/, + }), + ); + + await expect( + canvas.getByRole("heading", { name: "Previous readout" }), + ).toBeVisible(); + await expect(canvas.getByText("Historical")).toBeVisible(); + }, +}; + +export const MultipleReportTypes: Story = { + render: () => ( + + ), + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await expect( + canvas.getByRole("heading", { name: "Quality summary" }), + ).toBeVisible(); + + await userEvent.click( + canvas.getByRole("combobox", { name: "Report type" }), + ); + await userEvent.click( + await screen.findByRole("option", { name: "Experiment Readout" }), + ); + + await expect( + canvas.getByRole("heading", { name: "Experiment readout" }), + ).toBeVisible(); + await expect( + canvas.getByRole("combobox", { name: "Report version" }), + ).toBeVisible(); + }, +}; diff --git a/apps/portal/src/components/ExperimentReportCard.test.tsx b/apps/portal/src/components/ExperimentReportCard.test.tsx new file mode 100644 index 000000000..8856de763 --- /dev/null +++ b/apps/portal/src/components/ExperimentReportCard.test.tsx @@ -0,0 +1,143 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// @vitest-environment happy-dom +import { cleanup, render, screen } from "@testing-library/react"; +import type { ComponentProps } from "react"; +import { afterEach, describe, expect, it } from "vitest"; +import { MemoryRouter } from "react-router-dom"; +import { + groupExperimentReports, + resolveExperimentReportSelection, +} from "@/lib/experiment-reports"; +import type { Report } from "@/types"; +import { ExperimentReportCard } from "./ExperimentReportCard"; + +const completedReport: Report = { + _id: "report-completed", + id: "report-completed", + experimentId: "experiment-1", + experimentExecutionId: "execution-2", + status: "completed", + content: "# Completed report\n\nExact evidence.", + reporter: { + id: "pp-readout", + name: "Experiment Readout", + gitHash: "abc", + model: "gpt-5.4", + agentId: "copilot-sdk", + agentVersion: "1.0.0", + }, + structuredArtifact: { + schemaId: "scope.experiment-readout", + schemaVersion: "1.0", + contentType: "application/json", + url: "reports/report-completed/structured.json", + }, + createdAt: "2026-10-16T10:00:00.000Z", +}; + +const failedReport: Report = { + ...completedReport, + _id: "report-failed", + id: "report-failed", + status: "failed", + content: undefined, + structuredArtifact: undefined, + error: "Generation failed.", +}; + +const baseProps: ComponentProps = { + experimentId: "experiment-1", + experimentExecutionId: "execution-2", + attemptNumber: 2, + groups: [], +}; + +function reportProps( + report: Report, +): ComponentProps { + const groups = groupExperimentReports([report]); + return { + ...baseProps, + groups, + selection: resolveExperimentReportSelection(groups), + structuredUrl: + report.status === "completed" ? "/structured.json" : undefined, + }; +} + +function renderCard(props: ComponentProps) { + return render( + + + , + ); +} + +function expectStableShell(container: HTMLElement) { + const panel = container.querySelector( + '[data-slot="experiment-report-panel"]', + ); + const header = container.querySelector( + '[data-slot="experiment-report-header"]', + ); + const toolbar = container.querySelector( + '[data-slot="experiment-report-toolbar"]', + ); + const selectors = container.querySelector( + '[data-slot="experiment-report-selectors"]', + ); + const actions = container.querySelector( + '[data-slot="experiment-report-actions"]', + ); + const body = container.querySelector('[data-slot="experiment-report-body"]'); + + expect(panel).not.toBeNull(); + expect(header).not.toBeNull(); + expect(toolbar).not.toBeNull(); + expect(selectors).not.toBeNull(); + expect(actions).not.toBeNull(); + expect(body).not.toBeNull(); + expect(header?.nextElementSibling).toBe(body); + expect(header?.contains(toolbar)).toBe(true); +} + +afterEach(() => cleanup()); + +describe("ExperimentReportCard", () => { + it.each([ + ["completed", reportProps(completedReport)], + ["failed", reportProps(failedReport)], + ["processing", { ...baseProps, isProcessing: true }], + ["blocked", { ...baseProps, isBlocked: true }], + ["no artifact", baseProps], + ])("keeps the stable report shell for %s", (_state, props) => { + const { container } = renderCard(props); + + expectStableShell(container); + expect(screen.getByRole("heading", { name: "Reports" })).toBeTruthy(); + expect(screen.getByText(/Attempt 2/)).toBeTruthy(); + }); + + it("keeps report actions aligned while disabling unavailable artifacts", () => { + const { rerender } = renderCard(reportProps(completedReport)); + + expect(screen.getByRole("link", { name: "Structured JSON" })).toBeTruthy(); + expect(screen.getByRole("link", { name: "Direct link" })).toBeTruthy(); + + rerender( + + + , + ); + + expect( + screen + .getByRole("button", { name: "Structured JSON" }) + .hasAttribute("disabled"), + ).toBe(true); + expect(screen.getByRole("link", { name: "Direct link" })).toBeTruthy(); + expect(screen.getByText("Report generation failed")).toBeTruthy(); + }); +}); diff --git a/apps/portal/src/components/ExperimentReportCard.tsx b/apps/portal/src/components/ExperimentReportCard.tsx new file mode 100644 index 000000000..833f425a3 --- /dev/null +++ b/apps/portal/src/components/ExperimentReportCard.tsx @@ -0,0 +1,237 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { Download, ExternalLink } from "lucide-react"; +import { Link } from "react-router-dom"; +import { MarkdownRenderer } from "@/components/MarkdownRenderer"; +import { ReportStatusBadge } from "@/components/ReportStatusBadge"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import type { + ExperimentReportGroup, + ExperimentReportSelection, +} from "@/lib/experiment-reports"; +import { formatDate } from "@/lib/utils"; + +interface ExperimentReportCardProps { + experimentId: string; + experimentExecutionId: string; + attemptNumber?: number; + groups: ExperimentReportGroup[]; + selection?: ExperimentReportSelection; + structuredUrl?: string; + isLoading?: boolean; + error?: string; + isProcessing?: boolean; + isBlocked?: boolean; + onReportTypeChange?: (typeKey: string) => void; + onReportVersionChange?: (reportId: string) => void; + onRetry?: () => void; +} + +export function ExperimentReportCard({ + experimentId, + experimentExecutionId, + attemptNumber, + groups, + selection, + structuredUrl, + isLoading, + error, + isProcessing, + isBlocked, + onReportTypeChange, + onReportVersionChange, + onRetry, +}: ExperimentReportCardProps) { + const report = selection?.report; + const emptyStateClassName = + "flex min-h-44 flex-col items-start justify-center gap-3 rounded-lg border border-dashed bg-muted/20 p-6"; + + return ( + + +
+ Reports +

+ Attempt {attemptNumber ?? "…"} ·{" "} + {experimentExecutionId} +

+
+
+
+ {groups.length > 1 && selection && ( +
+ + +
+ )} + {selection && selection.group.reports.length > 1 && ( +
+ + +
+ )} +
+
+ {report ? ( + <> + + {selection?.isLatest ? "Latest" : "Historical"} + + + {structuredUrl ? ( + + ) : ( + + )} + + + ) : isLoading ? ( + Loading + ) : error ? ( + Unavailable + ) : isBlocked ? ( + Blocked + ) : isProcessing ? ( + Processing + ) : ( + No report + )} +
+
+
+ + {isLoading ? ( +
+

Loading report

+

+ The selected report version is being retrieved. +

+
+ ) : error ? ( +
+

Report unavailable

+

{error}

+ {onRetry && ( + + )} +
+ ) : report?.status === "failed" ? ( +
+

Report generation failed

+

+ {report.error ?? "Readout generation failed."} +

+
+ ) : report?.content ? ( +
+ {report.content} +
+ ) : ( +
+

+ {isBlocked + ? "Report blocked" + : isProcessing + ? "Report processing" + : "No report available"} +

+

+ {isBlocked + ? "Required processing must recover before a report can be published." + : isProcessing + ? "The report is being generated." + : "No built-in report is available for this execution."} +

+
+ )} +
+
+ ); +} diff --git a/apps/portal/src/components/ui/stories.play.test.tsx b/apps/portal/src/components/ui/stories.play.test.tsx index 6ed917da2..06dbf5816 100644 --- a/apps/portal/src/components/ui/stories.play.test.tsx +++ b/apps/portal/src/components/ui/stories.play.test.tsx @@ -5,6 +5,7 @@ import { describe, it, afterEach } from "vitest"; import { cleanup, render, within } from "@testing-library/react"; import { composeStories } from "@storybook/react-vite"; +import { MemoryRouter } from "react-router-dom"; import * as alert from "./alert.stories"; import * as alertDialog from "./alert-dialog.stories"; @@ -24,11 +25,12 @@ import * as tabs from "./tabs.stories"; import * as textarea from "./textarea.stories"; import * as tooltip from "./tooltip.stories"; import * as sonner from "./sonner.stories"; +import * as experimentReports from "../ExperimentReportCard.stories"; const modules: Record> = { alert, alertDialog, checkbox, dialog, dropdownMenu, label, popover, scrollArea, select, separator, sheet, skeleton, switch: sw, table, tabs, textarea, - tooltip, sonner, + tooltip, sonner, experimentReports, }; afterEach(() => cleanup()); @@ -41,7 +43,11 @@ for (const [name, mod] of Object.entries(modules)) { it(`${storyName} renders and play passes`, async () => { // eslint-disable-next-line @typescript-eslint/no-explicit-any const S = Story as any; - const { container } = render(); + const { container } = render( + + + , + ); // The story play functions only depend on `canvas` (userEvent/screen/expect // are imported directly in the story files). Invoke the raw play with a // canvas bound to the rendered container to avoid needing the vitest addon. diff --git a/apps/portal/src/lib/api.ts b/apps/portal/src/lib/api.ts index b8787adf1..036f273b0 100644 --- a/apps/portal/src/lib/api.ts +++ b/apps/portal/src/lib/api.ts @@ -354,6 +354,29 @@ export const api = { return parseExperimentExecution(value); }, + listExperimentReports: ( + id: string, + experimentExecutionId: string, + ): Promise => + request( + `/experiments/${encodeURIComponent(id)}/executions/${encodeURIComponent(experimentExecutionId)}/reports`, + undefined, + { scoped: true }, + ), + + experimentReportStructuredUrl: ( + id: string, + experimentExecutionId: string, + reportId: string, + ): string => { + const projectId = getSelectedProjectId(); + if (!projectId) throw new ProjectRequiredError(); + return `${BASE}${withProjectId( + `/experiments/${encodeURIComponent(id)}/executions/${encodeURIComponent(experimentExecutionId)}/reports/${encodeURIComponent(reportId)}/structured`, + projectId, + )}`; + }, + retryExperiment: async ( id: string, experimentExecutionId: string, @@ -910,6 +933,9 @@ export const api = { return request(`/reports/${id}`); }, + reportStructuredUrl: (id: string): string => + `${BASE}/reports/${encodeURIComponent(id)}/structured`, + /** Get reports for a specific run */ getRunReports: (requestId: string): Promise => { return request(`/requests/${requestId}/reports`); diff --git a/apps/portal/src/lib/experiment-reports.ts b/apps/portal/src/lib/experiment-reports.ts new file mode 100644 index 000000000..e072c7e19 --- /dev/null +++ b/apps/portal/src/lib/experiment-reports.ts @@ -0,0 +1,94 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { Report } from "@/types"; + +export interface ExperimentReportGroup { + key: string; + label: string; + reports: Report[]; +} + +export interface ExperimentReportSelection { + group: ExperimentReportGroup; + report: Report; + isLatest: boolean; +} + +function reportTimestamp(report: Report): number { + const parsed = Date.parse(report.createdAt); + return Number.isNaN(parsed) ? 0 : parsed; +} + +export function experimentReportTypeKey(report: Report): string { + if (report.templateId) return `template:${report.templateId}`; + if (report.reporter?.id) return `reporter:${report.reporter.id}`; + return "report:untyped"; +} + +function reportTypeLabel(report: Report): string { + if (report.templateId && report.reporter?.name) { + return `${report.reporter.name} · ${report.templateId}`; + } + return report.reporter?.name ?? report.templateId ?? "Report"; +} + +export function groupExperimentReports( + reports: Report[], +): ExperimentReportGroup[] { + const byType = new Map(); + for (const report of reports) { + const key = experimentReportTypeKey(report); + const group = byType.get(key); + if (group) { + group.reports.push(report); + } else { + byType.set(key, { + key, + label: reportTypeLabel(report), + reports: [report], + }); + } + } + + const groups = [...byType.values()]; + for (const group of groups) { + group.reports.sort( + (left, right) => + reportTimestamp(right) - reportTimestamp(left) || + right._id.localeCompare(left._id), + ); + } + return groups.sort( + (left, right) => + reportTimestamp(right.reports[0]) - reportTimestamp(left.reports[0]) || + left.label.localeCompare(right.label), + ); +} + +export function resolveExperimentReportSelection( + groups: ExperimentReportGroup[], + requestedType?: string, + requestedReportId?: string, +): ExperimentReportSelection | undefined { + if (groups.length === 0) return undefined; + + const reportGroup = requestedReportId + ? groups.find((group) => + group.reports.some((report) => report._id === requestedReportId), + ) + : undefined; + const group = + groups.find((candidate) => candidate.key === requestedType) ?? + reportGroup ?? + groups[0]; + const report = + group.reports.find((candidate) => candidate._id === requestedReportId) ?? + group.reports[0]; + + return { + group, + report, + isLatest: report._id === group.reports[0]._id, + }; +} diff --git a/apps/portal/src/lib/experiments.ts b/apps/portal/src/lib/experiments.ts index e5c8be80d..f0be53b8f 100644 --- a/apps/portal/src/lib/experiments.ts +++ b/apps/portal/src/lib/experiments.ts @@ -47,7 +47,7 @@ export interface ExperimentExecutionView { requests: ExperimentRequestSummaryView[]; handlerProgress: Record; blocking: ExperimentBlockerView[]; - currentReport?: { reportId?: string; url?: string; updatedAt?: string }; + report?: { reportId: string; url: string; updatedAt: string }; createdAt: string; updatedAt?: string; startedAt?: string; @@ -183,16 +183,16 @@ export function parseExperimentExecution( execution.handlerSummary ?? {}; const handlerProgress = record(progressValue, "experiment handler progress"); - const reportValue = execution.currentReport; - const currentReport = + const reportValue = execution.report; + const report = reportValue === undefined ? undefined : (() => { const report = record(reportValue, "experiment report"); return { - reportId: optionalString(report, "reportId"), - url: optionalString(report, "url"), - updatedAt: optionalString(report, "updatedAt"), + reportId: requiredString(report, "reportId", "experiment report"), + url: requiredString(report, "url", "experiment report"), + updatedAt: requiredString(report, "updatedAt", "experiment report"), }; })(); @@ -214,7 +214,7 @@ export function parseExperimentExecution( requests: vector.requests, handlerProgress, blocking, - currentReport, + report, createdAt: requiredString(execution, "createdAt", "experiment execution"), updatedAt: optionalString(execution, "updatedAt"), startedAt: optionalString(execution, "startedAt"), diff --git a/apps/portal/src/pages/ExperimentDetail.play.test.tsx b/apps/portal/src/pages/ExperimentDetail.play.test.tsx index ebda77fc6..21468bd48 100644 --- a/apps/portal/src/pages/ExperimentDetail.play.test.tsx +++ b/apps/portal/src/pages/ExperimentDetail.play.test.tsx @@ -18,6 +18,8 @@ vi.mock("@/lib/api", async (importOriginal) => { getExperiment: vi.fn(), listExperimentExecutions: vi.fn(), getExperimentExecution: vi.fn(), + listExperimentReports: vi.fn(), + experimentReportStructuredUrl: vi.fn(() => "/structured.json"), retryExperiment: vi.fn(), }, }; @@ -109,6 +111,30 @@ beforeAll(() => { vi.mocked(api.getExperimentExecution).mockResolvedValue( historicalExecution, ); + vi.mocked(api.listExperimentReports).mockImplementation( + async (_experimentId, executionId) => + executionId === "execution-current" + ? [ + { + _id: "report-current", + id: "report-current", + experimentId: "experiment-demo", + experimentExecutionId: "execution-current", + status: "completed", + content: "# Current readout\n\nThe latest exact-vector analysis.", + reporter: { + id: "pp-readout", + name: "Experiment Readout", + gitHash: "abc", + model: "gpt-5.4", + agentId: "copilot-sdk", + agentVersion: "1.0.0", + }, + createdAt: "2026-10-16T10:30:00.000Z", + }, + ] + : [], + ); }); describe("ExperimentDetail stories", () => { diff --git a/apps/portal/src/pages/ExperimentDetail.stories.tsx b/apps/portal/src/pages/ExperimentDetail.stories.tsx index 35330ccff..679d4f019 100644 --- a/apps/portal/src/pages/ExperimentDetail.stories.tsx +++ b/apps/portal/src/pages/ExperimentDetail.stories.tsx @@ -70,6 +70,67 @@ const experiment = { createdAt: "2026-10-16T09:00:00.000Z", }; +const currentReports = [ + { + _id: "report-current", + id: "report-current", + experimentId: "experiment-demo", + experimentExecutionId: "execution-current", + status: "completed", + content: "# Current readout\n\nThe latest exact-vector analysis.", + reporter: { + id: "pp-readout", + name: "Experiment Readout", + gitHash: "abc", + model: "gpt-5.4", + agentId: "copilot-sdk", + agentVersion: "1.0.0", + }, + structuredArtifact: { + schemaId: "scope.experiment-readout", + schemaVersion: "1.0", + contentType: "application/json", + url: "reports/report-current/structured.json", + }, + createdAt: "2026-10-16T10:30:00.000Z", + }, + { + _id: "report-previous", + id: "report-previous", + experimentId: "experiment-demo", + experimentExecutionId: "execution-current", + status: "completed", + content: "# Previous readout\n\nThe first report version.", + reporter: { + id: "pp-readout", + name: "Experiment Readout", + gitHash: "abc", + model: "gpt-5.4", + agentId: "copilot-sdk", + agentVersion: "1.0.0", + }, + createdAt: "2026-10-16T10:15:00.000Z", + }, + { + _id: "report-quality", + id: "report-quality", + experimentId: "experiment-demo", + experimentExecutionId: "execution-current", + templateId: "quality-summary", + status: "completed", + content: "# Quality summary\n\nAn additional attached report.", + reporter: { + id: "quality-summary", + name: "Quality Summary", + gitHash: "def", + model: "gpt-5.4", + agentId: "copilot-sdk", + agentVersion: "1.0.0", + }, + createdAt: "2026-10-16T10:20:00.000Z", + }, +]; + function ExperimentDetailStory() { const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } }, @@ -109,6 +170,15 @@ const meta = { "/api/v1/experiments/experiment-demo/executions/execution-history", () => HttpResponse.json(historicalExecution), ), + http.get( + "/api/v1/experiments/experiment-demo/executions/:executionId/reports", + ({ params }) => + HttpResponse.json( + params.executionId === "execution-current" + ? currentReports + : [], + ), + ), ], }, }, @@ -120,6 +190,19 @@ type Story = StoryObj; export const AttemptSelectionAndRunsTable: Story = { play: async ({ canvas, canvasElement }) => { + await expect( + await screen.findByRole("heading", { name: "Current readout" }), + ).toBeVisible(); + const tabList = canvas.getByRole("tablist", { + name: "Experiment detail sections", + }); + const header = canvasElement.querySelector( + '[data-slot="experiment-detail-header"]', + ); + if (!header) throw new Error("Compact Experiment header is missing"); + await expect(header.nextElementSibling).toBe(tabList.parentElement); + + await userEvent.click(canvas.getByRole("tab", { name: "Runs" })); await expect( await screen.findByText("Runs in attempt 2"), ).toBeVisible(); @@ -143,10 +226,10 @@ export const AttemptSelectionAndRunsTable: Story = { ); await expect( - await screen.findByText(/Viewing processing attempt #1 \(historical\)/), + await screen.findByText("Historical attempt"), ).toBeVisible(); await expect( - screen.getAllByText("Inspect the historical benchmark task")[0], + (await screen.findAllByText("Inspect the historical benchmark task"))[0], ).toBeVisible(); await expect(screen.queryAllByText("Implement the current benchmark task")) .toHaveLength(0); diff --git a/apps/portal/src/pages/ExperimentDetail.test.tsx b/apps/portal/src/pages/ExperimentDetail.test.tsx index c53653b77..e84adb84c 100644 --- a/apps/portal/src/pages/ExperimentDetail.test.tsx +++ b/apps/portal/src/pages/ExperimentDetail.test.tsx @@ -3,8 +3,17 @@ // @vitest-environment happy-dom import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; -import { afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { + afterEach, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, +} from "vitest"; import { MemoryRouter, Route, @@ -15,6 +24,7 @@ import type { ExperimentExecutionView, ExperimentView, } from "@/lib/experiments"; +import type { Report } from "@/types"; vi.mock("@/lib/api", async (importOriginal) => { const original = await importOriginal(); @@ -25,12 +35,17 @@ vi.mock("@/lib/api", async (importOriginal) => { getExperiment: vi.fn(), listExperimentExecutions: vi.fn(), getExperimentExecution: vi.fn(), + listExperimentReports: vi.fn(), retryExperiment: vi.fn(), + experimentReportStructuredUrl: vi.fn( + (_experimentId: string, executionId: string, reportId: string) => + `/api/v1/experiments/experiment-1/executions/${executionId}/reports/${reportId}/structured?projectId=project-1`, + ), }, }; }); -import { api } from "@/lib/api"; +import { ApiError, api } from "@/lib/api"; import { ExperimentDetail } from "./ExperimentDetail"; class ResizeObserverMock { @@ -104,13 +119,71 @@ const experiment: ExperimentView = { createdAt: "2026-10-16T09:00:00.000Z", }; +const historicalReport: Report = { + _id: "report-1", + id: "report-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-history", + status: "completed", + content: "# Historical readout\n\nExact attempt evidence.", + reporter: { + id: "pp-readout", + name: "Experiment Readout", + gitHash: "abc", + model: "gpt-5.4", + agentId: "copilot-sdk", + agentVersion: "1.0.0", + }, + structuredArtifact: { + schemaId: "scope.experiment-readout", + schemaVersion: "1.0", + contentType: "application/json", + url: "reports/report-1/structured.json", + }, + createdAt: "2026-10-16T10:30:00.000Z", +}; + +const currentReport: Report = { + ...historicalReport, + _id: "report-current", + id: "report-current", + experimentExecutionId: "execution-current", + content: "# Current readout\n\nLatest exact-attempt evidence.", + createdAt: "2026-10-16T11:30:00.000Z", +}; + +const previousCurrentReport: Report = { + ...currentReport, + _id: "report-previous", + id: "report-previous", + content: "# Previous readout\n\nEarlier retry output.", + createdAt: "2026-10-16T11:00:00.000Z", +}; + +const qualityReport: Report = { + ...currentReport, + _id: "report-quality", + id: "report-quality", + templateId: "quality-summary", + content: "# Quality summary\n\nAn additional report type.", + reporter: { + ...currentReport.reporter!, + id: "quality-summary", + name: "Quality Summary", + }, + createdAt: "2026-10-16T11:15:00.000Z", +}; + function LocationProbe() { const location = useLocation(); return
{location.pathname}{location.search}
; } -function renderDetail(initialEntry = "/experiments/experiment-1") { - vi.mocked(api.getExperiment).mockResolvedValue(experiment); +function renderDetail( + initialEntry = "/experiments/experiment-1", + experimentValue = experiment, +) { + vi.mocked(api.getExperiment).mockResolvedValue(experimentValue); vi.mocked(api.listExperimentExecutions).mockResolvedValue({ items: [currentExecution, historicalExecution], }); @@ -130,7 +203,15 @@ function renderDetail(initialEntry = "/experiments/experiment-1") { - } /> + + + + + } + /> } /> @@ -149,6 +230,11 @@ beforeAll(() => { }); }); +beforeEach(() => { + vi.mocked(api.listExperimentReports).mockReset(); + vi.mocked(api.listExperimentReports).mockResolvedValue([]); +}); + afterEach(() => { cleanup(); localStorage.clear(); @@ -169,6 +255,14 @@ describe("ExperimentDetail", () => { if (!embeddedTable) throw new Error("Embedded runs table block is missing"); expect(embeddedTable.className).toContain("space-y-3"); expect(embeddedTable.parentElement?.className).toContain("p-6"); + const header = document.querySelector( + '[data-slot="experiment-detail-header"]', + ); + const tabList = screen.getByRole("tablist", { + name: "Experiment detail sections", + }); + expect(header?.nextElementSibling).toBe(tabList.parentElement); + expect(screen.queryByText("Execution attempt 2")).toBeNull(); fireEvent.click(screen.getByRole("button", { name: "Next page" })); expect((await screen.findAllByText("Current task 11")).length).toBeGreaterThan(0); @@ -188,31 +282,45 @@ describe("ExperimentDetail", () => { ); expect( - await screen.findByText(/Viewing processing attempt #1 \(historical\)/), + await screen.findByText("Historical attempt"), ).toBeTruthy(); expect(screen.getByText("Runs in attempt 1")).toBeTruthy(); expect(screen.getAllByText("Historical task").length).toBeGreaterThan(0); expect(screen.queryAllByText("Current task 1")).toHaveLength(0); expect( - screen.getByRole("button", { name: "Retry processing" }).hasAttribute( - "disabled", - ), - ).toBe(true); - + screen.queryByRole("button", { name: "Retry processing" }), + ).toBeNull(); fireEvent.click(screen.getAllByText("Historical task")[0]); expect((await screen.findByTestId("location")).textContent).toBe( "/runs/request-history?runId=run-history", ); + + }); + + it("keeps processing retry disabled for a historical attempt", async () => { + renderDetail( + "/experiments/experiment-1?executionId=execution-history&tab=processing", + ); + + expect( + ( + await screen.findByRole("button", { name: "Retry processing" }) + ).hasAttribute("disabled"), + ).toBe(true); }); it("returns from a historical attempt to the current attempt", async () => { + const user = userEvent.setup(); renderDetail( "/experiments/experiment-1?executionId=execution-history", ); - fireEvent.click(await screen.findByRole("button", { - name: "View current attempt", - })); + await user.click( + await screen.findByRole("combobox", { name: "Experiment attempt" }), + ); + await user.click( + screen.getByRole("option", { name: "Attempt 2 · current" }), + ); await waitFor(() => { expect(screen.getByText("Runs in attempt 2")).toBeTruthy(); @@ -220,4 +328,256 @@ describe("ExperimentDetail", () => { expect(screen.getAllByText("Current task 1").length).toBeGreaterThan(0); expect(screen.queryAllByText("Historical task")).toHaveLength(0); }); + + it("loads the report and structured artifact for the exact selected attempt", async () => { + const user = userEvent.setup(); + vi.mocked(api.listExperimentReports).mockImplementation( + async (_experimentId, executionId) => + executionId === historicalExecution.experimentExecutionId + ? [historicalReport] + : [], + ); + + renderDetail( + "/experiments/experiment-1?executionId=execution-history", + ); + + expect( + await screen.findByRole("heading", { name: "Historical readout" }), + ).toBeTruthy(); + expect(api.listExperimentReports).toHaveBeenCalledWith( + "experiment-1", + "execution-history", + ); + expect( + screen.getByRole("link", { name: "Structured JSON" }).getAttribute("href"), + ).toContain("/executions/execution-history/reports/report-1/structured"); + + await user.click( + screen.getByRole("combobox", { name: "Experiment attempt" }), + ); + await user.click( + screen.getByRole("option", { name: "Attempt 2 · current" }), + ); + await waitFor(() => { + expect(api.listExperimentReports).toHaveBeenCalledWith( + "experiment-1", + "execution-current", + ); + }); + }); + + it("retries report loading for the same exact attempt", async () => { + vi.mocked(api.listExperimentReports) + .mockRejectedValueOnce(new Error("Report service unavailable")) + .mockResolvedValueOnce([]); + + renderDetail(); + + expect(await screen.findByText("Report service unavailable")).toBeTruthy(); + fireEvent.click(screen.getByRole("button", { name: /^Retry$/ })); + await waitFor(() => { + expect(api.listExperimentReports).toHaveBeenCalledTimes(2); + }); + expect(api.listExperimentReports).toHaveBeenLastCalledWith( + "experiment-1", + "execution-current", + ); + expect( + await screen.findByText("Runs in attempt 2"), + ).toBeTruthy(); + }); + + it("defaults to Reports for one report without showing unnecessary selectors", async () => { + vi.mocked(api.listExperimentReports).mockResolvedValue([currentReport]); + + renderDetail(); + + expect( + await screen.findByRole("heading", { name: "Current readout" }), + ).toBeTruthy(); + expect( + screen.getByRole("tab", { name: "Reports" }).getAttribute( + "data-state", + ), + ).toBe("active"); + expect(screen.queryByRole("combobox", { name: "Report type" })).toBeNull(); + expect( + screen.queryByRole("combobox", { name: "Report version" }), + ).toBeNull(); + }); + + it("defaults to the latest retry version and selects historical versions", async () => { + const user = userEvent.setup(); + vi.mocked(api.listExperimentReports).mockResolvedValue([ + previousCurrentReport, + currentReport, + ]); + + renderDetail(); + + expect( + await screen.findByRole("heading", { name: "Current readout" }), + ).toBeTruthy(); + await user.click( + screen.getByRole("combobox", { name: "Report version" }), + ); + await user.click( + screen.getByRole("option", { name: /Version 1 · Historical/ }), + ); + + expect( + await screen.findByRole("heading", { name: "Previous readout" }), + ).toBeTruthy(); + expect(screen.getByTestId("location").textContent).toContain( + "reportType=reporter%3App-readout", + ); + expect(screen.getByTestId("location").textContent).toContain( + "reportId=report-previous", + ); + expect( + screen.getByRole("link", { name: "Direct link" }).getAttribute("href"), + ).toContain("/reports/report-previous?"); + }); + + it("selects one report type at a time and honors direct query selection", async () => { + const user = userEvent.setup(); + vi.mocked(api.listExperimentReports).mockResolvedValue([ + previousCurrentReport, + currentReport, + qualityReport, + ]); + + renderDetail( + "/experiments/experiment-1?tab=reports&reportType=reporter%3App-readout&reportId=report-previous", + ); + + expect( + await screen.findByRole("heading", { name: "Previous readout" }), + ).toBeTruthy(); + await user.click(screen.getByRole("combobox", { name: "Report type" })); + await user.click( + screen.getByRole("option", { name: /Quality Summary/ }), + ); + + expect( + await screen.findByRole("heading", { name: "Quality summary" }), + ).toBeTruthy(); + expect(screen.queryByRole("heading", { name: "Current readout" })).toBeNull(); + expect(screen.getByTestId("location").textContent).toContain( + "reportType=template%3Aquality-summary", + ); + expect(screen.getByTestId("location").textContent).toContain( + "reportId=report-quality", + ); + }); + + it("falls back safely from stale tab, report type, and report version links", async () => { + vi.mocked(api.listExperimentReports).mockResolvedValue([ + currentReport, + qualityReport, + ]); + + renderDetail( + "/experiments/experiment-1?tab=missing&reportType=missing&reportId=missing", + ); + + expect( + await screen.findByRole("heading", { name: "Current readout" }), + ).toBeTruthy(); + await waitFor(() => { + expect(screen.getByTestId("location").textContent).toBe( + "/experiments/experiment-1", + ); + }); + }); + + it("falls back safely from a stale attempt while preserving a valid tab", async () => { + vi.mocked(api.getExperimentExecution).mockRejectedValueOnce( + new ApiError("Experiment execution not found", 404), + ); + + renderDetail( + "/experiments/experiment-1?executionId=execution-missing&tab=runs&reportId=report-missing", + ); + + expect(await screen.findByText("Runs in attempt 2")).toBeTruthy(); + await waitFor(() => { + expect(screen.getByTestId("location").textContent).toBe( + "/experiments/experiment-1?tab=runs", + ); + }); + }); + + it("defaults to Processing while blocked and keeps retry recovery scoped there", async () => { + vi.mocked(api.retryExperiment).mockResolvedValue({ + experimentId: "experiment-1", + experimentExecutionId: "execution-retry", + attemptNumber: 3, + }); + const processingExperiment: ExperimentView = { + ...experiment, + readiness: "blocked", + execution: { + ...currentExecution, + readiness: "blocked", + handlerProgress: { + "pp-observations": { status: "done" }, + "pp-readout": { + status: "failed", + error: "A dependency must be retried.", + }, + }, + }, + }; + + renderDetail("/experiments/experiment-1", processingExperiment); + + expect( + ( + await screen.findByRole("tab", { name: "Processing" }) + ).getAttribute("data-state"), + ).toBe("active"); + expect(screen.getByText("pp-observations")).toBeTruthy(); + expect(screen.getByText("pp-readout")).toBeTruthy(); + fireEvent.click(screen.getByRole("button", { name: "Retry processing" })); + await waitFor(() => { + expect(api.retryExperiment).toHaveBeenCalledWith( + "experiment-1", + "execution-current", + ); + }); + }); + + it("preserves the selected tab across attempts and clears report history state", async () => { + const user = userEvent.setup(); + vi.mocked(api.listExperimentReports).mockImplementation( + async (_experimentId, executionId) => + executionId === "execution-current" + ? [previousCurrentReport, currentReport] + : [historicalReport], + ); + + renderDetail( + "/experiments/experiment-1?tab=reports&reportType=reporter%3App-readout&reportId=report-previous", + ); + + expect( + await screen.findByRole("heading", { name: "Previous readout" }), + ).toBeTruthy(); + await user.click( + screen.getByRole("combobox", { name: "Experiment attempt" }), + ); + await user.click( + screen.getByRole("option", { name: "Attempt 1 · historical" }), + ); + + expect( + await screen.findByRole("heading", { name: "Historical readout" }), + ).toBeTruthy(); + const location = screen.getByTestId("location").textContent ?? ""; + expect(location).toContain("executionId=execution-history"); + expect(location).toContain("tab=reports"); + expect(location).not.toContain("report-previous"); + }); }); diff --git a/apps/portal/src/pages/ExperimentDetail.tsx b/apps/portal/src/pages/ExperimentDetail.tsx index 2fb41eab7..51ff99ae4 100644 --- a/apps/portal/src/pages/ExperimentDetail.tsx +++ b/apps/portal/src/pages/ExperimentDetail.tsx @@ -11,6 +11,7 @@ import { useSearchParams, } from "react-router-dom"; import { toast } from "sonner"; +import { ExperimentReportCard } from "@/components/ExperimentReportCard"; import { ExperimentStatusCard } from "@/components/ExperimentStatusCard"; import { ShortId } from "@/components/ShortId"; import { Badge } from "@/components/ui/badge"; @@ -34,7 +35,17 @@ import { SelectTrigger, SelectValue, } from "@/components/ui/select"; +import { + Tabs, + TabsContent, + TabsList, + TabsTrigger, +} from "@/components/ui/tabs"; import { ApiError, api } from "@/lib/api"; +import { + groupExperimentReports, + resolveExperimentReportSelection, +} from "@/lib/experiment-reports"; import type { ExperimentExecutionView, ExperimentRequestSummaryView, @@ -57,6 +68,37 @@ const DEFAULT_PAGE_SIZE = 10; type RunSortColumn = (typeof RUN_COLUMN_IDS)[number]; type SortDirection = "asc" | "desc"; +type ExperimentDetailTab = "runs" | "reports" | "processing"; + +const EXPERIMENT_DETAIL_TABS: ExperimentDetailTab[] = [ + "runs", + "reports", + "processing", +]; + +function handlerStatus(value: unknown): string | undefined { + if ( + value && + typeof value === "object" && + "status" in value && + typeof value.status === "string" + ) { + return value.status; + } + return undefined; +} + +function handlerError(value: unknown): string | undefined { + if ( + value && + typeof value === "object" && + "error" in value && + typeof value.error === "string" + ) { + return value.error; + } + return undefined; +} function requestSortValue( request: ExperimentRequestSummaryView, @@ -121,6 +163,30 @@ function ExecutionSummary({
Created
{formatDate(execution.createdAt)}
+ {execution.version !== undefined && ( +
+
Version
+
{execution.version}
+
+ )} + {execution.startedAt && ( +
+
Started
+
{formatDate(execution.startedAt)}
+
+ )} + {execution.finishedAt && ( +
+
Finished
+
{formatDate(execution.finishedAt)}
+
+ )} + {execution.supersededReason && ( +
+
Superseded by
+
{execution.supersededReason}
+
+ )} ); @@ -149,6 +215,10 @@ export function ExperimentDetail() { const queryClient = useQueryClient(); const [searchParams, setSearchParams] = useSearchParams(); const selectedExecutionId = searchParams.get("executionId") ?? undefined; + const selectedReportType = searchParams.get("reportType") ?? undefined; + const selectedReportId = + searchParams.get("reportId") ?? searchParams.get("report") ?? undefined; + const requestedTab = searchParams.get("tab") ?? undefined; const [customizingColumns, setCustomizingColumns] = useState(false); const [page, setPage] = useState(1); const [pageSize, setPageSize] = useState(DEFAULT_PAGE_SIZE); @@ -185,6 +255,7 @@ export function ExperimentDetail() { }); const currentExecutionId = experimentQuery.data?.execution.experimentExecutionId; + const reportExecutionId = selectedExecutionId ?? currentExecutionId; const isViewingHistorical = Boolean(selectedExecutionId) && selectedExecutionId !== currentExecutionId; @@ -193,6 +264,25 @@ export function ExperimentDetail() { queryFn: () => api.getExperimentExecution(id!, selectedExecutionId!), enabled: Boolean(id && selectedExecutionId && isViewingHistorical), }); + const reportsQuery = useQuery({ + queryKey: ["experiment-reports", id, reportExecutionId], + queryFn: () => api.listExperimentReports(id!, reportExecutionId!), + enabled: Boolean(id && reportExecutionId), + refetchInterval: (query) => { + const reports = query.state.data; + if ( + reports && + reports.length > 0 && + reports.every( + (report) => + report.status === "completed" || report.status === "failed", + ) + ) { + return false; + } + return 5_000; + }, + }); const retry = useMutation({ mutationFn: () => api.retryExperiment( @@ -205,6 +295,9 @@ export function ExperimentDetail() { void queryClient.invalidateQueries({ queryKey: ["experiment-executions", id], }); + void queryClient.invalidateQueries({ + queryKey: ["experiment-reports", id], + }); }, onError: (error) => { toast.error(error instanceof Error ? error.message : "Retry failed"); @@ -244,6 +337,102 @@ export function ExperimentDetail() { experimentQuery.data && activeExecution ? selectedExperimentView(experimentQuery.data, activeExecution) : undefined; + const reportGroups = useMemo( + () => groupExperimentReports(reportsQuery.data ?? []), + [reportsQuery.data], + ); + const reportSelection = useMemo( + () => + resolveExperimentReportSelection( + reportGroups, + selectedReportType, + selectedReportId, + ), + [reportGroups, selectedReportId, selectedReportType], + ); + + useEffect(() => { + if ( + !selectedExecutionId || + !(selectedExecutionQuery.error instanceof ApiError) || + selectedExecutionQuery.error.status !== 404 + ) { + return; + } + const next = new URLSearchParams(searchParams); + next.delete("executionId"); + next.delete("reportType"); + next.delete("reportId"); + next.delete("report"); + setSearchParams(next, { replace: true }); + }, [ + searchParams, + selectedExecutionId, + selectedExecutionQuery.error, + setSearchParams, + ]); + + const defaultTab: ExperimentDetailTab = + reportSelection || reportsQuery.isError + ? "reports" + : activeExperiment && + ["waiting", "processing", "blocked"].includes( + activeExperiment.readiness ?? "", + ) + ? "processing" + : "runs"; + const activeTab: ExperimentDetailTab = EXPERIMENT_DETAIL_TABS.includes( + requestedTab as ExperimentDetailTab, + ) + ? (requestedTab as ExperimentDetailTab) + : defaultTab; + + useEffect(() => { + if ( + !reportsQuery.isSuccess || + (selectedExecutionQuery.error instanceof ApiError && + selectedExecutionQuery.error.status === 404) + ) { + return; + } + const next = new URLSearchParams(searchParams); + let changed = false; + + if ( + selectedReportType && + selectedReportType !== reportSelection?.group.key + ) { + next.delete("reportType"); + changed = true; + } + if (selectedReportId && selectedReportId !== reportSelection?.report._id) { + next.delete("reportId"); + next.delete("report"); + changed = true; + } else if (searchParams.has("report") && reportSelection) { + next.delete("report"); + next.set("reportId", reportSelection.report._id); + changed = true; + } + if ( + requestedTab && + !EXPERIMENT_DETAIL_TABS.includes(requestedTab as ExperimentDetailTab) + ) { + next.delete("tab"); + changed = true; + } + + if (changed) setSearchParams(next, { replace: true }); + }, [ + reportSelection, + reportsQuery.isSuccess, + searchParams, + selectedReportId, + selectedReportType, + selectedExecutionQuery.error, + requestedTab, + setSearchParams, + ]); const sortedRequests = useMemo(() => { if (!activeExecution) return []; @@ -378,6 +567,35 @@ export function ExperimentDetail() { } else { next.set("executionId", executionId); } + next.delete("reportType"); + next.delete("reportId"); + next.delete("report"); + setSearchParams(next, { replace: true }); + }; + + const selectReportType = (typeKey: string) => { + const group = reportGroups.find((candidate) => candidate.key === typeKey); + if (!group) return; + const next = new URLSearchParams(searchParams); + next.set("reportType", group.key); + next.set("reportId", group.reports[0]._id); + next.delete("report"); + setSearchParams(next, { replace: true }); + }; + + const selectReportVersion = (reportId: string) => { + if (!reportSelection) return; + const next = new URLSearchParams(searchParams); + next.set("reportType", reportSelection.group.key); + next.set("reportId", reportId); + next.delete("report"); + setSearchParams(next, { replace: true }); + }; + + const selectTab = (value: string) => { + if (!EXPERIMENT_DETAIL_TABS.includes(value as ExperimentDetailTab)) return; + const next = new URLSearchParams(searchParams); + next.set("tab", value); setSearchParams(next, { replace: true }); }; @@ -455,10 +673,13 @@ export function ExperimentDetail() { experiment.execution.experimentExecutionId; return ( -
-
+
+
-
-

- Created {formatDate(experiment.createdAt)} -

+ {experiment.metadata.description && ( +

+ {experiment.metadata.description} +

+ )}
@@ -508,44 +744,8 @@ export function ExperimentDetail() {
-
-
- - {experiment.metadata.description && ( -

- {experiment.metadata.description} -

- )} - - {isViewingHistorical && ( -
- - - Viewing processing attempt #{activeExecution?.attemptNumber ?? "…"}{" "} - (historical).{" "} - - -
- )} + {selectedExecutionQuery.isLoading && isViewingHistorical && (

@@ -563,116 +763,212 @@ export function ExperimentDetail() { )} {activeExperiment && activeExecution && ( - <> - - + + + Runs + Reports + Processing + - {activeExperiment.blocking.length > 0 && ( - - - - Blocking dependencies - - - - {activeExperiment.blocking.map((blocking, index) => ( -

- {blocking.requestId && ( - - {blocking.requestId.slice(0, 8)} - + + + + setCustomizingColumns((open) => !open)} + /> + } + > +
+ {customizingColumns && ( +
- ))} +
- )} +
- - - setCustomizingColumns((open) => !open)} - /> + + void reportsQuery.refetch()} + /> + + + +
+ +
+ + + + + Handler progress + + + {Object.entries(activeExecution.handlerProgress).length === + 0 ? ( +

+ No handlers are registered for this attempt. +

+ ) : ( + Object.entries(activeExecution.handlerProgress).map( + ([handlerId, progress]) => ( +
+ {handlerId} + + {handlerStatus(progress) ?? "unknown"} + + {handlerError(progress) && ( + + {handlerError(progress)} + + )} +
+ ), + ) + )} +
+
+ {activeExperiment.blocking.length > 0 && ( + + + + Blocking dependencies + + + + {activeExperiment.blocking.map((blocking, index) => ( +
+ {blocking.requestId && ( + + {blocking.requestId.slice(0, 8)} + + )} + {blocking.reason} + + {blocking.handlerId} + + {blocking.error && {blocking.error}} +
+ ))} +
+
+ )} +
+ )}
); diff --git a/apps/portal/src/pages/InsightDetail.tsx b/apps/portal/src/pages/InsightDetail.tsx index 02792b53b..dd98f3110 100644 --- a/apps/portal/src/pages/InsightDetail.tsx +++ b/apps/portal/src/pages/InsightDetail.tsx @@ -215,12 +215,21 @@ export function InsightDetail() { - - {formatId(report.requestId)} - + {report.experimentId && report.experimentExecutionId ? ( + + {formatId(report.experimentId)} + + ) : report.requestId ? ( + + {formatId(report.requestId)} + + ) : "–"} diff --git a/apps/portal/src/pages/ReportDetail.tsx b/apps/portal/src/pages/ReportDetail.tsx index 685e69792..7b53cf8f8 100644 --- a/apps/portal/src/pages/ReportDetail.tsx +++ b/apps/portal/src/pages/ReportDetail.tsx @@ -12,7 +12,7 @@ import { Badge } from "@/components/ui/badge"; import { ReportStatusBadge } from "@/components/ReportStatusBadge"; import { LogViewer } from "@/components/LogViewer"; import { useLogStream } from "@/hooks/use-log-stream"; -import { ArrowLeft, Copy, Check, ExternalLink, ClipboardCopy, Lightbulb } from "lucide-react"; +import { ArrowLeft, Copy, Check, Download, ExternalLink, ClipboardCopy, Lightbulb } from "lucide-react"; import { formatDate } from "@/lib/utils"; import { useEffect, useRef, useState } from "react"; import { MarkdownRenderer } from "@/components/MarkdownRenderer"; @@ -131,6 +131,25 @@ export function ReportDetail() {
+ {report.structuredArtifact && ( + + )} {report.templateId && ( )} - - Run: {report.requestId.slice(0, 8)}… - - + {report.experimentId && report.experimentExecutionId ? ( + + + Experiment: {report.experimentId.slice(0, 8)}… + + + + ) : report.requestId ? ( + + Run: {report.requestId.slice(0, 8)}… + + + ) : null} Created {formatDate(report.createdAt)}
@@ -330,13 +361,24 @@ export function ReportDetail() { {report._id}
- Run ID:{" "} - - {report.requestId} - + + {report.experimentId ? "Experiment execution:" : "Run ID:"} + {" "} + {report.experimentId && report.experimentExecutionId ? ( + + {report.experimentExecutionId} + + ) : report.requestId ? ( + + {report.requestId} + + ) : "—"}
Created:{" "} diff --git a/apps/portal/src/pages/ReportPreviewPanel.test.tsx b/apps/portal/src/pages/ReportPreviewPanel.test.tsx new file mode 100644 index 000000000..9e1e09374 --- /dev/null +++ b/apps/portal/src/pages/ReportPreviewPanel.test.tsx @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// @vitest-environment happy-dom +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { cleanup, render, screen } from "@testing-library/react"; +import { MemoryRouter, Route, Routes } from "react-router-dom"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { Report } from "@/types"; + +const report: Report = { + _id: "report-1", + id: "report-1", + experimentId: "experiment-1", + experimentExecutionId: "experiment-execution-1", + status: "completed", + content: "# Generated report", + reporter: { + id: "pp-readout", + name: "Experiment Readout", + gitHash: "development", + model: "gpt-5.4", + agentId: "unknown", + agentVersion: "unknown", + }, + createdAt: "2026-01-01T00:00:00.000Z", +}; + +vi.mock("@/lib/api", () => ({ + api: { + getReport: vi.fn(async () => report), + }, +})); + +import { ReportPreviewPanel } from "./ReportPreviewPanel"; + +afterEach(() => { + cleanup(); + vi.clearAllMocks(); +}); + +describe("ReportPreviewPanel experiment reports", () => { + it("identifies the built-in report without exposing its handler identifier", async () => { + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); + render( + + + + } /> + + + , + ); + + expect(await screen.findByText("built-in")).toBeDefined(); + expect(screen.queryByText("default template")).toBeNull(); + }); +}); diff --git a/apps/portal/src/pages/ReportPreviewPanel.tsx b/apps/portal/src/pages/ReportPreviewPanel.tsx index 477ca8331..9e31ae366 100644 --- a/apps/portal/src/pages/ReportPreviewPanel.tsx +++ b/apps/portal/src/pages/ReportPreviewPanel.tsx @@ -57,11 +57,21 @@ export function ReportPreviewPanel() { onClose={closePanel} headerActions={
- - - + {report.experimentId && report.experimentExecutionId ? ( + + + + ) : report.requestId ? ( + + + + ) : null}
diff --git a/apps/portal/src/pages/ReportsList.tsx b/apps/portal/src/pages/ReportsList.tsx index b382614c6..704474141 100644 --- a/apps/portal/src/pages/ReportsList.tsx +++ b/apps/portal/src/pages/ReportsList.tsx @@ -89,7 +89,7 @@ export function ReportsList() { const columnOptions: CustomizeColumnsOption[] = [ { id: "id", label: "ID", required: true }, { id: "task", label: "Task" }, - { id: "runId", label: "Run ID" }, + { id: "runId", label: "Subject" }, { id: "status", label: "Status" }, { id: "template", label: "Template" }, { id: "model", label: "Model" }, @@ -116,17 +116,32 @@ export function ReportsList() { }, { id: "runId", - header: "Run ID", + header: "Subject", width: "120px", hidden: columnVisibility.isHidden("runId"), - cell: (r) => ( + cell: (r) => r.experimentId && r.experimentExecutionId ? ( { e.stopPropagation(); navigate(`/runs/${r.requestId}`); }} + onClick={(event) => { + event.stopPropagation(); + navigate( + `/experiments/${r.experimentId}?execution=${encodeURIComponent(r.experimentExecutionId!)}`, + ); + }} > - {formatId(r.requestId)} + Experiment {formatId(r.experimentId)} - ), + ) : r.requestId ? ( + { + event.stopPropagation(); + navigate(`/runs/${r.requestId}`); + }} + > + Run {formatId(r.requestId)} + + ) : —, }, { id: "status", @@ -150,7 +165,9 @@ export function ReportsList() { {r.templateId} ) : ( - default + + {r.experimentId ? "built-in" : "default"} + ), }, { @@ -190,7 +207,7 @@ export function ReportsList() {
("grid"); const [reportsFilter, setReportsFilter] = useState<"latest" | "all">("latest"); @@ -705,9 +710,10 @@ export function RunDetail() { if (error || !run) { return (
- +
@@ -804,9 +810,14 @@ export function RunDetail() {
{/* Back link + header */}
- + diff --git a/apps/portal/src/types.ts b/apps/portal/src/types.ts index f510eee81..3ec950d5c 100644 --- a/apps/portal/src/types.ts +++ b/apps/portal/src/types.ts @@ -503,10 +503,18 @@ export interface Reporter { export interface Report { _id: string; id: string; - requestId: string; + requestId?: string; + experimentId?: string; + experimentExecutionId?: string; task?: string; reporter?: Reporter; content?: string; + structuredArtifact?: { + schemaId: string; + schemaVersion: string; + contentType: "application/json"; + url: string; + }; status: ReportStatus; error?: string; insightReferences?: InsightReference[]; diff --git a/apps/workers/post-processor/src/post-processor.test.ts b/apps/workers/post-processor/src/post-processor.test.ts index a0d0d3460..4d06a771a 100644 --- a/apps/workers/post-processor/src/post-processor.test.ts +++ b/apps/workers/post-processor/src/post-processor.test.ts @@ -48,6 +48,9 @@ function makeProcessor() { (processor as any).collection = collection; (processor as any).blobStorage = blobStorage; + (processor as any).db = { + collection: vi.fn().mockReturnValue({}), + }; const safeDeleteMessage = vi .spyOn(processor as any, "safeDeleteMessage") diff --git a/apps/workers/readout/Dockerfile b/apps/workers/readout/Dockerfile new file mode 100644 index 000000000..9a2a63675 --- /dev/null +++ b/apps/workers/readout/Dockerfile @@ -0,0 +1,60 @@ +FROM node:22.22.0-slim AS builder + +# The bundled Copilot CLI trusts the OS system CA store. Without this bundle, +# SDK token validation fails with an opaque "fetch failed" error. +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && rm -rf /var/lib/apt/lists/* +ARG NPM_CONFIG_REGISTRY=https://registry.npmjs.org/ +ENV npm_config_registry=${NPM_CONFIG_REGISTRY} \ + COREPACK_NPM_REGISTRY=${NPM_CONFIG_REGISTRY} +RUN corepack enable && (corepack prepare pnpm@10.29.1 --activate || npm install -g pnpm@10.29.1 --force) + +WORKDIR /app + +COPY package.json pnpm-workspace.yaml pnpm-lock.yaml ./ +COPY tsconfig.base.json ./ +COPY packages/shared/package.json ./packages/shared/ +COPY apps/workers/readout/package.json ./apps/workers/readout/ + +RUN pnpm install --frozen-lockfile + +COPY packages/shared/tsconfig.json ./packages/shared/ +COPY packages/shared/src ./packages/shared/src/ +COPY apps/workers/readout/tsconfig.json ./apps/workers/readout/ +COPY apps/workers/readout/src ./apps/workers/readout/src/ +COPY apps/workers/readout/handler.yaml ./apps/workers/readout/ + +RUN pnpm --filter shared build && pnpm --filter readout-handler build + +# --- Dev stage: hot reload with tsx watch --- +FROM builder AS dev +ENV NODE_ENV=development +ENV SERVICE_DIR=workers/readout +COPY scripts/dev-entrypoint.sh /app/scripts/dev-entrypoint.sh +EXPOSE 80 +ENV PORT=80 +CMD ["/app/scripts/dev-entrypoint.sh"] + +# --- Production stage --- +FROM node:22.22.0-slim + +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && rm -rf /var/lib/apt/lists/* +ARG NPM_CONFIG_REGISTRY=https://registry.npmjs.org/ +ENV npm_config_registry=${NPM_CONFIG_REGISTRY} \ + COREPACK_NPM_REGISTRY=${NPM_CONFIG_REGISTRY} +RUN corepack enable && (corepack prepare pnpm@10.29.1 --activate || npm install -g pnpm@10.29.1 --force) + +WORKDIR /app + +COPY package.json pnpm-workspace.yaml pnpm-lock.yaml ./ +COPY packages/shared/package.json ./packages/shared/ +COPY apps/workers/readout/package.json ./apps/workers/readout/ + +RUN pnpm install --frozen-lockfile --prod + +COPY --from=builder /app/packages/shared/dist ./packages/shared/dist/ +COPY --from=builder /app/apps/workers/readout/dist ./apps/workers/readout/dist/ +COPY --from=builder /app/apps/workers/readout/handler.yaml ./apps/workers/readout/handler.yaml + +WORKDIR /app/apps/workers/readout + +CMD ["node", "dist/index.js"] diff --git a/apps/workers/readout/handler.yaml b/apps/workers/readout/handler.yaml new file mode 100644 index 000000000..0c15c5000 --- /dev/null +++ b/apps/workers/readout/handler.yaml @@ -0,0 +1,12 @@ +# Experiment handler registration metadata. +# Read by the generic register-handler script and POSTed to the +# scheduler's POST /handlers/register endpoint on deploy. +_id: pp-readout +type: experiment-handler +version: 1 +queue: pp-readout-queue +selector: readout +autoBackfill: false +dependsOn: + - pp-observations +dependencyMode: all-runs diff --git a/apps/workers/readout/package.json b/apps/workers/readout/package.json new file mode 100644 index 000000000..8e4b87b7f --- /dev/null +++ b/apps/workers/readout/package.json @@ -0,0 +1,24 @@ +{ + "name": "readout-handler", + "version": "1.0.0", + "type": "module", + "main": "dist/index.js", + "scripts": { + "build": "tsc", + "dev": "tsx watch src/index.ts", + "start": "node dist/index.js", + "lint": "eslint src/" + }, + "dependencies": { + "shared": "workspace:*", + "@azure/storage-queue": "^12.25.0", + "@github/copilot-sdk": "1.0.1", + "dotenv": "^16.4.5", + "zod": "^4.3.6" + }, + "devDependencies": { + "@types/node": "^22.7.0", + "tsx": "^4.19.0", + "typescript": "^5.6.2" + } +} diff --git a/apps/workers/readout/src/index.ts b/apps/workers/readout/src/index.ts new file mode 100644 index 000000000..d96b3221f --- /dev/null +++ b/apps/workers/readout/src/index.ts @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import "dotenv/config"; +import { + ReadoutQueueProcessor, + type ReadoutQueueProcessorConfig, +} from "./readout-queue-processor.js"; + +const config: ReadoutQueueProcessorConfig = { + mongoUri: + process.env.MONGO_CONNECTION_STRING || + process.env.AZURE_COSMOS_CONNECTION_STRING || + "mongodb://localhost:27017", + mongoDatabase: process.env.MONGO_DATABASE || "requests-db", + mongoCollection: "experiments", + storageAccountName: + process.env.AZURE_STORAGE_ACCOUNT_NAME || "devstoreaccount1", + storageConnectionString: + process.env.STORAGE_CONNECTION_STRING || + process.env.AZURE_STORAGE_CONNECTION_STRING || + undefined, + queueName: process.env.AZURE_STORAGE_QUEUE_READOUT || "pp-readout-queue", + batchSize: Number(process.env.BATCH_SIZE) || 1, + pollIntervalMs: Number(process.env.POLL_INTERVAL_MS) || 5000, + redisHost: process.env.REDIS_HOST || "localhost", + redisPort: Number(process.env.REDIS_PORT) || 6379, + redisPassword: process.env.REDIS_PASSWORD || "", + model: process.env.READOUT_MODEL || "gpt-5.4", + sessionTimeoutMs: + Number(process.env.READOUT_SESSION_TIMEOUT_MS) || 5 * 60 * 1000, + tokenManagerUrl: process.env.TOKEN_MANAGER_URL, +}; + +const processor = new ReadoutQueueProcessor(config); +processor.start().catch((error) => { + console.error("[readout-handler] Fatal error:", error); + process.exit(1); +}); diff --git a/apps/workers/readout/src/prompt.ts b/apps/workers/readout/src/prompt.ts new file mode 100644 index 000000000..d60932615 --- /dev/null +++ b/apps/workers/readout/src/prompt.ts @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { ExperimentEvidence } from "./readout-builder.js"; + +export const READOUT_SYSTEM_PROMPT = `You synthesize an Experiment benchmark readout from an exact immutable execution vector of Scope runs. + +The user message contains JSON evidence for every request/run pair in that vector: request metadata, coding outcome, profile metadata, criteria definitions, run-level and per-iteration observations, and ATIF trajectories. + +Return raw JSON only with this shape: +{ + "title": "string", + "executiveSummary": "string", + "profiles": [ + { "profileId": "optional string", "requestIds": ["string"], "summary": "string" } + ], + "runs": [ + { "requestId": "string", "runId": "string", "trajectoryHighlights": ["string"] } + ], + "findings": [ + { + "id": "string", + "title": "string", + "description": "string", + "severity": "critical|high|medium|low|info", + "evidence": [ + { + "requestId": "string", + "runId": "string", + "iteration": 0, + "criterionId": "optional string", + "excerpt": "string" + } + ] + } + ], + "actions": [ + { + "id": "string", + "title": "string", + "priority": "critical|high|medium|low", + "description": "string", + "relatedFindingIds": ["string"] + } + ] +} + +Rules: +- Use only requestId, runId, iteration, criterionId, and profileId values present in the evidence. +- Treat failed coding outcomes as valid evidence. +- Treat observation results with passed:false as evidence, not as missing data. +- An empty selected-observation set is valid; summarize trajectory and outcome evidence without inventing observations. +- Include every exact-vector run once in runs and every request once in a profile group. +- Preserve disagreements and mixed results instead of averaging them away. +- Ground every finding in at least one evidence reference. +- Return no Markdown fences or commentary.`; + +export function composeReadoutUserPrompt(evidence: ExperimentEvidence): string { + return [ + "Generate the Experiment readout synthesis for this exact immutable execution evidence vector.", + JSON.stringify(evidence), + ].join("\n\n"); +} diff --git a/apps/workers/readout/src/readout-builder.test.ts b/apps/workers/readout/src/readout-builder.test.ts new file mode 100644 index 000000000..35379012b --- /dev/null +++ b/apps/workers/readout/src/readout-builder.test.ts @@ -0,0 +1,191 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, expect, it } from "vitest"; +import type { + CriteriaDocument, + RequestDocument, + RunState, +} from "shared"; +import { + composeExperimentReadout, + renderExperimentReadoutMarkdown, + type ExperimentEvidence, +} from "./readout-builder.js"; + +function evidence(): ExperimentEvidence { + const criterion: CriteriaDocument = { + id: "uses-current-api", + projectId: "project-1", + prompt: "Uses the current API", + kind: "observation", + subject: "run", + taxonomyElementId: "dimension:dependency-currency", + createdAt: new Date(), + }; + const request = { + _id: "request-1", + projectId: "project-1", + workerType: "copilot", + profileId: "profile-1", + profileVersionId: "profile-version-1", + observations: [criterion.id], + scenario: { task: "Update the API", criteria: [] }, + createdAt: new Date(), + priority: 0, + } satisfies RequestDocument; + const run = { + _id: "run-1", + attemptNumber: 1, + status: "done", + outcome: "failed", + observationResults: [{ + criterionId: criterion.id, + passed: false, + feedback: "The deprecated API remained.", + evaluated: true, + }], + turns: [{ + iteration: 1, + judgeFeedback: "", + snapshotUrl: "https://example.test/snapshot", + passed: false, + timestamp: new Date("2026-10-13T12:00:00.000Z"), + observationResults: [{ + criterionId: criterion.id, + passed: true, + feedback: "The first iteration used the current API.", + evaluated: true, + }], + }], + } satisfies RunState; + return { + experimentId: "experiment-1", + experimentExecutionId: "execution-1", + runs: [{ request, run, criteria: [criterion], trajectories: [] }], + }; +} + +describe("experiment readout composition", () => { + it("preserves mixed run/iteration observations, profiles, and provenance", () => { + const readout = composeExperimentReadout(evidence(), { + title: "Experiment readout", + executiveSummary: "The run failed after mixed observation evidence.", + profiles: [{ + profileId: "profile-1", + requestIds: ["request-1"], + summary: "One profiled run.", + }], + runs: [{ + requestId: "request-1", + runId: "run-1", + trajectoryHighlights: ["The agent changed approach between iterations."], + }], + findings: [{ + id: "finding-1", + title: "Regression", + description: "The final state regressed.", + severity: "high", + evidence: [{ + requestId: "request-1", + runId: "run-1", + iteration: 1, + criterionId: "uses-current-api", + excerpt: "The deprecated API remained.", + }], + }], + actions: [{ + id: "action-1", + title: "Prevent regression", + priority: "high", + description: "Add a final-state check.", + relatedFindingIds: ["finding-1"], + }], + }, new Date("2026-10-13T12:00:00.000Z")); + + expect(readout.runs[0].observations).toEqual([ + expect.objectContaining({ subject: "run", passed: false }), + expect.objectContaining({ subject: "iteration", passed: true }), + ]); + expect(readout.findings[0].evidence[0]).toMatchObject({ + experimentExecutionId: "execution-1", + requestId: "request-1", + runId: "run-1", + }); + expect(readout.profiles).toEqual([{ + profileId: "profile-1", + requestIds: ["request-1"], + summary: "One profiled run.", + }]); + expect(readout.summary.failedRuns).toBe(1); + expect(readout.summary.passedObservations).toBe(1); + expect(readout.summary.failedObservations).toBe(1); + expect(renderExperimentReadoutMarkdown(readout)).toContain( + "Schema: `scope.experiment-readout@1.0`", + ); + }); + + it("rejects evidence outside the exact vector", () => { + expect(() => composeExperimentReadout(evidence(), { + title: "Experiment readout", + executiveSummary: "Invalid evidence.", + profiles: [{ requestIds: ["request-1"], summary: "One run." }], + runs: [{ + requestId: "request-1", + runId: "run-1", + trajectoryHighlights: [], + }], + findings: [{ + id: "finding-1", + title: "Invalid", + description: "References a newer run.", + severity: "high", + evidence: [{ + requestId: "request-1", + runId: "run-newer", + excerpt: "Not in the vector.", + }], + }], + actions: [], + })).toThrow("non-vector run"); + }); + + it("rejects findings that cite a missing iteration", () => { + expect(() => composeExperimentReadout(evidence(), { + title: "Experiment readout", + executiveSummary: "Invalid iteration evidence.", + profiles: [], + runs: [], + findings: [{ + id: "finding-1", + title: "Invalid", + description: "References a missing iteration.", + severity: "high", + evidence: [{ + requestId: "request-1", + runId: "run-1", + iteration: 2, + excerpt: "Not in the vector.", + }], + }], + actions: [], + })).toThrow("unknown iteration 2"); + }); + + it("canonically retains every exact-vector profile when synthesis omits it", () => { + const readout = composeExperimentReadout(evidence(), { + title: "Experiment readout", + executiveSummary: "Profile synthesis was omitted.", + profiles: [], + runs: [], + findings: [], + actions: [], + }); + + expect(readout.profiles).toEqual([{ + profileId: "profile-1", + requestIds: ["request-1"], + summary: "1 exact-vector run.", + }]); + }); +}); diff --git a/apps/workers/readout/src/readout-builder.ts b/apps/workers/readout/src/readout-builder.ts new file mode 100644 index 000000000..aab24a996 --- /dev/null +++ b/apps/workers/readout/src/readout-builder.ts @@ -0,0 +1,303 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { z } from "zod"; +import { + EXPERIMENT_READOUT_SCHEMA_ID, + EXPERIMENT_READOUT_SCHEMA_VERSION, + experimentReadoutSchema, + type CriteriaDocument, + type CriterionResult, + type ExperimentReadout, + type RequestDocument, + type RunState, +} from "shared"; + +export interface ExactRunEvidence { + request: RequestDocument; + run: RunState; + criteria: CriteriaDocument[]; + trajectories: Array<{ iteration: number; trajectory: unknown }>; +} + +export interface ExperimentEvidence { + experimentId: string; + experimentExecutionId: string; + runs: ExactRunEvidence[]; +} + +const evidenceReferenceSchema = z.object({ + requestId: z.string().min(1), + runId: z.string().min(1), + iteration: z.number().int().min(0).optional(), + criterionId: z.string().min(1).optional(), + excerpt: z.string().min(1), +}); + +export const readoutSynthesisSchema = z.object({ + title: z.string().min(1), + executiveSummary: z.string().min(1), + profiles: z.array(z.object({ + profileId: z.string().optional(), + requestIds: z.array(z.string().min(1)).min(1), + summary: z.string().min(1), + })), + runs: z.array(z.object({ + requestId: z.string().min(1), + runId: z.string().min(1), + trajectoryHighlights: z.array(z.string().min(1)), + })), + findings: z.array(z.object({ + id: z.string().min(1), + title: z.string().min(1), + description: z.string().min(1), + severity: z.enum(["critical", "high", "medium", "low", "info"]), + evidence: z.array(evidenceReferenceSchema).min(1), + })), + actions: z.array(z.object({ + id: z.string().min(1), + title: z.string().min(1), + priority: z.enum(["critical", "high", "medium", "low"]), + description: z.string().min(1), + relatedFindingIds: z.array(z.string().min(1)), + })), +}); + +export type ReadoutSynthesis = z.infer; + +function observationsForRun(evidence: ExactRunEvidence) { + const criteriaById = new Map(evidence.criteria.map((criterion) => [criterion.id, criterion])); + const mapResult = ( + result: CriterionResult, + subject: "run" | "iteration", + ) => { + const criterion = criteriaById.get(result.criterionId); + return { + criterionId: result.criterionId, + ...(criterion?.taxonomyElementId + ? { taxonomyElementId: criterion.taxonomyElementId } + : {}), + subject, + passed: result.passed, + feedback: result.feedback, + }; + }; + return [ + ...(evidence.run.observationResults ?? []).map((result) => + mapResult(result, "run") + ), + ...(evidence.run.turns ?? []).flatMap((turn) => + (turn.observationResults ?? []).map((result) => + mapResult(result, "iteration") + ) + ), + ]; +} + +export function composeExperimentReadout( + evidence: ExperimentEvidence, + synthesis: ReadoutSynthesis, + generatedAt = new Date(), +): ExperimentReadout { + const exactRuns = new Map( + evidence.runs.map((entry) => [ + `${entry.request._id}:${entry.run._id}`, + entry, + ]), + ); + const highlights = new Map( + synthesis.runs.map((entry) => [ + `${entry.requestId}:${entry.runId}`, + entry.trajectoryHighlights, + ]), + ); + for (const key of highlights.keys()) { + if (!exactRuns.has(key)) { + throw new Error(`Readout synthesis referenced non-vector run ${key}`); + } + } + + const criterionIds = new Set( + evidence.runs.flatMap((entry) => entry.criteria.map((criterion) => criterion.id)), + ); + for (const finding of synthesis.findings) { + for (const reference of finding.evidence) { + const key = `${reference.requestId}:${reference.runId}`; + const exactRun = exactRuns.get(key); + if (!exactRun) { + throw new Error(`Readout finding ${finding.id} referenced non-vector run ${key}`); + } + if ( + reference.iteration !== undefined && + !(exactRun.run.turns ?? []).some( + (turn) => turn.iteration === reference.iteration, + ) + ) { + throw new Error( + `Readout finding ${finding.id} referenced unknown iteration ${reference.iteration} for ${key}`, + ); + } + if (reference.criterionId && !criterionIds.has(reference.criterionId)) { + throw new Error( + `Readout finding ${finding.id} referenced unknown criterion ${reference.criterionId}`, + ); + } + } + } + + const runs = evidence.runs.map((entry) => ({ + requestId: entry.request._id, + runId: entry.run._id, + task: entry.request.scenario.task, + workerType: entry.request.workerType, + ...(entry.request.model ? { model: entry.request.model } : {}), + ...(entry.request.profileId ? { profileId: entry.request.profileId } : {}), + ...(entry.request.profileVersionId + ? { profileVersionId: entry.request.profileVersionId } + : {}), + status: entry.run.status, + ...(entry.run.outcome ? { outcome: entry.run.outcome } : {}), + observations: observationsForRun(entry), + trajectoryHighlights: + highlights.get(`${entry.request._id}:${entry.run._id}`) ?? [], + })); + const allObservations = runs.flatMap((run) => run.observations); + const synthesisProfiles = new Map( + synthesis.profiles.map((profile) => [profile.profileId ?? "", profile]), + ); + const requestIdsByProfile = new Map(); + for (const entry of evidence.runs) { + const key = entry.request.profileId ?? ""; + requestIdsByProfile.set(key, [ + ...(requestIdsByProfile.get(key) ?? []), + entry.request._id, + ]); + } + const profiles = [...requestIdsByProfile].map(([profileId, requestIds]) => ({ + ...(profileId ? { profileId } : {}), + requestIds, + summary: + synthesisProfiles.get(profileId)?.summary ?? + `${requestIds.length} exact-vector run${requestIds.length === 1 ? "" : "s"}.`, + })); + const findingIds = new Set(synthesis.findings.map((finding) => finding.id)); + for (const action of synthesis.actions) { + const unknown = action.relatedFindingIds.find((id) => !findingIds.has(id)); + if (unknown) { + throw new Error(`Readout action ${action.id} referenced unknown finding ${unknown}`); + } + } + + return experimentReadoutSchema.parse({ + meta: { + schemaId: EXPERIMENT_READOUT_SCHEMA_ID, + schemaVersion: EXPERIMENT_READOUT_SCHEMA_VERSION, + generatedAt: generatedAt.toISOString(), + experimentId: evidence.experimentId, + experimentExecutionId: evidence.experimentExecutionId, + }, + summary: { + title: synthesis.title, + executiveSummary: synthesis.executiveSummary, + totalRuns: runs.length, + succeededRuns: runs.filter((run) => run.outcome === "succeeded").length, + failedRuns: runs.filter((run) => run.outcome === "failed").length, + passedObservations: allObservations.filter((result) => result.passed).length, + failedObservations: allObservations.filter((result) => !result.passed).length, + }, + profiles, + runs, + findings: synthesis.findings.map((finding) => ({ + ...finding, + evidence: finding.evidence.map((reference) => ({ + experimentExecutionId: evidence.experimentExecutionId, + ...reference, + })), + })), + actions: synthesis.actions, + }); +} + +export function renderExperimentReadoutMarkdown(readout: ExperimentReadout): string { + const lines = [ + `# ${readout.summary.title}`, + "", + readout.summary.executiveSummary, + "", + "## Summary", + "", + `- Runs: ${readout.summary.totalRuns}`, + `- Succeeded: ${readout.summary.succeededRuns}`, + `- Failed: ${readout.summary.failedRuns}`, + `- Passed observations: ${readout.summary.passedObservations}`, + `- Failed observations: ${readout.summary.failedObservations}`, + "", + "## Runs", + "", + ]; + for (const run of readout.runs) { + lines.push( + `### ${run.task || run.requestId}`, + "", + `- Request: \`${run.requestId}\``, + `- Run: \`${run.runId}\``, + `- Status: ${run.status}${run.outcome ? ` / ${run.outcome}` : ""}`, + `- Profile: ${run.profileId ?? "none"}`, + "", + ); + if (run.observations.length > 0) { + lines.push("| Observation | Subject | Result | Evidence |", "| --- | --- | --- | --- |"); + for (const observation of run.observations) { + lines.push( + `| \`${observation.criterionId}\` | ${observation.subject} | ${observation.passed ? "Passed" : "Failed"} | ${observation.feedback.replaceAll("|", "\\|")} |`, + ); + } + lines.push(""); + } + if (run.trajectoryHighlights.length > 0) { + lines.push( + "**Trajectory highlights**", + "", + ...run.trajectoryHighlights.map((highlight) => `- ${highlight}`), + "", + ); + } + } + lines.push("## Findings", ""); + if (readout.findings.length === 0) { + lines.push("No cross-run findings were identified.", ""); + } else { + for (const finding of readout.findings) { + lines.push( + `### ${finding.title}`, + "", + `**Severity:** ${finding.severity}`, + "", + finding.description, + "", + ...finding.evidence.map( + (evidence) => + `- \`${evidence.requestId}/${evidence.runId}\`${evidence.iteration === undefined ? "" : ` iteration ${evidence.iteration}`}: ${evidence.excerpt}`, + ), + "", + ); + } + } + lines.push("## Actions", ""); + if (readout.actions.length === 0) { + lines.push("No follow-up actions were generated.", ""); + } else { + for (const action of readout.actions) { + lines.push( + `- **${action.title}** (${action.priority}): ${action.description}`, + ); + } + lines.push(""); + } + lines.push( + "---", + `Schema: \`${readout.meta.schemaId}@${readout.meta.schemaVersion}\``, + `Experiment execution: \`${readout.meta.experimentExecutionId}\``, + ); + return lines.join("\n"); +} diff --git a/apps/workers/readout/src/readout-generator.ts b/apps/workers/readout/src/readout-generator.ts new file mode 100644 index 000000000..6ca486d2a --- /dev/null +++ b/apps/workers/readout/src/readout-generator.ts @@ -0,0 +1,102 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { CopilotClient, type PermissionHandler, SessionEvent } from "@github/copilot-sdk"; +import type { LogEvent } from "shared"; +import { + composeExperimentReadout, + readoutSynthesisSchema, + type ExperimentEvidence, +} from "./readout-builder.js"; +import { + READOUT_SYSTEM_PROMPT, + composeReadoutUserPrompt, +} from "./prompt.js"; + +const approveNoTools: PermissionHandler = () => ({ + kind: "denied-by-rules", + rules: [], +}); + +type ReadoutLog = ( + level: LogEvent["level"], + message: string, + data?: Record, +) => Promise | void; + +export async function generateExperimentReadout(input: { + evidence: ExperimentEvidence; + model: string; + githubToken: string; + timeoutMs: number; + log: ReadoutLog; +}) { + const client = new CopilotClient({ gitHubToken: input.githubToken }); + let latestMessage = ""; + let streamedResponse = ""; + try { + const session = await client.createSession({ + model: input.model, + streaming: true, + availableTools: [], + excludedTools: ["builtin:*", "mcp:*"], + onPermissionRequest: approveNoTools, + systemMessage: { mode: "replace", content: READOUT_SYSTEM_PROMPT }, + }); + session.on((event: SessionEvent) => { + if (event.type === "assistant.message_delta") { + streamedResponse += event.data.deltaContent; + } else if (event.type === "assistant.message") { + latestMessage = event.data.content; + } else if (event.type === "session.error") { + void input.log("error", event.data.message, { + errorType: event.data.errorType, + }); + } + }); + + let prompt = composeReadoutUserPrompt(input.evidence); + for (let attempt = 1; attempt <= 3; attempt += 1) { + streamedResponse = ""; + latestMessage = ""; + await input.log("info", `Generating Experiment readout (${attempt}/3)`); + const response = await session.sendAndWait({ prompt }, input.timeoutMs); + const text = + response?.data.content?.trim() || + latestMessage.trim() || + streamedResponse.trim(); + if (!text) throw new Error("Copilot SDK returned an empty readout response"); + const jsonText = text.replace(/^```(?:json)?\s*/i, "").replace(/\s*```$/, ""); + let parsed: unknown; + try { + parsed = JSON.parse(jsonText) as unknown; + } catch (error) { + if (attempt === 3) { + throw new Error(`Readout response was not valid JSON: ${String(error)}`); + } + prompt = "Return the complete corrected readout synthesis as raw JSON only."; + continue; + } + const result = readoutSynthesisSchema.safeParse(parsed); + if (result.success) { + return composeExperimentReadout(input.evidence, result.data); + } + if (attempt === 3) { + throw new Error( + `Readout synthesis failed schema validation: ${result.error.issues.map((issue) => `${issue.path.join(".")}: ${issue.message}`).join("; ")}`, + ); + } + prompt = [ + "Return the complete corrected readout synthesis as raw JSON only.", + `Validation errors: ${result.error.issues.map((issue) => `${issue.path.join(".")}: ${issue.message}`).join("; ")}`, + ].join("\n"); + } + throw new Error("Readout generation ended without a valid result"); + } finally { + await client.stop().catch(async (error: unknown) => { + await input.log("warn", "Failed to stop Copilot SDK client cleanly", { + error: error instanceof Error ? error.message : String(error), + }); + }); + } +} diff --git a/apps/workers/readout/src/readout-queue-processor.test.ts b/apps/workers/readout/src/readout-queue-processor.test.ts new file mode 100644 index 000000000..7f2b4194e --- /dev/null +++ b/apps/workers/readout/src/readout-queue-processor.test.ts @@ -0,0 +1,332 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { DequeuedMessageItem } from "@azure/storage-queue"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { + EXPERIMENT_STORAGE_PROFILE_VERSION, + ExperimentStore, + type CriteriaDocument, + type ExperimentDocument, + type RequestDocument, + type RunHistoryDocument, + type VisibilityHeartbeat, +} from "shared"; + +const generate = vi.hoisted(() => vi.fn()); +vi.mock("./readout-generator.js", () => ({ + generateExperimentReadout: generate, +})); + +import { ReadoutQueueProcessor } from "./readout-queue-processor.js"; + +const now = new Date("2026-10-13T12:00:00.000Z"); +const experiment: ExperimentDocument = { + _id: "experiment-1", + experimentId: "experiment-1", + projectId: "project-1", + metadata: { name: "Experiment" }, + storageProfileVersion: EXPERIMENT_STORAGE_PROFILE_VERSION, + stateVersion: 1, + revision: { + experimentRevisionId: "revision-1", + revisionNumber: 1, + memberCount: 1, + membershipFingerprint: "membership", + definitionFingerprint: "definition", + createdAt: now, + }, + execution: { + experimentExecutionId: "execution-1", + attemptNumber: 1, + version: 2, + requests: { + storage: "inline", + entryCount: 1, + payloadFingerprint: "payload", + payloadVersion: 1, + entries: [{ + requestId: "request-1", + runId: "run-1", + dependencies: [{ handlerId: "pp-observations", state: "ready" }], + blockers: [], + }], + }, + vectorFingerprint: "vector", + handlerSummary: { + "pp-readout": { + status: "processing", + version: 1, + claimToken: "claim-1", + updatedAt: now, + }, + }, + blocking: [], + createdAt: now, + updatedAt: now, + }, + createdAt: now, + updatedAt: now, +}; + +const payload = { + type: "readout", + projectId: experiment.projectId, + experimentId: experiment.experimentId, + experimentExecutionId: experiment.execution.experimentExecutionId, + handlerId: "pp-readout", + handlerVersion: 1, + claimToken: "claim-1", + executionVersion: 1, +}; +type ReadoutPayload = typeof payload; + +const message = { messageId: "message-1" } as DequeuedMessageItem; +const heartbeat = { + stop: vi.fn().mockResolvedValue(undefined), +} as unknown as VisibilityHeartbeat; + +function processor() { + const instance = new ReadoutQueueProcessor({ + mongoUri: "mongodb://localhost:27017", + mongoDatabase: "scope", + mongoCollection: "experiments", + storageAccountName: "devstoreaccount1", + storageConnectionString: "UseDevelopmentStorage=true", + queueName: "pp-readout-queue", + batchSize: 1, + pollIntervalMs: 100, + redisHost: "localhost", + redisPort: 6379, + redisPassword: "", + model: "test-model", + sessionTimeoutMs: 1_000, + }); + const insertOne = vi.fn().mockResolvedValue({ acknowledged: true }); + const uploadJson = vi.fn().mockResolvedValue( + "https://storage.example/reports/report/structured.json", + ); + const safeDeleteMessage = vi.fn().mockResolvedValue(undefined); + Object.assign(instance as object, { + collection: {}, + db: { + collection: vi.fn((name: string) => + name === "reports" ? { insertOne } : {} + ), + }, + loadEvidence: vi.fn().mockResolvedValue({ + experimentId: experiment.experimentId, + experimentExecutionId: experiment.execution.experimentExecutionId, + runs: [], + }), + tokenClient: { + acquireToken: vi.fn().mockResolvedValue("token"), + }, + blobStorage: { uploadJson }, + safeDeleteMessage, + }); + return { instance, insertOne, uploadJson, safeDeleteMessage }; +} + +async function handle(instance: ReadoutQueueProcessor, log = vi.fn()) { + await (instance as unknown as { + handleRequest( + experiment: ExperimentDocument, + message: DequeuedMessageItem, + heartbeat: VisibilityHeartbeat, + log: ReturnType, + payload: ReadoutPayload, + ): Promise; + }).handleRequest(experiment, message, heartbeat, log, payload); +} + +describe("readout queue orchestration", () => { + beforeEach(() => { + vi.restoreAllMocks(); + generate.mockReset(); + generate.mockResolvedValue({ + meta: { + schemaId: "scope.experiment-readout", + schemaVersion: "1.0", + generatedAt: now.toISOString(), + experimentId: experiment.experimentId, + experimentExecutionId: experiment.execution.experimentExecutionId, + }, + summary: { + title: "Experiment readout", + executiveSummary: "Summary", + totalRuns: 1, + succeededRuns: 0, + failedRuns: 1, + passedObservations: 0, + failedObservations: 1, + }, + profiles: [], + runs: [], + findings: [], + actions: [], + }); + }); + + it("discards stale or duplicate exact-execution messages before generation", async () => { + vi.spyOn(ExperimentStore.prototype, "claimHandler").mockResolvedValue(null); + const { instance, insertOne, safeDeleteMessage } = processor(); + + await handle(instance); + + expect(generate).not.toHaveBeenCalled(); + expect(insertOne).not.toHaveBeenCalled(); + expect(safeDeleteMessage).toHaveBeenCalledOnce(); + }); + + it("publishes the report reference with final handler completion in one CAS", async () => { + vi.spyOn(ExperimentStore.prototype, "claimHandler").mockResolvedValue(experiment); + vi.spyOn(ExperimentStore.prototype, "get").mockResolvedValue(experiment); + const complete = vi + .spyOn(ExperimentStore.prototype, "completeHandler") + .mockResolvedValue(experiment); + const { instance, insertOne } = processor(); + + await handle(instance); + + expect(insertOne).toHaveBeenCalledWith(expect.objectContaining({ + experimentId: experiment.experimentId, + experimentExecutionId: experiment.execution.experimentExecutionId, + status: "completed", + })); + expect(complete).toHaveBeenCalledOnce(); + expect(complete).toHaveBeenCalledWith(expect.objectContaining({ + expectedExecutionVersion: experiment.execution.version, + status: "done", + report: expect.objectContaining({ + reportId: expect.any(String), + }), + })); + }); + + it("keeps an inserted report historical when work is superseded in flight", async () => { + vi.spyOn(ExperimentStore.prototype, "claimHandler").mockResolvedValue(experiment); + vi.spyOn(ExperimentStore.prototype, "get") + .mockResolvedValueOnce(experiment) + .mockResolvedValueOnce(experiment) + .mockResolvedValueOnce(experiment) + .mockResolvedValueOnce(experiment) + .mockResolvedValueOnce(experiment) + .mockResolvedValueOnce(null); + const complete = vi.spyOn(ExperimentStore.prototype, "completeHandler"); + const { instance, insertOne } = processor(); + const log = vi.fn().mockResolvedValue(undefined); + + await handle(instance, log); + + expect(insertOne).toHaveBeenCalledOnce(); + expect(complete).not.toHaveBeenCalled(); + expect(log).toHaveBeenCalledWith( + "warn", + "Readout report remained historical after supersession", + expect.objectContaining({ + experimentId: experiment.experimentId, + experimentExecutionId: experiment.execution.experimentExecutionId, + }), + ); + }); + + it("loads the archived run named by the immutable exact execution vector", async () => { + const { instance } = processor(); + delete (instance as unknown as Record).loadEvidence; + vi.spyOn(ExperimentStore.prototype, "readExecutionRequests") + .mockResolvedValue([{ + requestId: "request-1", + runId: "run-1", + dependencies: [{ handlerId: "pp-observations", state: "ready" }], + blockers: [], + }]); + const request = { + _id: "request-1", + projectId: "project-1", + workerType: "coder-acp-copilot", + observations: ["observation-criterion"], + scenario: { task: "Task", criteria: ["task-criterion"] }, + createdAt: now, + priority: 0, + run: { + _id: "run-newer", + attemptNumber: 2, + status: "done", + outcome: "succeeded", + }, + } satisfies RequestDocument; + const archivedRun = { + _id: "run-1", + requestId: "request-1", + projectId: "project-1", + attemptNumber: 1, + status: "done", + outcome: "failed", + observationResults: [{ + criterionId: "result-criterion", + evaluated: true, + passed: false, + feedback: "Archived evidence", + }], + } satisfies RunHistoryDocument; + const criteria = [ + { + id: "task-criterion", + prompt: "Task criterion", + projectId: "project-1", + createdAt: now, + }, + { + id: "observation-criterion", + prompt: "Configured observation", + projectId: "project-1", + createdAt: now, + }, + { + id: "result-criterion", + prompt: "Archived observation result", + projectId: "project-1", + createdAt: now, + }, + ] satisfies CriteriaDocument[]; + const historyFindOne = vi.fn().mockResolvedValue(archivedRun); + const criteriaFind = vi.fn(() => ({ + toArray: vi.fn().mockResolvedValue(criteria), + })); + Object.assign(instance as object, { + db: { + collection: vi.fn((name: string) => { + if (name === "requests") { + return { + find: vi.fn(() => ({ + toArray: vi.fn().mockResolvedValue([request]), + })), + }; + } + if (name === "runs") return { findOne: historyFindOne }; + if (name === "criteria") return { find: criteriaFind }; + return {}; + }), + }, + }); + + const evidence = await (instance as unknown as { + loadEvidence(value: typeof payload): Promise<{ + runs: Array<{ run: RunHistoryDocument }>; + }>; + }).loadEvidence(payload); + + expect(historyFindOne).toHaveBeenCalledWith({ + _id: "run-1", + requestId: "request-1", + projectId: "project-1", + }); + expect(evidence.runs[0]?.run._id).toBe("run-1"); + expect(evidence.runs[0]?.run.outcome).toBe("failed"); + expect(evidence.runs[0]?.run.observationResults?.[0]).toMatchObject({ + passed: false, + feedback: "Archived evidence", + }); + }); +}); diff --git a/apps/workers/readout/src/readout-queue-processor.ts b/apps/workers/readout/src/readout-queue-processor.ts new file mode 100644 index 000000000..9b73497d4 --- /dev/null +++ b/apps/workers/readout/src/readout-queue-processor.ts @@ -0,0 +1,379 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { DequeuedMessageItem } from "@azure/storage-queue"; +import { randomUUID } from "node:crypto"; +import { + BaseQueueProcessor, + BlobStorage, + type BaseQueueProcessorConfig, + type CriteriaDocument, + EXPERIMENT_READOUT_SCHEMA_ID, + EXPERIMENT_READOUT_SCHEMA_VERSION, + type ExperimentDocument, + type ExperimentExecutionHistoryDocument, + type ExperimentQueueMessagePayload, + ExperimentStore, + type LogEvent, + type ReportDocument, + type RequestDocument, + type RunHistoryDocument, + type RunState, + TokenManagerClient, + type VisibilityHeartbeat, + withRetry, +} from "shared"; +import { + renderExperimentReadoutMarkdown, + type ExactRunEvidence, + type ExperimentEvidence, +} from "./readout-builder.js"; +import { generateExperimentReadout } from "./readout-generator.js"; + +const HANDLER_ID = "pp-readout"; +const HANDLER_VERSION = 1; + +export interface ReadoutQueueProcessorConfig extends BaseQueueProcessorConfig { + model: string; + sessionTimeoutMs: number; + tokenManagerUrl?: string; +} + +interface ReadoutQueueMessage extends ExperimentQueueMessagePayload { + type: "readout"; + projectId: string; + handlerId: string; + handlerVersion: number; + claimToken: string; + executionVersion: number; +} + +function isReadoutMessage( + payload: Record | undefined, +): payload is Record & ReadoutQueueMessage { + return payload?.type === "readout" && + typeof payload.projectId === "string" && + typeof payload.experimentId === "string" && + typeof payload.experimentExecutionId === "string" && + payload.handlerId === HANDLER_ID && + payload.handlerVersion === HANDLER_VERSION && + typeof payload.claimToken === "string" && + typeof payload.executionVersion === "number"; +} + +export class ReadoutQueueProcessor extends BaseQueueProcessor { + private readonly readoutConfig: ReadoutQueueProcessorConfig; + private readonly tokenClient: TokenManagerClient; + private readonly blobStorage: BlobStorage; + + constructor(config: ReadoutQueueProcessorConfig) { + super(config, "readout-handler"); + this.readoutConfig = config; + this.tokenClient = new TokenManagerClient(config.tokenManagerUrl); + this.blobStorage = new BlobStorage({ + storageAccountName: config.storageAccountName, + storageConnectionString: config.storageConnectionString, + }); + } + + protected override extractDocumentId(payload: Record): string { + return typeof payload.experimentId === "string" ? payload.experimentId : ""; + } + + protected override async handleRequest( + _experiment: ExperimentDocument, + message: DequeuedMessageItem, + heartbeat: VisibilityHeartbeat, + log: ( + level: LogEvent["level"], + message: string, + data?: Record, + ) => Promise, + payload?: Record, + ): Promise { + if (!isReadoutMessage(payload)) { + await log("warn", "Discarding unsupported readout queue payload", { + payload, + }); + await this.safeDeleteMessage(message.messageId, heartbeat.stop()); + return; + } + + const store = this.experimentStore(); + const claimed = await withRetry(() => + store.claimHandler({ + projectId: payload.projectId, + experimentId: payload.experimentId, + experimentExecutionId: payload.experimentExecutionId, + expectedExecutionVersion: payload.executionVersion, + handlerId: payload.handlerId, + handlerVersion: payload.handlerVersion, + claimToken: payload.claimToken, + }) + ); + if (!claimed) { + await log("info", "Discarding stale or duplicate readout message", { + experimentId: payload.experimentId, + experimentExecutionId: payload.experimentExecutionId, + }); + await this.safeDeleteMessage(message.messageId, heartbeat.stop()); + return; + } + + try { + let completionVersion = await this.currentClaimVersion(payload); + if (completionVersion === null) return; + const evidence = await this.loadEvidence(payload); + + completionVersion = await this.currentClaimVersion(payload); + if (completionVersion === null) return; + const githubToken = await this.tokenClient.acquireToken("copilot-sdk"); + + completionVersion = await this.currentClaimVersion(payload); + if (completionVersion === null) return; + const readout = await generateExperimentReadout({ + evidence, + model: this.readoutConfig.model, + githubToken, + timeoutMs: this.readoutConfig.sessionTimeoutMs, + log, + }); + const markdown = renderExperimentReadoutMarkdown(readout); + + completionVersion = await this.currentClaimVersion(payload); + if (completionVersion === null) return; + const reportId = randomUUID(); + const structuredUrl = await this.blobStorage.uploadJson( + `reports/${reportId}/structured.json`, + readout, + ); + + completionVersion = await this.currentClaimVersion(payload); + if (completionVersion === null) return; + const now = new Date(); + const report: ReportDocument = { + _id: reportId, + projectId: payload.projectId, + experimentId: payload.experimentId, + experimentExecutionId: payload.experimentExecutionId, + reporter: { + id: HANDLER_ID, + name: "Experiment Readout", + gitHash: process.env.GIT_COMMIT || "unknown", + model: this.readoutConfig.model, + agentId: "copilot-sdk", + agentVersion: process.env.COPILOT_SDK_VERSION || "unknown", + }, + content: markdown, + structuredArtifact: { + schemaId: EXPERIMENT_READOUT_SCHEMA_ID, + schemaVersion: EXPERIMENT_READOUT_SCHEMA_VERSION, + contentType: "application/json", + url: structuredUrl, + }, + status: "completed", + createdAt: now, + updatedAt: now, + }; + await withRetry(() => + this.db.collection("reports").insertOne(report) + ); + + completionVersion = await this.currentClaimVersion(payload); + const completed = completionVersion === null + ? null + : await withRetry(() => + store.completeHandler({ + projectId: payload.projectId, + experimentId: payload.experimentId, + experimentExecutionId: payload.experimentExecutionId, + expectedExecutionVersion: completionVersion, + handlerId: payload.handlerId, + handlerVersion: payload.handlerVersion, + claimToken: payload.claimToken, + status: "done", + report: { + reportId, + url: `/api/v1/reports/${reportId}`, + updatedAt: now, + }, + }) + ); + if (!completed) { + await log("warn", "Readout report remained historical after supersession", { + experimentId: payload.experimentId, + experimentExecutionId: payload.experimentExecutionId, + reportId, + }); + } else { + await log("info", "Experiment readout completed", { + experimentId: payload.experimentId, + experimentExecutionId: payload.experimentExecutionId, + reportId, + }); + } + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + const completionVersion = await this.currentClaimVersion(payload); + if (completionVersion !== null) { + await withRetry(() => + store.completeHandler({ + projectId: payload.projectId, + experimentId: payload.experimentId, + experimentExecutionId: payload.experimentExecutionId, + expectedExecutionVersion: completionVersion, + handlerId: payload.handlerId, + handlerVersion: payload.handlerVersion, + claimToken: payload.claimToken, + status: "failed", + error: errorMessage, + }) + ); + } + await log("error", `Experiment readout failed: ${errorMessage}`, { + experimentId: payload.experimentId, + experimentExecutionId: payload.experimentExecutionId, + }); + } finally { + await this.safeDeleteMessage(message.messageId, heartbeat.stop()); + } + } + + private experimentStore(): ExperimentStore { + return new ExperimentStore({ + experiments: this.collection, + executions: this.db.collection( + "experiment-executions", + ), + chunks: this.db.collection("experiment-chunks"), + }); + } + + private async currentClaimVersion( + payload: ReadoutQueueMessage, + ): Promise { + const experiment = await withRetry(() => + this.experimentStore().get(payload.projectId, payload.experimentId) + ); + const execution = experiment?.execution; + const handler = execution?.handlerSummary[payload.handlerId]; + return execution?.experimentExecutionId === payload.experimentExecutionId && + !execution.frozenAt && + !execution.rollover && + handler?.status === "processing" && + handler.version === payload.handlerVersion && + handler.claimToken === payload.claimToken + ? execution.version + : null; + } + + private async loadEvidence( + payload: ReadoutQueueMessage, + ): Promise { + const refs = await withRetry(() => + this.experimentStore().readExecutionRequests({ + projectId: payload.projectId, + experimentId: payload.experimentId, + experimentExecutionId: payload.experimentExecutionId, + }) + ); + if (!refs) { + throw new Error( + `Experiment execution ${payload.experimentId}/${payload.experimentExecutionId} was not found`, + ); + } + const requests = await withRetry(() => + this.db.collection("requests").find({ + _id: { $in: refs.map((ref) => ref.requestId) }, + projectId: payload.projectId, + }).toArray() + ); + const requestById = new Map(requests.map((request) => [request._id, request])); + const history = this.db.collection("runs"); + const exactRuns: Array<{ request: RequestDocument; run: RunState }> = []; + for (const ref of refs) { + const request = requestById.get(ref.requestId); + if (!request) { + throw new Error(`Exact-vector request ${ref.requestId} was not found`); + } + const run = request.run?._id === ref.runId + ? request.run + : await withRetry(() => + history.findOne({ + _id: ref.runId, + requestId: ref.requestId, + projectId: payload.projectId, + }) + ); + if (!run) { + throw new Error( + `Exact-vector run ${ref.requestId}/${ref.runId} was not found`, + ); + } + exactRuns.push({ request, run }); + } + + const criterionIds = [ + ...new Set( + exactRuns.flatMap(({ request, run }) => [ + ...request.scenario.criteria, + ...(request.observations ?? []), + ...(run.observationResults ?? []).map((result) => result.criterionId), + ...(run.turns ?? []).flatMap((turn) => + (turn.observationResults ?? []).map((result) => result.criterionId) + ), + ]), + ), + ]; + const criteria = criterionIds.length === 0 + ? [] + : await withRetry(() => + this.db.collection("criteria").find({ + projectId: payload.projectId, + id: { $in: criterionIds }, + }).toArray() + ); + const criteriaById = new Map( + criteria.map((criterion) => [criterion.id, criterion]), + ); + + const runs: ExactRunEvidence[] = []; + for (const { request, run } of exactRuns) { + const runCriterionIds = new Set([ + ...request.scenario.criteria, + ...(request.observations ?? []), + ...(run.observationResults ?? []).map((result) => result.criterionId), + ...(run.turns ?? []).flatMap((turn) => + (turn.observationResults ?? []).map((result) => result.criterionId) + ), + ]); + const trajectories = await Promise.all( + (run.turns ?? []).flatMap((turn) => + turn.atifUrl + ? [ + withRetry(() => this.blobStorage.downloadJson(turn.atifUrl!)) + .then((trajectory) => ({ + iteration: turn.iteration, + trajectory, + })), + ] + : [] + ), + ); + runs.push({ + request, + run, + criteria: [...runCriterionIds].flatMap((id) => { + const criterion = criteriaById.get(id); + return criterion ? [criterion] : []; + }), + trajectories, + }); + } + return { + experimentId: payload.experimentId, + experimentExecutionId: payload.experimentExecutionId, + runs, + }; + } +} diff --git a/apps/workers/readout/tsconfig.json b/apps/workers/readout/tsconfig.json new file mode 100644 index 000000000..bf012ecbb --- /dev/null +++ b/apps/workers/readout/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] +} diff --git a/apps/workers/report-generator/src/report-queue-processor.ts b/apps/workers/report-generator/src/report-queue-processor.ts index 7ecaac8f8..1a0523a97 100644 --- a/apps/workers/report-generator/src/report-queue-processor.ts +++ b/apps/workers/report-generator/src/report-queue-processor.ts @@ -88,6 +88,11 @@ export class ReportQueueProcessor extends BaseQueueProcessor { const generationStart = Date.now(); const reportId = doc._id; const requestId = doc.requestId; + if (!requestId) { + throw new Error( + `Report ${reportId} has no requestId and cannot be processed by report-generator`, + ); + } if (!ReportQueueProcessor.coldStartTracked) { ReportQueueProcessor.coldStartTracked = true; diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 411c8fc2e..0a22780d0 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -217,6 +217,22 @@ services: - action: rebuild path: ./pnpm-lock.yaml + readout-handler: + build: + target: dev + develop: + watch: + - action: sync + path: ./apps/workers/readout/src + target: /app/apps/workers/readout/src + - action: sync + path: ./packages/shared/src + target: /app/packages/shared/src + - action: rebuild + path: ./apps/workers/readout/package.json + - action: rebuild + path: ./pnpm-lock.yaml + register-handler-post-processor: build: target: dev @@ -225,6 +241,10 @@ services: build: target: dev + register-handler-readout: + build: + target: dev + post-processor: build: target: dev diff --git a/docker-compose.yml b/docker-compose.yml index 027888cd2..8fc96f9f7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,7 @@ # docker compose --profile vscode-web up # + VS Code Web worker # docker compose --profile claude-code --profile copilot up # multiple workers # docker compose --profile report up # + Report Generator worker +# docker compose --profile readout up # + Experiment readout worker # docker compose up mongodb redis azurite # infra only (run app services natively) # # Host-native debugging: @@ -869,6 +870,51 @@ services: api: condition: service_started + readout-handler: + build: + context: . + dockerfile: apps/workers/readout/Dockerfile + args: + <<: *npm-build-args + profiles: [readout, all] + environment: + <<: *common-env + AZURE_STORAGE_QUEUE_READOUT: pp-readout-queue + READOUT_MODEL: ${READOUT_MODEL:-gpt-5.4} + GITHUB_TOKEN: ${GITHUB_TOKEN:-} + TOKEN_MANAGER_URL: http://token-manager:80 + GIT_COMMIT: ${GIT_COMMIT:-development} + READOUT_SESSION_TIMEOUT_MS: ${READOUT_SESSION_TIMEOUT_MS:-300000} + depends_on: + mongodb: + condition: service_healthy + required: false + redis: + condition: service_healthy + azurite: + condition: service_healthy + scheduler: + condition: service_started + register-handler-readout: + condition: service_completed_successfully + + register-handler-readout: + build: + context: . + dockerfile: apps/workers/readout/Dockerfile + args: + <<: *npm-build-args + profiles: [readout, all] + environment: + <<: *common-env + HANDLER_YAML_PATH: apps/workers/readout/handler.yaml + SCHEDULER_URL: http://scheduler:8080 + depends_on: + scheduler: + condition: service_started + restart: "no" + entrypoint: ["npx", "tsx", "packages/shared/src/scripts/register-handler.ts"] + register-handler-post-processor: build: context: . @@ -892,6 +938,8 @@ services: build: context: . dockerfile: apps/workers/post-processor/Dockerfile + args: + <<: *npm-build-args profiles: [post-processor] environment: <<: *common-env diff --git a/docs/architecture/experiments.md b/docs/architecture/experiments.md index bf3d6533c..2217258fb 100644 --- a/docs/architecture/experiments.md +++ b/docs/architecture/experiments.md @@ -269,15 +269,43 @@ to Experiment Detail after creation. The Experiments list uses the shared full-height, edge-to-edge `ListLayout`, `FilterRail`, `DataTable`, column customization, and cursor-aware `Pagination` interaction model used by Runs while retaining Experiment-specific status, revision, attempt, and run-count -columns. Detail selects the current processing attempt by default and supports -direct historical links via `?executionId=`. -A header attempt selector renders exactly one current or historical execution -at a time, including its state, blockers, and a shared -sortable/customizable/paginated Runs `DataTable`. Run rows link to the exact -stored request/run pair. Retry remains current-attempt-only. Run list and -detail views link `experimentId` back to the Experiment. A historical -correlation that has no Experiment document is view-only and cannot trigger -retry. +columns. + +Experiment Detail keeps the Experiment identity, concise status, and global +attempt selector in a compact header. The horizontal **Runs**, **Reports**, and +**Processing** tabs follow that header directly so their active content is +available without scrolling through overview cards. The selected attempt +scopes all three tabs: + +- **Runs** renders the exact stored request/run vector in an inset, + sortable, customizable, paginated shared `DataTable`. Rows link to the exact + request/run pair. +- **Reports** groups append-only report documents by persisted `templateId` or + reporter identity. It shows a type selector only when multiple report types + exist and a version selector only when a type has history. The newest type + and newest version are the defaults, and exactly one version is rendered. + Markdown, structured JSON, the global report-detail link, loading/error + retry, and status use the exact + `experimentId + experimentExecutionId + reportId` identity. Completed, + processing, failed, blocked, and no-artifact states share one stable panel + shell: identity, selectors, badges, and actions keep their positions while + only the content region changes. Artifact-only actions are disabled when no + artifact exists, while addressable report versions retain their direct link. +- **Processing** owns handler progress, blockers, errors, execution metadata, + and processing retry/recovery. Retry remains current-attempt-only. + +Browser history preserves the selected attempt, tab, report type, and report +version through `executionId`, `tab`, `reportType`, and `reportId` query +parameters. Missing parameters retain backward-compatible defaults. Stale +values fall back safely to the current data and are removed from the URL. +Detail defaults to **Reports** when a report document exists, **Processing** +while an artifact-free attempt is active or blocked, and **Runs** otherwise. +Public UI terminology remains generic **Reports**; `pp-readout` is only the +internal handler identifier. This projection does not add Experiment report +template attachment, selection, or triggering; those remain deferred to PR D. +Run list and detail views link `experimentId` back to the Experiment. A +historical correlation that has no Experiment document is view-only and cannot +trigger retry. ## Permanent request creation and composition deprecation diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index 06453f18a..7ecfbe15c 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -80,6 +80,7 @@ flowchart TB | `workers/coder-acp-copilot` | GitHub Copilot agent via Agent Client Protocol (ACP) | | `workers/report-generator` | Copilot SDK worker that generates post-run markdown reports | | `workers/post-processor` | DAG-based post-run handlers: `pp-atif` (trajectory ATIF) and `pp-observations` (boolean observations via the judge, per-iteration or whole-run by criterion `subject`) | +| `workers/readout` | Copilot SDK submission handler that generates schema-validated structured JSON and derived Markdown for an exact submission attempt | | `gateway` | AI Gateway — shared Rust TLS-intercepting proxy with plugin architecture (HAR capture, future: token refresh, rate limiting) | | `evaluations/static-prompts` | Developer-run static prompt quality and user-controlled prompt red-team suites | diff --git a/docs/architecture/post-processing.md b/docs/architecture/post-processing.md index cf00f2bb2..5a73d2960 100644 --- a/docs/architecture/post-processing.md +++ b/docs/architecture/post-processing.md @@ -34,7 +34,23 @@ Two paths trigger post-processing: - Updates `run.turns.$.atifUrl` in the request document 4. On success, stamps `run.handlerStatus["pp-atif"] = { status: "done", version: N }` (while legacy `run.postProcessorVersion` / `run.postProcessorStatus` may still be mirrored during the transition) -5. Triggers any downstream handlers once the scheduler sees the dependency graph is satisfied. (Reports are **not** a DAG handler — see [Report Triggering](#report-triggering-on-dag-drain).) +5. Triggers the downstream taxonomy handler once the scheduler sees the dependency graph is satisfied. (Reports are **not** a DAG handler — see [Report Triggering](#report-triggering-on-dag-drain).) + +## Taxonomy Generation + +The taxonomy handler (`apps/workers/taxonomy/`) consumes messages from the dedicated `pp-taxonomy-queue` and produces a structured JSON artifact for a completed run. It uses the GitHub Copilot SDK with a tool-backed session to fetch run metadata and ATIF trajectories from the API, then validates the generated JSON against the shared taxonomy Zod schema. + +### Flow + +1. Queue message arrives: `{ type: "taxonomy", requestId, runId }` +2. Worker fetches the request document from MongoDB and marks `run.handlerStatus["pp-taxonomy"].status = "processing"` +3. Copilot SDK session calls: + - `get_run_data` to fetch the request + active run from the API + - `get_atif_trajectory` to fetch per-iteration ATIF JSON from the API (`GET /api/v1/requests/:id/runs/:runId/atif?iteration=N`) +4. The worker validates the assistant response with `taxonomySchema.safeParse()` +5. If validation fails, the worker feeds the schema errors back into the same session and retries up to 3 total attempts +6. On success, the worker uploads `taxonomy.json` to `{requestId}/runs/{runId}/taxonomy.json` +7. The worker stamps `run.handlerStatus["pp-taxonomy"] = { status: "done", version: 1 }` and calls `POST /notify/handler-complete` ### Blob Storage Layout @@ -45,16 +61,52 @@ Two paths trigger post-processing: atif.trajectory.json iteration-2/ atif.trajectory.json + taxonomy.json ``` +The completed run stores the taxonomy blob URL on `run.taxonomyUrl`. + ### API Endpoints | Endpoint | Description | | --- | --- | | `GET /api/v1/requests/:id/atif?iteration=N` | Download ATIF for the latest run | | `GET /api/v1/requests/:id/runs/:runId/atif?iteration=N` | Download ATIF for a specific run | +| `GET /api/v1/requests/:id/taxonomy` | Download taxonomy JSON for the latest run | +| `GET /api/v1/requests/:id/runs/:runId/taxonomy` | Download taxonomy JSON for a specific run | + +The `iteration` query parameter is mandatory only for the ATIF endpoints. ATIF files are also included in archive exports as `iteration-{N}.atif.trajectory.json`. + +### Local troubleshooting: taxonomy CLI + +The Copilot SDK generation loop lives in `apps/workers/taxonomy/src/taxonomy-generator.ts` (`generateTaxonomy()`), shared by both the queue processor and a standalone developer CLI (`src/cli.ts`). The CLI invokes generation directly against an existing `requestId`, streams the session events live, and saves the validated taxonomy to a local file — **it never writes to MongoDB, blob storage, or the scheduler** (read-only). It is a developer tool and is intentionally not part of the Scope CLI. + +Because both taxonomy tools (`get_run_data` and `get_atif_trajectory`) read through the API, the CLI needs only an API URL and a Copilot token — no storage connection string. This lets it run against any environment (local or a remote deployment such as integration) given a reachable API. + +```bash +# Local stack (use your worktree's host-mapped API port): +SCOPE_MT_API_URL=http://localhost:3001 \ +GITHUB_TOKEN=$(gh auth token) \ +pnpm taxonomy [--run ] [--model ] [--timeout ] [-o out.json] [--stdout] [--quiet] + +# Integration (read-only against the deployed API): +SCOPE_MT_API_URL=https://msscope-int.azurewebsites.net \ +GITHUB_TOKEN=$(gh auth token) \ +pnpm taxonomy -o taxonomy-int.json +``` + +The root `pnpm taxonomy` script proxies to `pnpm --filter taxonomy-handler taxonomy` (i.e. `tsx src/cli.ts`). + +| Option | Description | +| --- | --- | +| `--run, -r ` | Run id used in the prompt/logging (default: active run resolved from the API) | +| `--model ` | Override `TAXONOMY_MODEL` (e.g. A/B a reasoning model vs the `gpt-5.4` default) | +| `--timeout ` | Override `SESSION_TIMEOUT_MS` (per-attempt session timeout) | +| `--output, -o ` | Output file path (default: `taxonomy-.json` in the cwd) | +| `--stdout` | Also print the taxonomy JSON to stdout | +| `--quiet` | Suppress live session-event streaming on stderr | -The `iteration` query parameter is mandatory for the ATIF endpoints. ATIF files are also included in archive exports as `iteration-{N}.atif.trajectory.json`. +Live progress (tool calls, errors, deltas) is written to stderr so the saved JSON file stays clean. The CLI exits non-zero on failure (empty response, JSON/schema validation failure after 3 attempts, timeout, or model rate limit) and surfaces the underlying error message — making it well suited to diagnosing the oversized-`get_atif_trajectory` paging loop and Copilot model rate limits. ## Taxonomy Observations (`pp-taxonomy`) @@ -92,7 +144,11 @@ Gate criteria are always `subject: "iteration"` (the API rejects `"run"` for gat - **Runs exactly once per run:** the scheduler dispatches `pp-taxonomy` a single time (it never sets `message.iteration`), so the run-subject branch needs no idempotency guard. 5. Stamps `run.handlerStatus["pp-taxonomy"] = { status: "done", version: N }` and notifies the scheduler. -Dimension grouping is **derived at read time** by joining each result's `criterionId` → criterion `taxonomyElementId` (the three R&A Readout dimensions; unset = unclassified). It never feeds back to the agent and never affects pass/fail. (#1053 later extends this worker with a richer scorecard artifact.) +Dimension grouping is **derived at read time** by joining each result's +`criterionId` → criterion `taxonomyElementId` (the three R&A Readout +dimensions; unset = unclassified). It never feeds back to the agent and never +affects pass/fail. Experiment readout synthesis consumes these classifications +after every exact-vector dependency is ready. ## Handler Interface @@ -165,7 +221,7 @@ The `AtifHandler` has two complementary test suites: The post-processor uses a version-based re-processing scheme: -- Each handler is registered independently in the `services` collection using a `post-process-handler` document keyed by handler ID (for example `pp-atif`) +- Each handler is registered independently in the `services` collection using a `post-process-handler` document keyed by handler ID (for example `pp-atif`, `pp-taxonomy`) - Each registration stores the handler `version`, queue name, selector, `autoBackfill` behavior, and `dependsOn` DAG edges - The scheduler compares each request's per-handler status/version state (for example `run.handlerStatus["pp-atif"]`) against the registered handler entry for that selector - Requests with missing or outdated handler state are re-dispatched for that specific handler once its dependencies are satisfied @@ -174,17 +230,18 @@ This allows deploying handler improvements and having them automatically applied ## Handler Registration -Each post-process handler declares its DAG metadata in a `handler.yaml` file colocated with the worker (e.g. `apps/workers/post-processor/handler.yaml`). This mirrors the coding-agent `agent.yaml` registration pattern. +Each post-process handler declares its DAG metadata in a `handler.yaml` file colocated with the worker (e.g. `apps/workers/taxonomy/handler.yaml`). This mirrors the coding-agent `agent.yaml` registration pattern. ```yaml -# apps/workers/post-processor/handler.yaml -_id: pp-atif +# apps/workers/taxonomy/handler.yaml +_id: pp-taxonomy type: post-process-handler version: 1 -queue: post-processor-queue -selector: atif -autoBackfill: true -dependsOn: [] +queue: pp-taxonomy-queue +selector: taxonomy +autoBackfill: false +dependsOn: + - pp-atif ``` On deploy, a generic registration script — `packages/shared/src/scripts/register-handler.ts` — reads the `handler.yaml` pointed to by `HANDLER_YAML_PATH`, waits for the scheduler's `/health`, and POSTs the parsed document to `${SCHEDULER_URL}/handlers/register` (with retry). The scheduler validates the document, runs a cycle check against the existing topology, and upserts it into the `services` collection via `HandlerDispatcher.registerHandler`. **Workers never write to the `services` collection directly** — the scheduler is the sole owner. @@ -209,11 +266,33 @@ flowchart LR Experiment handlers use the same `services` registration endpoint but declare `type: experiment-handler` and `dependencyMode: all-runs`. Their dependencies -name run-level DAG handlers. The coordinator projects those dependencies for -the current Experiment's exact request/run vector and dispatches only its -embedded current execution. Built-in readout processing is not part of this -foundation; see [Experiments](experiments.md) for the queue and rollover -contracts. +name run-level DAG handlers. The coordinator projects those dependencies for the current Experiment's exact +request/run vector and dispatches only its embedded current execution. + +The built-in `pp-readout` handler has `dependencyMode: all-runs` and depends on +`pp-observations`. A `done` observation handler is evidence even when the +coding outcome is `failed` or an observation has `passed: false`; failed or +dependency-blocked observations prevent dispatch, while unrelated handler +failures do not. Its queue message is attempt-scoped and includes the exact +`experimentId`, `experimentExecutionId`, execution version, handler version, +and claim token. + +The public surface remains generic: +`GET /api/v1/experiments/:id/executions/:experimentExecutionId/reports` returns +the exact execution's reports newest first. The global report endpoint resolves +the same append-only document, and the nested `/structured` route proxies its +schema-validated JSON artifact. The worker derives Markdown from that JSON and +atomically publishes the execution report pointer with final handler completion. +Internal handler, queue, and schema identifiers retain `readout` because they +identify this synthesis implementation rather than a public report type. + +Experiment-handler logs are persisted under the exact execution and streamed +through +`GET /api/v1/experiments/:id/executions/:experimentExecutionId/logs`. Queue +processors use `experimentExecutionId` as the log attempt key, so superseded +executions remain independently replayable and cannot inherit logs from a newer +current attempt. See [Experiments](experiments.md#built-in-readout) for the +full stale-work, storage, and final compare-and-swap contract. To **add or change a handler**, edit its `handler.yaml` (bump `version`, adjust `dependsOn`, etc.) — no code or migration changes are needed for registration. @@ -233,7 +312,7 @@ active template's trigger and creates the properly-templated `ReportDocument`(s) ```mermaid flowchart LR - RT["run done"] --> DAG["scheduler dispatches DAG
(pp-atif → …)"] + RT["run done"] --> DAG["scheduler dispatches DAG
(pp-atif → pp-taxonomy)"] DAG -->|"all handlers terminal"| DR{DAG drained?} DR -->|yes| TRIG["POST /api/v1/reports/trigger
{ requestId }"] TRIG --> API["API evaluates each template trigger
→ ReportDocument + report-queue {reportId}"] @@ -246,7 +325,7 @@ flowchart LR registered handler is terminal. A run with **zero** registered handlers is vacuously drained, so reports trigger immediately on run completion. -**Trigger regardless of handler success.** A flaky handler must not block report +**Trigger regardless of handler success.** A flaky taxonomy must not block report generation (reports analyze the run/ATIF), so the scheduler triggers reports whenever the DAG drains — not only when every handler succeeded. @@ -290,18 +369,20 @@ stateDiagram-v2 | Environment Variable | Default | Description | | --- | --- | --- | | `AZURE_STORAGE_QUEUE_POSTPROCESSOR` | `post-processor-queue` | Queue name for ATIF post-processor messages | +| `AZURE_STORAGE_QUEUE_TAXONOMY` | `pp-taxonomy-queue` | Queue name for taxonomy-generation jobs | | `SCHEDULER_PP_POLL_INTERVAL_MS` | `30000` | Scheduler backfill polling interval | | `BATCH_SIZE` | `1` | Messages to process per poll (worker-side) | | `POLL_INTERVAL_MS` | `5000` | Worker queue polling interval | -| `SCOPE_MT_API_URL` / `API_BASE_URL` | `http://localhost:3001` | API URL used by report Copilot tools | -| `SESSION_TIMEOUT_MS` | `300000` | Timeout for report Copilot SDK sessions | +| `SCOPE_MT_API_URL` / `API_BASE_URL` | `http://localhost:3001` | API URL used by report/taxonomy Copilot tools | +| `TAXONOMY_MODEL` | `gpt-5.4` | Copilot SDK model used by the taxonomy handler | +| `SESSION_TIMEOUT_MS` | `300000` | Timeout for report/taxonomy Copilot SDK sessions | | `SCHEDULER_URL` | _(unset)_ | Scheduler base URL — used by handlers to notify completion and by the register-handler script to reach `/handlers/register` | | `API_URL` | `http://api:80` | API base URL the **scheduler** uses to call `POST /api/v1/reports/trigger` when a run's handler DAG drains | | `HANDLER_YAML_PATH` | _(unset)_ | Path to a handler's `handler.yaml`, read by the generic `register-handler` script | ## Infrastructure -- **Queues:** Azure Storage Queues (`post-processor-queue` for ATIF, `report-queue` for reports) +- **Queues:** Azure Storage Queues (`post-processor-queue` for ATIF, `pp-taxonomy-queue` for taxonomy, `report-queue` for reports) - **Blob Storage:** `snapshots` container (shared with HAR, tool-calls, and other artifacts) - **Database:** MongoDB `requests` collection (`run.handlerStatus` map keyed by handler ID, with deprecated `run.postProcessorStatus` / `run.postProcessorVersion` compatibility fields during migration) - **KEDA:** ScaledObject scales the worker to zero when queue is empty diff --git a/docs/architecture/prompt-evaluations.md b/docs/architecture/prompt-evaluations.md index 1bdc0636a..4c932d956 100644 --- a/docs/architecture/prompt-evaluations.md +++ b/docs/architecture/prompt-evaluations.md @@ -30,7 +30,7 @@ families: | Prompt-feature extraction | Positive, negative, ambiguous, ancestor/descendant, and empty-feature cases | | Judge instructions | Bundled and independent strategies; file, tool-history, response, conflicting, missing, and prior-pass evidence | | Developer feedback | Default persona, custom persona, descendant guard on/off, and multiple root failures | -| Run reports | Default and appended system prompts; override as a control where the default prompt is intentionally absent | +| Run reports and Experiment readouts | Default and appended run-report system prompts; override as a control where the default prompt is intentionally absent; exact-execution Experiment readout synthesis | Judge and report tool descriptions are static AI-facing instructions, but they are evaluated through their owning end-to-end judge and report families rather diff --git a/evaluations/static-prompts/adapter/adapter.test.ts b/evaluations/static-prompts/adapter/adapter.test.ts index 4cc89ef9f..37e09aa77 100644 --- a/evaluations/static-prompts/adapter/adapter.test.ts +++ b/evaluations/static-prompts/adapter/adapter.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it } from "vitest"; import { REPORT_SYSTEM_PROMPT } from "../../../packages/shared/src/index.js"; +import { READOUT_SYSTEM_PROMPT } from "../../../apps/workers/readout/src/prompt.js"; import { buildCriteriaAuthoringRequest, buildCriteriaDependencySuggestionRequest, @@ -66,7 +67,12 @@ describe("static prompt adapter registry", () => { }, { family: "run-report", - variants: ["default", "append", "override-control"], + variants: [ + "default", + "append", + "override-control", + "experiment-readout", + ], }, ]); expect(listPromptTargets().map(({ family }) => family)).toEqual([ @@ -95,6 +101,11 @@ describe("static prompt adapter registry", () => { family: "run-report", variant: "override-control", }); + expect(targets).toContainEqual({ + adapterId: "run-report/experiment-readout", + family: "run-report", + variant: "experiment-readout", + }); expect(JSON.parse(JSON.stringify(targets))).toEqual(targets); }); }); @@ -126,6 +137,7 @@ describe("API prompt production contracts", () => { expect(result.output).toEqual({ prompt: "Use captured build output.", suggestedId: "builds_cleanly", + suggestedSubject: "run", }); }); @@ -422,4 +434,32 @@ describe("session prompt adapters", () => { ]), ); }); + + it("composes the exact experiment-readout prompt without tools", async () => { + const evidence = { + experimentId: "experiment-1", + experimentExecutionId: "execution-1", + runs: [], + }; + const result = await getPromptTarget( + "run-report", + "experiment-readout", + ).run( + { evidence }, + "experiment-readout", + fakeContext(() => "{}"), + ); + + expect(result.request.messages).toEqual([ + { role: "system", content: READOUT_SYSTEM_PROMPT }, + { + role: "user", + content: [ + "Generate the Experiment readout synthesis for this exact immutable execution evidence vector.", + JSON.stringify(evidence), + ].join("\n\n"), + }, + ]); + expect(result.request.tools).toEqual([]); + }); }); diff --git a/evaluations/static-prompts/adapter/targets/reports.ts b/evaluations/static-prompts/adapter/targets/reports.ts index 696ff31b3..b8d25e28d 100644 --- a/evaluations/static-prompts/adapter/targets/reports.ts +++ b/evaluations/static-prompts/adapter/targets/reports.ts @@ -4,6 +4,10 @@ import type { ReportTemplateSystemPrompt } from "../../../../packages/shared/src/index.js"; import { createReportTools } from "../../../../apps/workers/report-generator/src/tools.js"; import { resolveReportPrompts } from "../../../../apps/workers/report-generator/src/report-queue-processor.js"; +import { + READOUT_SYSTEM_PROMPT, + composeReadoutUserPrompt, +} from "../../../../apps/workers/readout/src/prompt.js"; import type { CompletionOptions, ComposedPromptRequest, @@ -213,6 +217,32 @@ export function composeReportRequest( variant: string, defaultModel: string, ): ComposedPromptRequest { + if (variant === "experiment-readout") { + const value = record(input); + const evidence = record(value.evidence, "input.evidence"); + return { + messages: [ + { role: "system", content: READOUT_SYSTEM_PROMPT }, + { + role: "user", + content: composeReadoutUserPrompt( + evidence as unknown as Parameters[0], + ), + }, + ], + model: + optionalString(value.model, "input.model") ?? defaultModel, + tools: [], + metadata: { + adapterId: "run-report/experiment-readout", + productionSources: [ + "apps/workers/readout/src/prompt.ts#READOUT_SYSTEM_PROMPT", + "apps/workers/readout/src/readout-generator.ts#composeReadoutUserPrompt", + ], + fidelity: "exact", + }, + }; + } return composeParsedReportRequest( parseReportInput(input, variant), variant, @@ -222,8 +252,18 @@ export function composeReportRequest( export const reportAdapter: PromptTargetAdapter = { family: "run-report", - variants: ["default", "append", "override-control"], + variants: ["default", "append", "override-control", "experiment-readout"], async run(input, variant, context) { + if (variant === "experiment-readout") { + const request = composeReportRequest(input, variant, context.model); + const completion = await context.complete(request); + return { + request, + rawResponse: completion.content, + output: completion.content.trim(), + invocationMetadata: completion.metadata, + }; + } const parsed = parseReportInput(input, variant); const request = composeParsedReportRequest(parsed, variant, context.model); const completion = await context.complete( diff --git a/evaluations/static-prompts/datasets/manifest.json b/evaluations/static-prompts/datasets/manifest.json index 06fcc0e6c..0908d6b9b 100644 --- a/evaluations/static-prompts/datasets/manifest.json +++ b/evaluations/static-prompts/datasets/manifest.json @@ -36,7 +36,7 @@ "prompt-feature-extraction": 30, "judge-instructions": 30, "developer-feedback": 25, - "run-report": 30 + "run-report": 31 } }, "files": [ @@ -90,6 +90,14 @@ "run-report" ] }, + { + "path": "v1/readouts.jsonl", + "sha256": "1099b5cf2e076dfbba9e5284ec0e9e2dd326a4f49c6a7475cc655a2efb738886", + "rows": 1, + "families": [ + "run-report" + ] + }, { "path": "v1/task-prompts.jsonl", "sha256": "58c8fd827afddcc197bef8013817d2aea0fd68667bcc0e71bb4ad6055c3aed06", diff --git a/evaluations/static-prompts/datasets/v1/readouts.jsonl b/evaluations/static-prompts/datasets/v1/readouts.jsonl new file mode 100644 index 000000000..36d3dd633 --- /dev/null +++ b/evaluations/static-prompts/datasets/v1/readouts.jsonl @@ -0,0 +1 @@ +{"schemaVersion":1,"id":"run-report-0123456789abcdef","family":"run-report","variant":"experiment-readout","smoke":true,"input":{"evidence":{"experimentId":"experiment-1","experimentExecutionId":"execution-1","runs":[{"request":{"_id":"request-1","projectId":"project-1","workerType":"coder-acp-copilot","profileId":"profile-1","observations":["uses-current-api"],"scenario":{"task":"Update the API","criteria":[]},"createdAt":"2026-10-13T12:00:00.000Z","priority":0},"run":{"_id":"run-1","attemptNumber":1,"status":"done","outcome":"failed","observationResults":[{"criterionId":"uses-current-api","passed":false,"feedback":"The deprecated API remained.","evaluated":true}],"turns":[]},"criteria":[{"id":"uses-current-api","projectId":"project-1","prompt":"Use the current API","kind":"observation","subject":"run","taxonomyElementId":"dimension:dependency-currency","createdAt":"2026-10-13T12:00:00.000Z"}],"trajectories":[]}]}},"expected":{"referenceReport":"{\"title\":\"Experiment readout\",\"executiveSummary\":\"The exact run failed and did not satisfy the selected observation.\",\"profiles\":[{\"profileId\":\"profile-1\",\"requestIds\":[\"request-1\"],\"summary\":\"One profiled run.\"}],\"runs\":[{\"requestId\":\"request-1\",\"runId\":\"run-1\",\"trajectoryHighlights\":[]}],\"findings\":[{\"id\":\"finding-1\",\"title\":\"Deprecated API remained\",\"description\":\"The final run evidence failed the selected observation.\",\"severity\":\"high\",\"evidence\":[{\"requestId\":\"request-1\",\"runId\":\"run-1\",\"criterionId\":\"uses-current-api\",\"excerpt\":\"The deprecated API remained.\"}]}],\"actions\":[{\"id\":\"action-1\",\"title\":\"Update the API usage\",\"priority\":\"high\",\"description\":\"Replace the deprecated API and verify the final state.\",\"relatedFindingIds\":[\"finding-1\"]}]}"},"evaluators":["generation_success","output_schema","non_empty_report","no_fabricated_references","relevance","coherence","groundedness","task_adherence","report_quality"],"tags":["experiment-readout","failed"],"provenance":{"kind":"synthetic-reviewed","sourceEndpoints":[],"projectId":"synthetic-project","sourceEntityIds":["experiment-1","execution-1","request-1","run-1"],"harvestedAt":"2026-10-13T12:00:00.000Z","selectionSeed":"scope-static-prompts-v1","sourceHash":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","note":"Reviewed synthetic exact-vector case covering failed coding outcome and passed:false observation evidence."},"review":{"status":"approved","method":"manual-contract-review-v1"}} diff --git a/evaluations/static-prompts/evaluation-manifest.yaml b/evaluations/static-prompts/evaluation-manifest.yaml index e35c81865..73f5a7f0f 100644 --- a/evaluations/static-prompts/evaluation-manifest.yaml +++ b/evaluations/static-prompts/evaluation-manifest.yaml @@ -19,4 +19,4 @@ qualityFamilies: - id: developer-feedback variants: [default, persona, descendant-guard] - id: run-report - variants: [default, append, override-control] + variants: [default, append, override-control, experiment-readout] diff --git a/evaluations/static-prompts/scripts/dataset-adapter-contract.test.ts b/evaluations/static-prompts/scripts/dataset-adapter-contract.test.ts index 7a8771043..78f9f8278 100644 --- a/evaluations/static-prompts/scripts/dataset-adapter-contract.test.ts +++ b/evaluations/static-prompts/scripts/dataset-adapter-contract.test.ts @@ -57,6 +57,17 @@ function fakeResponse(row: QualityCaseRow, request: ComposedPromptRequest): stri case "developer-feedback": return "Implement the missing requirement and rerun the relevant check."; case "run-report": + if (row.variant === "experiment-readout") { + expect(request.tools).toEqual([]); + return JSON.stringify({ + title: "Experiment readout", + executiveSummary: "The exact vector was reviewed.", + profiles: [], + runs: [], + findings: [], + actions: [], + }); + } expect(request.tools?.length).toBeGreaterThan(0); return "# Run report\n\nThe run evidence was reviewed."; } diff --git a/evaluations/static-prompts/scripts/validate-coverage.test.ts b/evaluations/static-prompts/scripts/validate-coverage.test.ts index 66cd57fca..f525c4e76 100644 --- a/evaluations/static-prompts/scripts/validate-coverage.test.ts +++ b/evaluations/static-prompts/scripts/validate-coverage.test.ts @@ -6,6 +6,6 @@ import { validateCoverage } from "./validate-coverage.js"; it("covers every quality manifest family and variant", async () => { await expect(validateCoverage()).resolves.toEqual({ - qualityTargets: 15, + qualityTargets: 16, }); }); diff --git a/evaluations/static-prompts/scripts/validate-dataset.ts b/evaluations/static-prompts/scripts/validate-dataset.ts index 395c09cb8..45d15a335 100644 --- a/evaluations/static-prompts/scripts/validate-dataset.ts +++ b/evaluations/static-prompts/scripts/validate-dataset.ts @@ -181,9 +181,13 @@ function validateFamilyContract(row: DatasetCase, prefix: string, errors: string assert(typeof input.includeDescendantGuard === "boolean", `${prefix}: input.includeDescendantGuard must be boolean`, errors); break; case "run-report": - assert(stringField(input, "requestId"), `${prefix}: input.requestId is required`, errors); - assert(stringField(input, "reportId"), `${prefix}: input.reportId is required`, errors); - assert(stringField(input, "userPrompt"), `${prefix}: input.userPrompt is required`, errors); + if (row.variant === "experiment-readout") { + assert(isRecord(input.evidence), `${prefix}: input.evidence is required`, errors); + } else { + assert(stringField(input, "requestId"), `${prefix}: input.requestId is required`, errors); + assert(stringField(input, "reportId"), `${prefix}: input.reportId is required`, errors); + assert(stringField(input, "userPrompt"), `${prefix}: input.userPrompt is required`, errors); + } if (row.variant === "append" || row.variant === "override-control") { assert(stringField(input, "systemPromptContent"), `${prefix}: input.systemPromptContent is required for ${row.variant}`, errors); } @@ -345,9 +349,13 @@ export async function validateDataset( assert( syntheticCases.every((row) => row.family === "judge-instructions" - || (row.family === "run-report" && (row.variant === "default" || row.variant === "override-control")) + || (row.family === "run-report" && ( + row.variant === "default" + || row.variant === "override-control" + || row.variant === "experiment-readout" + )) || (row.family === "developer-feedback" && row.variant === "persona")), - "synthetic cases are only permitted for unavailable judge evidence, feedback persona, and report override controls", + "synthetic cases are only permitted for unavailable judge evidence, feedback persona, report override controls, and experiment readouts", errors, ); const hasTag = (family: QualityFamily, tag: string): boolean => diff --git a/package.json b/package.json index 635bb66ae..d12b00825 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "dev:coder-acp-claude-code": "pnpm --filter coder-acp-claude-code dev", "dev:coder-acp-copilot": "pnpm --filter coder-acp-copilot dev", "dev:report-generator": "pnpm --filter report-generator dev", + "dev:readout": "pnpm --filter readout-handler dev", "docker:up": "scripts/worktree-env.sh && scripts/dev-compose.sh up --build", "docker:up:claude-code": "scripts/worktree-env.sh && SCOPE_REGISTER_CLAUDE_CODE_AGENT=true scripts/dev-compose.sh --env-file .env --env-file apps/workers/coder-acp-claude-code/versions.env --profile claude-code --profile report up --build", "docker:up:copilot": "scripts/worktree-env.sh && SCOPE_REGISTER_COPILOT_AGENT=true scripts/dev-compose.sh --env-file .env --env-file apps/workers/coder-acp-copilot/versions.env --profile copilot --profile report up --build", @@ -26,9 +27,9 @@ "docker:up:portal": "scripts/worktree-env.sh && scripts/dev-compose.sh --profile auth --profile portal up --build", "docker:dev": "scripts/worktree-env.sh && COMPOSE_EXPERIMENTAL_WATCH_POLL=1 scripts/dev-compose.sh -f docker-compose.yml -f docker-compose.dev.yml up --build --watch", "docker:dev:claude-code": "scripts/worktree-env.sh && SCOPE_REGISTER_CLAUDE_CODE_AGENT=true GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) COMPOSE_EXPERIMENTAL_WATCH_POLL=1 scripts/dev-compose.sh --env-file .env --env-file apps/workers/coder-acp-claude-code/versions.env -f docker-compose.yml -f docker-compose.dev.yml --profile claude-code --profile report --profile post-processor --profile auth --profile portal up --build --watch", - "docker:dev:copilot": "scripts/worktree-env.sh && SCOPE_REGISTER_COPILOT_AGENT=true GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) COMPOSE_EXPERIMENTAL_WATCH_POLL=1 scripts/dev-compose.sh --env-file .env --env-file apps/workers/coder-acp-copilot/versions.env -f docker-compose.yml -f docker-compose.dev.yml --profile copilot --profile report --profile post-processor --profile auth --profile portal up --build --watch", + "docker:dev:copilot": "scripts/worktree-env.sh && SCOPE_REGISTER_COPILOT_AGENT=true GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) COMPOSE_EXPERIMENTAL_WATCH_POLL=1 scripts/dev-compose.sh --env-file .env --env-file apps/workers/coder-acp-copilot/versions.env -f docker-compose.yml -f docker-compose.dev.yml --profile copilot --profile report --profile post-processor --profile readout --profile auth --profile portal up --build --watch", "docker:dev:portal": "scripts/worktree-env.sh && GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) COMPOSE_EXPERIMENTAL_WATCH_POLL=1 scripts/dev-compose.sh -f docker-compose.yml -f docker-compose.dev.yml --profile auth --profile portal up --build --watch", - "docker:dev:all": "scripts/worktree-env.sh && SCOPE_REGISTER_COPILOT_AGENT=true SCOPE_REGISTER_CLAUDE_CODE_AGENT=true GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) COMPOSE_EXPERIMENTAL_WATCH_POLL=1 scripts/dev-compose.sh --env-file .env --env-file apps/workers/coder-acp-copilot/versions.env --env-file apps/workers/coder-acp-claude-code/versions.env -f docker-compose.yml -f docker-compose.dev.yml --profile claude-code --profile copilot --profile report --profile post-processor --profile auth --profile portal up --build --watch", + "docker:dev:all": "scripts/worktree-env.sh && SCOPE_REGISTER_COPILOT_AGENT=true SCOPE_REGISTER_CLAUDE_CODE_AGENT=true GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) COMPOSE_EXPERIMENTAL_WATCH_POLL=1 scripts/dev-compose.sh --env-file .env --env-file apps/workers/coder-acp-copilot/versions.env --env-file apps/workers/coder-acp-claude-code/versions.env -f docker-compose.yml -f docker-compose.dev.yml --profile claude-code --profile copilot --profile report --profile post-processor --profile readout --profile auth --profile portal up --build --watch", "docker:dev:build": "scripts/worktree-env.sh && GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) scripts/dev-compose.sh --env-file .env --env-file apps/workers/coder-acp-copilot/versions.env --env-file apps/workers/coder-acp-claude-code/versions.env -f docker-compose.yml -f docker-compose.dev.yml --profile claude-code --profile copilot --profile report --profile auth --profile portal build", "docker:down": "scripts/worktree-env.sh && scripts/dev-compose.sh --env-file .env --env-file apps/workers/coder-acp-copilot/versions.env --env-file apps/workers/coder-acp-claude-code/versions.env --profile claude-code --profile copilot --profile report --profile portal down --volumes", "docker:build": "scripts/worktree-env.sh && scripts/dev-compose.sh --env-file .env --env-file apps/workers/coder-acp-copilot/versions.env --env-file apps/workers/coder-acp-claude-code/versions.env --profile claude-code --profile copilot --profile report build", diff --git a/packages/shared/src/experiments/experiment-store.test.ts b/packages/shared/src/experiments/experiment-store.test.ts index fe823a92a..4561cfd01 100644 --- a/packages/shared/src/experiments/experiment-store.test.ts +++ b/packages/shared/src/experiments/experiment-store.test.ts @@ -326,12 +326,35 @@ describe("ExperimentStore CAS contracts", () => { handlerVersion: 1, claimToken: "claim-1", status: "done", + report: { + reportId: "report-1", + url: "/api/v1/reports/report-1", + updatedAt: new Date("2026-01-01T00:01:00.000Z"), + }, }); expect(completed?.execution.handlerSummary.report).toMatchObject({ status: "done", version: 1, claimToken: "claim-1", }); + expect(completed?.execution.report).toEqual({ + reportId: "report-1", + url: "/api/v1/reports/report-1", + updatedAt: new Date("2026-01-01T00:01:00.000Z"), + }); + expect(experiments.current().execution).toMatchObject({ + handlerSummary: { + report: { + status: "done", + version: 1, + claimToken: "claim-1", + }, + }, + report: { + reportId: "report-1", + url: "/api/v1/reports/report-1", + }, + }); }); it("serializes request-state writers against the freezer", async () => { diff --git a/packages/shared/src/experiments/experiment-store.ts b/packages/shared/src/experiments/experiment-store.ts index da21e733d..d209f6dc9 100644 --- a/packages/shared/src/experiments/experiment-store.ts +++ b/packages/shared/src/experiments/experiment-store.ts @@ -23,6 +23,7 @@ import { type ExperimentHandlerStatus, type ExperimentMetadata, type ExperimentPage, + type ExperimentReportReference, type ExperimentRequestState, type ExperimentRequestStateUpdateResult, type ExperimentRevision, @@ -285,6 +286,29 @@ export class ExperimentStore { return raw ? parseHistory(raw) : null; } + async readExecutionRequests(input: { + projectId: string; + experimentId: string; + experimentExecutionId: string; + }): Promise { + const experiment = await this.get(input.projectId, input.experimentId); + const execution = + experiment?.execution.experimentExecutionId === + input.experimentExecutionId + ? experiment.execution + : await this.getExecution(input); + if (!execution) return null; + return this.payloads.readRequests( + { + projectId: input.projectId, + experimentId: input.experimentId, + ownerId: execution.experimentExecutionId, + generation: execution.attemptNumber, + }, + execution.requests, + ); + } + async listExecutionHistory(input: { projectId: string; experimentId: string; @@ -358,6 +382,7 @@ export class ExperimentStore { claimToken: string; status: Extract; error?: string; + report?: ExperimentReportReference; now?: Date; }): Promise { assertHandlerId(input.handlerId); @@ -387,6 +412,7 @@ export class ExperimentStore { }, "execution.updatedAt": now, "execution.finishedAt": now, + ...(input.report ? { "execution.report": input.report } : {}), updatedAt: now, }, $inc: { "execution.version": 1, stateVersion: 1 }, diff --git a/packages/shared/src/experiments/schemas.ts b/packages/shared/src/experiments/schemas.ts index d54676419..16e107895 100644 --- a/packages/shared/src/experiments/schemas.ts +++ b/packages/shared/src/experiments/schemas.ts @@ -121,6 +121,14 @@ const ExperimentExecutionBaseSchema = z message: "handlerSummary is limited to 64 handlers", }), blocking: z.array(ExperimentBlockerSchema).max(32), + report: z + .object({ + reportId: IdSchema, + url: z.string().min(1).max(2_048), + updatedAt: z.coerce.date(), + }) + .strict() + .optional(), createdAt: z.coerce.date(), updatedAt: z.coerce.date(), startedAt: z.coerce.date().optional(), diff --git a/packages/shared/src/experiments/types.ts b/packages/shared/src/experiments/types.ts index 9f9ea1f88..db1be2e7b 100644 --- a/packages/shared/src/experiments/types.ts +++ b/packages/shared/src/experiments/types.ts @@ -109,6 +109,12 @@ export interface ExperimentRequestStateUpdateClaim { claimedAt: Date; } +export interface ExperimentReportReference { + reportId: string; + url: string; + updatedAt: Date; +} + export interface ExperimentExecution { experimentExecutionId: string; attemptNumber: number; @@ -117,6 +123,7 @@ export interface ExperimentExecution { vectorFingerprint: string; handlerSummary: Record; blocking: ExperimentBlocker[]; + report?: ExperimentReportReference; createdAt: Date; updatedAt: Date; startedAt?: Date; diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 94bfa7c85..1b6abd4b1 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -29,5 +29,6 @@ export * from "./har/index.js"; export * from "./utils/index.js"; export * from "./schemas/index.js"; export * from "./auth/index.js"; +export * from "./schemas/readout.js"; export * from "./cursor.js"; export * from "./run-duration.js"; diff --git a/packages/shared/src/logging/log-publisher.test.ts b/packages/shared/src/logging/log-publisher.test.ts index d332ec8ad..bd8808355 100644 --- a/packages/shared/src/logging/log-publisher.test.ts +++ b/packages/shared/src/logging/log-publisher.test.ts @@ -68,6 +68,7 @@ describe("LogPublisher", () => { expect(parsed.message).toBe("hello"); expect(parsed.level).toBe("info"); expect(parsed.source).toBe("test-worker"); + expect(parsed.data.runId).toBe("attempt-1"); }); it("persists the log event via appendLogEvent under the per-attempt path", async () => { diff --git a/packages/shared/src/logging/log-publisher.ts b/packages/shared/src/logging/log-publisher.ts index 88c0dbe68..ef85f9184 100644 --- a/packages/shared/src/logging/log-publisher.ts +++ b/packages/shared/src/logging/log-publisher.ts @@ -87,9 +87,13 @@ export class LogPublisher { // The Redis channel is keyed on requestId so SSE listeners receive logs // for whichever attempt is currently running. const channel = `logs:${requestId}`; + const redisLogEvent: LogEvent = { + ...logEvent, + data: { ...data, runId }, + }; try { await this.redisBreaker.execute(() => - this.redis.publish(channel, JSON.stringify(logEvent)) + this.redis.publish(channel, JSON.stringify(redisLogEvent)) ); } catch (error) { // Silently ignore - circuit breaker handles logging state changes diff --git a/packages/shared/src/queue/base-queue-processor.test.ts b/packages/shared/src/queue/base-queue-processor.test.ts index 00d485be4..b4567994a 100644 --- a/packages/shared/src/queue/base-queue-processor.test.ts +++ b/packages/shared/src/queue/base-queue-processor.test.ts @@ -162,8 +162,10 @@ describe("BaseQueueProcessor visibility heartbeat lifecycle", () => { vi.useRealTimers(); }); - function makeMessage(): DequeuedMessageItem { - const payload = Buffer.from(JSON.stringify({ requestId: "doc-1" })).toString("base64"); + function makeMessage( + messagePayload: Record = { requestId: "doc-1" }, + ): DequeuedMessageItem { + const payload = Buffer.from(JSON.stringify(messagePayload)).toString("base64"); return { messageId: "msg-1", popReceipt: "receipt-0", @@ -171,7 +173,16 @@ describe("BaseQueueProcessor visibility heartbeat lifecycle", () => { } as unknown as DequeuedMessageItem; } - function setupProcessor(handleRequest: (heartbeat: VisibilityHeartbeat) => Promise | void) { + function setupProcessor( + handleRequest: ( + heartbeat: VisibilityHeartbeat, + log: ( + level: LogEvent["level"], + message: string, + data?: Record, + ) => Promise, + ) => Promise | void, + ) { class HBProcessor extends BaseQueueProcessor<{ _id: string }> { public capturedHeartbeat: VisibilityHeartbeat | undefined; public handleRequestCalls = 0; @@ -181,11 +192,24 @@ describe("BaseQueueProcessor visibility heartbeat lifecycle", () => { _doc: { _id: string }, _message: DequeuedMessageItem, heartbeat: VisibilityHeartbeat, + log: ( + level: LogEvent["level"], + message: string, + data?: Record, + ) => Promise, ): Promise { this.capturedHeartbeat = heartbeat; this.popReceiptAtHandleStart = heartbeat.popReceipt; this.handleRequestCalls++; - await handleRequest(heartbeat); + await handleRequest(heartbeat, log); + } + + protected override extractDocumentId( + payload: Record, + ): string { + return typeof payload.experimentId === "string" + ? payload.experimentId + : super.extractDocumentId(payload); } protected override async cleanup(): Promise { @@ -203,11 +227,12 @@ describe("BaseQueueProcessor visibility heartbeat lifecycle", () => { findOne: vi.fn().mockResolvedValue({ _id: "doc-1", run: { _id: "run-1" } }), updateOne: vi.fn().mockResolvedValue({ matchedCount: 1 }), }; - (proc as any).logPublisher = { + const logPublisher = { publish: vi.fn().mockResolvedValue(undefined), evictRun: vi.fn(), }; - return { proc, updateMessage, deleteMessage }; + (proc as any).logPublisher = logPublisher; + return { proc, updateMessage, deleteMessage, logPublisher }; } it("starts the heartbeat before handleRequest runs", async () => { @@ -223,6 +248,26 @@ describe("BaseQueueProcessor visibility heartbeat lifecycle", () => { expect(updateMessage).not.toHaveBeenCalled(); }); + it("persists Experiment worker logs under the exact execution", async () => { + const { proc, logPublisher } = setupProcessor(async (_heartbeat, log) => { + await log("info", "Experiment processing started"); + }); + await (proc as any).processMessage( + makeMessage({ + experimentId: "doc-1", + experimentExecutionId: "experiment-execution-1", + }), + ); + + expect(logPublisher.publish).toHaveBeenCalledWith( + "doc-1", + "experiment-execution-1", + "info", + "Experiment processing started", + undefined, + ); + }); + it("the heartbeat passed to handleRequest exposes the live pop receipt", async () => { let observedAfterTick: string | undefined; const { proc } = setupProcessor(async (heartbeat) => { diff --git a/packages/shared/src/queue/base-queue-processor.ts b/packages/shared/src/queue/base-queue-processor.ts index 08bc35d51..1ec8107cc 100644 --- a/packages/shared/src/queue/base-queue-processor.ts +++ b/packages/shared/src/queue/base-queue-processor.ts @@ -242,8 +242,13 @@ export abstract class BaseQueueProcessor { + it("accepts a one-request readout with empty selected observations", () => { + expect(experimentReadoutSchema.parse({ + meta: { + schemaId: EXPERIMENT_READOUT_SCHEMA_ID, + schemaVersion: EXPERIMENT_READOUT_SCHEMA_VERSION, + generatedAt: "2026-10-13T12:00:00.000Z", + experimentId: "experiment-1", + experimentExecutionId: "execution-1", + }, + summary: { + title: "Experiment readout", + executiveSummary: "The single run completed without selected observations.", + totalRuns: 1, + succeededRuns: 1, + failedRuns: 0, + passedObservations: 0, + failedObservations: 0, + }, + profiles: [{ + requestIds: ["request-1"], + summary: "One unprofiled request.", + }], + runs: [{ + requestId: "request-1", + runId: "run-1", + task: "Update the endpoint", + workerType: "copilot", + status: "done", + outcome: "succeeded", + observations: [], + trajectoryHighlights: [], + }], + findings: [], + actions: [], + })).toBeTruthy(); + }); +}); diff --git a/packages/shared/src/schemas/readout.ts b/packages/shared/src/schemas/readout.ts new file mode 100644 index 000000000..1ddae6780 --- /dev/null +++ b/packages/shared/src/schemas/readout.ts @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { z } from "zod"; + +export const EXPERIMENT_READOUT_SCHEMA_ID = "scope.experiment-readout"; +export const EXPERIMENT_READOUT_SCHEMA_VERSION = "1.0"; + +export const readoutObservationSchema = z.object({ + criterionId: z.string().min(1), + taxonomyElementId: z.string().optional(), + subject: z.enum(["run", "iteration"]), + passed: z.boolean(), + feedback: z.string(), +}); + +export const readoutEvidenceSchema = z.object({ + experimentExecutionId: z.string().min(1), + requestId: z.string().min(1), + runId: z.string().min(1), + iteration: z.number().int().min(0).optional(), + criterionId: z.string().min(1).optional(), + excerpt: z.string().min(1), +}); + +export const readoutRunSchema = z.object({ + requestId: z.string().min(1), + runId: z.string().min(1), + task: z.string(), + workerType: z.string().min(1), + model: z.string().optional(), + profileId: z.string().optional(), + profileVersionId: z.string().optional(), + status: z.string(), + outcome: z.string().optional(), + observations: z.array(readoutObservationSchema), + trajectoryHighlights: z.array(z.string().min(1)), +}); + +export const readoutFindingSchema = z.object({ + id: z.string().min(1), + title: z.string().min(1), + description: z.string().min(1), + severity: z.enum(["critical", "high", "medium", "low", "info"]), + evidence: z.array(readoutEvidenceSchema).min(1), +}); + +export const readoutActionSchema = z.object({ + id: z.string().min(1), + title: z.string().min(1), + priority: z.enum(["critical", "high", "medium", "low"]), + description: z.string().min(1), + relatedFindingIds: z.array(z.string().min(1)), +}); + +export const experimentReadoutSchema = z.object({ + meta: z.object({ + schemaId: z.literal(EXPERIMENT_READOUT_SCHEMA_ID), + schemaVersion: z.literal(EXPERIMENT_READOUT_SCHEMA_VERSION), + generatedAt: z.string().datetime({ offset: true }), + experimentId: z.string().min(1), + experimentExecutionId: z.string().min(1), + }), + summary: z.object({ + title: z.string().min(1), + executiveSummary: z.string().min(1), + totalRuns: z.number().int().min(1), + succeededRuns: z.number().int().min(0), + failedRuns: z.number().int().min(0), + passedObservations: z.number().int().min(0), + failedObservations: z.number().int().min(0), + }), + profiles: z.array(z.object({ + profileId: z.string().optional(), + requestIds: z.array(z.string().min(1)).min(1), + summary: z.string().min(1), + })), + runs: z.array(readoutRunSchema).min(1), + findings: z.array(readoutFindingSchema), + actions: z.array(readoutActionSchema), +}); + +export type ExperimentReadout = z.infer; +export type ReadoutRun = z.infer; diff --git a/packages/shared/src/schemas/report.ts b/packages/shared/src/schemas/report.ts index c9aa697db..7f73a7179 100644 --- a/packages/shared/src/schemas/report.ts +++ b/packages/shared/src/schemas/report.ts @@ -33,13 +33,25 @@ export const ReportStatusSchema = z.enum([ "failed", ]); +export const StructuredReportArtifactSchema = z + .object({ + schemaId: z.string(), + schemaVersion: z.string(), + contentType: z.literal("application/json"), + url: z.string(), + }) + .openapi("StructuredReportArtifact"); + export const ReportResponseSchema = z .object({ _id: z.string(), - requestId: z.string(), + requestId: z.string().optional(), + experimentId: z.string().optional(), + experimentExecutionId: z.string().optional(), templateId: z.string().optional(), reporter: ReporterSchema.optional(), content: z.string().optional(), + structuredArtifact: StructuredReportArtifactSchema.optional(), status: ReportStatusSchema, error: z.string().optional(), insightReferences: z.array(InsightReferenceSchema).optional(), @@ -47,6 +59,16 @@ export const ReportResponseSchema = z updatedAt: z.coerce.date().optional(), projectId: z.string(), }) + .refine( + (report) => + report.requestId !== undefined || + (report.experimentId !== undefined && + report.experimentExecutionId !== undefined), + { + message: + "Report must identify either a request or an exact Experiment execution", + }, + ) .openapi("ReportResponse"); export const CreateReportInputSchema = z diff --git a/packages/shared/src/scripts/register-handler.test.ts b/packages/shared/src/scripts/register-handler.test.ts index 162916404..f1723f5ba 100644 --- a/packages/shared/src/scripts/register-handler.test.ts +++ b/packages/shared/src/scripts/register-handler.test.ts @@ -51,4 +51,18 @@ describe("parseHandlerDocument", () => { it("rejects a non-object input", () => { expect(() => parseHandlerDocument("nope")).toThrow(/mapping/); }); + + it("accepts the pp-readout Experiment handler contract", () => { + const readout = { + _id: "pp-readout", + type: "experiment-handler", + version: 1, + queue: "pp-readout-queue", + selector: "readout", + autoBackfill: false, + dependsOn: ["pp-observations"], + dependencyMode: "all-runs", + }; + expect(parseHandlerDocument(readout)).toEqual(readout); + }); }); diff --git a/packages/shared/src/types/types.ts b/packages/shared/src/types/types.ts index 5d34c7363..28f5b5696 100644 --- a/packages/shared/src/types/types.ts +++ b/packages/shared/src/types/types.ts @@ -858,14 +858,24 @@ export interface Reporter { agentVersion: string; // @github/copilot-sdk package version } +export interface StructuredReportArtifact { + schemaId: string; + schemaVersion: string; + contentType: "application/json"; + url: string; +} + /** Report document stored in MongoDB */ export interface ReportDocument { _id: string; // UUID - requestId: string; // FK → RequestDocument._id + requestId?: string; // FK → RequestDocument._id for run reports + experimentId?: string; // FK → ExperimentDocument._id for Experiment reports + experimentExecutionId?: string; // Exact immutable Experiment execution projectId: string; // FK → ProjectDocument._id (denormalized from request) templateId?: string; // FK → ReportTemplateDocument.id (slug of template that generated this report) reporter?: Reporter; // Set by the worker when it picks up the job content?: string; // Generated markdown report + structuredArtifact?: StructuredReportArtifact; status: ReportStatus; error?: string; insightReferences?: InsightReference[]; // Insights discovered/referenced by this report diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d4a29c36a..75f306a3e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -759,6 +759,34 @@ importers: specifier: ^5.6.2 version: 5.9.3 + apps/workers/readout: + dependencies: + '@azure/storage-queue': + specifier: ^12.25.0 + version: 12.29.0 + '@github/copilot-sdk': + specifier: 1.0.1 + version: 1.0.1(typescript@5.9.3) + dotenv: + specifier: 16.6.1 + version: 16.6.1 + shared: + specifier: workspace:* + version: link:../../../packages/shared + zod: + specifier: ^4.3.6 + version: 4.3.6 + devDependencies: + '@types/node': + specifier: ^22.7.0 + version: 22.19.10 + tsx: + specifier: ^4.19.0 + version: 4.21.0 + typescript: + specifier: ^5.6.2 + version: 5.9.3 + apps/workers/report-generator: dependencies: '@azure/storage-queue': @@ -4782,7 +4810,7 @@ packages: engines: {node: '>=12'} dotenv@16.6.1: - resolution: {integrity: sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==} + resolution: {integrity: sha1-dz8OaVJ6gxXHKF1e5zxEWdIKgCA=, tarball: https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/dotenv/-/dotenv-16.6.1.tgz} engines: {node: '>=12'} dunder-proto@1.0.1: diff --git a/scripts/readout-container.test.ts b/scripts/readout-container.test.ts new file mode 100644 index 000000000..80d07532f --- /dev/null +++ b/scripts/readout-container.test.ts @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { readFile } from "node:fs/promises"; +import { describe, expect, it } from "vitest"; + +const root = new URL("../", import.meta.url); + +async function read(path: string): Promise { + return readFile(new URL(path, root), "utf8"); +} + +describe("readout container configuration", () => { + it("installs the system CA bundle in development and production stages", async () => { + const dockerfile = await read("apps/workers/readout/Dockerfile"); + const installs = dockerfile.match( + /apt-get install -y --no-install-recommends ca-certificates/g, + ); + + expect(installs).toHaveLength(2); + }); + + it("forwards the configured npm registry to readout image builds", async () => { + const compose = await read("docker-compose.yml"); + + for (const service of ["readout-handler", "register-handler-readout"]) { + expect(compose).toMatch( + new RegExp( + `\\n ${service}:\\n[\\s\\S]*?dockerfile: apps/workers/readout/Dockerfile\\n[\\s\\S]*?args:\\n\\s+<<: \\*npm-build-args`, + ), + ); + } + }); + + it("uses the development image for both post-processor registration jobs", async () => { + const compose = await read("docker-compose.dev.yml"); + + for (const service of [ + "register-handler-post-processor", + "register-handler-taxonomy", + ]) { + expect(compose).toMatch( + new RegExp(`\\n ${service}:\\n\\s+build:\\n\\s+target: dev`), + ); + } + }); +}); diff --git a/website/src/content/docs/guides/creating-experiments.mdx b/website/src/content/docs/guides/creating-experiments.mdx index 871788dad..423e5e7fb 100644 --- a/website/src/content/docs/guides/creating-experiments.mdx +++ b/website/src/content/docs/guides/creating-experiments.mdx @@ -35,9 +35,23 @@ Experiment-specific status, revision, attempt, and run-count columns. After creation, the Portal opens Experiment Detail, where you can inspect one processing attempt at a time. The current attempt is selected by default; use the header selector to open a historical attempt, whose URL can be shared -directly. Each selected attempt shows its blockers, handler progress, and a -sortable, customizable, paginated table of exact linked runs. Empty Experiments -remain useful as durable placeholders. +directly. The compact header is followed by three tabs: + +- **Runs** shows the selected attempt's exact linked runs in a sortable, + customizable, paginated table. +- **Reports** shows one report at a time. The latest report is the default. + When retries have created previous versions, use the compact version selector + to open one historical version. A report-type selector appears only when the + Experiment has more than one report type. Structured JSON and the report + detail link remain scoped to the selected attempt and version. +- **Processing** shows handler progress, blockers, errors, execution metadata, + and retry controls. Processing retry is available only for the current + attempt. + +The selected attempt, tab, report type, and report version are preserved in +the URL. Old links without these values continue to use the latest defaults, +and stale selections fall back to available data. Empty Experiments remain +useful as durable placeholders. ## Create from the CLI diff --git a/website/src/openapi/scope-openapi.json b/website/src/openapi/scope-openapi.json index 8dc2e06b4..a59a41a8d 100644 --- a/website/src/openapi/scope-openapi.json +++ b/website/src/openapi/scope-openapi.json @@ -1 +1 @@ -{"openapi":"3.1.0","info":{"title":"Scope API","version":"1.0.0","description":"REST API for the Scope platform — benchmarking AI coding agents"},"servers":[{"url":"/","description":"Current server"}],"components":{"securitySchemes":{"bearerAuth":{"type":"http","scheme":"bearer","bearerFormat":"JWT","description":"Use the unchanged IdP access token, without the Bearer prefix."}},"schemas":{"KeyResponse":{"type":"object","properties":{},"additionalProperties":{}},"KeyInput":{"type":"object","properties":{},"additionalProperties":{}},"ValidateKeyInput":{"type":"object","properties":{"token":{"type":"string"}},"required":["token"]},"AccountResponse":{"type":"object","properties":{},"additionalProperties":{}},"AccountInput":{"type":"object","properties":{},"additionalProperties":{}},"UserMeResponse":{"type":"object","properties":{"id":{"type":"string"},"role":{"type":"string"},"email":{"type":"string"},"displayName":{"type":"string"},"idp":{"type":"string"},"idpTenant":{"type":"string"}},"required":["id"]},"ProjectResponse":{"type":"object","properties":{"_id":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"creator":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"}},"required":["_id","name","createdAt"]},"CreateProjectInput":{"type":"object","properties":{"name":{"type":"string","minLength":1},"description":{"type":"string"},"creator":{"type":"string"}},"required":["name"]},"UpdateProjectInput":{"type":"object","properties":{"name":{"type":"string","minLength":1},"description":{"type":"string"}}},"CreateExperimentResponse":{"type":"object","properties":{"experiment":{"$ref":"#/components/schemas/ExperimentResponse"},"requestIds":{"type":"array","items":{"type":"string"}}},"required":["experiment","requestIds"]},"ExperimentResponse":{"type":"object","properties":{"experimentId":{"type":"string"},"projectId":{"type":"string"},"metadata":{"type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":256},"description":{"type":"string","maxLength":4096},"createdBy":{"type":"string"}},"required":["name"],"additionalProperties":false},"revision":{"type":"object","properties":{"experimentRevisionId":{"type":"string"},"revisionNumber":{"type":"integer"},"memberCount":{"type":"integer"},"createdAt":{"type":["string","null"],"format":"date-time"}},"required":["experimentRevisionId","revisionNumber","memberCount","createdAt"]},"currentExecution":{"type":"object","properties":{"experimentExecutionId":{"type":"string"},"attemptNumber":{"type":"integer"},"readiness":{"type":"string","enum":["empty","waiting","blocked","ready","processing","done","failed"]},"blockers":{"type":"array","items":{"type":"object","properties":{"requestId":{"type":"string"},"runId":{"type":"string"},"handlerId":{"type":"string"},"reason":{"type":"string"},"error":{"type":"string"}},"required":["handlerId","reason"]}},"progress":{"type":"object","additionalProperties":{"type":"object","properties":{"status":{"type":"string","enum":["queued","processing","done","failed"]},"updatedAt":{"type":["string","null"],"format":"date-time"},"error":{"type":"string"}},"required":["status","updatedAt"]}},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"startedAt":{"type":["string","null"],"format":"date-time"},"finishedAt":{"type":["string","null"],"format":"date-time"},"discardedAt":{"type":["string","null"],"format":"date-time"},"discardReason":{"type":"string"},"report":{"type":"object","properties":{"reportId":{"type":"string"},"url":{"type":"string"},"updatedAt":{"type":["string","null"],"format":"date-time"}}}},"required":["experimentExecutionId","attemptNumber","readiness","blockers","progress","createdAt","updatedAt"]},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"}},"required":["experimentId","projectId","metadata","revision","currentExecution","createdAt","updatedAt"]},"CreateExperimentInput":{"type":"object","properties":{"mode":{"type":"string","enum":["compose","empty"]},"metadata":{"type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":256},"description":{"type":"string","maxLength":4096}},"required":["name"],"additionalProperties":false},"requests":{"type":"array","items":{"type":"object","properties":{"scenario":{"$ref":"#/components/schemas/Scenario"},"model":{"type":"string"},"reasoningEffort":{"type":"string"},"maxIterations":{"type":"integer","minimum":1,"maximum":50},"personaInstructions":{"type":"string"},"persona":{"$ref":"#/components/schemas/Persona"},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"codebaseRevisionId":{"type":"string"},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceBindingSpec"}},"extensions":{"type":"array","items":{"type":"string"}},"profileId":{"type":"string"},"priority":{"type":"integer"},"agentsMd":{"type":"string"},"agentsMdParentIds":{"type":"array","items":{"type":"string"}},"gates":{"type":"array","items":{"$ref":"#/components/schemas/GateConfig"}},"observations":{"type":"array","items":{"type":"string"}},"workerType":{"type":"string","minLength":1}},"required":["scenario","workerType"]},"maxItems":250000,"default":[]}},"required":["mode","metadata"]},"Scenario":{"type":"object","properties":{"version":{"type":"string","enum":["v1","v2"]},"task":{"type":"string"},"criteria":{"type":"array","items":{"type":"string"}}},"required":["task","criteria"]},"Persona":{"type":"object","properties":{"personality":{"type":"string","enum":["demanding","friendly"]},"experience":{"type":"string","enum":["junior","senior"]},"verbosity":{"type":"string","enum":["brief","moderate"]},"type":{"type":"string","enum":["traditional","ai_assisted","vibe"]}},"required":["personality","experience","verbosity","type"]},"ResourceBindingSpec":{"anyOf":[{"type":"string"},{"type":"object","properties":{"ref":{"type":"string"},"params":{"type":"object","additionalProperties":{"type":"string"}}},"required":["ref"]}]},"GateConfig":{"type":"object","properties":{"gate":{"type":"string","enum":["select","build","test","run","deploy"]},"promptId":{"type":"string"},"promptText":{"type":"string"},"criteria":{"type":"array","items":{"type":"string"}},"maxIterations":{"type":"integer","minimum":1,"maximum":50}},"required":["gate","criteria"]},"ExperimentListResponse":{"type":"object","properties":{"data":{"type":"array","items":{"$ref":"#/components/schemas/ExperimentResponse"}},"nextCursor":{"type":"string"}},"required":["data"]},"ExperimentExecutionListResponse":{"type":"object","properties":{"data":{"type":"array","items":{"$ref":"#/components/schemas/ExperimentExecutionResponse"}},"nextCursor":{"type":"string"}},"required":["data"]},"ExperimentExecutionResponse":{"type":"object","properties":{"experimentExecutionId":{"type":"string"},"attemptNumber":{"type":"integer"},"readiness":{"type":"string","enum":["empty","waiting","blocked","ready","processing","done","failed"]},"blockers":{"type":"array","items":{"type":"object","properties":{"requestId":{"type":"string"},"runId":{"type":"string"},"handlerId":{"type":"string"},"reason":{"type":"string"},"error":{"type":"string"}},"required":["handlerId","reason"]}},"progress":{"type":"object","additionalProperties":{"type":"object","properties":{"status":{"type":"string","enum":["queued","processing","done","failed"]},"updatedAt":{"type":["string","null"],"format":"date-time"},"error":{"type":"string"}},"required":["status","updatedAt"]}},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"startedAt":{"type":["string","null"],"format":"date-time"},"finishedAt":{"type":["string","null"],"format":"date-time"},"discardedAt":{"type":["string","null"],"format":"date-time"},"discardReason":{"type":"string"},"supersededAt":{"type":["string","null"],"format":"date-time"},"supersededByExperimentExecutionId":{"type":"string"},"supersededReason":{"type":"string","enum":["rollover","retry"]}},"required":["experimentExecutionId","attemptNumber","readiness","blockers","progress","createdAt","updatedAt"]},"RetryExperimentResponse":{"type":"object","properties":{"experimentId":{"type":"string"},"experimentExecutionId":{"type":"string"},"attemptNumber":{"type":"integer"},"readiness":{"type":"string","enum":["empty","waiting","blocked","ready","processing","done","failed"]},"blockers":{"type":"array","items":{"type":"object","properties":{"requestId":{"type":"string"},"runId":{"type":"string"},"handlerId":{"type":"string"},"reason":{"type":"string"},"error":{"type":"string"}},"required":["handlerId","reason"]}}},"required":["experimentId","experimentExecutionId","attemptNumber","readiness","blockers"]},"RequestResponse":{"type":"object","properties":{"_id":{"type":"string"},"scenario":{"$ref":"#/components/schemas/Scenario"},"workerType":{"type":"string"},"model":{"type":"string"},"reasoningEffort":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"maxIterations":{"type":"number"},"personaInstructions":{"type":"string"},"persona":{"$ref":"#/components/schemas/Persona"},"deletedAt":{"type":["string","null"],"format":"date-time"},"taskPromptId":{"type":"string"},"agentsMdPromptId":{"type":"string"},"agentsMdParentIds":{"type":"array","items":{"type":"string"}},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"codebaseRevisionId":{"type":"string"},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceBinding"}},"extensions":{"type":"array","items":{"type":"string"}},"agentVersion":{"type":"string"},"profileId":{"type":"string"},"profileVersionId":{"type":"string"},"submissionId":{"type":"string"},"experimentId":{"type":"string"},"experimentExpectedRequestCount":{"type":"integer","minimum":0},"experimentRequestIndex":{"type":"integer","minimum":0},"experimentInitialRunId":{"type":"string"},"experimentFinalizedAt":{"type":["string","null"],"format":"date-time"},"priority":{"type":"integer","default":0},"gates":{"type":"array","items":{"$ref":"#/components/schemas/GateConfig"}},"observations":{"type":"array","items":{"type":"string"}},"gateSummaries":{"type":"array","items":{"$ref":"#/components/schemas/GateRunSummary"}},"run":{"$ref":"#/components/schemas/RunState"},"projectId":{"type":"string"}},"required":["_id","scenario","workerType","createdAt","projectId"]},"ResourceBinding":{"type":"object","properties":{"ref":{"type":"string"},"revisionId":{"type":"string"},"params":{"type":"object","additionalProperties":{"type":"string"}}},"required":["ref","revisionId","params"]},"GateRunSummary":{"type":"object","properties":{"gate":{"type":"string","enum":["select","build","test","run","deploy"]},"status":{"type":"string","enum":["passed","failed","skipped"]},"iterations":{"type":"integer","minimum":0}},"required":["gate","status","iterations"]},"RunState":{"type":"object","properties":{"_id":{"type":"string"},"attemptNumber":{"type":"integer","minimum":1},"status":{"type":"string","enum":["pending","queued","processing","paused","done"]},"queuedQueueName":{"type":"string"},"outcome":{"type":"string","enum":["succeeded","failed","finished"]},"result":{"type":"string"},"error":{"type":"string"},"logsUrl":{"type":"string"},"updatedAt":{"type":["string","null"],"format":"date-time"},"startedAt":{"type":["string","null"],"format":"date-time"},"finishedAt":{"type":["string","null"],"format":"date-time"},"durationMs":{"type":"number"},"turns":{"type":"array","items":{"$ref":"#/components/schemas/ConversationTurn"}},"workerVersion":{"type":"string"},"os":{"type":"object","properties":{"platform":{"type":"string"},"release":{"type":"string"},"arch":{"type":"string"}},"required":["platform","release","arch"]},"lastHeartbeatAt":{"type":["string","null"],"format":"date-time"},"worker":{"type":"object","properties":{"instanceId":{"type":"string"},"podName":{"type":"string"}},"required":["instanceId"]},"harUrl":{"type":"string"},"videoUrls":{"type":"array","items":{"type":"string"}},"setupVideoUrls":{"type":"array","items":{"type":"string"}},"tokenUsage":{"$ref":"#/components/schemas/TokenUsage"},"aiCallCount":{"type":"number"},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceRunOutcome"}},"mcpRegistered":{"type":"boolean"},"rawChatUrl":{"type":"string"},"rawChatFormat":{"type":"string"},"pausedAt":{"type":["string","null"],"format":"date-time"},"resumedAt":{"type":["string","null"],"format":"date-time"},"observationResults":{"type":"array","items":{"$ref":"#/components/schemas/CriterionResult"}}},"required":["_id","attemptNumber","status"]},"ConversationTurn":{"type":"object","properties":{"iteration":{"type":"number"},"gate":{"type":"string","enum":["select","build","test","run","deploy"]},"codingAgentResponse":{"type":"string"},"judgeFeedback":{"type":"string"},"snapshotUrl":{"type":"string"},"passed":{"type":"boolean"},"timestamp":{"type":["string","null"],"format":"date-time"},"criteriaResults":{"type":"array","items":{"$ref":"#/components/schemas/CriterionResult"}},"observationResults":{"type":"array","items":{"$ref":"#/components/schemas/CriterionResult"}},"harUrl":{"type":"string"},"videoUrls":{"type":"array","items":{"type":"string"}},"tokenUsage":{"$ref":"#/components/schemas/TokenUsage"},"startedAt":{"type":["string","null"],"format":"date-time"},"durationMs":{"type":"number"},"toolCalls":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"arguments":{"type":"object","additionalProperties":{}},"response":{"type":"string"},"timestamp":{"type":"string"}},"required":["id","name","arguments"]}},"toolCallsUrl":{"type":"string"},"toolCallCount":{"type":"number"},"aiCallCount":{"type":"number"},"rawChatUrl":{"type":"string"},"rawChatFormat":{"type":"string"},"chatResultUrl":{"type":"string"},"chatResultFormat":{"type":"string"}},"required":["iteration","judgeFeedback","snapshotUrl","passed","timestamp"]},"CriterionResult":{"type":"object","properties":{"criterionId":{"type":"string"},"passed":{"type":"boolean"},"feedback":{"type":"string"},"evaluated":{"type":"boolean"}},"required":["criterionId","passed","feedback","evaluated"]},"TokenUsage":{"type":"object","properties":{"promptTokens":{"type":"number"},"completionTokens":{"type":"number"},"totalTokens":{"type":"number"}},"required":["promptTokens","completionTokens","totalTokens"]},"ResourceRunOutcome":{"type":"object","properties":{"ref":{"type":"string"},"slug":{"type":"string"},"revisionId":{"type":"string"},"setupSucceeded":{"type":"boolean"},"published":{"type":"array","items":{"type":"string"}},"params":{"type":"object","additionalProperties":{"type":"string"}},"setupDurationMs":{"type":"number"},"error":{"type":"string"},"teardownRan":{"type":"boolean"}},"required":["ref","slug","revisionId","setupSucceeded","published"]},"CreateRequestInput":{"type":"object","properties":{"scenario":{"$ref":"#/components/schemas/Scenario"},"model":{"type":"string"},"reasoningEffort":{"type":"string"},"maxIterations":{"type":"integer","minimum":1,"maximum":50},"personaInstructions":{"type":"string"},"persona":{"$ref":"#/components/schemas/Persona"},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"codebaseRevisionId":{"type":"string"},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceBindingSpec"}},"extensions":{"type":"array","items":{"type":"string"}},"profileId":{"type":"string"},"profileVariations":{"type":"array","items":{"type":"string"},"deprecated":true,"description":"Deprecated for composition. Use POST /api/v1/experiments and its requests array."},"priority":{"type":"integer"},"agentsMd":{"type":"string"},"agentsMdParentIds":{"type":"array","items":{"type":"string"}},"gates":{"type":"array","items":{"$ref":"#/components/schemas/GateConfig"}},"observations":{"type":"array","items":{"type":"string"}}},"required":["scenario"]},"RunFacetsResponse":{"type":"object","properties":{"total":{"type":"number"},"facets":{"type":"object","properties":{"workerType":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"status":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"outcome":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"model":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"os":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"priority":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"agentVersion":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"profileId":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}}},"required":["workerType","status","outcome","model","os","priority","agentVersion","profileId"]}},"required":["total","facets"]},"RunFacetBucket":{"type":"object","properties":{"value":{"type":"string"},"count":{"type":"number"}},"required":["value","count"]},"PaginatedRunsResponse":{"type":"object","properties":{"data":{"type":"array","items":{"$ref":"#/components/schemas/RequestResponse"}},"limit":{"type":"number"},"estimatedTotal":{"type":"number"},"cursors":{"$ref":"#/components/schemas/Cursors"}},"required":["data","limit","estimatedTotal","cursors"]},"Cursors":{"type":"object","properties":{"next":{"type":["string","null"]},"prev":{"type":["string","null"]}},"required":["next","prev"]},"PaginatedRunGroupsResponse":{"type":"object","properties":{"data":{"type":"array","items":{"$ref":"#/components/schemas/RunGroup"}},"limit":{"type":"number"},"estimatedTotal":{"type":"number"},"cursors":{"$ref":"#/components/schemas/Cursors"}},"required":["data","limit","estimatedTotal","cursors"]},"RunGroup":{"type":"object","properties":{"key":{"type":"string"},"label":{"type":"string"},"runIds":{"type":"array","items":{"type":"string"}},"aggregates":{"$ref":"#/components/schemas/GroupAggregates"},"uniform":{"$ref":"#/components/schemas/GroupUniformValues"}},"required":["key","label","runIds","aggregates","uniform"]},"GroupAggregates":{"type":"object","properties":{"count":{"type":"number"},"turns":{"$ref":"#/components/schemas/AggregateStats"},"duration":{"$ref":"#/components/schemas/AggregateStats"},"promptTokens":{"$ref":"#/components/schemas/AggregateStats"},"completionTokens":{"$ref":"#/components/schemas/AggregateStats"},"statusCounts":{"type":"object","additionalProperties":{"type":"number"}},"outcomeCounts":{"type":"object","additionalProperties":{"type":"number"}}},"required":["count","turns","duration","promptTokens","completionTokens","statusCounts","outcomeCounts"]},"AggregateStats":{"type":["object","null"],"properties":{"min":{"type":"number"},"max":{"type":"number"},"mean":{"type":"number"},"stdDev":{"type":"number"}},"required":["min","max","mean","stdDev"]},"GroupUniformValues":{"type":"object","properties":{"workerType":{"type":"string"},"agentVersion":{"type":"string"},"model":{"type":"string"},"platform":{"type":"string"},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"codebaseRevisionId":{"type":"string"},"extensions":{"type":"array","items":{"type":"string"}},"status":{"type":"string","enum":["pending","queued","processing","paused","done"]},"submissionId":{"type":"string"},"experimentId":{"type":"string"},"task":{"type":"string"}}},"BulkResubmitInput":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1},"count":{"type":"integer","minimum":1,"maximum":10,"default":1},"overrides":{"type":"object","properties":{"profileId":{"type":["string","null"]},"workerType":{"type":"string"},"agentVersion":{"type":"string"},"model":{"type":["string","null"]},"reasoningEffort":{"type":["string","null"]},"maxIterations":{"type":["number","null"]},"mcpServers":{"type":["array","null"],"items":{"type":"string"}},"skillRevisions":{"type":["array","null"],"items":{"type":"string"}},"extensions":{"type":["array","null"],"items":{"type":"string"}}}}},"required":["ids"]},"ReportResponse":{"type":"object","properties":{"_id":{"type":"string"},"requestId":{"type":"string"},"templateId":{"type":"string"},"reporter":{"$ref":"#/components/schemas/Reporter"},"content":{"type":"string"},"status":{"type":"string","enum":["pending","generating","completed","failed"]},"error":{"type":"string"},"insightReferences":{"type":"array","items":{"$ref":"#/components/schemas/InsightReference"}},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","requestId","status","createdAt","projectId"]},"Reporter":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"gitHash":{"type":"string"},"model":{"type":"string"},"agentId":{"type":"string"},"agentVersion":{"type":"string"}},"required":["id","name","gitHash","model","agentId","agentVersion"]},"InsightReference":{"type":"object","properties":{"insightId":{"type":"string"},"referencedAt":{"type":["string","null"],"format":"date-time"},"isNew":{"type":"boolean"}},"required":["insightId","referencedAt","isNew"]},"CreateCriteriaInput":{"type":"object","properties":{"id":{"type":"string","pattern":"^[a-z][a-z0-9_]*$"},"prompt":{"type":"string"},"dependsOn":{"type":"array","items":{"type":"string"}},"gates":{"type":"array","items":{"type":"string","enum":["select","build","test","run","deploy"]}},"kind":{"type":"string","enum":["gate","observation"],"default":"gate"},"taxonomyElementId":{"type":"string","enum":["dimension:idiomatic-use","dimension:dependency-currency","dimension:configuration-correctness"]},"subject":{"type":"string","enum":["run","iteration"]}},"required":["id","prompt"]},"CriteriaResponse":{"type":"object","properties":{"id":{"type":"string"},"prompt":{"type":"string"},"dependsOn":{"type":"array","items":{"type":"string"}},"gates":{"type":"array","items":{"type":"string","enum":["select","build","test","run","deploy"]}},"kind":{"type":"string","enum":["gate","observation"]},"taxonomyElementId":{"type":"string","enum":["dimension:idiomatic-use","dimension:dependency-currency","dimension:configuration-correctness"]},"subject":{"type":"string","enum":["run","iteration"]},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["id","prompt","createdAt","projectId"]},"CriteriaGraph":{"type":"object","properties":{"nodes":{"type":"array","items":{"$ref":"#/components/schemas/CriteriaGraphNode"}},"edges":{"type":"array","items":{"$ref":"#/components/schemas/CriteriaGraphEdge"}}},"required":["nodes","edges"]},"CriteriaGraphNode":{"type":"object","properties":{"id":{"type":"string"},"prompt":{"type":"string"},"dependsOn":{"type":"array","items":{"type":"string"}},"gates":{"type":"array","items":{"type":"string","enum":["select","build","test","run","deploy"]}},"kind":{"type":"string","enum":["gate","observation"]},"taxonomyElementId":{"type":"string","enum":["dimension:idiomatic-use","dimension:dependency-currency","dimension:configuration-correctness"]},"subject":{"type":"string","enum":["run","iteration"]}},"required":["id","prompt"]},"CriteriaGraphEdge":{"type":"object","properties":{"from":{"type":"string"},"to":{"type":"string"}},"required":["from","to"]},"UpdateCriteriaInput":{"type":"object","properties":{"prompt":{"type":"string"},"dependsOn":{"type":"array","items":{"type":"string"}},"gates":{"type":"array","items":{"type":"string","enum":["select","build","test","run","deploy"]}},"kind":{"type":"string","enum":["gate","observation"]},"taxonomyElementId":{"type":"string","enum":["dimension:idiomatic-use","dimension:dependency-currency","dimension:configuration-correctness"]},"subject":{"type":"string","enum":["run","iteration"]}}},"CreatePromptFeatureInput":{"type":"object","properties":{"id":{"type":"string","pattern":"^[a-z][a-z0-9_]*$"},"prompt":{"type":"string"},"type":{"$ref":"#/components/schemas/PromptType"}},"required":["id","prompt"]},"PromptType":{"type":"string","enum":["select","build","test","run","deploy","agents.md"]},"PromptFeatureResult":{"type":"object","properties":{"featureId":{"type":"string"},"detected":{"type":"boolean"},"evaluated":{"type":"boolean"}},"required":["featureId","detected","evaluated"]},"SuggestedPromptFeature":{"type":"object","properties":{"suggestedId":{"type":"string"},"behavior":{"type":"string"},"prompt":{"type":"string"}},"required":["suggestedId","behavior","prompt"]},"PromptFeatureResponse":{"type":"object","properties":{"id":{"type":"string"},"prompt":{"type":"string"},"type":{"$ref":"#/components/schemas/PromptType"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["id","prompt","createdAt","projectId"]},"UpdatePromptFeatureInput":{"type":"object","properties":{"prompt":{"type":"string"}}},"TaskPromptResponse":{"type":"object","properties":{"_id":{"type":"string"},"keyId":{"type":"string"},"type":{"$ref":"#/components/schemas/PromptType"},"text":{"type":"string"},"contentBlobUrl":{"type":"string"},"features":{"type":"array","items":{"$ref":"#/components/schemas/PromptFeatureResult"}},"featuresExtractedAt":{"type":["string","null"],"format":"date-time"},"createdAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","keyId","createdAt","projectId"]},"CreateTaskPromptInput":{"type":"object","properties":{"text":{"type":"string"},"type":{"$ref":"#/components/schemas/PromptType"}},"required":["text"]},"PatchTaskPromptFeatureInput":{"type":"object","properties":{"detected":{"type":"boolean"}},"required":["detected"]},"CreateReportInput":{"type":"object","properties":{"requestId":{"type":"string"},"templateId":{"type":"string"}},"required":["requestId"]},"BulkCreateReportsInput":{"type":"object","properties":{"requestIds":{"type":"array","items":{"type":"string"}},"templateId":{"type":"string"}},"required":["requestIds"]},"BulkReportStatusInput":{"type":"object","properties":{"reportIds":{"type":"array","items":{"type":"string"}}},"required":["reportIds"]},"BulkReportSummaryResponse":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/ReportSummary"}},"ReportSummary":{"type":"object","properties":{"total":{"type":"number"},"pending":{"type":"number"},"generating":{"type":"number"},"completed":{"type":"number"},"failed":{"type":"number"}},"required":["total","pending","generating","completed","failed"]},"BulkReportSummaryInput":{"type":"object","properties":{"requestIds":{"type":"array","items":{"type":"string"}}},"required":["requestIds"]},"TriggerReportsInput":{"type":"object","properties":{"requestId":{"type":"string"}},"required":["requestId"]},"BulkTriggerReportsInput":{"type":"object","properties":{"requestIds":{"type":"array","items":{"type":"string"}}},"required":["requestIds"]},"InsightResponse":{"type":"object","properties":{"_id":{"type":"string"},"title":{"type":"string"},"description":{"type":"string"},"category":{"type":"string"},"tags":{"type":"array","items":{"type":"string"}},"upvotes":{"type":"number"},"downvotes":{"type":"number"},"blocked":{"type":"boolean"},"referenceCount":{"type":"number"},"createdBy":{"type":"string","enum":["agent","user"]},"sourceReportId":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","title","description","upvotes","downvotes","blocked","referenceCount","createdBy","createdAt","projectId"]},"ProfileWithVersionResponse":{"type":"object","properties":{"_id":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"latestVersion":{"type":"number"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"version":{"$ref":"#/components/schemas/ProfileVersionResponse"},"projectId":{"type":"string"}},"required":["_id","name","latestVersion","createdAt","version","projectId"]},"ProfileVersionResponse":{"type":"object","properties":{"_id":{"type":"string"},"profileId":{"type":"string"},"version":{"type":"number"},"workerType":{"type":"string"},"model":{"type":"string"},"reasoningEffort":{"type":"string"},"agentVersion":{"type":"string"},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceBindingSpec"}},"extensions":{"type":"array","items":{"type":"string"}},"createdAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","profileId","version","workerType","model","createdAt","projectId"]},"CreateProfileInput":{"type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":128},"description":{"type":"string","maxLength":512},"workerType":{"type":"string"},"model":{"type":"string"},"reasoningEffort":{"type":"string"},"agentVersion":{"type":"string"},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceBindingSpec"}},"extensions":{"type":"array","items":{"type":"string"}}},"required":["name","workerType","model"]},"ProfileResponse":{"type":"object","properties":{"_id":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"latestVersion":{"type":"number"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","name","latestVersion","createdAt","projectId"]},"UpdateProfileIdentity":{"type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":128},"description":{"type":"string","maxLength":512}}},"ReportTemplateResponse":{"type":"object","properties":{"_id":{"type":"string"},"id":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"userPrompt":{"type":"string"},"systemPrompt":{"$ref":"#/components/schemas/ReportTemplateSystemPrompt"},"trigger":{"$ref":"#/components/schemas/ReportTrigger"},"model":{"type":"string"},"timeoutMs":{"type":"integer","exclusiveMinimum":0},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","id","name","userPrompt","createdAt","projectId"]},"ReportTemplateSystemPrompt":{"type":"object","properties":{"mode":{"type":"string","enum":["append","override"]},"content":{"type":"string"}},"required":["mode","content"]},"ReportTrigger":{"oneOf":[{"type":"object","properties":{"type":{"type":"string","enum":["always"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["criteria"]},"criteriaIds":{"type":"array","items":{"type":"string"}},"match":{"type":"string","enum":["any","all"]}},"required":["type","criteriaIds"]},{"type":"object","properties":{"type":{"type":"string","enum":["taskPrompt"]},"taskPromptIds":{"type":"array","items":{"type":"string"}}},"required":["type","taskPromptIds"]},{"type":"object","properties":{"type":{"type":"string","enum":["promptFeature"]},"featureIds":{"type":"array","items":{"type":"string"}},"match":{"type":"string","enum":["any","all"]}},"required":["type","featureIds"]}]},"CreateReportTemplateInput":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"userPrompt":{"type":"string"},"systemPrompt":{"$ref":"#/components/schemas/ReportTemplateSystemPrompt"},"trigger":{"$ref":"#/components/schemas/ReportTrigger"},"model":{"type":"string"},"timeoutMs":{"type":"integer","exclusiveMinimum":0}},"required":["id","name","userPrompt"]},"UpdateReportTemplateInput":{"type":"object","properties":{"name":{"type":"string"},"description":{"type":"string"},"userPrompt":{"type":"string"},"systemPrompt":{"allOf":[{"$ref":"#/components/schemas/ReportTemplateSystemPrompt"},{"type":["object","null"]}]},"trigger":{"$ref":"#/components/schemas/ReportTrigger"},"model":{"type":["string","null"]},"timeoutMs":{"type":["integer","null"],"exclusiveMinimum":0}}},"AgentResponse":{"type":"object","properties":{"_id":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"modelProvider":{"type":"string"},"supportedModels":{"type":"array","items":{"type":"string"}},"defaultModel":{"type":"string"},"available":{"type":"boolean"},"capabilities":{"$ref":"#/components/schemas/AgentCapabilities"},"versions":{"type":"array","items":{"$ref":"#/components/schemas/AgentVersion"}},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"}},"required":["_id","name","supportedModels","createdAt"]},"AgentCapabilities":{"type":"object","properties":{"supportsReasoningEffort":{"type":"boolean"},"supportsMcpServers":{"type":"boolean"},"supportsSkills":{"type":"boolean"},"supportsExtensions":{"type":"boolean"},"supportsResources":{"type":"boolean"}}},"AgentVersion":{"type":"object","properties":{"agentVersion":{"type":"string","minLength":1},"workerVersion":{"type":"string","minLength":1},"components":{"type":"object","additionalProperties":{"type":"string"}},"gitCommit":{"type":"string","minLength":1},"buildTime":{"type":"string","minLength":1},"imageTag":{"type":"string","minLength":1},"queueName":{"type":"string"},"status":{"type":"string","enum":["active","retired"]},"createdAt":{"type":["string","null"],"format":"date-time"}},"required":["agentVersion","workerVersion","components","gitCommit","buildTime","imageTag","status","createdAt"]},"CreateAgentInput":{"type":"object","properties":{"_id":{"type":"string","minLength":1},"name":{"type":"string","minLength":1},"description":{"type":"string"},"modelProvider":{"type":"string"},"supportedModels":{"type":"array","items":{"type":"string"}},"defaultModel":{"type":"string"},"available":{"type":"boolean"},"capabilities":{"$ref":"#/components/schemas/AgentCapabilities"}},"required":["_id","name"]},"UpdateAgentInput":{"type":"object","properties":{"name":{"type":"string"},"description":{"type":"string"},"modelProvider":{"type":"string"},"supportedModels":{"type":"array","items":{"type":"string"}},"defaultModel":{"type":"string"},"available":{"type":"boolean"},"capabilities":{"$ref":"#/components/schemas/AgentCapabilities"}}},"RegisterAgentVersionInput":{"type":"object","properties":{"agentVersion":{"type":"string","minLength":1},"workerVersion":{"type":"string","minLength":1},"components":{"type":"object","additionalProperties":{"type":"string"}},"gitCommit":{"type":"string","minLength":1},"buildTime":{"type":"string","minLength":1},"imageTag":{"type":"string","minLength":1},"queueName":{"type":"string","minLength":1}},"required":["agentVersion","workerVersion","components","gitCommit","buildTime","imageTag","queueName"]},"PatchAgentVersionInput":{"type":"object","properties":{"status":{"type":"string","enum":["active","retired"]}},"required":["status"]},"ModelResponse":{"type":"object","properties":{"_id":{"type":"string"},"modelId":{"type":"string"},"provider":{"type":"string"},"agentId":{"type":"string"},"firstSeenAt":{"type":["string","null"],"format":"date-time"},"lastSeenAt":{"type":["string","null"],"format":"date-time"},"disappearedAt":{"type":["string","null"],"format":"date-time"},"providerAvailableFrom":{"type":["string","null"],"format":"date-time"},"providerEndOfLife":{"type":["string","null"],"format":"date-time"},"metadata":{"type":"object","additionalProperties":{}},"capabilities":{"$ref":"#/components/schemas/ModelCapabilities"}},"required":["_id","modelId","provider","agentId","firstSeenAt","lastSeenAt"]},"ModelCapabilities":{"type":"object","properties":{"reasoningEffort":{"type":"array","items":{"type":"string"}},"toolCalls":{"type":"boolean"},"vision":{"type":"boolean"},"streaming":{"type":"boolean"},"adaptiveThinking":{"type":"boolean"},"maxThinkingBudget":{"type":"number"}}},"McpServerResponse":{"type":"object","properties":{"_id":{"type":"string"},"slug":{"type":"string"},"name":{"type":"string"},"type":{"type":"string","enum":["sse","http","stdio"]},"url":{"type":"string"},"command":{"type":"string"},"args":{"type":"array","items":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"headers":{"type":"array","items":{"$ref":"#/components/schemas/McpServerHeader"}},"sessionMode":{"type":"string","enum":["stateful","stateless"]},"version":{"type":"string"},"description":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","slug","name","type","createdAt","projectId"]},"McpServerHeader":{"type":"object","properties":{"name":{"type":"string"},"value":{"type":"string"}},"required":["name","value"]},"UpdateMcpServerInput":{"type":"object","properties":{"name":{"type":"string"},"type":{"type":"string","enum":["sse","http","stdio"]},"url":{"type":"string"},"command":{"type":"string"},"args":{"type":"array","items":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"headers":{"type":"array","items":{"$ref":"#/components/schemas/McpServerHeader"}},"sessionMode":{"type":"string","enum":["stateful","stateless"]},"version":{"type":"string"},"description":{"type":"string"}}},"SkillResponse":{"type":"object","properties":{"_id":{"type":"string"},"slug":{"type":"string"},"source":{"type":"string"},"skillName":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"origin":{"type":"string","enum":["skills-sh","manual"]},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","slug","source","skillName","name","origin","createdAt","projectId"]},"SkillSearchResult":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"source":{"type":"string"},"description":{"type":"string"},"internal":{"type":"boolean"},"installs":{"type":"number"}},"required":["id","name","source","internal"]},"SkillDiscoveryResult":{"type":"object","properties":{"skillName":{"type":"string"},"skillPath":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"existsInLibrary":{"type":"boolean"},"currentRevisionCommitSha":{"type":"string"},"latestUpstreamCommitSha":{"type":"string"},"updateAvailable":{"type":"boolean"},"lastImportedAt":{"type":"string"}},"required":["skillName","skillPath"]},"SkillRevisionResponse":{"type":"object","properties":{"_id":{"type":"string"},"ref":{"type":"string"},"source":{"type":"string"},"skillName":{"type":"string"},"skillPath":{"type":"string"},"commitHash":{"type":"string"},"commitTimestamp":{"type":["string","null"],"format":"date-time"},"name":{"type":"string"},"description":{"type":"string"},"license":{"type":"string"},"compatibility":{"type":"string"},"allowedTools":{"type":"string"},"metadata":{"type":"object","additionalProperties":{"type":"string"}},"content":{"type":"string"},"archiveUrl":{"type":"string"},"validationWarnings":{"type":"array","items":{"type":"string"}},"resolvedAt":{"type":["string","null"],"format":"date-time"},"createdAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","ref","source","skillName","skillPath","commitHash","commitTimestamp","name","description","content","archiveUrl","resolvedAt","createdAt","projectId"]},"CreateSkillInput":{"type":"object","properties":{"source":{"type":"string"},"skillName":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"origin":{"type":"string","enum":["skills-sh","manual"]}},"required":["source","skillName","name","origin"]},"CodebaseResponse":{"type":"object","properties":{"_id":{"type":"string"},"slug":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"sourceType":{"type":"string","enum":["git","archive"]},"source":{"type":"string"},"defaultBranch":{"type":"string"},"revisionCounter":{"type":"number"},"latestRevisionId":{"type":"string"},"creator":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","slug","name","sourceType","revisionCounter","createdAt","projectId"]},"UpdateCodebaseInput":{"type":"object","properties":{"name":{"type":"string","minLength":1},"description":{"type":"string"},"defaultBranch":{"type":"string"}}},"CodebaseRevisionResponse":{"type":"object","properties":{"_id":{"type":"string"},"codebaseId":{"type":"string"},"slug":{"type":"string"},"revisionNumber":{"type":"number"},"ref":{"type":"string"},"sourceType":{"type":"string","enum":["git","archive"]},"source":{"type":"string"},"requestedRef":{"type":"string"},"resolvedCommitSha":{"type":"string"},"commitTimestamp":{"type":["string","null"],"format":"date-time"},"originalFilename":{"type":"string"},"contentSha256":{"type":"string"},"archiveUrl":{"type":"string"},"sizeBytes":{"type":"number"},"fileCount":{"type":"number"},"creator":{"type":"string"},"resolvedAt":{"type":["string","null"],"format":"date-time"},"createdAt":{"type":["string","null"],"format":"date-time"},"deduplicated":{"type":"boolean"},"projectId":{"type":"string"}},"required":["_id","codebaseId","slug","revisionNumber","ref","sourceType","archiveUrl","resolvedAt","createdAt","projectId"]},"ResolveCodebaseRevisionInput":{"type":"object","properties":{"requestedRef":{"type":"string"},"creator":{"type":"string"}}},"ResourceResponse":{"type":"object","properties":{"_id":{"type":"string"},"slug":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"revisionCounter":{"type":"number"},"latestRevisionId":{"type":"string"},"latestRevisionNumber":{"type":"number"},"creator":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","slug","name","revisionCounter","createdAt","projectId"]},"CreateResourceInput":{"type":"object","properties":{"name":{"type":"string","minLength":1},"slug":{"type":"string","minLength":1},"description":{"type":"string"},"setup":{"$ref":"#/components/schemas/ResourceScript"},"teardown":{"$ref":"#/components/schemas/ResourceScript"},"exports":{"type":"array","items":{"type":"string","pattern":"^[A-Za-z_][A-Za-z0-9_]*$"}},"parameters":{"type":"array","items":{"$ref":"#/components/schemas/ResourceParameter"}},"creator":{"type":"string"}},"required":["name","setup"]},"ResourceScript":{"type":"object","properties":{"sh":{"type":"string","minLength":1}}},"ResourceParameter":{"type":"object","properties":{"name":{"type":"string","pattern":"^[A-Za-z_][A-Za-z0-9_]*$"},"description":{"type":"string"},"required":{"type":"boolean","default":false},"default":{"type":"string"},"example":{"type":"string"}},"required":["name"]},"UpdateResourceInput":{"type":"object","properties":{"name":{"type":"string","minLength":1},"description":{"type":"string"}}},"ResourceRevisionResponse":{"type":"object","properties":{"_id":{"type":"string"},"resourceId":{"type":"string"},"slug":{"type":"string"},"revisionNumber":{"type":"number"},"ref":{"type":"string"},"setup":{"$ref":"#/components/schemas/ResourceScript"},"teardown":{"$ref":"#/components/schemas/ResourceScript"},"exports":{"type":"array","items":{"type":"string"}},"parameters":{"type":"array","items":{"$ref":"#/components/schemas/ResourceParameter"}},"contentSha256":{"type":"string"},"creator":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"deduplicated":{"type":"boolean"},"projectId":{"type":"string"}},"required":["_id","resourceId","slug","revisionNumber","ref","setup","exports","contentSha256","createdAt","projectId"]},"CreateResourceRevisionInput":{"type":"object","properties":{"setup":{"$ref":"#/components/schemas/ResourceScript"},"teardown":{"$ref":"#/components/schemas/ResourceScript"},"exports":{"type":"array","items":{"type":"string","pattern":"^[A-Za-z_][A-Za-z0-9_]*$"}},"parameters":{"type":"array","items":{"$ref":"#/components/schemas/ResourceParameter"}},"creator":{"type":"string"}},"required":["setup"]},"ExtensionResponse":{"type":"object","properties":{"_id":{"type":"string"},"slug":{"type":"string"},"publisher":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"origin":{"type":"string","enum":["marketplace","manual"]},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","slug","publisher","name","origin","createdAt","projectId"]},"ExtensionSearchResult":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"publisher":{"type":"string"},"description":{"type":"string"},"internal":{"type":"boolean"},"version":{"type":"string"}},"required":["id","name","publisher","internal"]},"CreateExtensionInput":{"type":"object","properties":{"_id":{"type":"string","pattern":"^[a-zA-Z0-9-]+\\.[a-zA-Z0-9-]+$"},"publisher":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"origin":{"type":"string","enum":["marketplace","manual"]}},"required":["_id","publisher","name","origin"]},"UpdateExtensionInput":{"type":"object","properties":{"name":{"type":"string"},"description":{"type":"string"}}},"ExtensionVersionInfo":{"type":"object","properties":{"version":{"type":"string"},"preRelease":{"type":"boolean"},"lastUpdated":{"type":"string"}},"required":["version","preRelease","lastUpdated"]},"CreateInsightInput":{"type":"object","properties":{"title":{"type":"string"},"description":{"type":"string"},"category":{"type":"string"},"tags":{"type":"array","items":{"type":"string"}},"createdBy":{"type":"string","enum":["agent","user"]},"sourceReportId":{"type":"string"}},"required":["title","description"]},"UpdateInsightInput":{"type":"object","properties":{"title":{"type":"string"},"description":{"type":"string"},"category":{"type":"string"},"tags":{"type":"array","items":{"type":"string"}}}},"FeatureFlagResponse":{"type":"object","properties":{"key":{"type":"string"},"label":{"type":"string"},"enabled":{"type":"boolean"},"updatedAt":{"type":["string","null"],"format":"date-time"}},"required":["key","label","enabled","updatedAt"]},"UpdateFeatureFlagInput":{"type":"object","properties":{"enabled":{"type":"boolean"}},"required":["enabled"]}},"parameters":{}},"paths":{"/api/v1/keys/preview":{"post":{"tags":["Keys"],"summary":"Preview key","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyInput"}}}},"responses":{"200":{"description":"Key preview","content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyResponse"}}}}}}},"/api/v1/keys":{"post":{"tags":["Keys"],"summary":"Create key","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyInput"}}}},"responses":{"201":{"description":"Key created","content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyResponse"}}}}}},"get":{"tags":["Keys"],"summary":"List keys","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/KeyResponse"}}}}}}}},"/api/v1/keys/{id}":{"get":{"tags":["Keys"],"summary":"Get key","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyResponse"}}}}}},"put":{"tags":["Keys"],"summary":"Update key","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyInput"}}}},"responses":{"200":{"description":"Key updated","content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyResponse"}}}}}},"delete":{"tags":["Keys"],"summary":"Delete key","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Key deleted","content":{"application/json":{"schema":{"type":"object","properties":{"success":{"type":"boolean"}},"required":["success"]}}}}}}},"/api/v1/keys/{id}/validate":{"post":{"tags":["Keys"],"summary":"Validate key","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ValidateKeyInput"}}}},"responses":{"200":{"description":"Validation result","content":{"application/json":{"schema":{"type":"object","properties":{},"additionalProperties":{},"description":"Key validation result"}}}}}}},"/api/v1/accounts":{"post":{"tags":["Accounts"],"summary":"Create account","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/AccountInput"}}}},"responses":{"201":{"description":"Account created","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AccountResponse"}}}}}},"get":{"tags":["Accounts"],"summary":"List accounts","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/AccountResponse"}}}}}}}},"/api/v1/accounts/{id}":{"get":{"tags":["Accounts"],"summary":"Get account","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AccountResponse"}}}}}},"put":{"tags":["Accounts"],"summary":"Update account","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/AccountInput"}}}},"responses":{"200":{"description":"Account updated","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AccountResponse"}}}}}},"delete":{"tags":["Accounts"],"summary":"Delete account","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Account deleted","content":{"application/json":{"schema":{"type":"object","properties":{"success":{"type":"boolean"}},"required":["success"]}}}}}}},"/api/v1/users/me":{"get":{"tags":["Users"],"summary":"Get the authenticated user's identity","description":"Read the existing authenticated Scope identity without enrollment or profile writes. Responses must not be HTTP-cached.","security":[{"bearerAuth":[]}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/UserMeResponse"}}}},"401":{"description":"Not authenticated"},"403":{"description":"User is not enrolled or is disabled"},"503":{"description":"Authentication service unavailable"}}},"post":{"tags":["Users"],"summary":"Enroll the authenticated user","description":"JIT-enroll the authenticated user, refresh their profile and lastLoginAt, apply bootstrap-admin rules, and warm the access cache. Use this only after an explicit IdP login; do not prefetch, poll, or automatically retry transient failures.","security":[{"bearerAuth":[]}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/UserMeResponse"}}}},"401":{"description":"Not authenticated"},"403":{"description":"User is disabled"},"503":{"description":"Authentication service unavailable"}}}},"/health":{"get":{"tags":["Health"],"summary":"Liveness probe","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"status":{"type":"string"},"version":{"type":"string"}},"required":["status","version"]}}}}}}},"/ready":{"get":{"tags":["Health"],"summary":"Readiness probe","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"status":{"type":"string"},"migrations":{}},"required":["status"]}}}},"503":{"description":"Service is not ready"}}}},"/about":{"get":{"tags":["System"],"summary":"API metadata","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"name":{"type":"string"},"version":{"type":"string"},"buildTime":{"type":"string"},"environment":{"type":"string"},"description":{"type":"string"},"workers":{"type":"array","items":{"type":"string"}}},"required":["name","version","buildTime","environment","description","workers"]}}}}}}},"/api/v1/version":{"get":{"tags":["System"],"summary":"Version info","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"commit":{"type":"string"},"buildTime":{"type":"string"},"environment":{"type":"string"},"strictAgentCapabilities":{"type":"boolean"}},"required":["commit","buildTime","environment","strictAgentCapabilities"]}}}}}}},"/api/v1/projects":{"get":{"tags":["Projects"],"summary":"List all projects","parameters":[{"schema":{"type":"string","enum":["true","false"],"description":"When true, include soft-deleted projects in the result"},"required":false,"description":"When true, include soft-deleted projects in the result","name":"includeDeleted","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ProjectResponse"}}}}}}},"post":{"tags":["Projects"],"summary":"Create a project","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateProjectInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProjectResponse"}}}},"400":{"description":"Invalid input"}}}},"/api/v1/projects/{id}":{"get":{"tags":["Projects"],"summary":"Get a project","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProjectResponse"}}}},"404":{"description":"Project not found"}}},"patch":{"tags":["Projects"],"summary":"Update a project","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateProjectInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProjectResponse"}}}},"404":{"description":"Project not found"}}},"delete":{"tags":["Projects"],"summary":"Soft-delete a project","responses":{"204":{"description":"Success"},"404":{"description":"Project not found"}}}},"/api/v1/projects/{id}/restore":{"post":{"tags":["Projects"],"summary":"Restore a soft-deleted project","responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProjectResponse"}}}},"404":{"description":"Project not found"}}}},"/api/v1/experiments":{"post":{"tags":["Experiments"],"summary":"Create an experiment and zero or more new requests","description":"Canonical composition endpoint. Request and run IDs are minted before insertion; the exact revision/execution is then finalized idempotently.","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateExperimentInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateExperimentResponse"}}}},"409":{"description":"Experiment identity conflict"}}},"get":{"tags":["Experiments"],"summary":"List experiments","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"},{"schema":{"type":"integer","minimum":1,"maximum":100,"default":50},"required":false,"name":"limit","in":"query"},{"schema":{"type":"string","minLength":1},"required":false,"name":"after","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ExperimentListResponse"}}}}}}},"/api/v1/experiments/{id}":{"get":{"tags":["Experiments"],"summary":"Get an experiment","parameters":[{"schema":{"type":"string","minLength":1},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ExperimentResponse"}}}},"404":{"description":"Experiment not found"}}}},"/api/v1/experiments/{id}/executions":{"get":{"tags":["Experiments"],"summary":"List current and historical experiment executions","parameters":[{"schema":{"type":"string","minLength":1},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"},{"schema":{"type":"integer","minimum":1,"maximum":100,"default":50},"required":false,"name":"limit","in":"query"},{"schema":{"type":"string","minLength":1},"required":false,"name":"after","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ExperimentExecutionListResponse"}}}},"404":{"description":"Experiment not found"}}}},"/api/v1/experiments/{id}/executions/{experimentExecutionId}":{"get":{"tags":["Experiments"],"summary":"Get one exact experiment execution","parameters":[{"schema":{"type":"string","minLength":1},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1},"required":true,"name":"experimentExecutionId","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ExperimentExecutionResponse"}}}},"404":{"description":"Experiment execution not found"}}}},"/api/v1/experiments/{id}/retry":{"post":{"tags":["Experiments"],"summary":"Retry current experiment processing","parameters":[{"schema":{"type":"string","minLength":1},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"experimentExecutionId":{"type":"string","minLength":1}},"required":["experimentExecutionId"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/RetryExperimentResponse"}}}},"404":{"description":"Experiment not found"},"409":{"description":"Execution is stale or cannot be retried"}}}},"/api/v1/requests":{"post":{"tags":["Requests"],"summary":"Submit request(s)","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"allOf":[{"$ref":"#/components/schemas/CreateRequestInput"},{"type":"object","properties":{"count":{"type":"number","minimum":1,"maximum":10,"default":1,"deprecated":true,"description":"Deprecated for composition. Use POST /api/v1/experiments and its requests array."},"skills":{"type":"array","items":{"type":"string"}},"agentVersion":{"type":"string"},"codebase":{"type":"string"}}}]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"anyOf":[{"$ref":"#/components/schemas/RequestResponse"},{"type":"array","items":{"$ref":"#/components/schemas/RequestResponse"}}]}}}}}},"get":{"tags":["Requests"],"summary":"List requests","parameters":[{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"worker","in":"query"},{"schema":{"type":"string"},"required":false,"name":"taskPromptId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"criteria","in":"query"},{"schema":{"type":"string"},"required":false,"name":"submissionId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"experimentId","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"profileId","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"status","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"outcome","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"model","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"os","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"priority","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"agentVersion","in":"query"},{"schema":{"type":"string"},"required":false,"name":"search","in":"query"},{"schema":{"type":"string","enum":["task","submissionId","experimentId","profile"]},"required":false,"name":"groupBy","in":"query"},{"schema":{"type":"integer","minimum":1,"maximum":100},"required":false,"name":"limit","in":"query"},{"schema":{"type":"string"},"required":false,"name":"after","in":"query"},{"schema":{"type":"string"},"required":false,"name":"before","in":"query"},{"schema":{"type":"string","enum":["true","false"]},"required":false,"name":"last","in":"query"},{"schema":{"type":"string","enum":["created","updated","priority","worker","status","id","duration","createdAt"]},"required":false,"name":"sortBy","in":"query"},{"schema":{"type":"string","enum":["asc","desc"]},"required":false,"name":"sortDir","in":"query"},{"schema":{"type":["string","null"],"format":"date-time"},"required":false,"name":"createdAfter","in":"query"},{"schema":{"type":["string","null"],"format":"date-time"},"required":false,"name":"createdBefore","in":"query"},{"schema":{"type":["integer","null"],"minimum":0},"required":false,"name":"turns","in":"query"},{"schema":{"type":"string","enum":["eq","gte","lte"]},"required":false,"name":"turnsOp","in":"query"},{"schema":{"type":["integer","null"],"minimum":0},"required":false,"name":"maxIterations","in":"query"},{"schema":{"type":"string","enum":["eq","gte","lte"]},"required":false,"name":"maxIterationsOp","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"anyOf":[{"$ref":"#/components/schemas/PaginatedRunsResponse"},{"$ref":"#/components/schemas/PaginatedRunGroupsResponse"}]}}}}}}},"/api/v1/requests/facets":{"get":{"tags":["Requests"],"summary":"List run filter facets","description":"Returns every distinct value and its full-dataset count per categorical filter dimension for the Runs list rail. Counts are absolute over all non-deleted runs **in the given project**: they intentionally ignore the active search, date range, iteration, and categorical selections so every selectable value stays visible with a stable count. Requires ?projectId=. Computed with parallel $group aggregations (Cosmos has no $facet) and cached per-project for a short TTL.","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/RunFacetsResponse"}}}}}}},"/api/v1/requests/{id}":{"get":{"tags":["Requests"],"summary":"Get request","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/RequestResponse"}}}},"404":{"description":"Not found"}}},"delete":{"tags":["Requests"],"summary":"Soft-delete request","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}},"404":{"description":"Not found"}}}},"/api/v1/analysis":{"get":{"tags":["Requests"],"summary":"Compute pass@k / success@T metrics","description":"Aggregates pass@k / success@T metrics over a single project's done runs. Requires ?projectId=.","parameters":[{"schema":{"type":"string"},"required":false,"name":"worker","in":"query"},{"schema":{"type":"string"},"required":false,"name":"taskPromptId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"criteria","in":"query"},{"schema":{"type":"string"},"required":false,"name":"features","in":"query"},{"schema":{"type":"string"},"required":false,"name":"submissionId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"experimentId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"k","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{},"additionalProperties":{},"description":"Analysis metrics"}}}}}}},"/api/v1/requests/bulk-resubmit":{"post":{"tags":["Requests"],"summary":"Bulk resubmit requests","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/BulkResubmitInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/RequestResponse"}}}}}}}},"/api/v1/requests/bulk":{"delete":{"tags":["Requests"],"summary":"Bulk soft-delete requests","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"}}},"required":["ids"]}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"deleted":{"type":"number"}},"required":["deleted"]}}}}}}},"/api/v1/requests/archive":{"post":{"tags":["Requests"],"summary":"Download batch archive of multiple runs","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1,"maxItems":100}},"required":["ids"]}}}},"responses":{"201":{"description":"Gzipped batch archive containing individual run archives"},"400":{"description":"Invalid input"},"404":{"description":"One or more runs not found"}}}},"/api/v1/runs/upload":{"post":{"tags":["Requests"],"summary":"Import run archive","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"201":{"description":"Success"}}}},"/api/v1/runs/upload-batch":{"post":{"tags":["Requests"],"summary":"Import batch run archive","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"207":{"description":"Multi-status: 201 if all runs imported, 400 if none imported, 207 if partial"},"400":{"description":"No archive uploaded, empty archive, or all runs failed"}}}},"/api/v1/requests/{id}/reports":{"get":{"tags":["Reports"],"summary":"Get reports for request","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ReportResponse"}}}}},"404":{"description":"Run not found"}}}},"/api/v1/requests/{id}/runs":{"get":{"tags":["Requests"],"summary":"List attempts for a request","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/RunState"}}}}},"404":{"description":"Request not found"}}}},"/api/v1/requests/{id}/runs/{runId}":{"get":{"tags":["Requests"],"summary":"Get a single attempt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/RunState"}}}},"404":{"description":"Request or run not found"}}}},"/api/v1/requests/bulk-retry":{"post":{"tags":["Requests"],"summary":"Bulk retry requests (start new attempts)","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1},"force":{"type":"boolean"}},"required":["ids"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"retried":{"type":"integer"},"skipped":{"type":"integer"},"results":{"type":"array","items":{"type":"object","properties":{"requestId":{"type":"string"},"runId":{"type":"string"},"attemptNumber":{"type":"integer"},"error":{"type":"string"}},"required":["requestId"]}}},"required":["retried","skipped","results"]}}}}}}},"/api/v1/requests/{id}/retry":{"post":{"tags":["Requests"],"summary":"Retry a request (start a new attempt)","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"force":{"type":"boolean"}}}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"requestId":{"type":"string"},"runId":{"type":"string"},"attemptNumber":{"type":"integer"}},"required":["requestId","runId","attemptNumber"]}}}},"404":{"description":"Request not found"},"409":{"description":"Conflict — request is not in a retryable state, or a concurrent retry won the race"},"422":{"description":"Cannot retry — current run not yet terminal"}}}},"/api/v1/requests/{id}/priority":{"post":{"tags":["Requests"],"summary":"Set priority on a single request","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"priority":{"type":"integer"}},"required":["priority"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"priority":{"type":"number"}},"required":["id","priority"]}}}}}}},"/api/v1/requests/bulk-priority":{"post":{"tags":["Requests"],"summary":"Set priority on multiple requests","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1,"maxItems":100},"priority":{"type":"integer"}},"required":["ids","priority"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"updated":{"type":"number"},"skipped":{"type":"number"}},"required":["updated","skipped"]}}}}}}},"/api/v1/requests/{id}/pause":{"post":{"tags":["Requests"],"summary":"Pause a single request","responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"status":{"type":"string"}},"required":["id","status"]}}}}}}},"/api/v1/requests/{id}/resume":{"post":{"tags":["Requests"],"summary":"Resume a paused request","responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"status":{"type":"string"}},"required":["id","status"]}}}}}}},"/api/v1/requests/bulk-pause":{"post":{"tags":["Requests"],"summary":"Pause multiple requests","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1,"maxItems":100}},"required":["ids"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"updated":{"type":"number"},"skipped":{"type":"number"}},"required":["updated","skipped"]}}}}}}},"/api/v1/requests/bulk-resume":{"post":{"tags":["Requests"],"summary":"Resume multiple paused requests","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1,"maxItems":100}},"required":["ids"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"updated":{"type":"number"},"skipped":{"type":"number"}},"required":["updated","skipped"]}}}}}}},"/api/v1/requests/{id}/cancel":{"post":{"tags":["Requests"],"summary":"Cancel a request (marks as done/failed and signals worker to exit)","responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"previousStatus":{"type":"string"},"status":{"type":"string"},"outcome":{"type":"string"}},"required":["id","previousStatus","status","outcome"]}}}}}}},"/api/v1/requests/bulk-cancel":{"post":{"tags":["Requests"],"summary":"Cancel multiple requests","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1,"maxItems":100}},"required":["ids"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"cancelled":{"type":"number"},"skipped":{"type":"number"},"results":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string"},"cancelled":{"type":"boolean"},"previousStatus":{"type":"string"},"error":{"type":"string"}},"required":["id","cancelled"]}}},"required":["cancelled","skipped","results"]}}}}}}},"/api/v1/requests/{id}/logs":{"get":{"tags":["Requests"],"summary":"Stream request logs (SSE)","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Server-sent event stream of log entries"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/logs":{"get":{"tags":["Requests"],"summary":"Stream logs for a specific attempt (SSE)","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"}],"responses":{"200":{"description":"Server-sent event stream of log entries"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/har":{"get":{"tags":["Requests"],"summary":"Download HAR file","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"HAR-format JSON file"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/har":{"get":{"tags":["Requests"],"summary":"Download HAR file for a specific attempt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"}],"responses":{"200":{"description":"HAR-format JSON file"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/atif":{"get":{"tags":["Requests"],"summary":"Download ATIF trajectory file for a specific iteration","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","description":"The iteration number (1-based)"},"required":true,"description":"The iteration number (1-based)","name":"iteration","in":"query"}],"responses":{"200":{"description":"ATIF v1.7 trajectory JSON file"},"400":{"description":"Missing or invalid iteration"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/atif":{"get":{"tags":["Requests"],"summary":"Download ATIF trajectory file for a specific attempt and iteration","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"},{"schema":{"type":"string","description":"The iteration number (1-based)"},"required":true,"description":"The iteration number (1-based)","name":"iteration","in":"query"}],"responses":{"200":{"description":"ATIF v1.7 trajectory JSON file"},"400":{"description":"Missing or invalid iteration"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/video":{"get":{"tags":["Requests"],"summary":"Download session recording","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"WebM video recording (supports Range requests)"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/video":{"get":{"tags":["Requests"],"summary":"Download session recording for a specific attempt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"}],"responses":{"200":{"description":"WebM video recording (supports Range requests)"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/tool-calls":{"get":{"tags":["Requests"],"summary":"Download per-iteration tool-calls JSONL","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"JSONL stream — one ToolCall per line"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/tool-calls":{"get":{"tags":["Requests"],"summary":"Download per-iteration tool-calls JSONL for a specific attempt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"}],"responses":{"200":{"description":"JSONL stream — one ToolCall per line"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/snapshots/{iteration}":{"get":{"tags":["Requests"],"summary":"Download iteration snapshot","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"iteration","in":"path"}],"responses":{"200":{"description":"Gzipped snapshot archive"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/snapshots/{iteration}":{"get":{"tags":["Requests"],"summary":"Download iteration snapshot for a specific attempt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"},{"schema":{"type":"string"},"required":true,"name":"iteration","in":"path"}],"responses":{"200":{"description":"Gzipped snapshot archive"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/archive":{"get":{"tags":["Requests"],"summary":"Download full run archive","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Gzipped run archive"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/archive":{"get":{"tags":["Requests"],"summary":"Download full run archive for a specific attempt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"}],"responses":{"200":{"description":"Gzipped run archive"},"404":{"description":"Not found"}}}},"/api/v1/criteria/generate-prompt":{"post":{"tags":["Criteria"],"summary":"Generate criterion prompt from behavior","parameters":[{"schema":{"type":"string","minLength":1,"description":"Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.","example":"00000000-0000-0000-0000-000000000000"},"required":false,"description":"Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"behavior":{"type":"string"},"currentId":{"type":"string"},"gates":{"type":"array","items":{"type":"string"}}},"required":["behavior"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"prompt":{"type":"string"},"suggestedId":{"type":"string"},"suggestedParents":{"type":"array","items":{"type":"string"}},"suggestedChildren":{"type":"array","items":{"type":"string"}}},"required":["prompt","suggestedId","suggestedParents","suggestedChildren"]}}}},"400":{"description":"Empty behavior string"},"503":{"description":"LLM not configured"}}}},"/api/v1/criteria/seed":{"post":{"tags":["Criteria"],"summary":"Seed criteria in bulk","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"criteria":{"type":"array","items":{"$ref":"#/components/schemas/CreateCriteriaInput"}}},"required":["criteria"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"seeded":{"type":"number"},"errors":{"type":"array","items":{"type":"string"}}},"required":["seeded","errors"]}}}},"400":{"description":"Seed batch would introduce a dependency cycle"}}}},"/api/v1/criteria":{"get":{"tags":["Criteria"],"summary":"List criteria","parameters":[{"schema":{"type":"string"},"required":false,"name":"q","in":"query"},{"schema":{"type":"string","description":"Comma-separated criterion IDs to include"},"required":false,"description":"Comma-separated criterion IDs to include","name":"ids","in":"query"},{"schema":{"type":"string","enum":["true","false"],"description":"When true and ids is set, also include dependency ancestors"},"required":false,"description":"When true and ids is set, also include dependency ancestors","name":"ancestors","in":"query"},{"schema":{"type":"string","enum":["gate","observation"],"description":"Filter by criterion kind"},"required":false,"description":"Filter by criterion kind","name":"kind","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/CriteriaResponse"}}}}}}},"post":{"tags":["Criteria"],"summary":"Create criterion","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateCriteriaInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CriteriaResponse"}}}},"409":{"description":"Criterion already exists"}}}},"/api/v1/criteria/mdp":{"get":{"tags":["Criteria"],"summary":"Compute MDP transitions","parameters":[{"schema":{"type":"string"},"required":false,"name":"criteria","in":"query"},{"schema":{"type":"string"},"required":false,"name":"features","in":"query"},{"schema":{"type":"string"},"required":false,"name":"since","in":"query"},{"schema":{"type":"string"},"required":false,"name":"worker","in":"query"},{"schema":{"type":"string"},"required":false,"name":"taskPromptId","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{},"additionalProperties":{}}}}}}}},"/api/v1/criteria/graph":{"get":{"tags":["Criteria"],"summary":"Get criteria DAG","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CriteriaGraph"}}}}}}},"/api/v1/criteria/{id}":{"get":{"tags":["Criteria"],"summary":"Get criterion","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CriteriaResponse"}}}},"404":{"description":"Criterion not found"}}},"put":{"tags":["Criteria"],"summary":"Update criterion","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateCriteriaInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CriteriaResponse"}}}},"400":{"description":"Invalid dependency reference or self-reference"},"404":{"description":"Criterion not found"}}},"delete":{"tags":["Criteria"],"summary":"Soft-delete criterion","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"deleted":{"type":"boolean"}},"required":["id","deleted"]}}}},"404":{"description":"Criterion not found"},"409":{"description":"Criterion has dependents"}}}},"/api/v1/prompt-features/generate-prompt":{"post":{"tags":["Prompt Features"],"summary":"Generate prompt feature from behavior","parameters":[{"schema":{"type":"string","minLength":1,"description":"Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.","example":"00000000-0000-0000-0000-000000000000"},"required":false,"description":"Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"behavior":{"type":"string"},"currentId":{"type":"string"}},"required":["behavior"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"prompt":{"type":"string"}},"required":["prompt"]}}}},"400":{"description":"Empty behavior string"},"503":{"description":"LLM not configured"}}}},"/api/v1/prompt-features/seed":{"post":{"tags":["Prompt Features"],"summary":"Seed prompt features in bulk","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"features":{"type":"array","items":{"$ref":"#/components/schemas/CreatePromptFeatureInput"}}},"required":["features"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"seeded":{"type":"number"},"errors":{"type":"array","items":{"type":"string"}}},"required":["seeded","errors"]}}}}}}},"/api/v1/prompt-features/extract-from-text":{"post":{"tags":["Prompt Features"],"summary":"Extract features from text","parameters":[{"schema":{"type":"string","minLength":1,"description":"Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.","example":"00000000-0000-0000-0000-000000000000"},"required":false,"description":"Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"text":{"type":"string"},"model":{"type":"string"},"type":{"type":"string","enum":["select","agents.md"]}},"required":["text"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"features":{"type":"array","items":{"$ref":"#/components/schemas/PromptFeatureResult"}},"suggestedFeatures":{"type":"array","items":{"$ref":"#/components/schemas/SuggestedPromptFeature"}},"cached":{"type":"boolean"}},"required":["features","cached"]}}}},"400":{"description":"Empty text string"},"503":{"description":"LLM not configured"}}}},"/api/v1/prompt-features":{"get":{"tags":["Prompt Features"],"summary":"List features","parameters":[{"schema":{"type":"string"},"required":false,"name":"q","in":"query"},{"schema":{"type":"string","enum":["select","agents.md"]},"required":false,"name":"type","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/PromptFeatureResponse"}}}}}}},"post":{"tags":["Prompt Features"],"summary":"Create feature","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreatePromptFeatureInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/PromptFeatureResponse"}}}},"400":{"description":"Invalid input"},"409":{"description":"Feature already exists"}}}},"/api/v1/prompt-features/{id}":{"get":{"tags":["Prompt Features"],"summary":"Get feature","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/PromptFeatureResponse"}}}},"404":{"description":"Feature not found"}}},"put":{"tags":["Prompt Features"],"summary":"Update feature","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdatePromptFeatureInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/PromptFeatureResponse"}}}},"400":{"description":"Invalid input"},"404":{"description":"Feature not found"}}},"delete":{"tags":["Prompt Features"],"summary":"Soft-delete feature","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"deleted":{"type":"boolean"}},"required":["id","deleted"]}}}},"404":{"description":"Feature not found"}}}},"/api/v1/task-prompts/generate":{"post":{"tags":["Task Prompts"],"summary":"Generate task prompts","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"description":{"type":"string"},"existingPrompt":{"type":"string"}}}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"tasks":{"type":"array","items":{"type":"string"}}},"required":["tasks"]}}}},"503":{"description":"LLM not configured"}}}},"/api/v1/task-prompts":{"get":{"tags":["Task Prompts"],"summary":"List task prompts","parameters":[{"schema":{"type":["number","null"]},"required":false,"name":"limit","in":"query"},{"schema":{"type":["number","null"]},"required":false,"name":"offset","in":"query"},{"schema":{"type":"string"},"required":false,"name":"search","in":"query"},{"schema":{"$ref":"#/components/schemas/PromptType"},"required":false,"name":"type","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"items":{"type":"array","items":{"$ref":"#/components/schemas/TaskPromptResponse"}},"total":{"type":"number"},"limit":{"type":"number"},"offset":{"type":"number"}},"required":["items","total","limit","offset"]}}}}}},"post":{"tags":["Task Prompts"],"summary":"Create or find task prompt","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateTaskPromptInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/TaskPromptResponse"}}}},"400":{"description":"Empty text string"}}}},"/api/v1/task-prompts/{id}":{"get":{"tags":["Task Prompts"],"summary":"Get task prompt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/TaskPromptResponse"}}}},"404":{"description":"Task prompt not found"}}},"delete":{"tags":["Task Prompts"],"summary":"Soft-delete task prompt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"deleted":{"type":"boolean"}},"required":["deleted"]}}}},"404":{"description":"Task prompt not found"}}}},"/api/v1/task-prompts/{id}/content":{"get":{"tags":["Task Prompts"],"summary":"Get resolved task prompt content","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"text":{"type":"string"}},"required":["id","text"]}}}},"404":{"description":"Task prompt not found"}}}},"/api/v1/task-prompts/{id}/extract-features":{"post":{"tags":["Task Prompts"],"summary":"Extract features from task prompt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":false,"name":"force","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"model":{"type":"string"}}}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"taskPromptId":{"type":"string"},"features":{"type":"array","items":{"$ref":"#/components/schemas/PromptFeatureResult"}},"featuresExtractedAt":{"type":["string","null"],"format":"date-time"},"suggestedFeatures":{"type":"array","items":{"$ref":"#/components/schemas/SuggestedPromptFeature"}},"cached":{"type":"boolean"}},"required":["taskPromptId","features","cached"]}}}},"404":{"description":"Task prompt not found"},"503":{"description":"LLM not configured"}}}},"/api/v1/task-prompts/{id}/features/{featureId}":{"patch":{"tags":["Task Prompts"],"summary":"Toggle feature flag on task prompt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"featureId","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/PatchTaskPromptFeatureInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/TaskPromptResponse"}}}},"400":{"description":"Invalid detected value"},"404":{"description":"Task prompt or feature not found"}}}},"/api/v1/reports":{"post":{"tags":["Reports"],"summary":"Create report","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateReportInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ReportResponse"}}}},"400":{"description":"Invalid input"},"404":{"description":"Run or template not found"}}},"get":{"tags":["Reports"],"summary":"List reports","parameters":[{"schema":{"type":"string"},"required":false,"name":"requestId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ReportResponse"}}}}}}}},"/api/v1/reports/bulk-create":{"post":{"tags":["Reports"],"summary":"Bulk create reports","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/BulkCreateReportsInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ReportResponse"}}}}},"400":{"description":"Invalid input"}}}},"/api/v1/reports/bulk-status":{"post":{"tags":["Reports"],"summary":"Bulk get report statuses","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/BulkReportStatusInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ReportResponse"}}}}},"400":{"description":"Invalid input"}}}},"/api/v1/reports/bulk-summary":{"post":{"tags":["Reports"],"summary":"Bulk get report summary per run","description":"Returns aggregated report status counts per run. Only the latest report per template is counted (re-triggers are deduplicated).","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/BulkReportSummaryInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/BulkReportSummaryResponse"}}}},"400":{"description":"Invalid input"}}}},"/api/v1/reports/{id}":{"get":{"tags":["Reports"],"summary":"Get report","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ReportResponse"}}}},"404":{"description":"Report not found"}}}},"/api/v1/reports/{id}/logs":{"get":{"tags":["Reports"],"summary":"Stream report logs (SSE)","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Server-sent event stream of log entries"},"404":{"description":"Report not found"}}}},"/api/v1/reports/trigger":{"post":{"tags":["Reports"],"summary":"Trigger reports","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/TriggerReportsInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"triggered":{"type":"number"},"reports":{"type":"array","items":{"$ref":"#/components/schemas/ReportResponse"}}},"required":["triggered","reports"]}}}},"400":{"description":"Invalid input"},"404":{"description":"Run not found"}}}},"/api/v1/reports/bulk-trigger":{"post":{"tags":["Reports"],"summary":"Bulk trigger reports","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/BulkTriggerReportsInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"type":"object","properties":{},"additionalProperties":{}}}}}},"400":{"description":"Invalid input"}}}},"/api/v1/reports/{id}/insights":{"get":{"tags":["Reports"],"summary":"Get insights for report","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"allOf":[{"$ref":"#/components/schemas/InsightResponse"},{"type":"object","properties":{"referencedAt":{"type":["string","null"],"format":"date-time"},"isNew":{"type":"boolean"}}}]}}}}},"404":{"description":"Report not found"}}},"post":{"tags":["Reports"],"summary":"Link insight to report","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"insightId":{"type":"string"},"isNew":{"type":"boolean"}},"required":["insightId"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"insightId":{"type":"string"},"referencedAt":{"type":["string","null"],"format":"date-time"},"isNew":{"type":"boolean"}},"required":["insightId","referencedAt","isNew"]}}}},"404":{"description":"Report or insight not found"},"409":{"description":"Insight already referenced by this report"}}}},"/api/v1/profiles":{"post":{"tags":["Profiles"],"summary":"Create a new profile","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateProfileInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProfileWithVersionResponse"}}}}}},"get":{"tags":["Profiles"],"summary":"List profiles","parameters":[{"schema":{"type":"string"},"required":false,"name":"workerType","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ProfileWithVersionResponse"}}}}}}}},"/api/v1/profiles/{profileId}":{"get":{"tags":["Profiles"],"summary":"Get profile with latest version","parameters":[{"schema":{"type":"string"},"required":true,"name":"profileId","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProfileWithVersionResponse"}}}}}},"post":{"tags":["Profiles"],"summary":"Create new profile version","parameters":[{"schema":{"type":"string"},"required":true,"name":"profileId","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"workerType":{"type":"string"},"model":{"type":"string"},"reasoningEffort":{"type":"string"},"agentVersion":{"type":"string"},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceBindingSpec"}},"extensions":{"type":"array","items":{"type":"string"}}},"required":["workerType","model"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProfileVersionResponse"}}}}}},"put":{"tags":["Profiles"],"summary":"Update profile identity","parameters":[{"schema":{"type":"string"},"required":true,"name":"profileId","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateProfileIdentity"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProfileResponse"}}}}}},"delete":{"tags":["Profiles"],"summary":"Delete profile","parameters":[{"schema":{"type":"string"},"required":true,"name":"profileId","in":"path"}],"responses":{"204":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}}}}},"/api/v1/profiles/{profileId}/versions":{"get":{"tags":["Profiles"],"summary":"List profile versions","parameters":[{"schema":{"type":"string"},"required":true,"name":"profileId","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ProfileVersionResponse"}}}}}}}},"/api/v1/profiles/{profileId}/versions/{version}":{"get":{"tags":["Profiles"],"summary":"Get profile version","parameters":[{"schema":{"type":"string"},"required":true,"name":"profileId","in":"path"},{"schema":{"type":["number","null"]},"required":false,"name":"version","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProfileVersionResponse"}}}}}}},"/api/v1/report-templates/default-system-prompt":{"get":{"tags":["Report Templates"],"summary":"Get default system prompt","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"content":{"type":"string"}},"required":["content"]}}}}}}},"/api/v1/report-templates/available-models":{"get":{"tags":["Report Templates"],"summary":"List models available for report generation","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"type":"object","properties":{"modelId":{"type":"string"}},"required":["modelId"]}}}}}}}},"/api/v1/report-templates":{"get":{"tags":["Report Templates"],"summary":"List report templates","parameters":[{"schema":{"type":"string"},"required":false,"name":"q","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ReportTemplateResponse"}}}}}}},"post":{"tags":["Report Templates"],"summary":"Create report template","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateReportTemplateInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ReportTemplateResponse"}}}},"409":{"description":"Report template already exists"}}}},"/api/v1/report-templates/{id}":{"get":{"tags":["Report Templates"],"summary":"Get report template","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ReportTemplateResponse"}}}},"404":{"description":"Report template not found"}}},"put":{"tags":["Report Templates"],"summary":"Update report template","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateReportTemplateInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ReportTemplateResponse"}}}},"404":{"description":"Report template not found"}}},"delete":{"tags":["Report Templates"],"summary":"Delete report template","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"204":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}},"404":{"description":"Report template not found"}}}},"/api/v1/agents":{"get":{"tags":["Agents"],"summary":"List agents","parameters":[{"schema":{"type":"string"},"required":false,"name":"modelProvider","in":"query"},{"schema":{"type":"string","enum":["true","false"]},"required":false,"name":"includeDeleted","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/AgentResponse"}}}}}}},"post":{"tags":["Agents"],"summary":"Create or update agent (upsert)","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateAgentInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AgentResponse"}}}},"400":{"description":"Validation error"},"409":{"description":"Restored agent versions conflict with an active queue owner"}}}},"/api/v1/agents/{id}":{"get":{"tags":["Agents"],"summary":"Get agent","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","enum":["true","false"]},"required":false,"name":"includeDeleted","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AgentResponse"}}}},"404":{"description":"Agent not found"}}},"put":{"tags":["Agents"],"summary":"Update agent","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateAgentInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AgentResponse"}}}},"400":{"description":"Validation error"},"404":{"description":"Agent not found"}}},"delete":{"tags":["Agents"],"summary":"Delete agent","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"204":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}},"404":{"description":"Agent not found"}}}},"/api/v1/agents/{id}/versions":{"get":{"tags":["Agents"],"summary":"List agent versions","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":false,"name":"status","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/AgentVersion"}}}}},"404":{"description":"Agent not found"}}},"post":{"tags":["Agents"],"summary":"Register agent version (upsert)","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/RegisterAgentVersionInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AgentVersion"}}}},"400":{"description":"Validation error"},"404":{"description":"Agent not found"},"409":{"description":"Queue is assigned to another agent"},"503":{"description":"Concurrent registration did not stabilize"}}}},"/api/v1/agents/{id}/versions/{agentVersion}":{"patch":{"tags":["Agents"],"summary":"Patch agent version","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"agentVersion","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/PatchAgentVersionInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AgentVersion"}}}},"400":{"description":"Invalid status value"},"404":{"description":"Agent or version not found"},"409":{"description":"Queue is assigned to another agent"},"503":{"description":"Concurrent activation did not stabilize"}}}},"/api/v1/models":{"get":{"tags":["Models"],"summary":"List models","parameters":[{"schema":{"type":"string"},"required":false,"name":"agentId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"provider","in":"query"},{"schema":{"type":"string","enum":["active","disappeared"]},"required":false,"name":"status","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ModelResponse"}}}}}}}},"/api/v1/models/{id}":{"get":{"tags":["Models"],"summary":"Get model","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ModelResponse"}}}}}}},"/api/v1/models/sync":{"post":{"tags":["Models"],"summary":"Sync models from provider","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"agentId":{"type":"string"},"provider":{"type":"string"},"models":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string"},"providerAvailableFrom":{"type":"string"},"providerEndOfLife":{"type":"string"},"metadata":{"type":"object","additionalProperties":{}},"capabilities":{"type":"object","properties":{"reasoningEffort":{"type":"array","items":{"type":"string"}},"toolCalls":{"type":"boolean"},"vision":{"type":"boolean"},"streaming":{"type":"boolean"},"adaptiveThinking":{"type":"boolean"},"maxThinkingBudget":{"type":"number"}}}},"required":["id"]}},"scannedAt":{"type":"string"}},"required":["agentId","provider","models","scannedAt"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"added":{"type":"array","items":{"type":"string"}},"removed":{"type":"array","items":{"type":"string"}},"unchanged":{"type":"array","items":{"type":"string"}}},"required":["added","removed","unchanged"]}}}}}}},"/api/v1/mcp/servers":{"get":{"tags":["MCP Servers"],"summary":"List MCP servers","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/McpServerResponse"}}}}}}},"post":{"tags":["MCP Servers"],"summary":"Create MCP server","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"_id":{"type":"string","pattern":"^[a-z0-9]([a-z0-9-]*[a-z0-9])?$"},"name":{"type":"string"},"type":{"type":"string","enum":["sse","http","stdio"]},"url":{"type":"string"},"command":{"type":"string"},"args":{"type":"array","items":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"headers":{"type":"array","items":{"$ref":"#/components/schemas/McpServerHeader"}},"sessionMode":{"type":"string","enum":["stateful","stateless"]},"version":{"type":"string"},"description":{"type":"string"}},"required":["_id","name","type"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/McpServerResponse"}}}}}}},"/api/v1/mcp/servers/{id}":{"get":{"tags":["MCP Servers"],"summary":"Get MCP server","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/McpServerResponse"}}}}}},"put":{"tags":["MCP Servers"],"summary":"Update MCP server","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateMcpServerInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/McpServerResponse"}}}}}},"delete":{"tags":["MCP Servers"],"summary":"Delete MCP server","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"204":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}}}}},"/api/v1/skills":{"get":{"tags":["Skills"],"summary":"List all skills","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/SkillResponse"}}}}}}},"post":{"tags":["Skills"],"summary":"Create or import a skill","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateSkillInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SkillResponse"}}}}}}},"/api/v1/skills/search":{"get":{"tags":["Skills"],"summary":"Search skills (internal + external)","parameters":[{"schema":{"type":"string"},"required":true,"name":"q","in":"query"},{"schema":{"type":"string"},"required":false,"name":"limit","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/SkillSearchResult"}}}}},"400":{"description":"Missing query parameter"}}}},"/api/v1/skills/search/external":{"get":{"tags":["Skills"],"summary":"Search external skills registry","parameters":[{"schema":{"type":"string"},"required":true,"name":"q","in":"query"},{"schema":{"type":"string"},"required":false,"name":"limit","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Optional project scope. When provided, external results are de-duplicated only against skills already installed in that project."},"required":false,"description":"Optional project scope. When provided, external results are de-duplicated only against skills already installed in that project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/SkillSearchResult"}}}}},"400":{"description":"Missing query parameter"}}}},"/api/v1/skills/discover":{"get":{"tags":["Skills"],"summary":"Discover skills in a GitHub repository","parameters":[{"schema":{"type":"string"},"required":true,"name":"source","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/SkillDiscoveryResult"}}}}},"400":{"description":"Missing or malformed source parameter"},"404":{"description":"Repository not found"},"502":{"description":"GitHub API error"}}}},"/api/v1/skills/{id}(*)/revisions":{"get":{"tags":["Skills"],"summary":"List skill revisions","parameters":[{"schema":{"type":"string"},"required":false,"name":"limit","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/SkillRevisionResponse"}}}}},"404":{"description":"Skill not found"}}}},"/api/v1/skills/{id}(*)":{"get":{"tags":["Skills"],"summary":"Get skill by slug","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SkillResponse"}}}},"404":{"description":"Skill not found"}}},"delete":{"tags":["Skills"],"summary":"Soft-delete a skill","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"204":{"description":"Success"},"404":{"description":"Skill not found"}}}},"/api/v1/skill-revisions/by-ref/{ref}(*)/archive":{"get":{"tags":["Skill Revisions"],"summary":"Download skill revision archive","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Binary tar.gz archive"},"404":{"description":"Skill revision or archive not found"},"500":{"description":"Blob storage error"}}}},"/api/v1/skill-revisions/by-ref/{ref}(*)":{"get":{"tags":["Skill Revisions"],"summary":"Get skill revision by ref","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SkillRevisionResponse"}}}},"404":{"description":"Skill revision not found"}}}},"/api/v1/skill-revisions/{id}":{"get":{"tags":["Skill Revisions"],"summary":"Get skill revision by ID","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SkillRevisionResponse"}}}},"404":{"description":"Skill revision not found"}}}},"/api/v1/skills/{id}(*)/resolve":{"post":{"tags":["Skills"],"summary":"Trigger skill resolution","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SkillRevisionResponse"}}}},"404":{"description":"Skill not found"},"500":{"description":"Blob storage error"}}}},"/api/v1/codebases":{"get":{"tags":["Codebases"],"summary":"List all codebases","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/CodebaseResponse"}}}}}}},"post":{"tags":["Codebases"],"summary":"Create a codebase (archive codebases require the archive file)","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"201":{"description":"Success"},"400":{"description":"Invalid input or missing archive"}}}},"/api/v1/codebases/{id}":{"get":{"tags":["Codebases"],"summary":"Get a codebase","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CodebaseResponse"}}}},"404":{"description":"Codebase not found"}}},"patch":{"tags":["Codebases"],"summary":"Update a codebase","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateCodebaseInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CodebaseResponse"}}}},"404":{"description":"Codebase not found"}}},"delete":{"tags":["Codebases"],"summary":"Delete a codebase","responses":{"204":{"description":"Success"},"404":{"description":"Codebase not found"}}}},"/api/v1/codebases/{id}/revisions":{"get":{"tags":["Codebases"],"summary":"List codebase revisions","parameters":[{"schema":{"type":"string"},"required":false,"name":"limit","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/CodebaseRevisionResponse"}}}}},"404":{"description":"Codebase not found"}}},"post":{"tags":["Codebases"],"summary":"Resolve a new git codebase revision","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ResolveCodebaseRevisionInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CodebaseRevisionResponse"}}}},"400":{"description":"Not a git codebase"},"404":{"description":"Codebase not found"},"502":{"description":"GitHub resolution failed"}}}},"/api/v1/codebases/{id}/upload":{"post":{"tags":["Codebases"],"summary":"Upload a codebase archive as a new revision","responses":{"201":{"description":"Success"},"400":{"description":"Missing archive or not an archive codebase"},"404":{"description":"Codebase not found"}}}},"/api/v1/codebase-revisions/{id}/archive":{"get":{"tags":["Codebase Revisions"],"summary":"Download codebase revision archive","responses":{"200":{"description":"Binary tar.gz archive"},"404":{"description":"Revision or archive not found"},"500":{"description":"Blob storage error"}}}},"/api/v1/codebase-revisions/{id}":{"get":{"tags":["Codebase Revisions"],"summary":"Get codebase revision by id","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CodebaseRevisionResponse"}}}},"404":{"description":"Revision not found"}}}},"/api/v1/resources":{"get":{"tags":["Resources"],"summary":"List all resources","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ResourceResponse"}}}}}}},"post":{"tags":["Resources"],"summary":"Create a resource and its first revision","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateResourceInput"}}}},"responses":{"201":{"description":"Success"},"400":{"description":"Invalid input"},"409":{"description":"Resource slug or revision ref already exists in this project"}}}},"/api/v1/resources/{id}":{"get":{"tags":["Resources"],"summary":"Get a resource","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ResourceResponse"}}}},"404":{"description":"Resource not found"}}},"patch":{"tags":["Resources"],"summary":"Update a resource","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateResourceInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ResourceResponse"}}}},"404":{"description":"Resource not found"}}},"delete":{"tags":["Resources"],"summary":"Delete a resource","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"204":{"description":"Success"},"404":{"description":"Resource not found"}}}},"/api/v1/resources/{id}/revisions":{"get":{"tags":["Resource Revisions"],"summary":"List resource revisions","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"limit","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ResourceRevisionResponse"}}}}},"404":{"description":"Resource not found"}}},"post":{"tags":["Resource Revisions"],"summary":"Create a resource revision","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateResourceRevisionInput"}}}},"responses":{"201":{"description":"Success"},"404":{"description":"Resource not found"},"409":{"description":"Resource revision ref already exists in this project"}}}},"/api/v1/resources/{id}/revisions/latest":{"get":{"tags":["Resource Revisions"],"summary":"Get the latest resource revision","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ResourceRevisionResponse"}}}},"404":{"description":"Revision not found"}}}},"/api/v1/resources/{id}/revisions/{revisionNumber}":{"get":{"tags":["Resource Revisions"],"summary":"Get a resource revision by resource and revision number","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"revisionNumber","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ResourceRevisionResponse"}}}},"404":{"description":"Revision not found"}}}},"/api/v1/resources/revisions/{id}":{"get":{"tags":["Resource Revisions"],"summary":"Get resource revision by id","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ResourceRevisionResponse"}}}},"404":{"description":"Revision not found"}}}},"/api/v1/extensions":{"get":{"tags":["Extensions"],"summary":"List all extensions","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ExtensionResponse"}}}}}}},"post":{"tags":["Extensions"],"summary":"Create or import an extension","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateExtensionInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ExtensionResponse"}}}}}}},"/api/v1/extensions/search":{"get":{"tags":["Extensions"],"summary":"Search extensions (internal + marketplace)","parameters":[{"schema":{"type":"string"},"required":true,"name":"q","in":"query"},{"schema":{"type":"string"},"required":false,"name":"limit","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ExtensionSearchResult"}}}}},"400":{"description":"Missing query parameter"}}}},"/api/v1/extensions/{id}":{"get":{"tags":["Extensions"],"summary":"Get extension by ID","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ExtensionResponse"}}}}}},"put":{"tags":["Extensions"],"summary":"Update extension","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateExtensionInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ExtensionResponse"}}}}}},"delete":{"tags":["Extensions"],"summary":"Delete extension","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"204":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}}}}},"/api/v1/extensions/{id}/versions":{"get":{"tags":["Extensions"],"summary":"List available versions for an extension","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":false,"name":"preRelease","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ExtensionVersionInfo"}}}}}}}},"/api/v1/insights":{"get":{"tags":["Insights"],"summary":"List insights","parameters":[{"schema":{"type":"string"},"required":false,"name":"q","in":"query"},{"schema":{"type":"string"},"required":false,"name":"blocked","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/InsightResponse"}}}}}}},"post":{"tags":["Insights"],"summary":"Create insight","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateInsightInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}}}}},"/api/v1/insights/search":{"get":{"tags":["Insights"],"summary":"Search insights","parameters":[{"schema":{"type":"string"},"required":true,"name":"q","in":"query"},{"schema":{"type":"string"},"required":false,"name":"blocked","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/InsightResponse"}}}}}}}},"/api/v1/insights/{id}":{"get":{"tags":["Insights"],"summary":"Get insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}},"404":{"description":"Insight not found"}}},"put":{"tags":["Insights"],"summary":"Update insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateInsightInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}},"404":{"description":"Insight not found"}}},"delete":{"tags":["Insights"],"summary":"Delete insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"204":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}},"404":{"description":"Insight not found"}}}},"/api/v1/insights/{id}/upvote":{"post":{"tags":["Insights"],"summary":"Upvote insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}},"404":{"description":"Insight not found"}}}},"/api/v1/insights/{id}/downvote":{"post":{"tags":["Insights"],"summary":"Downvote insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}},"404":{"description":"Insight not found"}}}},"/api/v1/insights/{id}/block":{"post":{"tags":["Insights"],"summary":"Block insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}},"404":{"description":"Insight not found"}}}},"/api/v1/insights/{id}/unblock":{"post":{"tags":["Insights"],"summary":"Unblock insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}},"404":{"description":"Insight not found"}}}},"/api/v1/insights/{id}/reports":{"get":{"tags":["Insights"],"summary":"Get reports referencing insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ReportResponse"}}}}},"404":{"description":"Insight not found"}}}},"/api/v1/feature-flags":{"get":{"tags":["Feature Flags"],"summary":"List feature flags","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/FeatureFlagResponse"}}}}}}}},"/api/v1/feature-flags/{key}":{"put":{"tags":["Feature Flags"],"summary":"Update feature flag","parameters":[{"schema":{"type":"string"},"required":true,"name":"key","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateFeatureFlagInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/FeatureFlagResponse"}}}}}}}},"webhooks":{}} \ No newline at end of file +{"openapi":"3.1.0","info":{"title":"Scope API","version":"1.0.0","description":"REST API for the Scope platform — benchmarking AI coding agents"},"servers":[{"url":"/","description":"Current server"}],"components":{"securitySchemes":{"bearerAuth":{"type":"http","scheme":"bearer","bearerFormat":"JWT","description":"Use the unchanged IdP access token, without the Bearer prefix."}},"schemas":{"KeyResponse":{"type":"object","properties":{},"additionalProperties":{}},"KeyInput":{"type":"object","properties":{},"additionalProperties":{}},"ValidateKeyInput":{"type":"object","properties":{"token":{"type":"string"}},"required":["token"]},"AccountResponse":{"type":"object","properties":{},"additionalProperties":{}},"AccountInput":{"type":"object","properties":{},"additionalProperties":{}},"UserMeResponse":{"type":"object","properties":{"id":{"type":"string"},"role":{"type":"string"},"email":{"type":"string"},"displayName":{"type":"string"},"idp":{"type":"string"},"idpTenant":{"type":"string"}},"required":["id"]},"ProjectResponse":{"type":"object","properties":{"_id":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"creator":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"}},"required":["_id","name","createdAt"]},"CreateProjectInput":{"type":"object","properties":{"name":{"type":"string","minLength":1},"description":{"type":"string"},"creator":{"type":"string"}},"required":["name"]},"UpdateProjectInput":{"type":"object","properties":{"name":{"type":"string","minLength":1},"description":{"type":"string"}}},"CreateExperimentResponse":{"type":"object","properties":{"experiment":{"$ref":"#/components/schemas/ExperimentResponse"},"requestIds":{"type":"array","items":{"type":"string"}}},"required":["experiment","requestIds"]},"ExperimentResponse":{"type":"object","properties":{"experimentId":{"type":"string"},"projectId":{"type":"string"},"metadata":{"type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":256},"description":{"type":"string","maxLength":4096},"createdBy":{"type":"string"}},"required":["name"],"additionalProperties":false},"revision":{"type":"object","properties":{"experimentRevisionId":{"type":"string"},"revisionNumber":{"type":"integer"},"memberCount":{"type":"integer"},"createdAt":{"type":["string","null"],"format":"date-time"}},"required":["experimentRevisionId","revisionNumber","memberCount","createdAt"]},"currentExecution":{"type":"object","properties":{"experimentExecutionId":{"type":"string"},"attemptNumber":{"type":"integer"},"readiness":{"type":"string","enum":["empty","waiting","blocked","ready","processing","done","failed"]},"blockers":{"type":"array","items":{"type":"object","properties":{"requestId":{"type":"string"},"runId":{"type":"string"},"handlerId":{"type":"string"},"reason":{"type":"string"},"error":{"type":"string"}},"required":["handlerId","reason"]}},"progress":{"type":"object","additionalProperties":{"type":"object","properties":{"status":{"type":"string","enum":["queued","processing","done","failed"]},"updatedAt":{"type":["string","null"],"format":"date-time"},"error":{"type":"string"}},"required":["status","updatedAt"]}},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"startedAt":{"type":["string","null"],"format":"date-time"},"finishedAt":{"type":["string","null"],"format":"date-time"},"discardedAt":{"type":["string","null"],"format":"date-time"},"discardReason":{"type":"string"},"report":{"type":"object","properties":{"reportId":{"type":"string"},"url":{"type":"string"},"updatedAt":{"type":["string","null"],"format":"date-time"}},"required":["reportId","url","updatedAt"]}},"required":["experimentExecutionId","attemptNumber","readiness","blockers","progress","createdAt","updatedAt"]},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"}},"required":["experimentId","projectId","metadata","revision","currentExecution","createdAt","updatedAt"]},"CreateExperimentInput":{"type":"object","properties":{"mode":{"type":"string","enum":["compose","empty"]},"metadata":{"type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":256},"description":{"type":"string","maxLength":4096}},"required":["name"],"additionalProperties":false},"requests":{"type":"array","items":{"type":"object","properties":{"scenario":{"$ref":"#/components/schemas/Scenario"},"model":{"type":"string"},"reasoningEffort":{"type":"string"},"maxIterations":{"type":"integer","minimum":1,"maximum":50},"personaInstructions":{"type":"string"},"persona":{"$ref":"#/components/schemas/Persona"},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"codebaseRevisionId":{"type":"string"},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceBindingSpec"}},"extensions":{"type":"array","items":{"type":"string"}},"profileId":{"type":"string"},"priority":{"type":"integer"},"agentsMd":{"type":"string"},"agentsMdParentIds":{"type":"array","items":{"type":"string"}},"gates":{"type":"array","items":{"$ref":"#/components/schemas/GateConfig"}},"observations":{"type":"array","items":{"type":"string"}},"workerType":{"type":"string","minLength":1}},"required":["scenario","workerType"]},"maxItems":250000,"default":[]}},"required":["mode","metadata"]},"Scenario":{"type":"object","properties":{"version":{"type":"string","enum":["v1","v2"]},"task":{"type":"string"},"criteria":{"type":"array","items":{"type":"string"}}},"required":["task","criteria"]},"Persona":{"type":"object","properties":{"personality":{"type":"string","enum":["demanding","friendly"]},"experience":{"type":"string","enum":["junior","senior"]},"verbosity":{"type":"string","enum":["brief","moderate"]},"type":{"type":"string","enum":["traditional","ai_assisted","vibe"]}},"required":["personality","experience","verbosity","type"]},"ResourceBindingSpec":{"anyOf":[{"type":"string"},{"type":"object","properties":{"ref":{"type":"string"},"params":{"type":"object","additionalProperties":{"type":"string"}}},"required":["ref"]}]},"GateConfig":{"type":"object","properties":{"gate":{"type":"string","enum":["select","build","test","run","deploy"]},"promptId":{"type":"string"},"promptText":{"type":"string"},"criteria":{"type":"array","items":{"type":"string"}},"maxIterations":{"type":"integer","minimum":1,"maximum":50}},"required":["gate","criteria"]},"ExperimentListResponse":{"type":"object","properties":{"data":{"type":"array","items":{"$ref":"#/components/schemas/ExperimentResponse"}},"nextCursor":{"type":"string"}},"required":["data"]},"ExperimentExecutionListResponse":{"type":"object","properties":{"data":{"type":"array","items":{"$ref":"#/components/schemas/ExperimentExecutionResponse"}},"nextCursor":{"type":"string"}},"required":["data"]},"ExperimentExecutionResponse":{"type":"object","properties":{"experimentExecutionId":{"type":"string"},"attemptNumber":{"type":"integer"},"readiness":{"type":"string","enum":["empty","waiting","blocked","ready","processing","done","failed"]},"blockers":{"type":"array","items":{"type":"object","properties":{"requestId":{"type":"string"},"runId":{"type":"string"},"handlerId":{"type":"string"},"reason":{"type":"string"},"error":{"type":"string"}},"required":["handlerId","reason"]}},"progress":{"type":"object","additionalProperties":{"type":"object","properties":{"status":{"type":"string","enum":["queued","processing","done","failed"]},"updatedAt":{"type":["string","null"],"format":"date-time"},"error":{"type":"string"}},"required":["status","updatedAt"]}},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"startedAt":{"type":["string","null"],"format":"date-time"},"finishedAt":{"type":["string","null"],"format":"date-time"},"discardedAt":{"type":["string","null"],"format":"date-time"},"discardReason":{"type":"string"},"supersededAt":{"type":["string","null"],"format":"date-time"},"supersededByExperimentExecutionId":{"type":"string"},"supersededReason":{"type":"string","enum":["rollover","retry"]},"report":{"type":"object","properties":{"reportId":{"type":"string"},"url":{"type":"string"},"updatedAt":{"type":["string","null"],"format":"date-time"}},"required":["reportId","url","updatedAt"]}},"required":["experimentExecutionId","attemptNumber","readiness","blockers","progress","createdAt","updatedAt"]},"ReportResponse":{"type":"object","properties":{"_id":{"type":"string"},"requestId":{"type":"string"},"experimentId":{"type":"string"},"experimentExecutionId":{"type":"string"},"templateId":{"type":"string"},"reporter":{"$ref":"#/components/schemas/Reporter"},"content":{"type":"string"},"structuredArtifact":{"$ref":"#/components/schemas/StructuredReportArtifact"},"status":{"type":"string","enum":["pending","generating","completed","failed"]},"error":{"type":"string"},"insightReferences":{"type":"array","items":{"$ref":"#/components/schemas/InsightReference"}},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","status","createdAt","projectId"]},"Reporter":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"gitHash":{"type":"string"},"model":{"type":"string"},"agentId":{"type":"string"},"agentVersion":{"type":"string"}},"required":["id","name","gitHash","model","agentId","agentVersion"]},"StructuredReportArtifact":{"type":"object","properties":{"schemaId":{"type":"string"},"schemaVersion":{"type":"string"},"contentType":{"type":"string","enum":["application/json"]},"url":{"type":"string"}},"required":["schemaId","schemaVersion","contentType","url"]},"InsightReference":{"type":"object","properties":{"insightId":{"type":"string"},"referencedAt":{"type":["string","null"],"format":"date-time"},"isNew":{"type":"boolean"}},"required":["insightId","referencedAt","isNew"]},"RetryExperimentResponse":{"type":"object","properties":{"experimentId":{"type":"string"},"experimentExecutionId":{"type":"string"},"attemptNumber":{"type":"integer"},"readiness":{"type":"string","enum":["empty","waiting","blocked","ready","processing","done","failed"]},"blockers":{"type":"array","items":{"type":"object","properties":{"requestId":{"type":"string"},"runId":{"type":"string"},"handlerId":{"type":"string"},"reason":{"type":"string"},"error":{"type":"string"}},"required":["handlerId","reason"]}}},"required":["experimentId","experimentExecutionId","attemptNumber","readiness","blockers"]},"RequestResponse":{"type":"object","properties":{"_id":{"type":"string"},"scenario":{"$ref":"#/components/schemas/Scenario"},"workerType":{"type":"string"},"model":{"type":"string"},"reasoningEffort":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"maxIterations":{"type":"number"},"personaInstructions":{"type":"string"},"persona":{"$ref":"#/components/schemas/Persona"},"deletedAt":{"type":["string","null"],"format":"date-time"},"taskPromptId":{"type":"string"},"agentsMdPromptId":{"type":"string"},"agentsMdParentIds":{"type":"array","items":{"type":"string"}},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"codebaseRevisionId":{"type":"string"},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceBinding"}},"extensions":{"type":"array","items":{"type":"string"}},"agentVersion":{"type":"string"},"profileId":{"type":"string"},"profileVersionId":{"type":"string"},"submissionId":{"type":"string"},"experimentId":{"type":"string"},"experimentExpectedRequestCount":{"type":"integer","minimum":0},"experimentRequestIndex":{"type":"integer","minimum":0},"experimentInitialRunId":{"type":"string"},"experimentFinalizedAt":{"type":["string","null"],"format":"date-time"},"priority":{"type":"integer","default":0},"gates":{"type":"array","items":{"$ref":"#/components/schemas/GateConfig"}},"observations":{"type":"array","items":{"type":"string"}},"gateSummaries":{"type":"array","items":{"$ref":"#/components/schemas/GateRunSummary"}},"run":{"$ref":"#/components/schemas/RunState"},"projectId":{"type":"string"}},"required":["_id","scenario","workerType","createdAt","projectId"]},"ResourceBinding":{"type":"object","properties":{"ref":{"type":"string"},"revisionId":{"type":"string"},"params":{"type":"object","additionalProperties":{"type":"string"}}},"required":["ref","revisionId","params"]},"GateRunSummary":{"type":"object","properties":{"gate":{"type":"string","enum":["select","build","test","run","deploy"]},"status":{"type":"string","enum":["passed","failed","skipped"]},"iterations":{"type":"integer","minimum":0}},"required":["gate","status","iterations"]},"RunState":{"type":"object","properties":{"_id":{"type":"string"},"attemptNumber":{"type":"integer","minimum":1},"status":{"type":"string","enum":["pending","queued","processing","paused","done"]},"queuedQueueName":{"type":"string"},"outcome":{"type":"string","enum":["succeeded","failed","finished"]},"result":{"type":"string"},"error":{"type":"string"},"logsUrl":{"type":"string"},"updatedAt":{"type":["string","null"],"format":"date-time"},"startedAt":{"type":["string","null"],"format":"date-time"},"finishedAt":{"type":["string","null"],"format":"date-time"},"durationMs":{"type":"number"},"turns":{"type":"array","items":{"$ref":"#/components/schemas/ConversationTurn"}},"workerVersion":{"type":"string"},"os":{"type":"object","properties":{"platform":{"type":"string"},"release":{"type":"string"},"arch":{"type":"string"}},"required":["platform","release","arch"]},"lastHeartbeatAt":{"type":["string","null"],"format":"date-time"},"worker":{"type":"object","properties":{"instanceId":{"type":"string"},"podName":{"type":"string"}},"required":["instanceId"]},"harUrl":{"type":"string"},"videoUrls":{"type":"array","items":{"type":"string"}},"setupVideoUrls":{"type":"array","items":{"type":"string"}},"tokenUsage":{"$ref":"#/components/schemas/TokenUsage"},"aiCallCount":{"type":"number"},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceRunOutcome"}},"mcpRegistered":{"type":"boolean"},"rawChatUrl":{"type":"string"},"rawChatFormat":{"type":"string"},"pausedAt":{"type":["string","null"],"format":"date-time"},"resumedAt":{"type":["string","null"],"format":"date-time"},"observationResults":{"type":"array","items":{"$ref":"#/components/schemas/CriterionResult"}}},"required":["_id","attemptNumber","status"]},"ConversationTurn":{"type":"object","properties":{"iteration":{"type":"number"},"gate":{"type":"string","enum":["select","build","test","run","deploy"]},"codingAgentResponse":{"type":"string"},"judgeFeedback":{"type":"string"},"snapshotUrl":{"type":"string"},"passed":{"type":"boolean"},"timestamp":{"type":["string","null"],"format":"date-time"},"criteriaResults":{"type":"array","items":{"$ref":"#/components/schemas/CriterionResult"}},"observationResults":{"type":"array","items":{"$ref":"#/components/schemas/CriterionResult"}},"harUrl":{"type":"string"},"videoUrls":{"type":"array","items":{"type":"string"}},"tokenUsage":{"$ref":"#/components/schemas/TokenUsage"},"startedAt":{"type":["string","null"],"format":"date-time"},"durationMs":{"type":"number"},"toolCalls":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"arguments":{"type":"object","additionalProperties":{}},"response":{"type":"string"},"timestamp":{"type":"string"}},"required":["id","name","arguments"]}},"toolCallsUrl":{"type":"string"},"toolCallCount":{"type":"number"},"aiCallCount":{"type":"number"},"rawChatUrl":{"type":"string"},"rawChatFormat":{"type":"string"},"chatResultUrl":{"type":"string"},"chatResultFormat":{"type":"string"}},"required":["iteration","judgeFeedback","snapshotUrl","passed","timestamp"]},"CriterionResult":{"type":"object","properties":{"criterionId":{"type":"string"},"passed":{"type":"boolean"},"feedback":{"type":"string"},"evaluated":{"type":"boolean"}},"required":["criterionId","passed","feedback","evaluated"]},"TokenUsage":{"type":"object","properties":{"promptTokens":{"type":"number"},"completionTokens":{"type":"number"},"totalTokens":{"type":"number"}},"required":["promptTokens","completionTokens","totalTokens"]},"ResourceRunOutcome":{"type":"object","properties":{"ref":{"type":"string"},"slug":{"type":"string"},"revisionId":{"type":"string"},"setupSucceeded":{"type":"boolean"},"published":{"type":"array","items":{"type":"string"}},"params":{"type":"object","additionalProperties":{"type":"string"}},"setupDurationMs":{"type":"number"},"error":{"type":"string"},"teardownRan":{"type":"boolean"}},"required":["ref","slug","revisionId","setupSucceeded","published"]},"CreateRequestInput":{"type":"object","properties":{"scenario":{"$ref":"#/components/schemas/Scenario"},"model":{"type":"string"},"reasoningEffort":{"type":"string"},"maxIterations":{"type":"integer","minimum":1,"maximum":50},"personaInstructions":{"type":"string"},"persona":{"$ref":"#/components/schemas/Persona"},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"codebaseRevisionId":{"type":"string"},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceBindingSpec"}},"extensions":{"type":"array","items":{"type":"string"}},"profileId":{"type":"string"},"profileVariations":{"type":"array","items":{"type":"string"},"deprecated":true,"description":"Deprecated for composition. Use POST /api/v1/experiments and its requests array."},"priority":{"type":"integer"},"agentsMd":{"type":"string"},"agentsMdParentIds":{"type":"array","items":{"type":"string"}},"gates":{"type":"array","items":{"$ref":"#/components/schemas/GateConfig"}},"observations":{"type":"array","items":{"type":"string"}}},"required":["scenario"]},"RunFacetsResponse":{"type":"object","properties":{"total":{"type":"number"},"facets":{"type":"object","properties":{"workerType":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"status":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"outcome":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"model":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"os":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"priority":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"agentVersion":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"profileId":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}}},"required":["workerType","status","outcome","model","os","priority","agentVersion","profileId"]}},"required":["total","facets"]},"RunFacetBucket":{"type":"object","properties":{"value":{"type":"string"},"count":{"type":"number"}},"required":["value","count"]},"PaginatedRunsResponse":{"type":"object","properties":{"data":{"type":"array","items":{"$ref":"#/components/schemas/RequestResponse"}},"limit":{"type":"number"},"estimatedTotal":{"type":"number"},"cursors":{"$ref":"#/components/schemas/Cursors"}},"required":["data","limit","estimatedTotal","cursors"]},"Cursors":{"type":"object","properties":{"next":{"type":["string","null"]},"prev":{"type":["string","null"]}},"required":["next","prev"]},"PaginatedRunGroupsResponse":{"type":"object","properties":{"data":{"type":"array","items":{"$ref":"#/components/schemas/RunGroup"}},"limit":{"type":"number"},"estimatedTotal":{"type":"number"},"cursors":{"$ref":"#/components/schemas/Cursors"}},"required":["data","limit","estimatedTotal","cursors"]},"RunGroup":{"type":"object","properties":{"key":{"type":"string"},"label":{"type":"string"},"runIds":{"type":"array","items":{"type":"string"}},"aggregates":{"$ref":"#/components/schemas/GroupAggregates"},"uniform":{"$ref":"#/components/schemas/GroupUniformValues"}},"required":["key","label","runIds","aggregates","uniform"]},"GroupAggregates":{"type":"object","properties":{"count":{"type":"number"},"turns":{"$ref":"#/components/schemas/AggregateStats"},"duration":{"$ref":"#/components/schemas/AggregateStats"},"promptTokens":{"$ref":"#/components/schemas/AggregateStats"},"completionTokens":{"$ref":"#/components/schemas/AggregateStats"},"statusCounts":{"type":"object","additionalProperties":{"type":"number"}},"outcomeCounts":{"type":"object","additionalProperties":{"type":"number"}}},"required":["count","turns","duration","promptTokens","completionTokens","statusCounts","outcomeCounts"]},"AggregateStats":{"type":["object","null"],"properties":{"min":{"type":"number"},"max":{"type":"number"},"mean":{"type":"number"},"stdDev":{"type":"number"}},"required":["min","max","mean","stdDev"]},"GroupUniformValues":{"type":"object","properties":{"workerType":{"type":"string"},"agentVersion":{"type":"string"},"model":{"type":"string"},"platform":{"type":"string"},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"codebaseRevisionId":{"type":"string"},"extensions":{"type":"array","items":{"type":"string"}},"status":{"type":"string","enum":["pending","queued","processing","paused","done"]},"submissionId":{"type":"string"},"experimentId":{"type":"string"},"task":{"type":"string"}}},"BulkResubmitInput":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1},"count":{"type":"integer","minimum":1,"maximum":10,"default":1},"overrides":{"type":"object","properties":{"profileId":{"type":["string","null"]},"workerType":{"type":"string"},"agentVersion":{"type":"string"},"model":{"type":["string","null"]},"reasoningEffort":{"type":["string","null"]},"maxIterations":{"type":["number","null"]},"mcpServers":{"type":["array","null"],"items":{"type":"string"}},"skillRevisions":{"type":["array","null"],"items":{"type":"string"}},"extensions":{"type":["array","null"],"items":{"type":"string"}}}}},"required":["ids"]},"CreateCriteriaInput":{"type":"object","properties":{"id":{"type":"string","pattern":"^[a-z][a-z0-9_]*$"},"prompt":{"type":"string"},"dependsOn":{"type":"array","items":{"type":"string"}},"gates":{"type":"array","items":{"type":"string","enum":["select","build","test","run","deploy"]}},"kind":{"type":"string","enum":["gate","observation"],"default":"gate"},"taxonomyElementId":{"type":"string","enum":["dimension:idiomatic-use","dimension:dependency-currency","dimension:configuration-correctness"]},"subject":{"type":"string","enum":["run","iteration"]}},"required":["id","prompt"]},"CriteriaResponse":{"type":"object","properties":{"id":{"type":"string"},"prompt":{"type":"string"},"dependsOn":{"type":"array","items":{"type":"string"}},"gates":{"type":"array","items":{"type":"string","enum":["select","build","test","run","deploy"]}},"kind":{"type":"string","enum":["gate","observation"]},"taxonomyElementId":{"type":"string","enum":["dimension:idiomatic-use","dimension:dependency-currency","dimension:configuration-correctness"]},"subject":{"type":"string","enum":["run","iteration"]},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["id","prompt","createdAt","projectId"]},"CriteriaGraph":{"type":"object","properties":{"nodes":{"type":"array","items":{"$ref":"#/components/schemas/CriteriaGraphNode"}},"edges":{"type":"array","items":{"$ref":"#/components/schemas/CriteriaGraphEdge"}}},"required":["nodes","edges"]},"CriteriaGraphNode":{"type":"object","properties":{"id":{"type":"string"},"prompt":{"type":"string"},"dependsOn":{"type":"array","items":{"type":"string"}},"gates":{"type":"array","items":{"type":"string","enum":["select","build","test","run","deploy"]}},"kind":{"type":"string","enum":["gate","observation"]},"taxonomyElementId":{"type":"string","enum":["dimension:idiomatic-use","dimension:dependency-currency","dimension:configuration-correctness"]},"subject":{"type":"string","enum":["run","iteration"]}},"required":["id","prompt"]},"CriteriaGraphEdge":{"type":"object","properties":{"from":{"type":"string"},"to":{"type":"string"}},"required":["from","to"]},"UpdateCriteriaInput":{"type":"object","properties":{"prompt":{"type":"string"},"dependsOn":{"type":"array","items":{"type":"string"}},"gates":{"type":"array","items":{"type":"string","enum":["select","build","test","run","deploy"]}},"kind":{"type":"string","enum":["gate","observation"]},"taxonomyElementId":{"type":"string","enum":["dimension:idiomatic-use","dimension:dependency-currency","dimension:configuration-correctness"]},"subject":{"type":"string","enum":["run","iteration"]}}},"CreatePromptFeatureInput":{"type":"object","properties":{"id":{"type":"string","pattern":"^[a-z][a-z0-9_]*$"},"prompt":{"type":"string"},"type":{"$ref":"#/components/schemas/PromptType"}},"required":["id","prompt"]},"PromptType":{"type":"string","enum":["select","build","test","run","deploy","agents.md"]},"PromptFeatureResult":{"type":"object","properties":{"featureId":{"type":"string"},"detected":{"type":"boolean"},"evaluated":{"type":"boolean"}},"required":["featureId","detected","evaluated"]},"SuggestedPromptFeature":{"type":"object","properties":{"suggestedId":{"type":"string"},"behavior":{"type":"string"},"prompt":{"type":"string"}},"required":["suggestedId","behavior","prompt"]},"PromptFeatureResponse":{"type":"object","properties":{"id":{"type":"string"},"prompt":{"type":"string"},"type":{"$ref":"#/components/schemas/PromptType"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["id","prompt","createdAt","projectId"]},"UpdatePromptFeatureInput":{"type":"object","properties":{"prompt":{"type":"string"}}},"TaskPromptResponse":{"type":"object","properties":{"_id":{"type":"string"},"keyId":{"type":"string"},"type":{"$ref":"#/components/schemas/PromptType"},"text":{"type":"string"},"contentBlobUrl":{"type":"string"},"features":{"type":"array","items":{"$ref":"#/components/schemas/PromptFeatureResult"}},"featuresExtractedAt":{"type":["string","null"],"format":"date-time"},"createdAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","keyId","createdAt","projectId"]},"CreateTaskPromptInput":{"type":"object","properties":{"text":{"type":"string"},"type":{"$ref":"#/components/schemas/PromptType"}},"required":["text"]},"PatchTaskPromptFeatureInput":{"type":"object","properties":{"detected":{"type":"boolean"}},"required":["detected"]},"CreateReportInput":{"type":"object","properties":{"requestId":{"type":"string"},"templateId":{"type":"string"}},"required":["requestId"]},"BulkCreateReportsInput":{"type":"object","properties":{"requestIds":{"type":"array","items":{"type":"string"}},"templateId":{"type":"string"}},"required":["requestIds"]},"BulkReportStatusInput":{"type":"object","properties":{"reportIds":{"type":"array","items":{"type":"string"}}},"required":["reportIds"]},"BulkReportSummaryResponse":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/ReportSummary"}},"ReportSummary":{"type":"object","properties":{"total":{"type":"number"},"pending":{"type":"number"},"generating":{"type":"number"},"completed":{"type":"number"},"failed":{"type":"number"}},"required":["total","pending","generating","completed","failed"]},"BulkReportSummaryInput":{"type":"object","properties":{"requestIds":{"type":"array","items":{"type":"string"}}},"required":["requestIds"]},"TriggerReportsInput":{"type":"object","properties":{"requestId":{"type":"string"}},"required":["requestId"]},"BulkTriggerReportsInput":{"type":"object","properties":{"requestIds":{"type":"array","items":{"type":"string"}}},"required":["requestIds"]},"InsightResponse":{"type":"object","properties":{"_id":{"type":"string"},"title":{"type":"string"},"description":{"type":"string"},"category":{"type":"string"},"tags":{"type":"array","items":{"type":"string"}},"upvotes":{"type":"number"},"downvotes":{"type":"number"},"blocked":{"type":"boolean"},"referenceCount":{"type":"number"},"createdBy":{"type":"string","enum":["agent","user"]},"sourceReportId":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","title","description","upvotes","downvotes","blocked","referenceCount","createdBy","createdAt","projectId"]},"ProfileWithVersionResponse":{"type":"object","properties":{"_id":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"latestVersion":{"type":"number"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"version":{"$ref":"#/components/schemas/ProfileVersionResponse"},"projectId":{"type":"string"}},"required":["_id","name","latestVersion","createdAt","version","projectId"]},"ProfileVersionResponse":{"type":"object","properties":{"_id":{"type":"string"},"profileId":{"type":"string"},"version":{"type":"number"},"workerType":{"type":"string"},"model":{"type":"string"},"reasoningEffort":{"type":"string"},"agentVersion":{"type":"string"},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceBindingSpec"}},"extensions":{"type":"array","items":{"type":"string"}},"createdAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","profileId","version","workerType","model","createdAt","projectId"]},"CreateProfileInput":{"type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":128},"description":{"type":"string","maxLength":512},"workerType":{"type":"string"},"model":{"type":"string"},"reasoningEffort":{"type":"string"},"agentVersion":{"type":"string"},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceBindingSpec"}},"extensions":{"type":"array","items":{"type":"string"}}},"required":["name","workerType","model"]},"ProfileResponse":{"type":"object","properties":{"_id":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"latestVersion":{"type":"number"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","name","latestVersion","createdAt","projectId"]},"UpdateProfileIdentity":{"type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":128},"description":{"type":"string","maxLength":512}}},"ReportTemplateResponse":{"type":"object","properties":{"_id":{"type":"string"},"id":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"userPrompt":{"type":"string"},"systemPrompt":{"$ref":"#/components/schemas/ReportTemplateSystemPrompt"},"trigger":{"$ref":"#/components/schemas/ReportTrigger"},"model":{"type":"string"},"timeoutMs":{"type":"integer","exclusiveMinimum":0},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","id","name","userPrompt","createdAt","projectId"]},"ReportTemplateSystemPrompt":{"type":"object","properties":{"mode":{"type":"string","enum":["append","override"]},"content":{"type":"string"}},"required":["mode","content"]},"ReportTrigger":{"oneOf":[{"type":"object","properties":{"type":{"type":"string","enum":["always"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["criteria"]},"criteriaIds":{"type":"array","items":{"type":"string"}},"match":{"type":"string","enum":["any","all"]}},"required":["type","criteriaIds"]},{"type":"object","properties":{"type":{"type":"string","enum":["taskPrompt"]},"taskPromptIds":{"type":"array","items":{"type":"string"}}},"required":["type","taskPromptIds"]},{"type":"object","properties":{"type":{"type":"string","enum":["promptFeature"]},"featureIds":{"type":"array","items":{"type":"string"}},"match":{"type":"string","enum":["any","all"]}},"required":["type","featureIds"]}]},"CreateReportTemplateInput":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"userPrompt":{"type":"string"},"systemPrompt":{"$ref":"#/components/schemas/ReportTemplateSystemPrompt"},"trigger":{"$ref":"#/components/schemas/ReportTrigger"},"model":{"type":"string"},"timeoutMs":{"type":"integer","exclusiveMinimum":0}},"required":["id","name","userPrompt"]},"UpdateReportTemplateInput":{"type":"object","properties":{"name":{"type":"string"},"description":{"type":"string"},"userPrompt":{"type":"string"},"systemPrompt":{"allOf":[{"$ref":"#/components/schemas/ReportTemplateSystemPrompt"},{"type":["object","null"]}]},"trigger":{"$ref":"#/components/schemas/ReportTrigger"},"model":{"type":["string","null"]},"timeoutMs":{"type":["integer","null"],"exclusiveMinimum":0}}},"AgentResponse":{"type":"object","properties":{"_id":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"modelProvider":{"type":"string"},"supportedModels":{"type":"array","items":{"type":"string"}},"defaultModel":{"type":"string"},"available":{"type":"boolean"},"capabilities":{"$ref":"#/components/schemas/AgentCapabilities"},"versions":{"type":"array","items":{"$ref":"#/components/schemas/AgentVersion"}},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"}},"required":["_id","name","supportedModels","createdAt"]},"AgentCapabilities":{"type":"object","properties":{"supportsReasoningEffort":{"type":"boolean"},"supportsMcpServers":{"type":"boolean"},"supportsSkills":{"type":"boolean"},"supportsExtensions":{"type":"boolean"},"supportsResources":{"type":"boolean"}}},"AgentVersion":{"type":"object","properties":{"agentVersion":{"type":"string","minLength":1},"workerVersion":{"type":"string","minLength":1},"components":{"type":"object","additionalProperties":{"type":"string"}},"gitCommit":{"type":"string","minLength":1},"buildTime":{"type":"string","minLength":1},"imageTag":{"type":"string","minLength":1},"queueName":{"type":"string"},"status":{"type":"string","enum":["active","retired"]},"createdAt":{"type":["string","null"],"format":"date-time"}},"required":["agentVersion","workerVersion","components","gitCommit","buildTime","imageTag","status","createdAt"]},"CreateAgentInput":{"type":"object","properties":{"_id":{"type":"string","minLength":1},"name":{"type":"string","minLength":1},"description":{"type":"string"},"modelProvider":{"type":"string"},"supportedModels":{"type":"array","items":{"type":"string"}},"defaultModel":{"type":"string"},"available":{"type":"boolean"},"capabilities":{"$ref":"#/components/schemas/AgentCapabilities"}},"required":["_id","name"]},"UpdateAgentInput":{"type":"object","properties":{"name":{"type":"string"},"description":{"type":"string"},"modelProvider":{"type":"string"},"supportedModels":{"type":"array","items":{"type":"string"}},"defaultModel":{"type":"string"},"available":{"type":"boolean"},"capabilities":{"$ref":"#/components/schemas/AgentCapabilities"}}},"RegisterAgentVersionInput":{"type":"object","properties":{"agentVersion":{"type":"string","minLength":1},"workerVersion":{"type":"string","minLength":1},"components":{"type":"object","additionalProperties":{"type":"string"}},"gitCommit":{"type":"string","minLength":1},"buildTime":{"type":"string","minLength":1},"imageTag":{"type":"string","minLength":1},"queueName":{"type":"string","minLength":1}},"required":["agentVersion","workerVersion","components","gitCommit","buildTime","imageTag","queueName"]},"PatchAgentVersionInput":{"type":"object","properties":{"status":{"type":"string","enum":["active","retired"]}},"required":["status"]},"ModelResponse":{"type":"object","properties":{"_id":{"type":"string"},"modelId":{"type":"string"},"provider":{"type":"string"},"agentId":{"type":"string"},"firstSeenAt":{"type":["string","null"],"format":"date-time"},"lastSeenAt":{"type":["string","null"],"format":"date-time"},"disappearedAt":{"type":["string","null"],"format":"date-time"},"providerAvailableFrom":{"type":["string","null"],"format":"date-time"},"providerEndOfLife":{"type":["string","null"],"format":"date-time"},"metadata":{"type":"object","additionalProperties":{}},"capabilities":{"$ref":"#/components/schemas/ModelCapabilities"}},"required":["_id","modelId","provider","agentId","firstSeenAt","lastSeenAt"]},"ModelCapabilities":{"type":"object","properties":{"reasoningEffort":{"type":"array","items":{"type":"string"}},"toolCalls":{"type":"boolean"},"vision":{"type":"boolean"},"streaming":{"type":"boolean"},"adaptiveThinking":{"type":"boolean"},"maxThinkingBudget":{"type":"number"}}},"McpServerResponse":{"type":"object","properties":{"_id":{"type":"string"},"slug":{"type":"string"},"name":{"type":"string"},"type":{"type":"string","enum":["sse","http","stdio"]},"url":{"type":"string"},"command":{"type":"string"},"args":{"type":"array","items":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"headers":{"type":"array","items":{"$ref":"#/components/schemas/McpServerHeader"}},"sessionMode":{"type":"string","enum":["stateful","stateless"]},"version":{"type":"string"},"description":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","slug","name","type","createdAt","projectId"]},"McpServerHeader":{"type":"object","properties":{"name":{"type":"string"},"value":{"type":"string"}},"required":["name","value"]},"UpdateMcpServerInput":{"type":"object","properties":{"name":{"type":"string"},"type":{"type":"string","enum":["sse","http","stdio"]},"url":{"type":"string"},"command":{"type":"string"},"args":{"type":"array","items":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"headers":{"type":"array","items":{"$ref":"#/components/schemas/McpServerHeader"}},"sessionMode":{"type":"string","enum":["stateful","stateless"]},"version":{"type":"string"},"description":{"type":"string"}}},"SkillResponse":{"type":"object","properties":{"_id":{"type":"string"},"slug":{"type":"string"},"source":{"type":"string"},"skillName":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"origin":{"type":"string","enum":["skills-sh","manual"]},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","slug","source","skillName","name","origin","createdAt","projectId"]},"SkillSearchResult":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"source":{"type":"string"},"description":{"type":"string"},"internal":{"type":"boolean"},"installs":{"type":"number"}},"required":["id","name","source","internal"]},"SkillDiscoveryResult":{"type":"object","properties":{"skillName":{"type":"string"},"skillPath":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"existsInLibrary":{"type":"boolean"},"currentRevisionCommitSha":{"type":"string"},"latestUpstreamCommitSha":{"type":"string"},"updateAvailable":{"type":"boolean"},"lastImportedAt":{"type":"string"}},"required":["skillName","skillPath"]},"SkillRevisionResponse":{"type":"object","properties":{"_id":{"type":"string"},"ref":{"type":"string"},"source":{"type":"string"},"skillName":{"type":"string"},"skillPath":{"type":"string"},"commitHash":{"type":"string"},"commitTimestamp":{"type":["string","null"],"format":"date-time"},"name":{"type":"string"},"description":{"type":"string"},"license":{"type":"string"},"compatibility":{"type":"string"},"allowedTools":{"type":"string"},"metadata":{"type":"object","additionalProperties":{"type":"string"}},"content":{"type":"string"},"archiveUrl":{"type":"string"},"validationWarnings":{"type":"array","items":{"type":"string"}},"resolvedAt":{"type":["string","null"],"format":"date-time"},"createdAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","ref","source","skillName","skillPath","commitHash","commitTimestamp","name","description","content","archiveUrl","resolvedAt","createdAt","projectId"]},"CreateSkillInput":{"type":"object","properties":{"source":{"type":"string"},"skillName":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"origin":{"type":"string","enum":["skills-sh","manual"]}},"required":["source","skillName","name","origin"]},"CodebaseResponse":{"type":"object","properties":{"_id":{"type":"string"},"slug":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"sourceType":{"type":"string","enum":["git","archive"]},"source":{"type":"string"},"defaultBranch":{"type":"string"},"revisionCounter":{"type":"number"},"latestRevisionId":{"type":"string"},"creator":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","slug","name","sourceType","revisionCounter","createdAt","projectId"]},"UpdateCodebaseInput":{"type":"object","properties":{"name":{"type":"string","minLength":1},"description":{"type":"string"},"defaultBranch":{"type":"string"}}},"CodebaseRevisionResponse":{"type":"object","properties":{"_id":{"type":"string"},"codebaseId":{"type":"string"},"slug":{"type":"string"},"revisionNumber":{"type":"number"},"ref":{"type":"string"},"sourceType":{"type":"string","enum":["git","archive"]},"source":{"type":"string"},"requestedRef":{"type":"string"},"resolvedCommitSha":{"type":"string"},"commitTimestamp":{"type":["string","null"],"format":"date-time"},"originalFilename":{"type":"string"},"contentSha256":{"type":"string"},"archiveUrl":{"type":"string"},"sizeBytes":{"type":"number"},"fileCount":{"type":"number"},"creator":{"type":"string"},"resolvedAt":{"type":["string","null"],"format":"date-time"},"createdAt":{"type":["string","null"],"format":"date-time"},"deduplicated":{"type":"boolean"},"projectId":{"type":"string"}},"required":["_id","codebaseId","slug","revisionNumber","ref","sourceType","archiveUrl","resolvedAt","createdAt","projectId"]},"ResolveCodebaseRevisionInput":{"type":"object","properties":{"requestedRef":{"type":"string"},"creator":{"type":"string"}}},"ResourceResponse":{"type":"object","properties":{"_id":{"type":"string"},"slug":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"revisionCounter":{"type":"number"},"latestRevisionId":{"type":"string"},"latestRevisionNumber":{"type":"number"},"creator":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","slug","name","revisionCounter","createdAt","projectId"]},"CreateResourceInput":{"type":"object","properties":{"name":{"type":"string","minLength":1},"slug":{"type":"string","minLength":1},"description":{"type":"string"},"setup":{"$ref":"#/components/schemas/ResourceScript"},"teardown":{"$ref":"#/components/schemas/ResourceScript"},"exports":{"type":"array","items":{"type":"string","pattern":"^[A-Za-z_][A-Za-z0-9_]*$"}},"parameters":{"type":"array","items":{"$ref":"#/components/schemas/ResourceParameter"}},"creator":{"type":"string"}},"required":["name","setup"]},"ResourceScript":{"type":"object","properties":{"sh":{"type":"string","minLength":1}}},"ResourceParameter":{"type":"object","properties":{"name":{"type":"string","pattern":"^[A-Za-z_][A-Za-z0-9_]*$"},"description":{"type":"string"},"required":{"type":"boolean","default":false},"default":{"type":"string"},"example":{"type":"string"}},"required":["name"]},"UpdateResourceInput":{"type":"object","properties":{"name":{"type":"string","minLength":1},"description":{"type":"string"}}},"ResourceRevisionResponse":{"type":"object","properties":{"_id":{"type":"string"},"resourceId":{"type":"string"},"slug":{"type":"string"},"revisionNumber":{"type":"number"},"ref":{"type":"string"},"setup":{"$ref":"#/components/schemas/ResourceScript"},"teardown":{"$ref":"#/components/schemas/ResourceScript"},"exports":{"type":"array","items":{"type":"string"}},"parameters":{"type":"array","items":{"$ref":"#/components/schemas/ResourceParameter"}},"contentSha256":{"type":"string"},"creator":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"deduplicated":{"type":"boolean"},"projectId":{"type":"string"}},"required":["_id","resourceId","slug","revisionNumber","ref","setup","exports","contentSha256","createdAt","projectId"]},"CreateResourceRevisionInput":{"type":"object","properties":{"setup":{"$ref":"#/components/schemas/ResourceScript"},"teardown":{"$ref":"#/components/schemas/ResourceScript"},"exports":{"type":"array","items":{"type":"string","pattern":"^[A-Za-z_][A-Za-z0-9_]*$"}},"parameters":{"type":"array","items":{"$ref":"#/components/schemas/ResourceParameter"}},"creator":{"type":"string"}},"required":["setup"]},"ExtensionResponse":{"type":"object","properties":{"_id":{"type":"string"},"slug":{"type":"string"},"publisher":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"origin":{"type":"string","enum":["marketplace","manual"]},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","slug","publisher","name","origin","createdAt","projectId"]},"ExtensionSearchResult":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"publisher":{"type":"string"},"description":{"type":"string"},"internal":{"type":"boolean"},"version":{"type":"string"}},"required":["id","name","publisher","internal"]},"CreateExtensionInput":{"type":"object","properties":{"_id":{"type":"string","pattern":"^[a-zA-Z0-9-]+\\.[a-zA-Z0-9-]+$"},"publisher":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"origin":{"type":"string","enum":["marketplace","manual"]}},"required":["_id","publisher","name","origin"]},"UpdateExtensionInput":{"type":"object","properties":{"name":{"type":"string"},"description":{"type":"string"}}},"ExtensionVersionInfo":{"type":"object","properties":{"version":{"type":"string"},"preRelease":{"type":"boolean"},"lastUpdated":{"type":"string"}},"required":["version","preRelease","lastUpdated"]},"CreateInsightInput":{"type":"object","properties":{"title":{"type":"string"},"description":{"type":"string"},"category":{"type":"string"},"tags":{"type":"array","items":{"type":"string"}},"createdBy":{"type":"string","enum":["agent","user"]},"sourceReportId":{"type":"string"}},"required":["title","description"]},"UpdateInsightInput":{"type":"object","properties":{"title":{"type":"string"},"description":{"type":"string"},"category":{"type":"string"},"tags":{"type":"array","items":{"type":"string"}}}},"FeatureFlagResponse":{"type":"object","properties":{"key":{"type":"string"},"label":{"type":"string"},"enabled":{"type":"boolean"},"updatedAt":{"type":["string","null"],"format":"date-time"}},"required":["key","label","enabled","updatedAt"]},"UpdateFeatureFlagInput":{"type":"object","properties":{"enabled":{"type":"boolean"}},"required":["enabled"]}},"parameters":{}},"paths":{"/api/v1/keys/preview":{"post":{"tags":["Keys"],"summary":"Preview key","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyInput"}}}},"responses":{"200":{"description":"Key preview","content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyResponse"}}}}}}},"/api/v1/keys":{"post":{"tags":["Keys"],"summary":"Create key","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyInput"}}}},"responses":{"201":{"description":"Key created","content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyResponse"}}}}}},"get":{"tags":["Keys"],"summary":"List keys","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/KeyResponse"}}}}}}}},"/api/v1/keys/{id}":{"get":{"tags":["Keys"],"summary":"Get key","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyResponse"}}}}}},"put":{"tags":["Keys"],"summary":"Update key","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyInput"}}}},"responses":{"200":{"description":"Key updated","content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyResponse"}}}}}},"delete":{"tags":["Keys"],"summary":"Delete key","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Key deleted","content":{"application/json":{"schema":{"type":"object","properties":{"success":{"type":"boolean"}},"required":["success"]}}}}}}},"/api/v1/keys/{id}/validate":{"post":{"tags":["Keys"],"summary":"Validate key","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ValidateKeyInput"}}}},"responses":{"200":{"description":"Validation result","content":{"application/json":{"schema":{"type":"object","properties":{},"additionalProperties":{},"description":"Key validation result"}}}}}}},"/api/v1/accounts":{"post":{"tags":["Accounts"],"summary":"Create account","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/AccountInput"}}}},"responses":{"201":{"description":"Account created","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AccountResponse"}}}}}},"get":{"tags":["Accounts"],"summary":"List accounts","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/AccountResponse"}}}}}}}},"/api/v1/accounts/{id}":{"get":{"tags":["Accounts"],"summary":"Get account","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AccountResponse"}}}}}},"put":{"tags":["Accounts"],"summary":"Update account","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/AccountInput"}}}},"responses":{"200":{"description":"Account updated","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AccountResponse"}}}}}},"delete":{"tags":["Accounts"],"summary":"Delete account","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Account deleted","content":{"application/json":{"schema":{"type":"object","properties":{"success":{"type":"boolean"}},"required":["success"]}}}}}}},"/api/v1/users/me":{"get":{"tags":["Users"],"summary":"Get the authenticated user's identity","description":"Read the existing authenticated Scope identity without enrollment or profile writes. Responses must not be HTTP-cached.","security":[{"bearerAuth":[]}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/UserMeResponse"}}}},"401":{"description":"Not authenticated"},"403":{"description":"User is not enrolled or is disabled"},"503":{"description":"Authentication service unavailable"}}},"post":{"tags":["Users"],"summary":"Enroll the authenticated user","description":"JIT-enroll the authenticated user, refresh their profile and lastLoginAt, apply bootstrap-admin rules, and warm the access cache. Use this only after an explicit IdP login; do not prefetch, poll, or automatically retry transient failures.","security":[{"bearerAuth":[]}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/UserMeResponse"}}}},"401":{"description":"Not authenticated"},"403":{"description":"User is disabled"},"503":{"description":"Authentication service unavailable"}}}},"/health":{"get":{"tags":["Health"],"summary":"Liveness probe","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"status":{"type":"string"},"version":{"type":"string"}},"required":["status","version"]}}}}}}},"/ready":{"get":{"tags":["Health"],"summary":"Readiness probe","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"status":{"type":"string"},"migrations":{}},"required":["status"]}}}},"503":{"description":"Service is not ready"}}}},"/about":{"get":{"tags":["System"],"summary":"API metadata","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"name":{"type":"string"},"version":{"type":"string"},"buildTime":{"type":"string"},"environment":{"type":"string"},"description":{"type":"string"},"workers":{"type":"array","items":{"type":"string"}}},"required":["name","version","buildTime","environment","description","workers"]}}}}}}},"/api/v1/version":{"get":{"tags":["System"],"summary":"Version info","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"commit":{"type":"string"},"buildTime":{"type":"string"},"environment":{"type":"string"},"strictAgentCapabilities":{"type":"boolean"}},"required":["commit","buildTime","environment","strictAgentCapabilities"]}}}}}}},"/api/v1/projects":{"get":{"tags":["Projects"],"summary":"List all projects","parameters":[{"schema":{"type":"string","enum":["true","false"],"description":"When true, include soft-deleted projects in the result"},"required":false,"description":"When true, include soft-deleted projects in the result","name":"includeDeleted","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ProjectResponse"}}}}}}},"post":{"tags":["Projects"],"summary":"Create a project","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateProjectInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProjectResponse"}}}},"400":{"description":"Invalid input"}}}},"/api/v1/projects/{id}":{"get":{"tags":["Projects"],"summary":"Get a project","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProjectResponse"}}}},"404":{"description":"Project not found"}}},"patch":{"tags":["Projects"],"summary":"Update a project","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateProjectInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProjectResponse"}}}},"404":{"description":"Project not found"}}},"delete":{"tags":["Projects"],"summary":"Soft-delete a project","responses":{"204":{"description":"Success"},"404":{"description":"Project not found"}}}},"/api/v1/projects/{id}/restore":{"post":{"tags":["Projects"],"summary":"Restore a soft-deleted project","responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProjectResponse"}}}},"404":{"description":"Project not found"}}}},"/api/v1/experiments":{"post":{"tags":["Experiments"],"summary":"Create an experiment and zero or more new requests","description":"Canonical composition endpoint. Request and run IDs are minted before insertion; the exact revision/execution is then finalized idempotently.","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateExperimentInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateExperimentResponse"}}}},"409":{"description":"Experiment identity conflict"}}},"get":{"tags":["Experiments"],"summary":"List experiments","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"},{"schema":{"type":"integer","minimum":1,"maximum":100,"default":50},"required":false,"name":"limit","in":"query"},{"schema":{"type":"string","minLength":1},"required":false,"name":"after","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ExperimentListResponse"}}}}}}},"/api/v1/experiments/{id}":{"get":{"tags":["Experiments"],"summary":"Get an experiment","parameters":[{"schema":{"type":"string","minLength":1},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ExperimentResponse"}}}},"404":{"description":"Experiment not found"}}}},"/api/v1/experiments/{id}/executions":{"get":{"tags":["Experiments"],"summary":"List current and historical experiment executions","parameters":[{"schema":{"type":"string","minLength":1},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"},{"schema":{"type":"integer","minimum":1,"maximum":100,"default":50},"required":false,"name":"limit","in":"query"},{"schema":{"type":"string","minLength":1},"required":false,"name":"after","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ExperimentExecutionListResponse"}}}},"404":{"description":"Experiment not found"}}}},"/api/v1/experiments/{id}/executions/{experimentExecutionId}":{"get":{"tags":["Experiments"],"summary":"Get one exact experiment execution","parameters":[{"schema":{"type":"string","minLength":1},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1},"required":true,"name":"experimentExecutionId","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ExperimentExecutionResponse"}}}},"404":{"description":"Experiment execution not found"}}}},"/api/v1/experiments/{id}/executions/{experimentExecutionId}/reports":{"get":{"tags":["Reports"],"summary":"List reports for one exact Experiment execution","parameters":[{"schema":{"type":"string","minLength":1},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1},"required":true,"name":"experimentExecutionId","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ReportResponse"}}}}},"404":{"description":"Experiment execution not found"}}}},"/api/v1/experiments/{id}/executions/{experimentExecutionId}/reports/{reportId}/structured":{"get":{"tags":["Reports"],"summary":"Download structured JSON for an exact Experiment report","parameters":[{"schema":{"type":"string","minLength":1},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1},"required":true,"name":"experimentExecutionId","in":"path"},{"schema":{"type":"string","minLength":1},"required":true,"name":"reportId","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success"},"404":{"description":"Experiment report or structured artifact not found"}}}},"/api/v1/experiments/{id}/executions/{experimentExecutionId}/logs":{"get":{"tags":["Experiments"],"summary":"Stream logs for one exact Experiment execution","parameters":[{"schema":{"type":"string","minLength":1},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1},"required":true,"name":"experimentExecutionId","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"},{"schema":{"type":"string","enum":["true","false"]},"required":false,"name":"fromStart","in":"query"}],"responses":{"200":{"description":"Server-sent event stream of log entries"},"404":{"description":"Experiment execution not found"}}}},"/api/v1/experiments/{id}/retry":{"post":{"tags":["Experiments"],"summary":"Retry current experiment processing","parameters":[{"schema":{"type":"string","minLength":1},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"experimentExecutionId":{"type":"string","minLength":1}},"required":["experimentExecutionId"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/RetryExperimentResponse"}}}},"404":{"description":"Experiment not found"},"409":{"description":"Execution is stale or cannot be retried"}}}},"/api/v1/requests":{"post":{"tags":["Requests"],"summary":"Submit request(s)","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"allOf":[{"$ref":"#/components/schemas/CreateRequestInput"},{"type":"object","properties":{"count":{"type":"number","minimum":1,"maximum":10,"default":1,"deprecated":true,"description":"Deprecated for composition. Use POST /api/v1/experiments and its requests array."},"skills":{"type":"array","items":{"type":"string"}},"agentVersion":{"type":"string"},"codebase":{"type":"string"}}}]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"anyOf":[{"$ref":"#/components/schemas/RequestResponse"},{"type":"array","items":{"$ref":"#/components/schemas/RequestResponse"}}]}}}}}},"get":{"tags":["Requests"],"summary":"List requests","parameters":[{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"worker","in":"query"},{"schema":{"type":"string"},"required":false,"name":"taskPromptId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"criteria","in":"query"},{"schema":{"type":"string"},"required":false,"name":"submissionId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"experimentId","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"profileId","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"status","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"outcome","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"model","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"os","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"priority","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"agentVersion","in":"query"},{"schema":{"type":"string"},"required":false,"name":"search","in":"query"},{"schema":{"type":"string","enum":["task","submissionId","experimentId","profile"]},"required":false,"name":"groupBy","in":"query"},{"schema":{"type":"integer","minimum":1,"maximum":100},"required":false,"name":"limit","in":"query"},{"schema":{"type":"string"},"required":false,"name":"after","in":"query"},{"schema":{"type":"string"},"required":false,"name":"before","in":"query"},{"schema":{"type":"string","enum":["true","false"]},"required":false,"name":"last","in":"query"},{"schema":{"type":"string","enum":["created","updated","priority","worker","status","id","duration","createdAt"]},"required":false,"name":"sortBy","in":"query"},{"schema":{"type":"string","enum":["asc","desc"]},"required":false,"name":"sortDir","in":"query"},{"schema":{"type":["string","null"],"format":"date-time"},"required":false,"name":"createdAfter","in":"query"},{"schema":{"type":["string","null"],"format":"date-time"},"required":false,"name":"createdBefore","in":"query"},{"schema":{"type":["integer","null"],"minimum":0},"required":false,"name":"turns","in":"query"},{"schema":{"type":"string","enum":["eq","gte","lte"]},"required":false,"name":"turnsOp","in":"query"},{"schema":{"type":["integer","null"],"minimum":0},"required":false,"name":"maxIterations","in":"query"},{"schema":{"type":"string","enum":["eq","gte","lte"]},"required":false,"name":"maxIterationsOp","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"anyOf":[{"$ref":"#/components/schemas/PaginatedRunsResponse"},{"$ref":"#/components/schemas/PaginatedRunGroupsResponse"}]}}}}}}},"/api/v1/requests/facets":{"get":{"tags":["Requests"],"summary":"List run filter facets","description":"Returns every distinct value and its full-dataset count per categorical filter dimension for the Runs list rail. Counts are absolute over all non-deleted runs **in the given project**: they intentionally ignore the active search, date range, iteration, and categorical selections so every selectable value stays visible with a stable count. Requires ?projectId=. Computed with parallel $group aggregations (Cosmos has no $facet) and cached per-project for a short TTL.","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/RunFacetsResponse"}}}}}}},"/api/v1/requests/{id}":{"get":{"tags":["Requests"],"summary":"Get request","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/RequestResponse"}}}},"404":{"description":"Not found"}}},"delete":{"tags":["Requests"],"summary":"Soft-delete request","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}},"404":{"description":"Not found"}}}},"/api/v1/analysis":{"get":{"tags":["Requests"],"summary":"Compute pass@k / success@T metrics","description":"Aggregates pass@k / success@T metrics over a single project's done runs. Requires ?projectId=.","parameters":[{"schema":{"type":"string"},"required":false,"name":"worker","in":"query"},{"schema":{"type":"string"},"required":false,"name":"taskPromptId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"criteria","in":"query"},{"schema":{"type":"string"},"required":false,"name":"features","in":"query"},{"schema":{"type":"string"},"required":false,"name":"submissionId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"experimentId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"k","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{},"additionalProperties":{},"description":"Analysis metrics"}}}}}}},"/api/v1/requests/bulk-resubmit":{"post":{"tags":["Requests"],"summary":"Bulk resubmit requests","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/BulkResubmitInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/RequestResponse"}}}}}}}},"/api/v1/requests/bulk":{"delete":{"tags":["Requests"],"summary":"Bulk soft-delete requests","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"}}},"required":["ids"]}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"deleted":{"type":"number"}},"required":["deleted"]}}}}}}},"/api/v1/requests/archive":{"post":{"tags":["Requests"],"summary":"Download batch archive of multiple runs","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1,"maxItems":100}},"required":["ids"]}}}},"responses":{"201":{"description":"Gzipped batch archive containing individual run archives"},"400":{"description":"Invalid input"},"404":{"description":"One or more runs not found"}}}},"/api/v1/runs/upload":{"post":{"tags":["Requests"],"summary":"Import run archive","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"201":{"description":"Success"}}}},"/api/v1/runs/upload-batch":{"post":{"tags":["Requests"],"summary":"Import batch run archive","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"207":{"description":"Multi-status: 201 if all runs imported, 400 if none imported, 207 if partial"},"400":{"description":"No archive uploaded, empty archive, or all runs failed"}}}},"/api/v1/requests/{id}/reports":{"get":{"tags":["Reports"],"summary":"Get reports for request","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ReportResponse"}}}}},"404":{"description":"Run not found"}}}},"/api/v1/requests/{id}/runs":{"get":{"tags":["Requests"],"summary":"List attempts for a request","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/RunState"}}}}},"404":{"description":"Request not found"}}}},"/api/v1/requests/{id}/runs/{runId}":{"get":{"tags":["Requests"],"summary":"Get a single attempt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/RunState"}}}},"404":{"description":"Request or run not found"}}}},"/api/v1/requests/bulk-retry":{"post":{"tags":["Requests"],"summary":"Bulk retry requests (start new attempts)","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1},"force":{"type":"boolean"}},"required":["ids"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"retried":{"type":"integer"},"skipped":{"type":"integer"},"results":{"type":"array","items":{"type":"object","properties":{"requestId":{"type":"string"},"runId":{"type":"string"},"attemptNumber":{"type":"integer"},"error":{"type":"string"}},"required":["requestId"]}}},"required":["retried","skipped","results"]}}}}}}},"/api/v1/requests/{id}/retry":{"post":{"tags":["Requests"],"summary":"Retry a request (start a new attempt)","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"force":{"type":"boolean"}}}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"requestId":{"type":"string"},"runId":{"type":"string"},"attemptNumber":{"type":"integer"}},"required":["requestId","runId","attemptNumber"]}}}},"404":{"description":"Request not found"},"409":{"description":"Conflict — request is not in a retryable state, or a concurrent retry won the race"},"422":{"description":"Cannot retry — current run not yet terminal"}}}},"/api/v1/requests/{id}/priority":{"post":{"tags":["Requests"],"summary":"Set priority on a single request","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"priority":{"type":"integer"}},"required":["priority"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"priority":{"type":"number"}},"required":["id","priority"]}}}}}}},"/api/v1/requests/bulk-priority":{"post":{"tags":["Requests"],"summary":"Set priority on multiple requests","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1,"maxItems":100},"priority":{"type":"integer"}},"required":["ids","priority"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"updated":{"type":"number"},"skipped":{"type":"number"}},"required":["updated","skipped"]}}}}}}},"/api/v1/requests/{id}/pause":{"post":{"tags":["Requests"],"summary":"Pause a single request","responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"status":{"type":"string"}},"required":["id","status"]}}}}}}},"/api/v1/requests/{id}/resume":{"post":{"tags":["Requests"],"summary":"Resume a paused request","responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"status":{"type":"string"}},"required":["id","status"]}}}}}}},"/api/v1/requests/bulk-pause":{"post":{"tags":["Requests"],"summary":"Pause multiple requests","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1,"maxItems":100}},"required":["ids"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"updated":{"type":"number"},"skipped":{"type":"number"}},"required":["updated","skipped"]}}}}}}},"/api/v1/requests/bulk-resume":{"post":{"tags":["Requests"],"summary":"Resume multiple paused requests","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1,"maxItems":100}},"required":["ids"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"updated":{"type":"number"},"skipped":{"type":"number"}},"required":["updated","skipped"]}}}}}}},"/api/v1/requests/{id}/cancel":{"post":{"tags":["Requests"],"summary":"Cancel a request (marks as done/failed and signals worker to exit)","responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"previousStatus":{"type":"string"},"status":{"type":"string"},"outcome":{"type":"string"}},"required":["id","previousStatus","status","outcome"]}}}}}}},"/api/v1/requests/bulk-cancel":{"post":{"tags":["Requests"],"summary":"Cancel multiple requests","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1,"maxItems":100}},"required":["ids"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"cancelled":{"type":"number"},"skipped":{"type":"number"},"results":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string"},"cancelled":{"type":"boolean"},"previousStatus":{"type":"string"},"error":{"type":"string"}},"required":["id","cancelled"]}}},"required":["cancelled","skipped","results"]}}}}}}},"/api/v1/requests/{id}/logs":{"get":{"tags":["Requests"],"summary":"Stream request logs (SSE)","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Server-sent event stream of log entries"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/logs":{"get":{"tags":["Requests"],"summary":"Stream logs for a specific attempt (SSE)","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"}],"responses":{"200":{"description":"Server-sent event stream of log entries"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/har":{"get":{"tags":["Requests"],"summary":"Download HAR file","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"HAR-format JSON file"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/har":{"get":{"tags":["Requests"],"summary":"Download HAR file for a specific attempt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"}],"responses":{"200":{"description":"HAR-format JSON file"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/atif":{"get":{"tags":["Requests"],"summary":"Download ATIF trajectory file for a specific iteration","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","description":"The iteration number (1-based)"},"required":true,"description":"The iteration number (1-based)","name":"iteration","in":"query"}],"responses":{"200":{"description":"ATIF v1.7 trajectory JSON file"},"400":{"description":"Missing or invalid iteration"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/atif":{"get":{"tags":["Requests"],"summary":"Download ATIF trajectory file for a specific attempt and iteration","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"},{"schema":{"type":"string","description":"The iteration number (1-based)"},"required":true,"description":"The iteration number (1-based)","name":"iteration","in":"query"}],"responses":{"200":{"description":"ATIF v1.7 trajectory JSON file"},"400":{"description":"Missing or invalid iteration"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/video":{"get":{"tags":["Requests"],"summary":"Download session recording","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"WebM video recording (supports Range requests)"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/video":{"get":{"tags":["Requests"],"summary":"Download session recording for a specific attempt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"}],"responses":{"200":{"description":"WebM video recording (supports Range requests)"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/tool-calls":{"get":{"tags":["Requests"],"summary":"Download per-iteration tool-calls JSONL","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"JSONL stream — one ToolCall per line"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/tool-calls":{"get":{"tags":["Requests"],"summary":"Download per-iteration tool-calls JSONL for a specific attempt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"}],"responses":{"200":{"description":"JSONL stream — one ToolCall per line"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/snapshots/{iteration}":{"get":{"tags":["Requests"],"summary":"Download iteration snapshot","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"iteration","in":"path"}],"responses":{"200":{"description":"Gzipped snapshot archive"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/snapshots/{iteration}":{"get":{"tags":["Requests"],"summary":"Download iteration snapshot for a specific attempt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"},{"schema":{"type":"string"},"required":true,"name":"iteration","in":"path"}],"responses":{"200":{"description":"Gzipped snapshot archive"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/archive":{"get":{"tags":["Requests"],"summary":"Download full run archive","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Gzipped run archive"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/archive":{"get":{"tags":["Requests"],"summary":"Download full run archive for a specific attempt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"}],"responses":{"200":{"description":"Gzipped run archive"},"404":{"description":"Not found"}}}},"/api/v1/criteria/generate-prompt":{"post":{"tags":["Criteria"],"summary":"Generate criterion prompt from behavior","parameters":[{"schema":{"type":"string","minLength":1,"description":"Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.","example":"00000000-0000-0000-0000-000000000000"},"required":false,"description":"Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"behavior":{"type":"string"},"currentId":{"type":"string"},"gates":{"type":"array","items":{"type":"string"}}},"required":["behavior"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"prompt":{"type":"string"},"suggestedId":{"type":"string"},"suggestedParents":{"type":"array","items":{"type":"string"}},"suggestedChildren":{"type":"array","items":{"type":"string"}}},"required":["prompt","suggestedId","suggestedParents","suggestedChildren"]}}}},"400":{"description":"Empty behavior string"},"503":{"description":"LLM not configured"}}}},"/api/v1/criteria/seed":{"post":{"tags":["Criteria"],"summary":"Seed criteria in bulk","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"criteria":{"type":"array","items":{"$ref":"#/components/schemas/CreateCriteriaInput"}}},"required":["criteria"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"seeded":{"type":"number"},"errors":{"type":"array","items":{"type":"string"}}},"required":["seeded","errors"]}}}},"400":{"description":"Seed batch would introduce a dependency cycle"}}}},"/api/v1/criteria":{"get":{"tags":["Criteria"],"summary":"List criteria","parameters":[{"schema":{"type":"string"},"required":false,"name":"q","in":"query"},{"schema":{"type":"string","description":"Comma-separated criterion IDs to include"},"required":false,"description":"Comma-separated criterion IDs to include","name":"ids","in":"query"},{"schema":{"type":"string","enum":["true","false"],"description":"When true and ids is set, also include dependency ancestors"},"required":false,"description":"When true and ids is set, also include dependency ancestors","name":"ancestors","in":"query"},{"schema":{"type":"string","enum":["gate","observation"],"description":"Filter by criterion kind"},"required":false,"description":"Filter by criterion kind","name":"kind","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/CriteriaResponse"}}}}}}},"post":{"tags":["Criteria"],"summary":"Create criterion","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateCriteriaInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CriteriaResponse"}}}},"409":{"description":"Criterion already exists"}}}},"/api/v1/criteria/mdp":{"get":{"tags":["Criteria"],"summary":"Compute MDP transitions","parameters":[{"schema":{"type":"string"},"required":false,"name":"criteria","in":"query"},{"schema":{"type":"string"},"required":false,"name":"features","in":"query"},{"schema":{"type":"string"},"required":false,"name":"since","in":"query"},{"schema":{"type":"string"},"required":false,"name":"worker","in":"query"},{"schema":{"type":"string"},"required":false,"name":"taskPromptId","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{},"additionalProperties":{}}}}}}}},"/api/v1/criteria/graph":{"get":{"tags":["Criteria"],"summary":"Get criteria DAG","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CriteriaGraph"}}}}}}},"/api/v1/criteria/{id}":{"get":{"tags":["Criteria"],"summary":"Get criterion","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CriteriaResponse"}}}},"404":{"description":"Criterion not found"}}},"put":{"tags":["Criteria"],"summary":"Update criterion","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateCriteriaInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CriteriaResponse"}}}},"400":{"description":"Invalid dependency reference or self-reference"},"404":{"description":"Criterion not found"}}},"delete":{"tags":["Criteria"],"summary":"Soft-delete criterion","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"deleted":{"type":"boolean"}},"required":["id","deleted"]}}}},"404":{"description":"Criterion not found"},"409":{"description":"Criterion has dependents"}}}},"/api/v1/prompt-features/generate-prompt":{"post":{"tags":["Prompt Features"],"summary":"Generate prompt feature from behavior","parameters":[{"schema":{"type":"string","minLength":1,"description":"Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.","example":"00000000-0000-0000-0000-000000000000"},"required":false,"description":"Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"behavior":{"type":"string"},"currentId":{"type":"string"}},"required":["behavior"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"prompt":{"type":"string"}},"required":["prompt"]}}}},"400":{"description":"Empty behavior string"},"503":{"description":"LLM not configured"}}}},"/api/v1/prompt-features/seed":{"post":{"tags":["Prompt Features"],"summary":"Seed prompt features in bulk","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"features":{"type":"array","items":{"$ref":"#/components/schemas/CreatePromptFeatureInput"}}},"required":["features"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"seeded":{"type":"number"},"errors":{"type":"array","items":{"type":"string"}}},"required":["seeded","errors"]}}}}}}},"/api/v1/prompt-features/extract-from-text":{"post":{"tags":["Prompt Features"],"summary":"Extract features from text","parameters":[{"schema":{"type":"string","minLength":1,"description":"Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.","example":"00000000-0000-0000-0000-000000000000"},"required":false,"description":"Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"text":{"type":"string"},"model":{"type":"string"},"type":{"type":"string","enum":["select","agents.md"]}},"required":["text"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"features":{"type":"array","items":{"$ref":"#/components/schemas/PromptFeatureResult"}},"suggestedFeatures":{"type":"array","items":{"$ref":"#/components/schemas/SuggestedPromptFeature"}},"cached":{"type":"boolean"}},"required":["features","cached"]}}}},"400":{"description":"Empty text string"},"503":{"description":"LLM not configured"}}}},"/api/v1/prompt-features":{"get":{"tags":["Prompt Features"],"summary":"List features","parameters":[{"schema":{"type":"string"},"required":false,"name":"q","in":"query"},{"schema":{"type":"string","enum":["select","agents.md"]},"required":false,"name":"type","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/PromptFeatureResponse"}}}}}}},"post":{"tags":["Prompt Features"],"summary":"Create feature","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreatePromptFeatureInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/PromptFeatureResponse"}}}},"400":{"description":"Invalid input"},"409":{"description":"Feature already exists"}}}},"/api/v1/prompt-features/{id}":{"get":{"tags":["Prompt Features"],"summary":"Get feature","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/PromptFeatureResponse"}}}},"404":{"description":"Feature not found"}}},"put":{"tags":["Prompt Features"],"summary":"Update feature","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdatePromptFeatureInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/PromptFeatureResponse"}}}},"400":{"description":"Invalid input"},"404":{"description":"Feature not found"}}},"delete":{"tags":["Prompt Features"],"summary":"Soft-delete feature","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"deleted":{"type":"boolean"}},"required":["id","deleted"]}}}},"404":{"description":"Feature not found"}}}},"/api/v1/task-prompts/generate":{"post":{"tags":["Task Prompts"],"summary":"Generate task prompts","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"description":{"type":"string"},"existingPrompt":{"type":"string"}}}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"tasks":{"type":"array","items":{"type":"string"}}},"required":["tasks"]}}}},"503":{"description":"LLM not configured"}}}},"/api/v1/task-prompts":{"get":{"tags":["Task Prompts"],"summary":"List task prompts","parameters":[{"schema":{"type":["number","null"]},"required":false,"name":"limit","in":"query"},{"schema":{"type":["number","null"]},"required":false,"name":"offset","in":"query"},{"schema":{"type":"string"},"required":false,"name":"search","in":"query"},{"schema":{"$ref":"#/components/schemas/PromptType"},"required":false,"name":"type","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"items":{"type":"array","items":{"$ref":"#/components/schemas/TaskPromptResponse"}},"total":{"type":"number"},"limit":{"type":"number"},"offset":{"type":"number"}},"required":["items","total","limit","offset"]}}}}}},"post":{"tags":["Task Prompts"],"summary":"Create or find task prompt","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateTaskPromptInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/TaskPromptResponse"}}}},"400":{"description":"Empty text string"}}}},"/api/v1/task-prompts/{id}":{"get":{"tags":["Task Prompts"],"summary":"Get task prompt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/TaskPromptResponse"}}}},"404":{"description":"Task prompt not found"}}},"delete":{"tags":["Task Prompts"],"summary":"Soft-delete task prompt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"deleted":{"type":"boolean"}},"required":["deleted"]}}}},"404":{"description":"Task prompt not found"}}}},"/api/v1/task-prompts/{id}/content":{"get":{"tags":["Task Prompts"],"summary":"Get resolved task prompt content","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"text":{"type":"string"}},"required":["id","text"]}}}},"404":{"description":"Task prompt not found"}}}},"/api/v1/task-prompts/{id}/extract-features":{"post":{"tags":["Task Prompts"],"summary":"Extract features from task prompt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":false,"name":"force","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"model":{"type":"string"}}}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"taskPromptId":{"type":"string"},"features":{"type":"array","items":{"$ref":"#/components/schemas/PromptFeatureResult"}},"featuresExtractedAt":{"type":["string","null"],"format":"date-time"},"suggestedFeatures":{"type":"array","items":{"$ref":"#/components/schemas/SuggestedPromptFeature"}},"cached":{"type":"boolean"}},"required":["taskPromptId","features","cached"]}}}},"404":{"description":"Task prompt not found"},"503":{"description":"LLM not configured"}}}},"/api/v1/task-prompts/{id}/features/{featureId}":{"patch":{"tags":["Task Prompts"],"summary":"Toggle feature flag on task prompt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"featureId","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/PatchTaskPromptFeatureInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/TaskPromptResponse"}}}},"400":{"description":"Invalid detected value"},"404":{"description":"Task prompt or feature not found"}}}},"/api/v1/reports":{"post":{"tags":["Reports"],"summary":"Create report","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateReportInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ReportResponse"}}}},"400":{"description":"Invalid input"},"404":{"description":"Run or template not found"}}},"get":{"tags":["Reports"],"summary":"List reports","parameters":[{"schema":{"type":"string"},"required":false,"name":"requestId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"experimentId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"experimentExecutionId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ReportResponse"}}}}}}}},"/api/v1/reports/bulk-create":{"post":{"tags":["Reports"],"summary":"Bulk create reports","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/BulkCreateReportsInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ReportResponse"}}}}},"400":{"description":"Invalid input"}}}},"/api/v1/reports/{id}/structured":{"get":{"tags":["Reports"],"summary":"Download a report's structured JSON artifact","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success"},"404":{"description":"Report or structured artifact not found"}}}},"/api/v1/reports/bulk-status":{"post":{"tags":["Reports"],"summary":"Bulk get report statuses","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/BulkReportStatusInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ReportResponse"}}}}},"400":{"description":"Invalid input"}}}},"/api/v1/reports/bulk-summary":{"post":{"tags":["Reports"],"summary":"Bulk get report summary per run","description":"Returns aggregated report status counts per run. Only the latest report per template is counted (re-triggers are deduplicated).","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/BulkReportSummaryInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/BulkReportSummaryResponse"}}}},"400":{"description":"Invalid input"}}}},"/api/v1/reports/{id}":{"get":{"tags":["Reports"],"summary":"Get report","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ReportResponse"}}}},"404":{"description":"Report not found"}}}},"/api/v1/reports/{id}/logs":{"get":{"tags":["Reports"],"summary":"Stream report logs (SSE)","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Server-sent event stream of log entries"},"404":{"description":"Report not found"}}}},"/api/v1/reports/trigger":{"post":{"tags":["Reports"],"summary":"Trigger reports","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/TriggerReportsInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"triggered":{"type":"number"},"reports":{"type":"array","items":{"$ref":"#/components/schemas/ReportResponse"}}},"required":["triggered","reports"]}}}},"400":{"description":"Invalid input"},"404":{"description":"Run not found"}}}},"/api/v1/reports/bulk-trigger":{"post":{"tags":["Reports"],"summary":"Bulk trigger reports","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/BulkTriggerReportsInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"type":"object","properties":{},"additionalProperties":{}}}}}},"400":{"description":"Invalid input"}}}},"/api/v1/reports/{id}/insights":{"get":{"tags":["Reports"],"summary":"Get insights for report","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"allOf":[{"$ref":"#/components/schemas/InsightResponse"},{"type":"object","properties":{"referencedAt":{"type":["string","null"],"format":"date-time"},"isNew":{"type":"boolean"}}}]}}}}},"404":{"description":"Report not found"}}},"post":{"tags":["Reports"],"summary":"Link insight to report","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"insightId":{"type":"string"},"isNew":{"type":"boolean"}},"required":["insightId"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"insightId":{"type":"string"},"referencedAt":{"type":["string","null"],"format":"date-time"},"isNew":{"type":"boolean"}},"required":["insightId","referencedAt","isNew"]}}}},"404":{"description":"Report or insight not found"},"409":{"description":"Insight already referenced by this report"}}}},"/api/v1/profiles":{"post":{"tags":["Profiles"],"summary":"Create a new profile","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateProfileInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProfileWithVersionResponse"}}}}}},"get":{"tags":["Profiles"],"summary":"List profiles","parameters":[{"schema":{"type":"string"},"required":false,"name":"workerType","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ProfileWithVersionResponse"}}}}}}}},"/api/v1/profiles/{profileId}":{"get":{"tags":["Profiles"],"summary":"Get profile with latest version","parameters":[{"schema":{"type":"string"},"required":true,"name":"profileId","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProfileWithVersionResponse"}}}}}},"post":{"tags":["Profiles"],"summary":"Create new profile version","parameters":[{"schema":{"type":"string"},"required":true,"name":"profileId","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"workerType":{"type":"string"},"model":{"type":"string"},"reasoningEffort":{"type":"string"},"agentVersion":{"type":"string"},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceBindingSpec"}},"extensions":{"type":"array","items":{"type":"string"}}},"required":["workerType","model"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProfileVersionResponse"}}}}}},"put":{"tags":["Profiles"],"summary":"Update profile identity","parameters":[{"schema":{"type":"string"},"required":true,"name":"profileId","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateProfileIdentity"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProfileResponse"}}}}}},"delete":{"tags":["Profiles"],"summary":"Delete profile","parameters":[{"schema":{"type":"string"},"required":true,"name":"profileId","in":"path"}],"responses":{"204":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}}}}},"/api/v1/profiles/{profileId}/versions":{"get":{"tags":["Profiles"],"summary":"List profile versions","parameters":[{"schema":{"type":"string"},"required":true,"name":"profileId","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ProfileVersionResponse"}}}}}}}},"/api/v1/profiles/{profileId}/versions/{version}":{"get":{"tags":["Profiles"],"summary":"Get profile version","parameters":[{"schema":{"type":"string"},"required":true,"name":"profileId","in":"path"},{"schema":{"type":["number","null"]},"required":false,"name":"version","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProfileVersionResponse"}}}}}}},"/api/v1/report-templates/default-system-prompt":{"get":{"tags":["Report Templates"],"summary":"Get default system prompt","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"content":{"type":"string"}},"required":["content"]}}}}}}},"/api/v1/report-templates/available-models":{"get":{"tags":["Report Templates"],"summary":"List models available for report generation","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"type":"object","properties":{"modelId":{"type":"string"}},"required":["modelId"]}}}}}}}},"/api/v1/report-templates":{"get":{"tags":["Report Templates"],"summary":"List report templates","parameters":[{"schema":{"type":"string"},"required":false,"name":"q","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ReportTemplateResponse"}}}}}}},"post":{"tags":["Report Templates"],"summary":"Create report template","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateReportTemplateInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ReportTemplateResponse"}}}},"409":{"description":"Report template already exists"}}}},"/api/v1/report-templates/{id}":{"get":{"tags":["Report Templates"],"summary":"Get report template","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ReportTemplateResponse"}}}},"404":{"description":"Report template not found"}}},"put":{"tags":["Report Templates"],"summary":"Update report template","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateReportTemplateInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ReportTemplateResponse"}}}},"404":{"description":"Report template not found"}}},"delete":{"tags":["Report Templates"],"summary":"Delete report template","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"204":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}},"404":{"description":"Report template not found"}}}},"/api/v1/agents":{"get":{"tags":["Agents"],"summary":"List agents","parameters":[{"schema":{"type":"string"},"required":false,"name":"modelProvider","in":"query"},{"schema":{"type":"string","enum":["true","false"]},"required":false,"name":"includeDeleted","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/AgentResponse"}}}}}}},"post":{"tags":["Agents"],"summary":"Create or update agent (upsert)","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateAgentInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AgentResponse"}}}},"400":{"description":"Validation error"},"409":{"description":"Restored agent versions conflict with an active queue owner"}}}},"/api/v1/agents/{id}":{"get":{"tags":["Agents"],"summary":"Get agent","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","enum":["true","false"]},"required":false,"name":"includeDeleted","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AgentResponse"}}}},"404":{"description":"Agent not found"}}},"put":{"tags":["Agents"],"summary":"Update agent","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateAgentInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AgentResponse"}}}},"400":{"description":"Validation error"},"404":{"description":"Agent not found"}}},"delete":{"tags":["Agents"],"summary":"Delete agent","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"204":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}},"404":{"description":"Agent not found"}}}},"/api/v1/agents/{id}/versions":{"get":{"tags":["Agents"],"summary":"List agent versions","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":false,"name":"status","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/AgentVersion"}}}}},"404":{"description":"Agent not found"}}},"post":{"tags":["Agents"],"summary":"Register agent version (upsert)","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/RegisterAgentVersionInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AgentVersion"}}}},"400":{"description":"Validation error"},"404":{"description":"Agent not found"},"409":{"description":"Queue is assigned to another agent"},"503":{"description":"Concurrent registration did not stabilize"}}}},"/api/v1/agents/{id}/versions/{agentVersion}":{"patch":{"tags":["Agents"],"summary":"Patch agent version","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"agentVersion","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/PatchAgentVersionInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AgentVersion"}}}},"400":{"description":"Invalid status value"},"404":{"description":"Agent or version not found"},"409":{"description":"Queue is assigned to another agent"},"503":{"description":"Concurrent activation did not stabilize"}}}},"/api/v1/models":{"get":{"tags":["Models"],"summary":"List models","parameters":[{"schema":{"type":"string"},"required":false,"name":"agentId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"provider","in":"query"},{"schema":{"type":"string","enum":["active","disappeared"]},"required":false,"name":"status","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ModelResponse"}}}}}}}},"/api/v1/models/{id}":{"get":{"tags":["Models"],"summary":"Get model","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ModelResponse"}}}}}}},"/api/v1/models/sync":{"post":{"tags":["Models"],"summary":"Sync models from provider","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"agentId":{"type":"string"},"provider":{"type":"string"},"models":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string"},"providerAvailableFrom":{"type":"string"},"providerEndOfLife":{"type":"string"},"metadata":{"type":"object","additionalProperties":{}},"capabilities":{"type":"object","properties":{"reasoningEffort":{"type":"array","items":{"type":"string"}},"toolCalls":{"type":"boolean"},"vision":{"type":"boolean"},"streaming":{"type":"boolean"},"adaptiveThinking":{"type":"boolean"},"maxThinkingBudget":{"type":"number"}}}},"required":["id"]}},"scannedAt":{"type":"string"}},"required":["agentId","provider","models","scannedAt"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"added":{"type":"array","items":{"type":"string"}},"removed":{"type":"array","items":{"type":"string"}},"unchanged":{"type":"array","items":{"type":"string"}}},"required":["added","removed","unchanged"]}}}}}}},"/api/v1/mcp/servers":{"get":{"tags":["MCP Servers"],"summary":"List MCP servers","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/McpServerResponse"}}}}}}},"post":{"tags":["MCP Servers"],"summary":"Create MCP server","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"_id":{"type":"string","pattern":"^[a-z0-9]([a-z0-9-]*[a-z0-9])?$"},"name":{"type":"string"},"type":{"type":"string","enum":["sse","http","stdio"]},"url":{"type":"string"},"command":{"type":"string"},"args":{"type":"array","items":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"headers":{"type":"array","items":{"$ref":"#/components/schemas/McpServerHeader"}},"sessionMode":{"type":"string","enum":["stateful","stateless"]},"version":{"type":"string"},"description":{"type":"string"}},"required":["_id","name","type"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/McpServerResponse"}}}}}}},"/api/v1/mcp/servers/{id}":{"get":{"tags":["MCP Servers"],"summary":"Get MCP server","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/McpServerResponse"}}}}}},"put":{"tags":["MCP Servers"],"summary":"Update MCP server","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateMcpServerInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/McpServerResponse"}}}}}},"delete":{"tags":["MCP Servers"],"summary":"Delete MCP server","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"204":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}}}}},"/api/v1/skills":{"get":{"tags":["Skills"],"summary":"List all skills","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/SkillResponse"}}}}}}},"post":{"tags":["Skills"],"summary":"Create or import a skill","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateSkillInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SkillResponse"}}}}}}},"/api/v1/skills/search":{"get":{"tags":["Skills"],"summary":"Search skills (internal + external)","parameters":[{"schema":{"type":"string"},"required":true,"name":"q","in":"query"},{"schema":{"type":"string"},"required":false,"name":"limit","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/SkillSearchResult"}}}}},"400":{"description":"Missing query parameter"}}}},"/api/v1/skills/search/external":{"get":{"tags":["Skills"],"summary":"Search external skills registry","parameters":[{"schema":{"type":"string"},"required":true,"name":"q","in":"query"},{"schema":{"type":"string"},"required":false,"name":"limit","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Optional project scope. When provided, external results are de-duplicated only against skills already installed in that project."},"required":false,"description":"Optional project scope. When provided, external results are de-duplicated only against skills already installed in that project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/SkillSearchResult"}}}}},"400":{"description":"Missing query parameter"}}}},"/api/v1/skills/discover":{"get":{"tags":["Skills"],"summary":"Discover skills in a GitHub repository","parameters":[{"schema":{"type":"string"},"required":true,"name":"source","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/SkillDiscoveryResult"}}}}},"400":{"description":"Missing or malformed source parameter"},"404":{"description":"Repository not found"},"502":{"description":"GitHub API error"}}}},"/api/v1/skills/{id}(*)/revisions":{"get":{"tags":["Skills"],"summary":"List skill revisions","parameters":[{"schema":{"type":"string"},"required":false,"name":"limit","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/SkillRevisionResponse"}}}}},"404":{"description":"Skill not found"}}}},"/api/v1/skills/{id}(*)":{"get":{"tags":["Skills"],"summary":"Get skill by slug","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SkillResponse"}}}},"404":{"description":"Skill not found"}}},"delete":{"tags":["Skills"],"summary":"Soft-delete a skill","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"204":{"description":"Success"},"404":{"description":"Skill not found"}}}},"/api/v1/skill-revisions/by-ref/{ref}(*)/archive":{"get":{"tags":["Skill Revisions"],"summary":"Download skill revision archive","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Binary tar.gz archive"},"404":{"description":"Skill revision or archive not found"},"500":{"description":"Blob storage error"}}}},"/api/v1/skill-revisions/by-ref/{ref}(*)":{"get":{"tags":["Skill Revisions"],"summary":"Get skill revision by ref","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SkillRevisionResponse"}}}},"404":{"description":"Skill revision not found"}}}},"/api/v1/skill-revisions/{id}":{"get":{"tags":["Skill Revisions"],"summary":"Get skill revision by ID","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SkillRevisionResponse"}}}},"404":{"description":"Skill revision not found"}}}},"/api/v1/skills/{id}(*)/resolve":{"post":{"tags":["Skills"],"summary":"Trigger skill resolution","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SkillRevisionResponse"}}}},"404":{"description":"Skill not found"},"500":{"description":"Blob storage error"}}}},"/api/v1/codebases":{"get":{"tags":["Codebases"],"summary":"List all codebases","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/CodebaseResponse"}}}}}}},"post":{"tags":["Codebases"],"summary":"Create a codebase (archive codebases require the archive file)","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"201":{"description":"Success"},"400":{"description":"Invalid input or missing archive"}}}},"/api/v1/codebases/{id}":{"get":{"tags":["Codebases"],"summary":"Get a codebase","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CodebaseResponse"}}}},"404":{"description":"Codebase not found"}}},"patch":{"tags":["Codebases"],"summary":"Update a codebase","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateCodebaseInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CodebaseResponse"}}}},"404":{"description":"Codebase not found"}}},"delete":{"tags":["Codebases"],"summary":"Delete a codebase","responses":{"204":{"description":"Success"},"404":{"description":"Codebase not found"}}}},"/api/v1/codebases/{id}/revisions":{"get":{"tags":["Codebases"],"summary":"List codebase revisions","parameters":[{"schema":{"type":"string"},"required":false,"name":"limit","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/CodebaseRevisionResponse"}}}}},"404":{"description":"Codebase not found"}}},"post":{"tags":["Codebases"],"summary":"Resolve a new git codebase revision","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ResolveCodebaseRevisionInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CodebaseRevisionResponse"}}}},"400":{"description":"Not a git codebase"},"404":{"description":"Codebase not found"},"502":{"description":"GitHub resolution failed"}}}},"/api/v1/codebases/{id}/upload":{"post":{"tags":["Codebases"],"summary":"Upload a codebase archive as a new revision","responses":{"201":{"description":"Success"},"400":{"description":"Missing archive or not an archive codebase"},"404":{"description":"Codebase not found"}}}},"/api/v1/codebase-revisions/{id}/archive":{"get":{"tags":["Codebase Revisions"],"summary":"Download codebase revision archive","responses":{"200":{"description":"Binary tar.gz archive"},"404":{"description":"Revision or archive not found"},"500":{"description":"Blob storage error"}}}},"/api/v1/codebase-revisions/{id}":{"get":{"tags":["Codebase Revisions"],"summary":"Get codebase revision by id","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CodebaseRevisionResponse"}}}},"404":{"description":"Revision not found"}}}},"/api/v1/resources":{"get":{"tags":["Resources"],"summary":"List all resources","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ResourceResponse"}}}}}}},"post":{"tags":["Resources"],"summary":"Create a resource and its first revision","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateResourceInput"}}}},"responses":{"201":{"description":"Success"},"400":{"description":"Invalid input"},"409":{"description":"Resource slug or revision ref already exists in this project"}}}},"/api/v1/resources/{id}":{"get":{"tags":["Resources"],"summary":"Get a resource","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ResourceResponse"}}}},"404":{"description":"Resource not found"}}},"patch":{"tags":["Resources"],"summary":"Update a resource","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateResourceInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ResourceResponse"}}}},"404":{"description":"Resource not found"}}},"delete":{"tags":["Resources"],"summary":"Delete a resource","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"204":{"description":"Success"},"404":{"description":"Resource not found"}}}},"/api/v1/resources/{id}/revisions":{"get":{"tags":["Resource Revisions"],"summary":"List resource revisions","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"limit","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ResourceRevisionResponse"}}}}},"404":{"description":"Resource not found"}}},"post":{"tags":["Resource Revisions"],"summary":"Create a resource revision","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateResourceRevisionInput"}}}},"responses":{"201":{"description":"Success"},"404":{"description":"Resource not found"},"409":{"description":"Resource revision ref already exists in this project"}}}},"/api/v1/resources/{id}/revisions/latest":{"get":{"tags":["Resource Revisions"],"summary":"Get the latest resource revision","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ResourceRevisionResponse"}}}},"404":{"description":"Revision not found"}}}},"/api/v1/resources/{id}/revisions/{revisionNumber}":{"get":{"tags":["Resource Revisions"],"summary":"Get a resource revision by resource and revision number","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"revisionNumber","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ResourceRevisionResponse"}}}},"404":{"description":"Revision not found"}}}},"/api/v1/resources/revisions/{id}":{"get":{"tags":["Resource Revisions"],"summary":"Get resource revision by id","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ResourceRevisionResponse"}}}},"404":{"description":"Revision not found"}}}},"/api/v1/extensions":{"get":{"tags":["Extensions"],"summary":"List all extensions","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ExtensionResponse"}}}}}}},"post":{"tags":["Extensions"],"summary":"Create or import an extension","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateExtensionInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ExtensionResponse"}}}}}}},"/api/v1/extensions/search":{"get":{"tags":["Extensions"],"summary":"Search extensions (internal + marketplace)","parameters":[{"schema":{"type":"string"},"required":true,"name":"q","in":"query"},{"schema":{"type":"string"},"required":false,"name":"limit","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ExtensionSearchResult"}}}}},"400":{"description":"Missing query parameter"}}}},"/api/v1/extensions/{id}":{"get":{"tags":["Extensions"],"summary":"Get extension by ID","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ExtensionResponse"}}}}}},"put":{"tags":["Extensions"],"summary":"Update extension","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateExtensionInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ExtensionResponse"}}}}}},"delete":{"tags":["Extensions"],"summary":"Delete extension","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"204":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}}}}},"/api/v1/extensions/{id}/versions":{"get":{"tags":["Extensions"],"summary":"List available versions for an extension","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":false,"name":"preRelease","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ExtensionVersionInfo"}}}}}}}},"/api/v1/insights":{"get":{"tags":["Insights"],"summary":"List insights","parameters":[{"schema":{"type":"string"},"required":false,"name":"q","in":"query"},{"schema":{"type":"string"},"required":false,"name":"blocked","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/InsightResponse"}}}}}}},"post":{"tags":["Insights"],"summary":"Create insight","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateInsightInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}}}}},"/api/v1/insights/search":{"get":{"tags":["Insights"],"summary":"Search insights","parameters":[{"schema":{"type":"string"},"required":true,"name":"q","in":"query"},{"schema":{"type":"string"},"required":false,"name":"blocked","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/InsightResponse"}}}}}}}},"/api/v1/insights/{id}":{"get":{"tags":["Insights"],"summary":"Get insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}},"404":{"description":"Insight not found"}}},"put":{"tags":["Insights"],"summary":"Update insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateInsightInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}},"404":{"description":"Insight not found"}}},"delete":{"tags":["Insights"],"summary":"Delete insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"204":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}},"404":{"description":"Insight not found"}}}},"/api/v1/insights/{id}/upvote":{"post":{"tags":["Insights"],"summary":"Upvote insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}},"404":{"description":"Insight not found"}}}},"/api/v1/insights/{id}/downvote":{"post":{"tags":["Insights"],"summary":"Downvote insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}},"404":{"description":"Insight not found"}}}},"/api/v1/insights/{id}/block":{"post":{"tags":["Insights"],"summary":"Block insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}},"404":{"description":"Insight not found"}}}},"/api/v1/insights/{id}/unblock":{"post":{"tags":["Insights"],"summary":"Unblock insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}},"404":{"description":"Insight not found"}}}},"/api/v1/insights/{id}/reports":{"get":{"tags":["Insights"],"summary":"Get reports referencing insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ReportResponse"}}}}},"404":{"description":"Insight not found"}}}},"/api/v1/feature-flags":{"get":{"tags":["Feature Flags"],"summary":"List feature flags","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/FeatureFlagResponse"}}}}}}}},"/api/v1/feature-flags/{key}":{"put":{"tags":["Feature Flags"],"summary":"Update feature flag","parameters":[{"schema":{"type":"string"},"required":true,"name":"key","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateFeatureFlagInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/FeatureFlagResponse"}}}}}}}},"webhooks":{}} \ No newline at end of file