From a1612d4bf7d3a77d6a6d91e6c30d0a367a411a61 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Tue, 13 Oct 2026 16:26:21 +0200 Subject: [PATCH 01/25] feat(submissions): add attempt state foundation Introduce stable submission identities, deterministic request/run vectors, resumable CAS rollover with immutable history, named-handler dependency evaluation, Cosmos-safe indexes, and attempt-scoped worker claim/completion helpers. Align the observation handler identity with the pp-observations dependency contract. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 99f84f04-d3b0-40f1-a254-4dfbd21e4d87 --- apps/judge/src/agent-trajectory.test.ts | 2 +- ...observation-trajectory.integration.test.ts | 2 +- apps/portal/src/lib/enrichment.test.ts | 28 +- apps/portal/src/lib/enrichment.ts | 10 +- apps/scheduler/src/handler-dispatcher.ts | 69 +- apps/workers/post-processor/Dockerfile | 2 +- .../post-processor/handler.taxonomy.yaml | 6 +- .../src/handlers/taxonomy-handler.test.ts | 6 +- .../src/handlers/taxonomy-handler.ts | 6 +- docker-compose.yml | 4 +- docs/architecture/criteria-provider.md | 2 +- docs/architecture/post-processing.md | 37 +- .../src/034-create-submission-indexes.test.ts | 52 ++ .../034-create-submission-indexes.ts | 67 ++ .../db-migrations/src/required-migrations.ts | 1 + .../shared/src/criteria/criteria-store.ts | 2 +- packages/shared/src/index.ts | 1 + packages/shared/src/judge/judge-client.ts | 4 +- packages/shared/src/schemas/index.ts | 1 + packages/shared/src/schemas/request.ts | 4 + packages/shared/src/schemas/submission.ts | 182 +++++ .../shared/src/scripts/register-handler.ts | 30 +- packages/shared/src/submissions/index.ts | 5 + .../submission-coordinator.test.ts | 693 ++++++++++++++++++ .../src/submissions/submission-coordinator.ts | 573 +++++++++++++++ .../src/submissions/submission-state.test.ts | 155 ++++ .../src/submissions/submission-state.ts | 184 +++++ packages/shared/src/types/types.ts | 130 +++- 28 files changed, 2167 insertions(+), 91 deletions(-) create mode 100644 packages/db-migrations/src/034-create-submission-indexes.test.ts create mode 100644 packages/db-migrations/src/migrations/034-create-submission-indexes.ts create mode 100644 packages/shared/src/schemas/submission.ts create mode 100644 packages/shared/src/submissions/index.ts create mode 100644 packages/shared/src/submissions/submission-coordinator.test.ts create mode 100644 packages/shared/src/submissions/submission-coordinator.ts create mode 100644 packages/shared/src/submissions/submission-state.test.ts create mode 100644 packages/shared/src/submissions/submission-state.ts diff --git a/apps/judge/src/agent-trajectory.test.ts b/apps/judge/src/agent-trajectory.test.ts index 8bb870184..ac4d713b8 100644 --- a/apps/judge/src/agent-trajectory.test.ts +++ b/apps/judge/src/agent-trajectory.test.ts @@ -48,7 +48,7 @@ async function call( /** * Realistic ATIF blob: system + user step, then agent steps that install, - * use, and uninstall `left-pad`. Mirrors the real ATIF shape pp-taxonomy + * use, and uninstall `left-pad`. Mirrors the real ATIF shape pp-observations * downloads (steps carry their own tool_calls AND the matching * observation.results, paired by source_call_id === tool_call_id). */ diff --git a/apps/judge/src/observation-trajectory.integration.test.ts b/apps/judge/src/observation-trajectory.integration.test.ts index 066c40da3..d8b89150d 100644 --- a/apps/judge/src/observation-trajectory.integration.test.ts +++ b/apps/judge/src/observation-trajectory.integration.test.ts @@ -34,7 +34,7 @@ const hasToken = !!process.env.GITHUB_TOKEN; const JUDGE_MODEL = process.env.JUDGE_MODEL || "gpt-5.4-mini"; /** - * Realistic ATIF trajectory (the exact shape pp-taxonomy downloads and forwards + * Realistic ATIF trajectory (the exact shape pp-observations downloads and forwards * via `fromAtif`): a system + user step, then agent steps that install * `left-pad`, use it, and uninstall it before finishing. Each agent step pairs * its `tool_calls[]` with the matching same-step `observation.results[]` by diff --git a/apps/portal/src/lib/enrichment.test.ts b/apps/portal/src/lib/enrichment.test.ts index 879bc5b11..9e4b19024 100644 --- a/apps/portal/src/lib/enrichment.test.ts +++ b/apps/portal/src/lib/enrichment.test.ts @@ -33,59 +33,59 @@ describe("deriveEnrichmentStatus", () => { expect(deriveEnrichmentStatus(run, false)).toBe("done"); }); - it("does NOT flip to done after pp-atif while pp-taxonomy is still pending (the bug)", () => { - // Observations configured, pp-atif done, pp-taxonomy not yet dispatched. + it("does NOT flip to done after pp-atif while pp-observations is still pending (the bug)", () => { + // Observations configured, pp-atif done, pp-observations not yet dispatched. const beforeDispatch = makeRun({ postProcessorStatus: "done", // legacy scalar already flipped — must be ignored handlerStatus: { "pp-atif": { status: "done" } }, }); expect(deriveEnrichmentStatus(beforeDispatch, true)).toBe("processing"); - // pp-taxonomy queued. + // pp-observations queued. const queued = makeRun({ postProcessorStatus: "done", - handlerStatus: { "pp-atif": { status: "done" }, "pp-taxonomy": { status: "queued" } }, + handlerStatus: { "pp-atif": { status: "done" }, "pp-observations": { status: "queued" } }, }); expect(deriveEnrichmentStatus(queued, true)).toBe("processing"); - // pp-taxonomy processing. + // pp-observations processing. const processing = makeRun({ postProcessorStatus: "done", - handlerStatus: { "pp-atif": { status: "done" }, "pp-taxonomy": { status: "processing" } }, + handlerStatus: { "pp-atif": { status: "done" }, "pp-observations": { status: "processing" } }, }); expect(deriveEnrichmentStatus(processing, true)).toBe("processing"); }); - it("flips to done only once both pp-atif and pp-taxonomy are done", () => { + it("flips to done only once both pp-atif and pp-observations are done", () => { const run = makeRun({ - handlerStatus: { "pp-atif": { status: "done" }, "pp-taxonomy": { status: "done" } }, + handlerStatus: { "pp-atif": { status: "done" }, "pp-observations": { status: "done" } }, }); expect(deriveEnrichmentStatus(run, true)).toBe("done"); }); it("reports failed when a handler failed and nothing is still in flight", () => { const run = makeRun({ - handlerStatus: { "pp-atif": { status: "done" }, "pp-taxonomy": { status: "failed" } }, + handlerStatus: { "pp-atif": { status: "done" }, "pp-observations": { status: "failed" } }, }); expect(deriveEnrichmentStatus(run, true)).toBe("failed"); }); it("prefers processing over failed while another handler is still running", () => { const run = makeRun({ - handlerStatus: { "pp-atif": { status: "processing" }, "pp-taxonomy": { status: "failed" } }, + handlerStatus: { "pp-atif": { status: "processing" }, "pp-observations": { status: "failed" } }, }); expect(deriveEnrichmentStatus(run, true)).toBe("processing"); }); - it("treats a no-op pp-taxonomy (no observations) as part of the DAG", () => { - // No observations configured, but pp-taxonomy still runs as a no-op handler. + it("treats a no-op pp-observations (no observations) as part of the DAG", () => { + // No observations configured, but pp-observations still runs as a no-op handler. const stillRunning = makeRun({ - handlerStatus: { "pp-atif": { status: "done" }, "pp-taxonomy": { status: "processing" } }, + handlerStatus: { "pp-atif": { status: "done" }, "pp-observations": { status: "processing" } }, }); expect(deriveEnrichmentStatus(stillRunning, false)).toBe("processing"); const allDone = makeRun({ - handlerStatus: { "pp-atif": { status: "done" }, "pp-taxonomy": { status: "done" } }, + handlerStatus: { "pp-atif": { status: "done" }, "pp-observations": { status: "done" } }, }); expect(deriveEnrichmentStatus(allDone, false)).toBe("done"); }); diff --git a/apps/portal/src/lib/enrichment.ts b/apps/portal/src/lib/enrichment.ts index 99f5bfebc..756a510d9 100644 --- a/apps/portal/src/lib/enrichment.ts +++ b/apps/portal/src/lib/enrichment.ts @@ -12,14 +12,14 @@ export type PostProcessorStatus = "queued" | "processing" | "done" | "failed"; * * Each post-processor handler overwrites that scalar on its own completion, so * it flips to "done" as soon as the FIRST handler (pp-atif) finishes — even - * while pp-taxonomy is still computing observations. The "Enriched" badge (and + * while pp-observations is still computing observations. The "Enriched" badge (and * the run-detail poller) must instead stay "Enriching…" until every handler in * the DAG is terminal. * - * Known topology: pp-atif always runs; pp-taxonomy is expected whenever the run + * Known topology: pp-atif always runs; pp-observations is expected whenever the run * has configured observations. We treat taxonomy as still-pending until - * pp-taxonomy reaches a terminal state, which also covers the brief window - * after pp-atif finishes but before the scheduler has dispatched pp-taxonomy + * pp-observations reaches a terminal state, which also covers the brief window + * after pp-atif finishes but before the scheduler has dispatched pp-observations * (so it isn't yet present in `handlerStatus`). * * Falls back to the legacy `postProcessorStatus` scalar for runs that predate @@ -36,7 +36,7 @@ export function deriveEnrichmentStatus( return run.postProcessorStatus; } const statuses = Object.values(handlerStatus).map((h) => h?.status); - const taxonomyStatus = handlerStatus["pp-taxonomy"]?.status; + const taxonomyStatus = handlerStatus["pp-observations"]?.status; const taxonomyPending = hasConfiguredObservations && taxonomyStatus !== "done" && taxonomyStatus !== "failed"; if (taxonomyPending || statuses.some((s) => s === "queued" || s === "processing")) { diff --git a/apps/scheduler/src/handler-dispatcher.ts b/apps/scheduler/src/handler-dispatcher.ts index 4ff1f8275..d744092c3 100644 --- a/apps/scheduler/src/handler-dispatcher.ts +++ b/apps/scheduler/src/handler-dispatcher.ts @@ -3,8 +3,13 @@ import { Collection, Db } from "mongodb"; import { QueueClient } from "@azure/storage-queue"; -import type { RequestDocument, HandlerServiceDocument, HandlerRunStatus } from "shared"; -import { DependencyGraph, withRetry } from "shared"; +import type { + RequestDocument, + HandlerServiceDocument, + HandlerRunStatus, + ServiceHandlerDocument, +} from "shared"; +import { DependencyGraph, isHandlerDagDrained, withRetry } from "shared"; import type { RetryOptions } from "shared"; import type { NotifyHandler } from "./notify-routes.js"; @@ -151,7 +156,27 @@ export class HandlerDispatcher implements NotifyHandler { await this.maybeTriggerReports(requestId, runId); } - async registerHandler(doc: HandlerServiceDocument): Promise { + async registerHandler(doc: ServiceHandlerDocument): Promise { + if (doc.type === "submission-handler") { + const postProcessHandlers = await this.loadHandlers(true); + for (const dependency of doc.dependsOn) { + if (!postProcessHandlers.some((handler) => handler._id === dependency)) { + throw new Error( + `Submission handler '${doc._id}' depends on unknown run handler '${dependency}'`, + ); + } + } + const { _id, ...fields } = doc; + await this.db.collection("services").updateOne( + { _id } as any, + { $set: { ...fields, dependsOn: [...fields.dependsOn], updatedAt: new Date() } } as any, + { upsert: true }, + ); + console.log( + `[HandlerDispatcher] Registered submission handler ${_id} (version=${doc.version}, queue=${doc.queue}, dependsOn=[${doc.dependsOn.join(", ")}])`, + ); + return; + } // Validate the resulting topology has no cycles before persisting. Read // fresh (bypass the cache) so validation reflects any concurrent registrations. const existing = await this.loadHandlers(true); @@ -183,43 +208,12 @@ export class HandlerDispatcher implements NotifyHandler { // ── Report trigger on DAG drain ───────────────────────────────────── - /** - * A handler is "terminal" for a run when it can produce no further work: - * it has succeeded (`done`), failed (`failed`), or is permanently blocked - * because one of its transitive dependencies failed (so it can never run). - */ - private isHandlerTerminal( - handlerId: string, - graph: DependencyGraph, - handlerStatus: Record | undefined, - ): boolean { - const status = handlerStatus?.[handlerId]?.status; - if (status === "done" || status === "failed") return true; - - // Not yet dispatched/processed — terminal only if blocked by a failed - // ancestor (any transitive dependency that failed). - if (status === undefined) { - for (const ancestorId of graph.getAncestors(handlerId)) { - if (handlerStatus?.[ancestorId]?.status === "failed") return true; - } - } - - // "queued" or "processing" — still active. - return false; - } - - /** - * The DAG is drained for a run when every registered handler is terminal. - * Vacuously true when there are no handlers. - */ private isDagDrained( handlers: HandlerServiceDocument[], - graph: DependencyGraph, + _graph: DependencyGraph, handlerStatus: Record | undefined, ): boolean { - return handlers.every((h) => - this.isHandlerTerminal(h._id, graph, handlerStatus), - ); + return isHandlerDagDrained(handlers, handlerStatus); } /** @@ -239,12 +233,11 @@ export class HandlerDispatcher implements NotifyHandler { if ((run as any).reportsTriggeredAt) return; const handlers = await this.loadHandlers(); - const graph = this.buildGraph(handlers); const handlerStatus = (run as any).handlerStatus as | Record | undefined; - if (!this.isDagDrained(handlers, graph, handlerStatus)) return; + if (!this.isDagDrained(handlers, this.buildGraph(handlers), handlerStatus)) return; // Atomically claim the trigger so concurrent notify/poll paths fire once. const claimed = await this.collection.findOneAndUpdate( diff --git a/apps/workers/post-processor/Dockerfile b/apps/workers/post-processor/Dockerfile index 50f1c7a7c..2f5427cfe 100644 --- a/apps/workers/post-processor/Dockerfile +++ b/apps/workers/post-processor/Dockerfile @@ -24,7 +24,7 @@ COPY packages/telemetry/src ./packages/telemetry/src/ COPY apps/workers/post-processor/tsconfig.json ./apps/workers/post-processor/ COPY apps/workers/post-processor/src ./apps/workers/post-processor/src/ # Copy every handler manifest (handler.yaml = pp-atif, handler.taxonomy.yaml = -# pp-taxonomy, plus any future handlers). The registration jobs read these by +# pp-observations, plus any future handlers). The registration jobs read these by # HANDLER_YAML_PATH, so all must be present in the image. COPY apps/workers/post-processor/handler*.yaml ./apps/workers/post-processor/ diff --git a/apps/workers/post-processor/handler.taxonomy.yaml b/apps/workers/post-processor/handler.taxonomy.yaml index 37411625f..849568456 100644 --- a/apps/workers/post-processor/handler.taxonomy.yaml +++ b/apps/workers/post-processor/handler.taxonomy.yaml @@ -2,16 +2,16 @@ # Read by the generic register-handler script and POSTed to the # scheduler's POST /handlers/register endpoint on deploy. # -# pp-taxonomy records per-iteration boolean observations (kind:"observation" +# pp-observations records boolean observations (kind:"observation" # criteria) by reusing the judge. It depends on pp-atif so each turn's ATIF # trajectory is available, giving the judge codebase (snapshot) + trajectory # (ATIF) access. Shares the post-processor-queue with pp-atif (same worker). # See issue #1156. -_id: pp-taxonomy +_id: pp-observations type: post-process-handler version: 1 queue: post-processor-queue -selector: taxonomy +selector: observations autoBackfill: true dependsOn: - pp-atif diff --git a/apps/workers/post-processor/src/handlers/taxonomy-handler.test.ts b/apps/workers/post-processor/src/handlers/taxonomy-handler.test.ts index 1f191caa9..1aeb0b2ec 100644 --- a/apps/workers/post-processor/src/handlers/taxonomy-handler.test.ts +++ b/apps/workers/post-processor/src/handlers/taxonomy-handler.test.ts @@ -32,7 +32,7 @@ function makeCtx( } as any; } -const msg: PostProcessorMessage = { type: "taxonomy", requestId: "req-1", runId: "run-1" }; +const msg: PostProcessorMessage = { type: "observations", requestId: "req-1", runId: "run-1" }; describe("TaxonomyHandler", () => { let judge: { evaluate: ReturnType }; @@ -41,8 +41,8 @@ describe("TaxonomyHandler", () => { judge = { evaluate: vi.fn() }; }); - it("has type 'taxonomy'", () => { - expect(new TaxonomyHandler(judge as unknown as JudgeClient).type).toBe("taxonomy"); + it("has type 'observations'", () => { + expect(new TaxonomyHandler(judge as unknown as JudgeClient).type).toBe("observations"); }); it("no-ops when no observations are selected", async () => { diff --git a/apps/workers/post-processor/src/handlers/taxonomy-handler.ts b/apps/workers/post-processor/src/handlers/taxonomy-handler.ts index 62cf41a85..2baa81365 100644 --- a/apps/workers/post-processor/src/handlers/taxonomy-handler.ts +++ b/apps/workers/post-processor/src/handlers/taxonomy-handler.ts @@ -26,7 +26,7 @@ interface CriterionMeta { } /** - * Taxonomy Handler (pp-taxonomy): records boolean observations. + * Taxonomy Handler (pp-observations): records boolean observations. * * Each selected observation criterion is routed by its `subject` (#1156): * @@ -40,12 +40,12 @@ interface CriterionMeta { * behavior (e.g. a dependency added in one turn and removed in another) visible. * * Reuses the judge service; never gates, never steers the agent. Empty/absent - * `observations` → no-op. pp-taxonomy is dispatched exactly once per run (the + * `observations` → no-op. pp-observations is dispatched exactly once per run (the * scheduler never sets `message.iteration`), so the run-subject branch runs once * with no idempotency guard. */ export class TaxonomyHandler implements PostProcessHandler { - readonly type = "taxonomy"; + readonly type = "observations"; readonly version = POST_PROCESSOR_VERSION; readonly autoBackfill = true; private readonly judge: JudgeClient; diff --git a/docker-compose.yml b/docker-compose.yml index 027888cd2..221dd5c03 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -886,7 +886,7 @@ services: restart: "no" entrypoint: ["npx", "tsx", "packages/shared/src/scripts/register-handler.ts"] - # Registers the pp-taxonomy handler (observation evaluator) — shares the + # Registers the pp-observations handler (observation evaluator) — shares the # post-processor worker + post-processor-queue, depends on pp-atif. Issue #1156. register-handler-taxonomy: build: @@ -918,7 +918,7 @@ services: AZURE_STORAGE_QUEUE_POSTPROCESSOR: post-processor-queue SCOPE_MT_API_URL: http://api:80 SCHEDULER_URL: http://scheduler:8080 - # pp-taxonomy (issue #1156) calls the judge service to evaluate + # pp-observations (issue #1156) calls the judge service to evaluate # observation criteria. Without this the JudgeClient falls back to # localhost:3002 and cannot reach the judge container. JUDGE_SERVICE_URL: http://judge:80 diff --git a/docs/architecture/criteria-provider.md b/docs/architecture/criteria-provider.md index af6db54f6..23ae0d8ec 100644 --- a/docs/architecture/criteria-provider.md +++ b/docs/architecture/criteria-provider.md @@ -181,7 +181,7 @@ bypassed or drift between callers: default): `kind:"observation"` ⇒ `"run"`, `kind:"gate"`/legacy ⇒ `"iteration"`. Gate criteria are rejected if `subject:"run"` is supplied — `"iteration"` is their only legal value. The field is threaded through `create`/`update`/`find` and both provider backends - so the `pp-taxonomy` worker and the CLI/Portal can read it. + so the `pp-observations` worker and the CLI/Portal can read it. - **Referential integrity on delete** — deleting a criterion that still has dependents is rejected. diff --git a/docs/architecture/post-processing.md b/docs/architecture/post-processing.md index dbeec1301..4518ef391 100644 --- a/docs/architecture/post-processing.md +++ b/docs/architecture/post-processing.md @@ -56,9 +56,9 @@ Two paths trigger post-processing: The `iteration` query parameter is mandatory for the ATIF endpoints. ATIF files are also included in archive exports as `iteration-{N}.atif.trajectory.json`. -## Taxonomy Observations (`pp-taxonomy`) +## Taxonomy Observations (`pp-observations`) -The `pp-taxonomy` handler records **observation criteria** — boolean outcomes + +The `pp-observations` handler records **observation criteria** — boolean outcomes + evidence about the codebase and the agent's trajectory, **without gating or steering** the coding agent (issue #1156). It is a DAG node `dependsOn: [pp-atif]`, because some observations are only derivable from the trajectory (ATIF), not the final snapshot. @@ -89,8 +89,8 @@ Gate criteria are always `subject: "iteration"` (the API rejects `"run"` for gat 4. **Run-subject:** evaluates **once** with `snapshotUrl` = final turn's snapshot, `atifUrls` = every turn's `atifUrl` (the judge downloads + merges them into one `AgentTrajectory`), and the full `conversationHistory`. Writes the verdicts to **`run.observationResults`** (run level). - In all judge calls there is no persona, no gate, no taxonomyElementId (the classifier lives on the criterion). ATIF supersedes per-turn `toolCallsUrl`, so only ATIF is sent. - Result shape/storage mirrors gate `criteriaResults`. No new blob, no `run.taxonomyUrl`. - - **Runs exactly once per run:** the scheduler dispatches `pp-taxonomy` a single time (it never sets `message.iteration`), so the run-subject branch needs no idempotency guard. -5. Stamps `run.handlerStatus["pp-taxonomy"] = { status: "done", version: N }` and notifies the scheduler. + - **Runs exactly once per run:** the scheduler dispatches `pp-observations` a single time (it never sets `message.iteration`), so the run-subject branch needs no idempotency guard. +5. Stamps `run.handlerStatus["pp-observations"] = { status: "done", version: N }` and notifies the scheduler. Dimension grouping is **derived at read time** by joining each result's `criterionId` → criterion `taxonomyElementId` (the three R&A Readout dimensions; unset = unclassified). It never feeds back to the agent and never affects pass/fail. (#1053 later extends this worker with a richer scorecard artifact.) @@ -203,7 +203,34 @@ flowchart LR **Where it runs:** - **Docker Compose:** one `register-handler-` init service per handler (e.g. `pp-atif`) runs the generic script via `tsx`, gated on the scheduler being up. The worker waits for its registration service to complete. -- **Kubernetes:** a `register-handler-post-processor` Job runs the compiled script (`node packages/shared/dist/scripts/register-handler.js`) on each deploy. `pp-atif` and `pp-taxonomy` are registered in K8s via their respective registration Jobs (`register-handler-taxonomy` points its `HANDLER_YAML_PATH` at `handler.taxonomy.yaml`). The scheduler exposes a `Service` (`scheduler.scoped.svc.cluster.local:8080`) so Jobs and workers can reach `/handlers/register` and the notify endpoints. +- **Kubernetes:** a `register-handler-post-processor` Job runs the compiled script (`node packages/shared/dist/scripts/register-handler.js`) on each deploy. `pp-atif` and `pp-observations` are registered in K8s via their respective registration Jobs (`register-handler-taxonomy` points its `HANDLER_YAML_PATH` at `handler.taxonomy.yaml`). The scheduler exposes a `Service` (`scheduler.scoped.svc.cluster.local:8080`) so Jobs and workers can reach `/handlers/register` and the notify endpoints. + +## Submission handlers + +Submission handlers use the same `services` registration endpoint but declare +`type: submission-handler` and `dependencyMode: all-runs`. Their dependencies +name run-level DAG handlers, not other submission handlers. The scheduler +reconciles each submission's exact request/run vector, projects those named +dependencies, and enqueues only the current embedded submission attempt. + +The first consumer is `pp-readout`, which depends on `pp-observations` for every +run in the vector. Its queue contract is: + +```json +{ + "type": "readout", + "submissionId": "…", + "submissionRunId": "…" +} +``` + +Workers must use the shared `claimSubmissionHandler`, +`isCurrentSubmissionRun`, and `completeSubmissionHandler` helpers. These helpers +scope every mutation to both identifiers, providing stale-message discard, +cooperative supersession checks, and stale completion rejection. Queue delivery +remains at-least-once; the embedded current attempt is the only dispatchable +state, and the scheduler poll repairs missed notifications, stale projections, +request-vector drift, and interrupted rollover/finalization. To **add or change a handler**, edit its `handler.yaml` (bump `version`, adjust `dependsOn`, etc.) — no code or migration changes are needed for registration. diff --git a/packages/db-migrations/src/034-create-submission-indexes.test.ts b/packages/db-migrations/src/034-create-submission-indexes.test.ts new file mode 100644 index 000000000..1fd59c8bd --- /dev/null +++ b/packages/db-migrations/src/034-create-submission-indexes.test.ts @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { Db } from "mongodb"; +import { describe, expect, it, vi } from "vitest"; +import { CreateSubmissionIndexes } from "./migrations/034-create-submission-indexes.js"; + +function collection() { + return { + createIndex: vi.fn(async () => "index"), + }; +} + +describe("migration 034: CreateSubmissionIndexes", () => { + it("creates submission, history, dispatch, and recovery indexes", async () => { + const submissions = collection(); + const submissionRuns = collection(); + const requests = collection(); + const db = { + collection: vi.fn((name: string) => { + if (name === "submissions") return submissions; + if (name === "submission-runs") return submissionRuns; + if (name === "requests") return requests; + throw new Error(`Unexpected collection ${name}`); + }), + } as unknown as Db; + + await new CreateSubmissionIndexes().up(db); + + expect(submissions.createIndex).toHaveBeenCalledWith( + { projectId: 1, createdAt: -1, _id: -1 }, + {}, + ); + expect(submissions.createIndex).toHaveBeenCalledWith( + { "run.handlerStatus.pp-readout.status": 1 }, + {}, + ); + expect(submissionRuns.createIndex).toHaveBeenCalledWith( + { submissionId: 1, attemptNumber: -1 }, + {}, + ); + expect(requests.createIndex).toHaveBeenCalledWith( + { submissionFinalizedAt: 1 }, + {}, + ); + expect(requests.createIndex).toHaveBeenCalledWith( + { submissionId: 1, projectId: 1, submissionRequestIndex: 1, _id: 1 }, + {}, + ); + }); +}); + diff --git a/packages/db-migrations/src/migrations/034-create-submission-indexes.ts b/packages/db-migrations/src/migrations/034-create-submission-indexes.ts new file mode 100644 index 000000000..8ceb836e9 --- /dev/null +++ b/packages/db-migrations/src/migrations/034-create-submission-indexes.ts @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { Db } from "mongodb"; +import type { MigrationInterface } from "mongo-migrate-ts"; +import { ensureIndex } from "../cosmos-index-helpers.js"; + +const TAG = "034"; + +export class CreateSubmissionIndexes implements MigrationInterface { + async up(db: Db): Promise { + const submissions = db.collection("submissions"); + await ensureIndex( + submissions, + { projectId: 1, createdAt: -1, _id: -1 }, + {}, + "submissions", + TAG, + ); + await ensureIndex( + submissions, + { "run.handlerStatus.pp-readout.status": 1 }, + {}, + "submissions", + TAG, + ); + + const submissionRuns = db.collection("submission-runs"); + await ensureIndex( + submissionRuns, + { submissionId: 1 }, + {}, + "submission-runs", + TAG, + ); + await ensureIndex( + submissionRuns, + { submissionId: 1, attemptNumber: -1 }, + {}, + "submission-runs", + TAG, + ); + + const requests = db.collection("requests"); + await ensureIndex( + requests, + { submissionFinalizedAt: 1 }, + {}, + "requests", + TAG, + ); + await ensureIndex( + requests, + { submissionId: 1, projectId: 1, submissionRequestIndex: 1, _id: 1 }, + {}, + "requests", + TAG, + ); + console.log(`[${TAG}] Submission indexes complete`); + } + + async down(_db: Db): Promise { + console.log( + `[${TAG}-down] Skipping index changes - drop submission indexes manually if needed`, + ); + } +} diff --git a/packages/db-migrations/src/required-migrations.ts b/packages/db-migrations/src/required-migrations.ts index 4d0e54062..bee754391 100644 --- a/packages/db-migrations/src/required-migrations.ts +++ b/packages/db-migrations/src/required-migrations.ts @@ -46,6 +46,7 @@ export const REQUIRED_MIGRATIONS: readonly string[] = [ "031-migrate-services-to-handler-dag.ts", "032-add-handler-status-index.ts", "033-backfill-handler-status.ts", + "034-create-submission-indexes.ts", "021-backfill-criteria-kind.ts", "022-backfill-criteria-subject.ts", ]; diff --git a/packages/shared/src/criteria/criteria-store.ts b/packages/shared/src/criteria/criteria-store.ts index 7413449c9..0b47231b4 100644 --- a/packages/shared/src/criteria/criteria-store.ts +++ b/packages/shared/src/criteria/criteria-store.ts @@ -439,7 +439,7 @@ export class CriteriaStore { /** * Resolve observation-criteria ids, asserting each is an active - * kind:"observation" criterion. Used by submit validation + pp-taxonomy. + * kind:"observation" criterion. Used by submit validation + pp-observations. * Throws CriteriaValidationError on a missing or non-observation id. */ async resolveObservations(ids: string[]): Promise { diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index a6febdece..fae65d02f 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -20,6 +20,7 @@ export * from "./codebases/index.js"; export * from "./resources/index.js"; export * from "./extensions/index.js"; export * from "./projects/index.js"; +export * from "./submissions/index.js"; export * from "./agent-version.js"; export * from "./resolve-agent-version.js"; export * from "./resolve-agent-target.js"; diff --git a/packages/shared/src/judge/judge-client.ts b/packages/shared/src/judge/judge-client.ts index 7a5f85850..633fd6e8f 100644 --- a/packages/shared/src/judge/judge-client.ts +++ b/packages/shared/src/judge/judge-client.ts @@ -18,11 +18,11 @@ export interface JudgeEvaluateRequest { /** Blob URL of this iteration's captured tool calls/outputs (build/test/run output). */ toolCallsUrl?: string; /** Blob URL of this iteration's ATIF trajectory JSON. Normalized superset of - * tool calls; sent by pp-taxonomy in place of toolCallsUrl so observations + * tool calls; sent by pp-observations in place of toolCallsUrl so observations * can be evaluated against the full agent trajectory. See issue #1156. */ atifUrl?: string; /** Blob URLs of EVERY iteration's ATIF trajectory JSON, in iteration order. - * Sent by pp-taxonomy for `subject:"run"` observations so the judge merges + * Sent by pp-observations for `subject:"run"` observations so the judge merges * them into one whole-run trajectory (cross-iteration behavior is visible in a * single evaluation). Takes precedence over `atifUrl`. See issue #1156. */ atifUrls?: string[]; diff --git a/packages/shared/src/schemas/index.ts b/packages/shared/src/schemas/index.ts index cb1df8b4c..52a7bed3f 100644 --- a/packages/shared/src/schemas/index.ts +++ b/packages/shared/src/schemas/index.ts @@ -22,3 +22,4 @@ export * from "./account.js"; export * from "./feature-flag.js"; export * from "./profile.js"; export * from "./project.js"; +export * from "./submission.js"; diff --git a/packages/shared/src/schemas/request.ts b/packages/shared/src/schemas/request.ts index 5271582a5..5aefb2466 100644 --- a/packages/shared/src/schemas/request.ts +++ b/packages/shared/src/schemas/request.ts @@ -181,6 +181,10 @@ export const RequestResponseSchema = z profileId: z.string().optional(), profileVersionId: z.string().optional(), submissionId: z.string().optional(), + submissionExpectedRequestCount: z.number().int().min(1).optional(), + submissionRequestIndex: z.number().int().min(0).optional(), + submissionInitialRunId: z.string().optional(), + submissionFinalizedAt: z.coerce.date().optional(), priority: z.number().int().default(0), gates: z.array(GateConfigSchema).optional(), observations: z.array(z.string()).optional(), diff --git a/packages/shared/src/schemas/submission.ts b/packages/shared/src/schemas/submission.ts new file mode 100644 index 000000000..3558ae603 --- /dev/null +++ b/packages/shared/src/schemas/submission.ts @@ -0,0 +1,182 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { z } from "zod"; +import { extendZodWithOpenApi } from "@asteasolutions/zod-to-openapi"; + +extendZodWithOpenApi(z); + +export const SubmissionRequestRefSchema = z + .object({ + requestId: z.string(), + runId: z.string(), + }) + .openapi("SubmissionRequestRef"); + +export const SubmissionDependencyStateSchema = z.enum([ + "waiting", + "ready", + "failed", + "blocked", +]); + +export const SubmissionDependencyProjectionSchema = z + .object({ + requestId: z.string(), + runId: z.string(), + handlerId: z.string(), + state: SubmissionDependencyStateSchema, + version: z.number().optional(), + updatedAt: z.coerce.date().optional(), + error: z.string().optional(), + }) + .openapi("SubmissionDependencyProjection"); + +export const SubmissionBlockingDetailSchema = z + .object({ + requestId: z.string(), + runId: z.string(), + handlerId: z.string(), + reason: z.enum([ + "request-missing", + "run-changed", + "dependency-not-ready", + "dependency-failed", + "dependency-blocked", + ]), + error: z.string().optional(), + }) + .openapi("SubmissionBlockingDetail"); + +export const SubmissionHandlerStatusSchema = z + .object({ + status: z.enum(["queued", "processing", "done", "failed"]), + version: z.number().optional(), + updatedAt: z.coerce.date().optional(), + error: z.string().optional(), + }) + .openapi("SubmissionHandlerStatus"); + +export const SubmissionReportPointerSchema = z + .object({ + reportId: z.string().optional(), + url: z.string().optional(), + updatedAt: z.coerce.date().optional(), + }) + .openapi("SubmissionReportPointer"); + +export const SubmissionRolloverClaimSchema = z.object({ + nextSubmissionRunId: z.string(), + vectorFingerprint: z.string(), + requests: z.array(SubmissionRequestRefSchema), + reason: z.enum(["request-vector-changed", "manual-retry"]), + claimedAt: z.coerce.date(), +}); + +export const SubmissionRunStateSchema = z + .object({ + _id: z.string(), + attemptNumber: z.number().int().min(1), + vectorFingerprint: z.string(), + requests: z.array(SubmissionRequestRefSchema), + createdAt: z.coerce.date(), + updatedAt: z.coerce.date().optional(), + startedAt: z.coerce.date().optional(), + finishedAt: z.coerce.date().optional(), + dependencyProjection: z.array(SubmissionDependencyProjectionSchema).default([]), + handlerStatus: z.record(z.string(), SubmissionHandlerStatusSchema).optional(), + blocking: z.array(SubmissionBlockingDetailSchema).default([]), + report: SubmissionReportPointerSchema.optional(), + reconcileAfter: z.coerce.date().optional(), + rollover: SubmissionRolloverClaimSchema.optional(), + supersededAt: z.coerce.date().optional(), + supersededBySubmissionRunId: z.string().optional(), + supersededReason: z.enum(["request-vector-changed", "manual-retry"]).optional(), + }) + .openapi("SubmissionRunState"); + +export const SubmissionDocumentSchema = z + .object({ + _id: z.string(), + submissionId: z.string(), + projectId: z.string(), + requestIds: z.array(z.string()).min(1), + createdAt: z.coerce.date(), + updatedAt: z.coerce.date().optional(), + run: SubmissionRunStateSchema, + }) + .openapi("SubmissionDocument"); + +export const SubmissionRunHistoryDocumentSchema = SubmissionRunStateSchema.extend({ + submissionId: z.string(), + projectId: z.string(), +}).openapi("SubmissionRunHistoryDocument"); + +export const SubmissionRunResponseSchema = SubmissionRunStateSchema.omit({ + reconcileAfter: true, + rollover: true, +}) + .extend({ + submissionId: z.string().optional(), + projectId: z.string().optional(), + }) + .openapi("SubmissionRunResponse"); + +export const SubmissionReadinessSchema = z.enum([ + "waiting", + "ready", + "blocked", + "processing", + "done", + "failed", +]); + +export const SubmissionRequestSummarySchema = z + .object({ + requestId: z.string(), + runId: z.string(), + task: z.string(), + status: z.string(), + outcome: z.string().optional(), + workerType: z.string(), + model: z.string().optional(), + profileId: z.string().optional(), + }) + .openapi("SubmissionRequestSummary"); + +export const SubmissionResponseSchema = z + .object({ + submissionId: z.string(), + projectId: z.string(), + requestIds: z.array(z.string()), + createdAt: z.coerce.date(), + updatedAt: z.coerce.date().optional(), + submissionRunId: z.string(), + attemptNumber: z.number().int(), + readiness: SubmissionReadinessSchema, + requests: z.array(SubmissionRequestSummarySchema), + dependencyProjection: z.array(SubmissionDependencyProjectionSchema), + blocking: z.array(SubmissionBlockingDetailSchema), + handlerProgress: z.record(z.string(), SubmissionHandlerStatusSchema).optional(), + report: SubmissionReportPointerSchema.optional(), + supersededAt: z.coerce.date().optional(), + supersededBySubmissionRunId: z.string().optional(), + supersededReason: z.enum(["request-vector-changed", "manual-retry"]).optional(), + }) + .openapi("SubmissionResponse"); + +export const SubmissionListResponseSchema = z + .object({ + submissions: z.array(SubmissionResponseSchema), + }) + .openapi("SubmissionListResponse"); + +export const SubmissionRetryResponseSchema = z + .object({ + submissionId: z.string(), + submissionRunId: z.string(), + attemptNumber: z.number().int(), + readiness: SubmissionReadinessSchema, + blocking: z.array(SubmissionBlockingDetailSchema), + }) + .openapi("SubmissionRetryResponse"); diff --git a/packages/shared/src/scripts/register-handler.ts b/packages/shared/src/scripts/register-handler.ts index c585e9eb1..730b860f6 100644 --- a/packages/shared/src/scripts/register-handler.ts +++ b/packages/shared/src/scripts/register-handler.ts @@ -21,7 +21,7 @@ */ import { readFile } from "node:fs/promises"; import { parse as parseYaml } from "yaml"; -import type { HandlerServiceDocument } from "../types/types.js"; +import type { ServiceHandlerDocument } from "../types/types.js"; const HEALTH_RETRIES = 30; const HEALTH_DELAY_MS = 3_000; @@ -33,7 +33,7 @@ function isNonEmptyString(value: unknown): value is string { } /** Validate the parsed YAML conforms to the HandlerServiceDocument shape. */ -export function parseHandlerDocument(raw: unknown): HandlerServiceDocument { +export function parseHandlerDocument(raw: unknown): ServiceHandlerDocument { if (typeof raw !== "object" || raw === null) { throw new Error("handler.yaml must contain a mapping"); } @@ -42,8 +42,10 @@ export function parseHandlerDocument(raw: unknown): HandlerServiceDocument { if (!isNonEmptyString(doc._id)) { throw new Error("handler.yaml: '_id' is required"); } - if (doc.type !== "post-process-handler") { - throw new Error("handler.yaml: 'type' must be 'post-process-handler'"); + if (doc.type !== "post-process-handler" && doc.type !== "submission-handler") { + throw new Error( + "handler.yaml: 'type' must be 'post-process-handler' or 'submission-handler'", + ); } if (typeof doc.version !== "number" || !Number.isFinite(doc.version)) { throw new Error("handler.yaml: 'version' must be a number"); @@ -62,15 +64,27 @@ export function parseHandlerDocument(raw: unknown): HandlerServiceDocument { throw new Error("handler.yaml: 'dependsOn' must be an array of strings"); } - return { + const common = { _id: doc._id, - type: "post-process-handler", version: doc.version, queue: doc.queue, selector: doc.selector, autoBackfill: doc.autoBackfill, dependsOn: dependsOn as string[], }; + if (doc.type === "submission-handler") { + if (doc.dependencyMode !== "all-runs") { + throw new Error( + "handler.yaml: submission handlers require dependencyMode: 'all-runs'", + ); + } + return { + ...common, + type: "submission-handler", + dependencyMode: "all-runs", + }; + } + return { ...common, type: "post-process-handler" }; } async function delay(ms: number): Promise { @@ -97,7 +111,7 @@ async function waitForScheduler(schedulerUrl: string): Promise { async function postRegistration( schedulerUrl: string, - doc: HandlerServiceDocument, + doc: ServiceHandlerDocument, ): Promise { for (let attempt = 1; attempt <= POST_RETRIES; attempt++) { try { @@ -128,7 +142,7 @@ async function postRegistration( export async function registerHandlerFromYaml( yamlPath: string, schedulerUrl: string, -): Promise { +): Promise { const content = await readFile(yamlPath, "utf-8"); const doc = parseHandlerDocument(parseYaml(content)); diff --git a/packages/shared/src/submissions/index.ts b/packages/shared/src/submissions/index.ts new file mode 100644 index 000000000..5aa35026a --- /dev/null +++ b/packages/shared/src/submissions/index.ts @@ -0,0 +1,5 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +export * from "./submission-state.js"; +export * from "./submission-coordinator.js"; diff --git a/packages/shared/src/submissions/submission-coordinator.test.ts b/packages/shared/src/submissions/submission-coordinator.test.ts new file mode 100644 index 000000000..d44b4c12e --- /dev/null +++ b/packages/shared/src/submissions/submission-coordinator.test.ts @@ -0,0 +1,693 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, expect, it, vi } from "vitest"; +import type { Collection } from "mongodb"; +import type { + RequestDocument, + ServiceHandlerDocument, + SubmissionDocument, + SubmissionRunHistoryDocument, +} from "../types/types.js"; +import { + SubmissionCoordinator, + claimSubmissionHandler, + completeSubmissionHandler, + ensureNativeSubmission, +} from "./submission-coordinator.js"; + +function queryResult(items: T[]) { + return { + project: () => queryResult(items), + sort: () => queryResult(items), + limit: () => queryResult(items), + toArray: async () => items, + }; +} + +describe("submission coordinator persistence", () => { + it("creates a native submission idempotently with the exact initial vector", async () => { + let stored: SubmissionDocument | null = null; + const updateOne = vi.fn(async ( + _filter: unknown, + update: { $setOnInsert: SubmissionDocument }, + ) => { + stored ??= update.$setOnInsert; + return { matchedCount: stored ? 1 : 0, modifiedCount: 0 }; + }); + const findOne = vi.fn(async () => stored); + const submissions = { + updateOne, + findOne, + } as unknown as Collection; + + const first = await ensureNativeSubmission({ + collections: { submissions }, + submissionId: "submission-1", + projectId: "project-1", + requests: [ + { requestId: "request-1", runId: "run-1" }, + { requestId: "request-2", runId: "run-2" }, + ], + createdAt: new Date("2026-01-01T00:00:00Z"), + }); + const second = await ensureNativeSubmission({ + collections: { submissions }, + submissionId: "submission-1", + projectId: "project-1", + requests: [ + { requestId: "request-1", runId: "run-1" }, + { requestId: "request-2", runId: "run-2" }, + ], + createdAt: new Date("2026-01-01T00:00:00Z"), + }); + + expect(first).toEqual(second); + expect(first.requestIds).toEqual(["request-1", "request-2"]); + expect(first.run.requests).toEqual([ + { requestId: "request-1", runId: "run-1" }, + { requestId: "request-2", runId: "run-2" }, + ]); + expect(first.run._id).toBe(second.run._id); + expect(updateOne).toHaveBeenCalledTimes(2); + }); + + it("rejects an idempotency key reused with a different immutable vector", async () => { + const stored: SubmissionDocument = { + _id: "submission-1", + submissionId: "submission-1", + projectId: "project-1", + requestIds: ["request-1"], + createdAt: new Date(), + run: { + _id: "submission-run-1", + attemptNumber: 1, + vectorFingerprint: "fingerprint", + requests: [{ requestId: "request-1", runId: "run-1" }], + createdAt: new Date(), + dependencyProjection: [], + blocking: [], + }, + }; + const submissions = { + updateOne: vi.fn(async () => ({ matchedCount: 1, modifiedCount: 0 })), + findOne: vi.fn(async () => stored), + } as unknown as Collection; + + await expect( + ensureNativeSubmission({ + collections: { submissions }, + submissionId: "submission-1", + projectId: "project-1", + requests: [{ requestId: "request-1", runId: "different-run" }], + }), + ).rejects.toThrow("different immutable request vector"); + }); + + it("scopes claims and completions to the exact embedded submission run", async () => { + const findOneAndUpdate = vi.fn(async (_filter: unknown, _update: unknown) => null); + const updateOne = vi.fn(async (_filter: unknown, _update: unknown) => ({ + modifiedCount: 0, + })); + const submissions = { + findOneAndUpdate, + updateOne, + } as unknown as Collection; + + expect( + await claimSubmissionHandler({ + submissions, + submissionId: "submission-1", + submissionRunId: "stale-run", + handlerId: "pp-readout", + version: 1, + }), + ).toBeNull(); + expect(findOneAndUpdate.mock.calls[0]?.[0]).toMatchObject({ + _id: "submission-1", + "run._id": "stale-run", + "run.rollover": { $exists: false }, + }); + + expect( + await completeSubmissionHandler({ + submissions, + submissionId: "submission-1", + submissionRunId: "stale-run", + handlerId: "pp-readout", + status: "done", + version: 1, + }), + ).toBe(false); + expect(updateOne.mock.calls[0]?.[0]).toMatchObject({ + _id: "submission-1", + "run._id": "stale-run", + "run.rollover": { $exists: false }, + "run.handlerStatus.pp-readout.status": "processing", + }); + }); + + it("defers reconciliation while a bulk retry replaces its request vector", async () => { + const submission: SubmissionDocument = { + _id: "submission-1", + submissionId: "submission-1", + projectId: "project-1", + requestIds: ["request-1"], + createdAt: new Date(), + run: { + _id: "submission-run-1", + attemptNumber: 1, + vectorFingerprint: "old-vector", + requests: [{ requestId: "request-1", runId: "run-1" }], + createdAt: new Date(), + reconcileAfter: new Date(Date.now() + 60_000), + dependencyProjection: [], + blocking: [], + }, + }; + const requestsFind = vi.fn(); + const archiveInsert = vi.fn(); + const coordinator = new SubmissionCoordinator({ + requests: { + find: requestsFind, + } as unknown as Collection, + submissions: { + findOne: vi.fn(async () => submission), + } as unknown as Collection, + submissionRuns: { + insertOne: archiveInsert, + } as unknown as Collection, + services: {} as Collection, + }); + + const result = await coordinator.reconcile("submission-1"); + + expect(result?.rolled).toBe(false); + expect(requestsFind).not.toHaveBeenCalled(); + expect(archiveInsert).not.toHaveBeenCalled(); + }); + + it("resumes an interrupted drift rollover without double-rolling a manual retry", async () => { + const requestDocs: RequestDocument[] = [ + { + _id: "request-1", + projectId: "project-1", + scenario: { task: "Task", criteria: ["criterion"] }, + workerType: "worker", + createdAt: new Date(), + priority: 0, + submissionId: "submission-1", + run: { + _id: "run-2", + attemptNumber: 2, + status: "done", + outcome: "failed", + handlerStatus: { + "pp-observations": { status: "done", version: 2 }, + unrelated: { status: "failed" }, + }, + }, + }, + ]; + let submission: SubmissionDocument = { + _id: "submission-1", + submissionId: "submission-1", + projectId: "project-1", + requestIds: ["request-1"], + createdAt: new Date(), + run: { + _id: "submission-run-1", + attemptNumber: 1, + vectorFingerprint: "old-vector", + requests: [{ requestId: "request-1", runId: "run-1" }], + createdAt: new Date(), + dependencyProjection: [], + blocking: [], + }, + }; + const history: SubmissionRunHistoryDocument[] = []; + const services: ServiceHandlerDocument[] = [ + { + _id: "pp-atif", + type: "post-process-handler", + version: 1, + queue: "post-processor", + selector: "atif", + autoBackfill: true, + dependsOn: [], + }, + { + _id: "pp-observations", + type: "post-process-handler", + version: 2, + queue: "post-processor", + selector: "observations", + autoBackfill: true, + dependsOn: ["pp-atif"], + }, + { + _id: "pp-readout", + type: "submission-handler", + version: 1, + queue: "readout", + selector: "readout", + autoBackfill: false, + dependsOn: ["pp-observations"], + dependencyMode: "all-runs", + }, + ]; + const submissions = { + findOne: vi.fn(async () => structuredClone(submission)), + updateOne: vi.fn(async () => ({ matchedCount: 1, modifiedCount: 1 })), + findOneAndUpdate: vi.fn(async ( + filter: Record, + update: { $set: Record }, + ) => { + if (filter["run._id"] && filter["run._id"] !== submission.run._id) return null; + if (update.$set["run.rollover"]) { + submission = { + ...submission, + updatedAt: update.$set.updatedAt as Date, + run: { + ...submission.run, + rollover: update.$set[ + "run.rollover" + ] as SubmissionDocument["run"]["rollover"], + updatedAt: update.$set["run.updatedAt"] as Date, + }, + }; + } else if (update.$set.run) { + rollAttempts++; + if (rollAttempts === 1) return null; + submission = { + ...submission, + run: update.$set.run as SubmissionDocument["run"], + updatedAt: update.$set.updatedAt as Date, + }; + } else { + submission = { + ...submission, + updatedAt: update.$set.updatedAt as Date, + run: { + ...submission.run, + dependencyProjection: update.$set[ + "run.dependencyProjection" + ] as SubmissionDocument["run"]["dependencyProjection"], + blocking: update.$set[ + "run.blocking" + ] as SubmissionDocument["run"]["blocking"], + updatedAt: update.$set["run.updatedAt"] as Date, + }, + }; + } + return structuredClone(submission); + }), + } as unknown as Collection; + let rollAttempts = 0; + const enqueue = vi.fn(async ( + _service: ServiceHandlerDocument, + _message: { submissionId: string; submissionRunId: string; type: string }, + ) => undefined); + const archiveUpdate = vi.fn(async ( + _filter: unknown, + update: { $setOnInsert: SubmissionRunHistoryDocument }, + ) => { + if (!history.some((item) => item._id === update.$setOnInsert._id)) { + history.push(update.$setOnInsert); + } + return { matchedCount: 1, modifiedCount: 1 }; + }); + const coordinator = new SubmissionCoordinator( + { + requests: { + find: vi.fn(() => queryResult(requestDocs)), + } as unknown as Collection, + submissions, + submissionRuns: { + updateOne: archiveUpdate, + } as unknown as Collection, + services: { + find: vi.fn(() => + queryResult(services.filter((service) => service.type === "post-process-handler")), + ), + findOne: vi.fn(async () => + services.find((service) => service._id === "pp-readout") ?? null, + ), + } as unknown as Collection, + }, + enqueue, + ); + + const result = await coordinator.retryCurrent( + "submission-1", + "submission-run-1", + ); + + expect(result?.rolled).toBe(true); + expect(result?.submission.run.attemptNumber).toBe(2); + expect(result?.submission.run.requests).toEqual([ + { requestId: "request-1", runId: "run-2" }, + ]); + expect(result?.submission.run.dependencyProjection[0]).toMatchObject({ + runId: "run-2", + state: "ready", + version: 2, + }); + expect(history).toHaveLength(1); + expect(archiveUpdate).toHaveBeenCalledTimes(2); + expect(history[0]).toMatchObject({ + _id: "submission-run-1", + supersededBySubmissionRunId: result?.submission.run._id, + supersededReason: "request-vector-changed", + }); + expect(enqueue).toHaveBeenCalledTimes(1); + expect(enqueue.mock.calls[0]?.[1]).toMatchObject({ + submissionId: "submission-1", + submissionRunId: result?.submission.run._id, + }); + }); + + it("links immutable history only to CAS-winning successors under concurrent vector drift", async () => { + const request = (runId: string, attemptNumber: number): RequestDocument => ({ + _id: "request-1", + projectId: "project-1", + scenario: { task: "Task", criteria: ["criterion"] }, + workerType: "worker", + createdAt: new Date(), + priority: 0, + submissionId: "submission-1", + run: { + _id: runId, + attemptNumber, + status: "done", + }, + }); + let submission: SubmissionDocument = { + _id: "submission-1", + submissionId: "submission-1", + projectId: "project-1", + requestIds: ["request-1"], + createdAt: new Date(), + run: { + _id: "submission-run-1", + attemptNumber: 1, + vectorFingerprint: "initial", + requests: [{ requestId: "request-1", runId: "run-1" }], + createdAt: new Date(), + dependencyProjection: [], + blocking: [], + }, + }; + const history = new Map(); + const submissions = { + findOne: vi.fn(async () => structuredClone(submission)), + updateOne: vi.fn(async () => ({ matchedCount: 1, modifiedCount: 1 })), + findOneAndUpdate: vi.fn(async ( + filter: Record, + update: { $set: Record }, + ) => { + if (filter["run._id"] !== submission.run._id) return null; + if (update.$set["run.rollover"]) { + if (submission.run.rollover) return null; + submission = { + ...submission, + run: { + ...submission.run, + rollover: update.$set[ + "run.rollover" + ] as SubmissionDocument["run"]["rollover"], + }, + }; + } else if (update.$set.run) { + if ( + filter["run.rollover.nextSubmissionRunId"] !== + submission.run.rollover?.nextSubmissionRunId + ) { + return null; + } + submission = { + ...submission, + run: update.$set.run as SubmissionDocument["run"], + }; + } else { + submission = { + ...submission, + run: { + ...submission.run, + dependencyProjection: update.$set[ + "run.dependencyProjection" + ] as SubmissionDocument["run"]["dependencyProjection"], + blocking: update.$set[ + "run.blocking" + ] as SubmissionDocument["run"]["blocking"], + }, + }; + } + return structuredClone(submission); + }), + } as unknown as Collection; + const submissionRuns = { + updateOne: vi.fn(async ( + filter: { _id: string }, + update: { $setOnInsert: SubmissionRunHistoryDocument }, + ) => { + if (!history.has(filter._id)) { + history.set(filter._id, structuredClone(update.$setOnInsert)); + } + return { matchedCount: 1, modifiedCount: 1 }; + }), + } as unknown as Collection; + const services = { + find: vi.fn(() => queryResult([])), + findOne: vi.fn(async () => null), + } as unknown as Collection; + let firstCoordinatorReads = 0; + const coordinatorA = new SubmissionCoordinator({ + requests: { + find: vi.fn(() => + queryResult([ + firstCoordinatorReads++ === 0 + ? request("run-2", 2) + : request("run-3", 3), + ]), + ), + } as unknown as Collection, + submissions, + submissionRuns, + services, + }); + const coordinatorB = new SubmissionCoordinator({ + requests: { + find: vi.fn(() => queryResult([request("run-3", 3)])), + } as unknown as Collection, + submissions, + submissionRuns, + services, + }); + + await Promise.all([ + coordinatorA.reconcile("submission-1"), + coordinatorB.reconcile("submission-1"), + ]); + + expect(submission.run.requests).toEqual([ + { requestId: "request-1", runId: "run-3" }, + ]); + expect(submission.run.rollover).toBeUndefined(); + expect(history.size).toBe(2); + const knownRunIds = new Set([...history.keys(), submission.run._id]); + for (const archived of history.values()) { + expect(knownRunIds.has(archived.supersededBySubmissionRunId!)).toBe(true); + } + }); + + it("repairs an interrupted claimed rollover on the next polling reconciliation", async () => { + const requestDocument: RequestDocument = { + _id: "request-1", + projectId: "project-1", + scenario: { task: "Task", criteria: ["criterion"] }, + workerType: "worker", + createdAt: new Date(), + priority: 0, + submissionId: "submission-1", + run: { + _id: "run-2", + attemptNumber: 2, + status: "done", + }, + }; + let submission: SubmissionDocument = { + _id: "submission-1", + submissionId: "submission-1", + projectId: "project-1", + requestIds: ["request-1"], + createdAt: new Date(), + run: { + _id: "submission-run-1", + attemptNumber: 1, + vectorFingerprint: "initial", + requests: [{ requestId: "request-1", runId: "run-1" }], + createdAt: new Date(), + dependencyProjection: [], + blocking: [], + }, + }; + const history = new Map(); + let interruptReplacement = true; + const submissions = { + findOne: vi.fn(async () => structuredClone(submission)), + updateOne: vi.fn(async () => ({ matchedCount: 1, modifiedCount: 1 })), + findOneAndUpdate: vi.fn(async ( + filter: Record, + update: { $set: Record }, + ) => { + if (filter["run._id"] !== submission.run._id) return null; + if (update.$set["run.rollover"]) { + if (submission.run.rollover) return null; + submission = { + ...submission, + run: { + ...submission.run, + rollover: update.$set[ + "run.rollover" + ] as SubmissionDocument["run"]["rollover"], + }, + }; + } else if (update.$set.run) { + if (interruptReplacement) { + interruptReplacement = false; + throw new Error("simulated process interruption"); + } + submission = { + ...submission, + run: update.$set.run as SubmissionDocument["run"], + }; + } else { + submission = { + ...submission, + run: { + ...submission.run, + dependencyProjection: update.$set[ + "run.dependencyProjection" + ] as SubmissionDocument["run"]["dependencyProjection"], + blocking: update.$set[ + "run.blocking" + ] as SubmissionDocument["run"]["blocking"], + }, + }; + } + return structuredClone(submission); + }), + } as unknown as Collection; + const submissionRuns = { + updateOne: vi.fn(async ( + filter: { _id: string }, + update: { $setOnInsert: SubmissionRunHistoryDocument }, + ) => { + if (!history.has(filter._id)) { + history.set(filter._id, structuredClone(update.$setOnInsert)); + } + return { matchedCount: 1, modifiedCount: 1 }; + }), + } as unknown as Collection; + const collections = { + requests: { + find: vi.fn(() => queryResult([requestDocument])), + } as unknown as Collection, + submissions, + submissionRuns, + services: { + find: vi.fn(() => queryResult([])), + findOne: vi.fn(async () => null), + } as unknown as Collection, + }; + + await expect( + new SubmissionCoordinator(collections).reconcile("submission-1"), + ).rejects.toThrow("simulated process interruption"); + expect(submission.run.rollover).toBeDefined(); + expect(history.size).toBe(1); + + const repaired = await new SubmissionCoordinator(collections).reconcile( + "submission-1", + ); + + expect(repaired?.submission.run.attemptNumber).toBe(2); + expect(repaired?.submission.run.requests).toEqual([ + { requestId: "request-1", runId: "run-2" }, + ]); + expect(repaired?.submission.run.rollover).toBeUndefined(); + expect(history.get("submission-run-1")?.supersededBySubmissionRunId).toBe( + repaired?.submission.run._id, + ); + }); + + it("recovers an interrupted final submission write only for complete native request groups", async () => { + const requests: RequestDocument[] = [ + { + _id: "request-1", + projectId: "project-1", + scenario: { task: "Task", criteria: ["criterion"] }, + workerType: "worker", + createdAt: new Date(), + priority: 0, + submissionId: "submission-1", + submissionExpectedRequestCount: 2, + submissionRequestIndex: 0, + submissionInitialRunId: "run-1", + run: { _id: "retried-run", attemptNumber: 2, status: "pending" }, + }, + { + _id: "request-2", + projectId: "project-1", + scenario: { task: "Task", criteria: ["criterion"] }, + workerType: "worker", + createdAt: new Date(), + priority: 0, + submissionId: "submission-1", + submissionExpectedRequestCount: 2, + submissionRequestIndex: 1, + submissionInitialRunId: "run-2", + run: { _id: "run-2", attemptNumber: 1, status: "pending" }, + }, + ]; + let stored: SubmissionDocument | null = null; + const updateMany = vi.fn(async () => ({ modifiedCount: 2 })); + const coordinator = new SubmissionCoordinator({ + requests: { + find: vi.fn(() => queryResult(requests)), + updateMany, + } as unknown as Collection, + submissions: { + findOne: vi.fn(async () => stored), + updateOne: vi.fn(async ( + _filter: unknown, + update: { $setOnInsert: SubmissionDocument }, + ) => { + stored ??= update.$setOnInsert; + return { upsertedCount: 1 }; + }), + } as unknown as Collection, + submissionRuns: {} as Collection, + services: {} as Collection, + }); + + expect(await coordinator.recoverMissingNativeSubmissions()).toBe(1); + expect(stored).not.toBeNull(); + const recovered = stored as SubmissionDocument | null; + expect(recovered?.requestIds).toEqual(["request-1", "request-2"]); + expect(recovered?.run.requests).toEqual([ + { requestId: "request-1", runId: "run-1" }, + { requestId: "request-2", runId: "run-2" }, + ]); + expect(await coordinator.recoverMissingNativeSubmissions()).toBe(1); + expect(updateMany).toHaveBeenCalledTimes(2); + expect(updateMany).toHaveBeenCalledWith( + expect.objectContaining({ + submissionId: "submission-1", + submissionFinalizedAt: { $exists: false }, + }), + expect.objectContaining({ $set: { submissionFinalizedAt: expect.any(Date) } }), + ); + }); +}); diff --git a/packages/shared/src/submissions/submission-coordinator.ts b/packages/shared/src/submissions/submission-coordinator.ts new file mode 100644 index 000000000..5720a744a --- /dev/null +++ b/packages/shared/src/submissions/submission-coordinator.ts @@ -0,0 +1,573 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { v5 as uuidv5 } from "uuid"; +import type { Collection, Document } from "mongodb"; +import type { + HandlerServiceDocument, + RequestDocument, + SubmissionDocument, + SubmissionHandlerServiceDocument, + SubmissionQueueMessagePayload, + SubmissionReportPointer, + SubmissionRunHistoryDocument, + SubmissionRunState, + ServiceHandlerDocument, +} from "../types/types.js"; +import { + OBSERVATIONS_HANDLER_ID, + READOUT_HANDLER_ID, + computeSubmissionVectorFingerprint, + deriveSubmissionReadiness, + projectSubmissionDependency, +} from "./submission-state.js"; + +const SUBMISSION_RUN_NAMESPACE = "6b0f8a6a-5cda-4cb4-bcc4-79bb0ac7f54d"; + +export interface SubmissionCollections { + requests: Collection; + submissions: Collection; + submissionRuns: Collection; + services: Collection; +} + +export type EnqueueSubmission = ( + service: SubmissionHandlerServiceDocument, + message: SubmissionQueueMessagePayload, +) => Promise; + +export interface ReconcileSubmissionResult { + submission: SubmissionDocument; + readiness: ReturnType; + rolled: boolean; + dispatched: boolean; +} + +function submissionRunId( + submissionId: string, + attemptNumber: number, + vectorFingerprint: string, +): string { + return uuidv5( + `${submissionId}:${attemptNumber}:${vectorFingerprint}`, + SUBMISSION_RUN_NAMESPACE, + ); +} + +function initialSubmissionRun( + submissionId: string, + requests: Array<{ requestId: string; runId: string }>, + createdAt: Date, +): SubmissionRunState { + const vectorFingerprint = computeSubmissionVectorFingerprint(requests); + return { + _id: submissionRunId(submissionId, 1, vectorFingerprint), + attemptNumber: 1, + vectorFingerprint, + requests, + createdAt, + dependencyProjection: [], + blocking: [], + }; +} + +export async function ensureNativeSubmission(input: { + collections: Pick; + submissionId: string; + projectId: string; + requests: Array<{ requestId: string; runId: string }>; + createdAt?: Date; +}): Promise { + const requestIds = input.requests.map((request) => request.requestId); + const createdAt = input.createdAt ?? new Date(); + const doc: SubmissionDocument = { + _id: input.submissionId, + submissionId: input.submissionId, + projectId: input.projectId, + requestIds, + createdAt, + run: initialSubmissionRun(input.submissionId, input.requests, createdAt), + }; + await input.collections.submissions.updateOne( + { _id: input.submissionId } as Document, + { $setOnInsert: doc } as Document, + { upsert: true }, + ); + const persisted = await input.collections.submissions.findOne({ + _id: input.submissionId, + } as Document); + if (!persisted) { + throw new Error(`Submission ${input.submissionId} was not persisted`); + } + const sameProject = persisted.projectId === input.projectId; + const sameRequests = + persisted.requestIds.length === requestIds.length && + persisted.requestIds.every((requestId, index) => requestId === requestIds[index]); + const sameInitialVector = + persisted.run.attemptNumber > 1 || + (persisted.run.requests.length === input.requests.length && + persisted.run.requests.every((request, index) => { + const expected = input.requests[index]; + return ( + request.requestId === expected?.requestId && + request.runId === expected.runId + ); + })); + if (!sameProject || !sameRequests || !sameInitialVector) { + throw new Error( + `Submission ${input.submissionId} already exists with a different immutable request vector`, + ); + } + return persisted; +} + +async function loadPostProcessHandlers( + services: Collection, +): Promise { + const documents = await services + .find({ type: "post-process-handler" }) + .toArray(); + return documents.filter( + (document): document is HandlerServiceDocument => + document.type === "post-process-handler", + ); +} + +async function loadReadoutService( + services: Collection, +): Promise { + const document = await services.findOne({ + _id: READOUT_HANDLER_ID, + type: "submission-handler", + }); + return document?.type === "submission-handler" ? document : null; +} + +async function archiveAndRoll(input: { + collections: SubmissionCollections; + submission: SubmissionDocument; + requests: Array<{ requestId: string; runId: string }>; + reason: "request-vector-changed" | "manual-retry"; +}): Promise { + let claimed = input.submission; + if (!claimed.run.rollover) { + const claimedAt = new Date(); + const nextAttemptNumber = claimed.run.attemptNumber + 1; + const vectorFingerprint = computeSubmissionVectorFingerprint(input.requests); + const nextSubmissionRunId = submissionRunId( + claimed._id, + nextAttemptNumber, + vectorFingerprint, + ); + const claimedDocument = await input.collections.submissions.findOneAndUpdate( + { + _id: claimed._id, + "run._id": claimed.run._id, + "run.vectorFingerprint": claimed.run.vectorFingerprint, + "run.rollover": { $exists: false }, + } as Document, + { + $set: { + "run.rollover": { + nextSubmissionRunId, + vectorFingerprint, + requests: input.requests, + reason: input.reason, + claimedAt, + }, + "run.updatedAt": claimedAt, + updatedAt: claimedAt, + }, + } as Document, + { returnDocument: "after" }, + ); + if (!claimedDocument) return null; + claimed = claimedDocument; + } + const rollover = claimed.run.rollover; + if (!rollover) return null; + const { rollover: _rollover, reconcileAfter: _reconcileAfter, ...archivedRun } = + claimed.run; + const archived: SubmissionRunHistoryDocument = { + ...archivedRun, + submissionId: claimed._id, + projectId: claimed.projectId, + supersededAt: rollover.claimedAt, + supersededBySubmissionRunId: rollover.nextSubmissionRunId, + supersededReason: rollover.reason, + }; + await input.collections.submissionRuns.updateOne( + { _id: archived._id } as Document, + { $setOnInsert: archived } as Document, + { upsert: true }, + ); + + const nextRun: SubmissionRunState = { + _id: rollover.nextSubmissionRunId, + attemptNumber: claimed.run.attemptNumber + 1, + vectorFingerprint: rollover.vectorFingerprint, + requests: rollover.requests, + createdAt: rollover.claimedAt, + dependencyProjection: [], + blocking: [], + }; + const updated = await input.collections.submissions.findOneAndUpdate( + { + _id: claimed._id, + "run._id": claimed.run._id, + "run.rollover.nextSubmissionRunId": rollover.nextSubmissionRunId, + } as Document, + { + $set: { + run: nextRun, + updatedAt: rollover.claimedAt, + }, + } as Document, + { returnDocument: "after" }, + ); + return updated; +} + +export class SubmissionCoordinator { + private recoveryAfterRequestId: string | null = null; + + constructor( + private readonly collections: SubmissionCollections, + private readonly enqueue?: EnqueueSubmission, + ) {} + + async recoverMissingNativeSubmissions(limit = 30): Promise { + const filter: Document = { + submissionId: { $exists: true }, + submissionExpectedRequestCount: { $exists: true }, + submissionFinalizedAt: { $exists: false }, + deletedAt: { $exists: false }, + ...(this.recoveryAfterRequestId + ? { _id: { $gt: this.recoveryAfterRequestId } } + : {}), + }; + const candidates = await this.collections.requests + .find(filter) + .project({ + _id: 1, + projectId: 1, + submissionId: 1, + submissionExpectedRequestCount: 1, + submissionRequestIndex: 1, + submissionInitialRunId: 1, + "run._id": 1, + createdAt: 1, + }) + .sort({ _id: 1 }) + .limit(limit) + .toArray(); + this.recoveryAfterRequestId = + candidates.length === limit ? candidates.at(-1)?._id ?? null : null; + let recovered = 0; + const visited = new Set(); + for (const candidate of candidates) { + const submissionId = candidate.submissionId; + const expected = candidate.submissionExpectedRequestCount; + if (!submissionId || !expected || visited.has(submissionId)) continue; + visited.add(submissionId); + const requests = await this.collections.requests + .find({ + submissionId, + projectId: candidate.projectId, + submissionExpectedRequestCount: expected, + deletedAt: { $exists: false }, + } as Document) + .sort({ submissionRequestIndex: 1, _id: 1 }) + .toArray(); + if ( + requests.length !== expected || + requests.some( + (request, index) => + request.submissionRequestIndex !== index || + !request.submissionInitialRunId, + ) + ) { + continue; + } + await ensureNativeSubmission({ + collections: this.collections, + submissionId, + projectId: candidate.projectId, + requests: requests.map((request) => ({ + requestId: request._id, + runId: request.submissionInitialRunId!, + })), + createdAt: requests[0]?.createdAt, + }); + await this.collections.requests.updateMany( + { + submissionId, + projectId: candidate.projectId, + submissionExpectedRequestCount: expected, + submissionFinalizedAt: { $exists: false }, + } as Document, + { $set: { submissionFinalizedAt: new Date() } } as Document, + ); + recovered++; + } + return recovered; + } + + async reconcile(submissionId: string): Promise { + let rolled = false; + for (let attempt = 0; attempt < 4; attempt++) { + const submission = await this.collections.submissions.findOne({ + _id: submissionId, + } as Document); + if (!submission) return null; + if ( + submission.run.reconcileAfter && + submission.run.reconcileAfter.getTime() > Date.now() + ) { + return { + submission, + readiness: deriveSubmissionReadiness(submission.run), + rolled, + dispatched: false, + }; + } + if (submission.run.reconcileAfter) { + await this.collections.submissions.updateOne( + { _id: submissionId, "run._id": submission.run._id } as Document, + { + $unset: { "run.reconcileAfter": "" }, + $set: { updatedAt: new Date() }, + } as Document, + ); + delete submission.run.reconcileAfter; + } + if (submission.run.rollover) { + const updated = await archiveAndRoll({ + collections: this.collections, + submission, + requests: submission.run.rollover.requests, + reason: submission.run.rollover.reason, + }); + rolled = true; + if (!updated) continue; + continue; + } + + const requestDocs = await this.collections.requests + .find({ + _id: { $in: submission.requestIds }, + projectId: submission.projectId, + deletedAt: { $exists: false }, + } as Document) + .toArray(); + const byId = new Map(requestDocs.map((request) => [request._id, request])); + const latestVector = submission.requestIds.flatMap((requestId) => { + const runId = byId.get(requestId)?.run?._id; + return runId ? [{ requestId, runId }] : []; + }); + const latestFingerprint = + latestVector.length === submission.requestIds.length + ? computeSubmissionVectorFingerprint(latestVector) + : submission.run.vectorFingerprint; + + if ( + latestVector.length === submission.requestIds.length && + latestFingerprint !== submission.run.vectorFingerprint + ) { + const updated = await archiveAndRoll({ + collections: this.collections, + submission, + requests: latestVector, + reason: "request-vector-changed", + }); + rolled = true; + if (!updated) continue; + continue; + } + + const handlers = await loadPostProcessHandlers(this.collections.services); + const projected = submission.run.requests.map((ref) => + projectSubmissionDependency({ + requestId: ref.requestId, + expectedRunId: ref.runId, + run: byId.get(ref.requestId)?.run, + handlers, + handlerId: OBSERVATIONS_HANDLER_ID, + }), + ); + const dependencyProjection = projected.map((result) => result.projection); + const blocking = projected.flatMap((result) => + result.blocking ? [result.blocking] : [], + ); + const now = new Date(); + const refreshed = await this.collections.submissions.findOneAndUpdate( + { _id: submissionId, "run._id": submission.run._id } as Document, + { + $set: { + "run.dependencyProjection": dependencyProjection, + "run.blocking": blocking, + "run.updatedAt": now, + updatedAt: now, + }, + } as Document, + { returnDocument: "after" }, + ); + if (!refreshed) continue; + + let dispatched = false; + const readiness = deriveSubmissionReadiness(refreshed.run); + if (readiness === "ready" && this.enqueue) { + const service = await loadReadoutService(this.collections.services); + const existing = refreshed.run.handlerStatus?.[READOUT_HANDLER_ID]; + const eligible = + service && + (!existing || + (service.autoBackfill && + existing.status === "done" && + (existing.version ?? 0) < service.version)); + if (service && eligible) { + await this.enqueue(service, { + type: service.selector, + submissionId, + submissionRunId: refreshed.run._id, + }); + const queuedAt = new Date(); + const marked = await this.collections.submissions.updateOne( + { + _id: submissionId, + "run._id": refreshed.run._id, + [`run.handlerStatus.${READOUT_HANDLER_ID}.status`]: { + $nin: ["queued", "processing", "failed"], + }, + } as Document, + { + $set: { + [`run.handlerStatus.${READOUT_HANDLER_ID}`]: { + status: "queued", + version: service.version, + updatedAt: queuedAt, + }, + "run.updatedAt": queuedAt, + updatedAt: queuedAt, + }, + } as Document, + ); + dispatched = marked.modifiedCount === 1; + } + } + const finalSubmission = dispatched + ? await this.collections.submissions.findOne({ _id: submissionId } as Document) + : refreshed; + if (!finalSubmission) return null; + return { + submission: finalSubmission, + readiness: deriveSubmissionReadiness(finalSubmission.run), + rolled, + dispatched, + }; + } + throw new Error(`Submission ${submissionId} could not be reconciled after concurrent updates`); + } + + async retryCurrent( + submissionId: string, + expectedSubmissionRunId: string, + ): Promise { + const reconciled = await this.reconcile(submissionId); + if (!reconciled) return null; + if (reconciled.submission.run._id !== expectedSubmissionRunId) { + return reconciled; + } + if (reconciled.readiness === "blocked" || reconciled.readiness === "waiting") { + return reconciled; + } + const rolled = await archiveAndRoll({ + collections: this.collections, + submission: reconciled.submission, + requests: reconciled.submission.run.requests, + reason: "manual-retry", + }); + if (!rolled) return this.reconcile(submissionId); + return this.reconcile(submissionId); + } +} + +export async function claimSubmissionHandler(input: { + submissions: Collection; + submissionId: string; + submissionRunId: string; + handlerId: string; + version: number; +}): Promise { + const now = new Date(); + return input.submissions.findOneAndUpdate( + { + _id: input.submissionId, + "run._id": input.submissionRunId, + "run.rollover": { $exists: false }, + [`run.handlerStatus.${input.handlerId}.status`]: { $in: ["queued"] }, + } as Document, + { + $set: { + [`run.handlerStatus.${input.handlerId}`]: { + status: "processing", + version: input.version, + updatedAt: now, + }, + "run.startedAt": now, + "run.updatedAt": now, + updatedAt: now, + }, + } as Document, + { returnDocument: "after" }, + ); +} + +export async function isCurrentSubmissionRun(input: { + submissions: Collection; + submissionId: string; + submissionRunId: string; +}): Promise { + return Boolean( + await input.submissions.findOne({ + _id: input.submissionId, + "run._id": input.submissionRunId, + "run.rollover": { $exists: false }, + } as Document), + ); +} + +export async function completeSubmissionHandler(input: { + submissions: Collection; + submissionId: string; + submissionRunId: string; + handlerId: string; + status: "done" | "failed"; + version: number; + error?: string; + report?: SubmissionReportPointer; +}): Promise { + const now = new Date(); + const set: Record = { + [`run.handlerStatus.${input.handlerId}`]: { + status: input.status, + version: input.version, + updatedAt: now, + ...(input.error ? { error: input.error } : {}), + }, + "run.updatedAt": now, + "run.finishedAt": now, + updatedAt: now, + }; + if (input.report) set["run.report"] = input.report; + const result = await input.submissions.updateOne( + { + _id: input.submissionId, + "run._id": input.submissionRunId, + "run.rollover": { $exists: false }, + [`run.handlerStatus.${input.handlerId}.status`]: "processing", + } as Document, + { $set: set } as Document, + ); + return result.modifiedCount === 1; +} diff --git a/packages/shared/src/submissions/submission-state.test.ts b/packages/shared/src/submissions/submission-state.test.ts new file mode 100644 index 000000000..854558c68 --- /dev/null +++ b/packages/shared/src/submissions/submission-state.test.ts @@ -0,0 +1,155 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, expect, it } from "vitest"; +import type { HandlerServiceDocument, RunState, SubmissionRunState } from "../types/types.js"; +import { + OBSERVATIONS_HANDLER_ID, + READOUT_HANDLER_ID, + computeSubmissionVectorFingerprint, + deriveSubmissionReadiness, + evaluateNamedHandlerState, + isHandlerDagDrained, + projectSubmissionDependency, +} from "./submission-state.js"; + +const handlers: HandlerServiceDocument[] = [ + { + _id: "pp-atif", + type: "post-process-handler", + version: 1, + queue: "post-processor", + selector: "atif", + autoBackfill: true, + dependsOn: [], + }, + { + _id: OBSERVATIONS_HANDLER_ID, + type: "post-process-handler", + version: 1, + queue: "post-processor", + selector: "observations", + autoBackfill: true, + dependsOn: ["pp-atif"], + }, +]; + +function run(handlerStatus: RunState["handlerStatus"]): RunState { + return { + _id: "run-1", + attemptNumber: 1, + status: "done", + outcome: "failed", + handlerStatus, + observationResults: [ + { criterionId: "observation-1", passed: false, feedback: "evidence", evaluated: true }, + ], + }; +} + +describe("submission state helpers", () => { + it("fingerprints vectors deterministically independent of request order", () => { + expect( + computeSubmissionVectorFingerprint([ + { requestId: "b", runId: "2" }, + { requestId: "a", runId: "1" }, + ]), + ).toBe( + computeSubmissionVectorFingerprint([ + { requestId: "a", runId: "1" }, + { requestId: "b", runId: "2" }, + ]), + ); + }); + + it("treats completed observations as ready even for failed coding outcomes and passed:false evidence", () => { + const result = projectSubmissionDependency({ + requestId: "request-1", + expectedRunId: "run-1", + run: run({ + [OBSERVATIONS_HANDLER_ID]: { status: "done", version: 1 }, + unrelated: { status: "failed", error: "ignored" }, + }), + handlers, + }); + + expect(result).toEqual({ + projection: { + requestId: "request-1", + runId: "run-1", + handlerId: OBSERVATIONS_HANDLER_ID, + state: "ready", + version: 1, + }, + drifted: false, + }); + }); + + it("blocks when observations fail or are blocked by a failed dependency", () => { + expect( + evaluateNamedHandlerState( + OBSERVATIONS_HANDLER_ID, + handlers, + { [OBSERVATIONS_HANDLER_ID]: { status: "failed" } }, + ), + ).toBe("failed"); + expect( + evaluateNamedHandlerState( + OBSERVATIONS_HANDLER_ID, + handlers, + { "pp-atif": { status: "failed" } }, + ), + ).toBe("blocked"); + }); + + it("reuses named-handler DAG semantics for existing report triggering", () => { + expect( + isHandlerDagDrained(handlers, { + "pp-atif": { status: "failed" }, + }), + ).toBe(true); + expect( + isHandlerDagDrained(handlers, { + "pp-atif": { status: "processing" }, + }), + ).toBe(false); + }); + + it("detects exact run drift", () => { + const result = projectSubmissionDependency({ + requestId: "request-1", + expectedRunId: "old-run", + run: run({ [OBSERVATIONS_HANDLER_ID]: { status: "done" } }), + handlers, + }); + expect(result.drifted).toBe(true); + expect(result.blocking?.reason).toBe("run-changed"); + }); + + it("derives submission readiness from dependencies and readout progress", () => { + const base: SubmissionRunState = { + _id: "submission-run-1", + attemptNumber: 1, + vectorFingerprint: "fingerprint", + requests: [{ requestId: "request-1", runId: "run-1" }], + createdAt: new Date(), + dependencyProjection: [{ + requestId: "request-1", + runId: "run-1", + handlerId: OBSERVATIONS_HANDLER_ID, + state: "ready", + }], + blocking: [], + }; + + expect(deriveSubmissionReadiness(base)).toBe("ready"); + expect(deriveSubmissionReadiness({ + ...base, + handlerStatus: { [READOUT_HANDLER_ID]: { status: "processing" } }, + })).toBe("processing"); + expect(deriveSubmissionReadiness({ + ...base, + handlerStatus: { [READOUT_HANDLER_ID]: { status: "done" } }, + })).toBe("done"); + }); +}); diff --git a/packages/shared/src/submissions/submission-state.ts b/packages/shared/src/submissions/submission-state.ts new file mode 100644 index 000000000..cbaf44452 --- /dev/null +++ b/packages/shared/src/submissions/submission-state.ts @@ -0,0 +1,184 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { + HandlerRunStatus, + HandlerServiceDocument, + RunState, + SubmissionBlockingDetail, + SubmissionDependencyProjection, + SubmissionHandlerState, + SubmissionReadiness, + SubmissionRequestRef, + SubmissionRunState, +} from "../types/types.js"; +import { DependencyGraph } from "../graph/dependency-graph.js"; + +export const OBSERVATIONS_HANDLER_ID = "pp-observations"; +export const READOUT_HANDLER_ID = "pp-readout"; + +export function computeSubmissionVectorFingerprint( + requests: readonly SubmissionRequestRef[], +): string { + const canonical = [...requests] + .sort((left, right) => left.requestId.localeCompare(right.requestId)) + .map(({ requestId, runId }) => `${requestId}:${runId}`) + .join("\n"); + let hash = 0xcbf29ce484222325n; + for (let index = 0; index < canonical.length; index++) { + hash ^= BigInt(canonical.charCodeAt(index)); + hash = BigInt.asUintN(64, hash * 0x100000001b3n); + } + return hash.toString(16).padStart(16, "0"); +} + +export function evaluateNamedHandlerState( + handlerId: string, + handlers: readonly HandlerServiceDocument[], + handlerStatus: Record | undefined, +): SubmissionHandlerState { + const status = handlerStatus?.[handlerId]; + if (status?.status === "done") return "ready"; + if (status?.status === "failed") return "failed"; + + const graph = new DependencyGraph( + handlers.map((handler) => ({ + id: handler._id, + dependsOn: handler.dependsOn, + })), + ); + + if (!handlers.some((handler) => handler._id === handlerId)) { + return "waiting"; + } + + for (const ancestorId of graph.getAncestors(handlerId)) { + if (handlerStatus?.[ancestorId]?.status === "failed") return "blocked"; + } + return "waiting"; +} + +function isTerminalHandlerState( + handlerId: string, + graph: DependencyGraph, + handlerStatus: Record | undefined, +): boolean { + const status = handlerStatus?.[handlerId]?.status; + if (status === "done" || status === "failed") return true; + if (status !== undefined) return false; + for (const ancestorId of graph.getAncestors(handlerId)) { + if (handlerStatus?.[ancestorId]?.status === "failed") return true; + } + return false; +} + +export function isHandlerDagDrained( + handlers: readonly HandlerServiceDocument[], + handlerStatus: Record | undefined, +): boolean { + const graph = new DependencyGraph( + handlers.map((handler) => ({ + id: handler._id, + dependsOn: handler.dependsOn, + })), + ); + return handlers.every((handler) => + isTerminalHandlerState(handler._id, graph, handlerStatus), + ); +} + +export function projectSubmissionDependency(input: { + requestId: string; + expectedRunId: string; + run?: RunState; + handlers: readonly HandlerServiceDocument[]; + handlerId?: string; +}): { + projection: SubmissionDependencyProjection; + blocking?: SubmissionBlockingDetail; + drifted: boolean; +} { + const handlerId = input.handlerId ?? OBSERVATIONS_HANDLER_ID; + if (!input.run) { + return { + projection: { + requestId: input.requestId, + runId: input.expectedRunId, + handlerId, + state: "blocked", + }, + blocking: { + requestId: input.requestId, + runId: input.expectedRunId, + handlerId, + reason: "request-missing", + }, + drifted: false, + }; + } + if (input.run._id !== input.expectedRunId) { + return { + projection: { + requestId: input.requestId, + runId: input.expectedRunId, + handlerId, + state: "blocked", + }, + blocking: { + requestId: input.requestId, + runId: input.expectedRunId, + handlerId, + reason: "run-changed", + }, + drifted: true, + }; + } + + const source = input.run.handlerStatus?.[handlerId]; + const state = evaluateNamedHandlerState( + handlerId, + input.handlers, + input.run.handlerStatus, + ); + const projection: SubmissionDependencyProjection = { + requestId: input.requestId, + runId: input.expectedRunId, + handlerId, + state, + ...(source?.version !== undefined ? { version: source.version } : {}), + ...(source?.updatedAt ? { updatedAt: source.updatedAt } : {}), + ...(source?.error ? { error: source.error } : {}), + }; + if (state !== "failed" && state !== "blocked") { + return { projection, drifted: false }; + } + return { + projection, + blocking: { + requestId: input.requestId, + runId: input.expectedRunId, + handlerId, + reason: state === "failed" ? "dependency-failed" : "dependency-blocked", + ...(source?.error ? { error: source.error } : {}), + }, + drifted: false, + }; +} + +export function deriveSubmissionReadiness(run: SubmissionRunState): SubmissionReadiness { + const readout = run.handlerStatus?.[READOUT_HANDLER_ID]; + if (readout?.status === "done") return "done"; + if (readout?.status === "failed") return "failed"; + if (readout?.status === "queued" || readout?.status === "processing") { + return "processing"; + } + if (run.blocking.length > 0) return "blocked"; + if ( + run.requests.length > 0 && + run.dependencyProjection.length === run.requests.length && + run.dependencyProjection.every((dependency) => dependency.state === "ready") + ) { + return "ready"; + } + return "waiting"; +} diff --git a/packages/shared/src/types/types.ts b/packages/shared/src/types/types.ts index f12651b0b..5f8315307 100644 --- a/packages/shared/src/types/types.ts +++ b/packages/shared/src/types/types.ts @@ -204,7 +204,7 @@ export interface ConversationTurn { timestamp: Date; criteriaResults?: CriterionResult[]; // Per-criterion breakdown from DAG evaluation /** Per-iteration observation results (kind:"observation" criteria), recorded - * by the pp-taxonomy post-processor. Same shape as criteriaResults but never + * by the pp-observations post-processor. Same shape as criteriaResults but never * gates the agent. Empty/absent when no observations were selected. #1156. */ observationResults?: CriterionResult[]; harUrl?: string; // Blob storage URL to the HAR file for this turn @@ -385,6 +385,18 @@ export interface RequestDocument { profileId?: string; // FK → ProfileDocument._id (the profile lineage) profileVersionId?: string; // FK → ProfileVersionDocument._id (exact version used) submissionId?: string; // FK → SubmissionDocument._id + /** Expected native request count for interrupted SubmissionDocument finalization. + * Historical/imported requests omit it and remain view-only if their parent + * submission document is missing. */ + submissionExpectedRequestCount?: number; + /** Stable position within the native submission's immutable request vector. */ + submissionRequestIndex?: number; + /** Run id minted for the submission's first attempt. It remains unchanged + * across request retries so recovery can rebuild the exact initial vector. */ + submissionInitialRunId?: string; + /** Set after the parent SubmissionDocument has been idempotently finalized. + * Missing on interrupted native writes so the scheduler can recover them. */ + submissionFinalizedAt?: Date; /** Scheduling priority. Higher = processed first. Default: 0. */ priority: number; /** @@ -396,7 +408,7 @@ export interface RequestDocument { gates?: GateConfig[]; /** * Observation-criteria ids (kind:"observation") selected at submit. The - * pp-taxonomy post-processor evaluates exactly this set after the run + * pp-observations post-processor evaluates exactly this set after the run * completes and writes per-turn `observationResults`. Empty/absent = none. * See issue #1156. */ @@ -532,7 +544,7 @@ export interface RunState { handlerStatus?: Record; /** Whole-run observation results — one entry per selected `subject:"run"` * observation criterion, evaluated once against the final snapshot + the - * merged trajectory across all iterations by pp-taxonomy. Per-iteration + * merged trajectory across all iterations by pp-observations. Per-iteration * (`subject:"iteration"`) observations live on `turn.observationResults` * instead. Empty/absent when no run-subject observations were selected. #1156. */ observationResults?: CriterionResult[]; @@ -567,6 +579,118 @@ export interface HandlerServiceDocument { dependsOn: string[]; } +export interface SubmissionHandlerServiceDocument { + _id: string; + type: "submission-handler"; + version: number; + queue: string; + selector: string; + autoBackfill: boolean; + dependsOn: string[]; + dependencyMode: "all-runs"; +} + +export type ServiceHandlerDocument = + | HandlerServiceDocument + | SubmissionHandlerServiceDocument; + +export interface SubmissionRequestRef { + requestId: string; + runId: string; +} + +export type SubmissionHandlerState = "waiting" | "ready" | "failed" | "blocked"; + +export interface SubmissionDependencyProjection { + requestId: string; + runId: string; + handlerId: string; + state: SubmissionHandlerState; + version?: number; + updatedAt?: Date; + error?: string; +} + +export interface SubmissionBlockingDetail { + requestId: string; + runId: string; + handlerId: string; + reason: + | "request-missing" + | "run-changed" + | "dependency-not-ready" + | "dependency-failed" + | "dependency-blocked"; + error?: string; +} + +export interface SubmissionReportPointer { + reportId?: string; + url?: string; + updatedAt?: Date; +} + +export interface SubmissionRolloverClaim { + nextSubmissionRunId: string; + vectorFingerprint: string; + requests: SubmissionRequestRef[]; + reason: "request-vector-changed" | "manual-retry"; + claimedAt: Date; +} + +export interface SubmissionRunState { + _id: string; + attemptNumber: number; + vectorFingerprint: string; + requests: SubmissionRequestRef[]; + createdAt: Date; + updatedAt?: Date; + startedAt?: Date; + finishedAt?: Date; + dependencyProjection: SubmissionDependencyProjection[]; + handlerStatus?: Record; + blocking: SubmissionBlockingDetail[]; + report?: SubmissionReportPointer; + /** Internal scheduler hold used by bulk request retry so one completed + * replacement batch produces one vector rollover. Expiry self-heals crashes. */ + reconcileAfter?: Date; + /** Internal resumable rollover claim. Its CAS winner exclusively determines + * the immutable archive's successor metadata. */ + rollover?: SubmissionRolloverClaim; + supersededAt?: Date; + supersededBySubmissionRunId?: string; + supersededReason?: "request-vector-changed" | "manual-retry"; +} + +export interface SubmissionDocument { + _id: string; + submissionId: string; + projectId: string; + requestIds: string[]; + createdAt: Date; + updatedAt?: Date; + run: SubmissionRunState; +} + +export interface SubmissionRunHistoryDocument extends SubmissionRunState { + submissionId: string; + projectId: string; +} + +export type SubmissionReadiness = + | "waiting" + | "ready" + | "blocked" + | "processing" + | "done" + | "failed"; + +export interface SubmissionQueueMessagePayload { + type: string; + submissionId: string; + submissionRunId: string; +} + /** * Field names whose values move from `RequestDocument` (top-level, legacy * shape) into `RequestDocument.run: RunState` (new shape introduced for From 36fb2247b036104b2acf62a1ceead4d671e01942 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Tue, 13 Oct 2026 16:26:26 +0200 Subject: [PATCH 02/25] feat(submissions): orchestrate native submission attempts Finalize every native submit as an exact submission vector, recover interrupted final writes, reconcile and archive vector drift, dispatch only current attempts, and expose project-scoped submission detail, history, and retry APIs with polling repair. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 99f84f04-d3b0-40f1-a254-4dfbd21e4d87 --- apps/api/src/endpoints.test.ts | 64 +++++ apps/api/src/index.ts | 14 ++ apps/api/src/route-context.ts | 6 + apps/api/src/routes/requests/index.ts | 97 +++++++- apps/api/src/routes/submissions.test.ts | 253 ++++++++++++++++++++ apps/api/src/routes/submissions.ts | 245 +++++++++++++++++++ apps/api/src/test-helpers.ts | 34 ++- apps/scheduler/src/index.ts | 44 +++- apps/scheduler/src/notify-routes.ts | 22 +- apps/scheduler/src/submission-dispatcher.ts | 112 +++++++++ docs/architecture/app-design.md | 76 +++++- docs/architecture/overview.md | 10 +- website/src/openapi/scope-openapi.json | 2 +- 13 files changed, 959 insertions(+), 20 deletions(-) create mode 100644 apps/api/src/routes/submissions.test.ts create mode 100644 apps/api/src/routes/submissions.ts create mode 100644 apps/scheduler/src/submission-dispatcher.ts diff --git a/apps/api/src/endpoints.test.ts b/apps/api/src/endpoints.test.ts index 6af7b8cb5..c105562c7 100644 --- a/apps/api/src/endpoints.test.ts +++ b/apps/api/src/endpoints.test.ts @@ -1955,6 +1955,44 @@ describe("API Endpoints", () => { expect(res.status).toBe(201); const doc = (mocks.collection.insertOne as any).mock.calls[0][0]; expect(doc).toHaveProperty("reasoningEffort", "high"); + expect(doc).toHaveProperty("submissionExpectedRequestCount", 1); + const submissionInsert = (mocks.submissionCollection.updateOne as any).mock.calls[0][1].$setOnInsert; + expect(submissionInsert.requestIds).toEqual([doc._id]); + expect(submissionInsert.run.requests).toEqual([ + { requestId: doc._id, runId: doc.run._id }, + ]); + }); + + it("creates one complete submission vector for count fan-out", async () => { + const res = await request(testServer()) + .post(`/api/v1/requests?worker=coder-acp-copilot&projectId=${TEST_PROJECT_ID}`) + .send({ + scenario: { task: "Build something", criteria: ["works"] }, + count: 3, + }); + + expect(res.status).toBe(201); + const docs = (mocks.collection.insertMany as any).mock.calls[0][0]; + expect(docs).toHaveLength(3); + expect(docs.every((doc: { submissionExpectedRequestCount: number }) => + doc.submissionExpectedRequestCount === 3)).toBe(true); + expect( + docs.map((doc: { submissionRequestIndex: number }) => doc.submissionRequestIndex), + ).toEqual([0, 1, 2]); + expect( + docs.every( + (doc: { submissionInitialRunId: string; run: { _id: string } }) => + doc.submissionInitialRunId === doc.run._id, + ), + ).toBe(true); + const submissionInsert = (mocks.submissionCollection.updateOne as any).mock.calls[0][1].$setOnInsert; + expect(submissionInsert.requestIds).toEqual(docs.map((doc: { _id: string }) => doc._id)); + expect(submissionInsert.run.requests).toEqual( + docs.map((doc: { _id: string; run: { _id: string } }) => ({ + requestId: doc._id, + runId: doc.run._id, + })), + ); }); it("profile reasoningEffort takes precedence over client-provided effort", async () => { @@ -2259,6 +2297,7 @@ describe("API Endpoints", () => { new Set(["base-profile", "var-profile"]), ); for (const doc of docs) { + expect(doc.submissionExpectedRequestCount).toBe(2); expect(Array.isArray(doc.gates)).toBe(true); const selectGate = doc.gates.find((g: { gate: string }) => g.gate === "select"); const buildGate = doc.gates.find((g: { gate: string }) => g.gate === "build"); @@ -2269,6 +2308,10 @@ describe("API Endpoints", () => { expect(buildGate.promptId).toBe("tp-build"); expect(buildGate.promptText).toBeUndefined(); } + const submissionInsert = (mocks.submissionCollection.updateOne as any).mock.calls[0][1].$setOnInsert; + expect(submissionInsert.requestIds).toEqual( + docs.map((doc: { _id: string }) => doc._id), + ); // Response surfaces the configured gate count. expect(res.body.gates).toBe(2); }); @@ -2426,6 +2469,9 @@ describe("API Endpoints", () => { expect(insertCall[0]).toHaveProperty("taskPromptId", "tp-123"); expect(insertCall[0]).toHaveProperty("model", "gpt-4o"); expect(insertCall[0]).toHaveProperty("maxIterations", 5); + expect(insertCall[0]).toHaveProperty("submissionExpectedRequestCount", 1); + const submissionInsert = (mocks.submissionCollection.updateOne as any).mock.calls[0][1].$setOnInsert; + expect(submissionInsert.requestIds).toEqual([insertCall[0]._id]); }); it("applies profile override: uses profile fields for worker, model, mcpServers, skillRevisions, extensions", async () => { @@ -2802,11 +2848,13 @@ describe("API Endpoints", () => { it("retries eligible requests and skips non-terminal ones", async () => { const doc1 = { _id: "req-1", + submissionId: "submission-1", workerType: "coder-acp-copilot", run: { _id: "run-1", attemptNumber: 1, status: "done", outcome: "failed" }, }; const doc2 = { _id: "req-2", + submissionId: "submission-1", workerType: "coder-acp-copilot", run: { _id: "run-2", attemptNumber: 1, status: "processing" }, }; @@ -2832,6 +2880,22 @@ describe("API Endpoints", () => { $set: expect.objectContaining({ agentVersion: "1.0.0" }), }), ); + expect(mocks.submissionCollection.updateMany).toHaveBeenNthCalledWith( + 1, + { _id: { $in: ["submission-1"] } }, + expect.objectContaining({ + $set: expect.objectContaining({ + "run.reconcileAfter": expect.any(Date), + }), + }), + ); + expect(mocks.submissionCollection.updateMany).toHaveBeenNthCalledWith( + 2, + { _id: { $in: ["submission-1"] } }, + expect.objectContaining({ + $unset: { "run.reconcileAfter": "" }, + }), + ); }); it("returns 400 when ids array is empty", async () => { diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 9803a0581..7f95ab2f5 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -47,6 +47,7 @@ import { registerInsightsRoutes } from "./routes/insights.js"; import { registerSecretsRoutes } from "./routes/secrets.js"; import { registerProfilesRoutes } from "./routes/profiles.js"; import { registerProjectsRoutes } from "./routes/projects.js"; +import { registerSubmissionsRoutes } from "./routes/submissions.js"; import { ProjectScopeError } from "./utils/project-scope.js"; import { registerUsersRoutes } from "./routes/users.js"; import { shutdownPortalCopilotClientPool } from "./copilot-client-pool.js"; @@ -59,6 +60,8 @@ import type { InsightDocument, RequestDocument, RunHistoryDocument, + SubmissionDocument, + SubmissionRunHistoryDocument, CodingAgentDocument, ModelDocument, McpServerDocument, @@ -97,6 +100,8 @@ let mongoClient: MongoClient; let db: Db; let collection: Collection; let runsCol: Collection; +let submissionCollection: Collection; +let submissionRunsCollection: Collection; let projectCollection: Collection; let projectStore: ProjectStore; let criteriaCollection: Collection; @@ -144,6 +149,8 @@ async function initializeClients(): Promise { db = mongoClient.db(mongoDatabase); collection = db.collection(mongoCollection); runsCol = db.collection("runs"); + submissionCollection = db.collection("submissions"); + submissionRunsCollection = db.collection("submission-runs"); projectCollection = db.collection("projects"); projectStore = new ProjectStore(projectCollection); criteriaCollection = db.collection("criteria"); @@ -277,6 +284,8 @@ const routeCtx: RouteContext = { get db() { return db; }, get requestCollection() { return collection; }, get runsCollection() { return runsCol; }, + get submissionCollection() { return submissionCollection; }, + get submissionRunsCollection() { return submissionRunsCollection; }, get projectCollection() { return projectCollection; }, get criteriaCollection() { return criteriaCollection; }, get promptFeatureCollection() { return promptFeatureCollection; }, @@ -338,6 +347,7 @@ app.use(createUserAccessMiddleware(() => userAccessResolver)); registerSecretsRoutes(routeCtx); registerSystemRoutes(routeCtx); registerProjectsRoutes(routeCtx); +registerSubmissionsRoutes(routeCtx); registerRequestsRoutes(routeCtx); registerRequestsCancelRoutes(routeCtx); registerRequestsLogsRoutes(routeCtx); @@ -424,6 +434,8 @@ export interface TestDependencies { db?: Db; collection?: Collection; runsCollection?: Collection; + submissionCollection?: Collection; + submissionRunsCollection?: Collection; criteriaCollection?: Collection; promptFeatureCollection?: Collection; reportCollection?: Collection; @@ -465,6 +477,8 @@ export function _injectTestDependencies(deps: TestDependencies): void { if (deps.db) db = deps.db; if (deps.collection) collection = deps.collection; if (deps.runsCollection) runsCol = deps.runsCollection; + if (deps.submissionCollection) submissionCollection = deps.submissionCollection; + if (deps.submissionRunsCollection) submissionRunsCollection = deps.submissionRunsCollection; if (deps.criteriaCollection) criteriaCollection = deps.criteriaCollection; if (deps.promptFeatureCollection) promptFeatureCollection = deps.promptFeatureCollection; if (deps.reportCollection) reportCollection = deps.reportCollection; diff --git a/apps/api/src/route-context.ts b/apps/api/src/route-context.ts index 580eae160..b675e8b9f 100644 --- a/apps/api/src/route-context.ts +++ b/apps/api/src/route-context.ts @@ -52,6 +52,8 @@ import type { ReportTemplateResponseSchema, RequestResponseSchema, RunHistoryDocumentSchema, + SubmissionDocumentSchema, + SubmissionRunHistoryDocumentSchema, } from "shared"; // ─── Document types (inferred from Zod schemas) ───────────────────────────── @@ -74,6 +76,8 @@ export type ReportTrigger = z.infer; export type ReportTemplateDocument = Omit, "_id">; export type RequestDocument = z.infer; export type RunHistoryDocument = z.infer; +export type SubmissionDocument = z.infer; +export type SubmissionRunHistoryDocument = z.infer; export type WorkerType = string; @@ -92,6 +96,8 @@ export interface RouteContext { db: Db; requestCollection: Collection; runsCollection: Collection; + submissionCollection: Collection; + submissionRunsCollection: Collection; projectCollection: Collection; criteriaCollection: Collection; promptFeatureCollection: Collection; diff --git a/apps/api/src/routes/requests/index.ts b/apps/api/src/routes/requests/index.ts index ffb48667d..070a583f3 100644 --- a/apps/api/src/routes/requests/index.ts +++ b/apps/api/src/routes/requests/index.ts @@ -37,6 +37,7 @@ import { isCriterionCompatibleWithGate, orderGates, computeTaskPromptId, + ensureNativeSubmission, } from "shared"; import type { ProfileDocument, @@ -844,6 +845,7 @@ apiRoute(ctx.app, ctx.registry, { const variationResults: Array<{ profileId: string; label?: string; ids: string[] }> = []; const agentsMdPromptId = await resolveAgentsMdPromptId(); const agentsMdFields = buildAgentsMdFields(agentsMdPromptId); + const expectedRequestCount = resolved.length * count; for (const r of resolved) { const newIds: string[] = []; @@ -878,6 +880,9 @@ apiRoute(ctx.app, ctx.registry, { profileVersionId: r.profileVersion.ref ?? r.profileVersion._id, ...(persistedVariationGates ? { gates: persistedVariationGates } : {}), submissionId, + submissionExpectedRequestCount: expectedRequestCount, + submissionRequestIndex: newDocs.length, + submissionInitialRunId: runId, ...agentsMdFields, run: { _id: runId, attemptNumber: 1, status: "pending", logsUrl: ctx.blobStorage.getLogsBlobUrl(`${requestId}/runs/${runId}/run.jsonl`) }, }; @@ -891,6 +896,20 @@ apiRoute(ctx.app, ctx.registry, { } await ctx.requestCollection.insertMany(newDocs); + await ensureNativeSubmission({ + collections: { submissions: ctx.submissionCollection }, + submissionId, + projectId, + requests: newDocs.map((doc) => ({ + requestId: doc._id, + runId: doc.run!._id, + })), + createdAt: newDocs[0]?.createdAt, + }); + await ctx.requestCollection.updateMany( + { _id: { $in: allNewIds }, submissionId }, + { $set: { submissionFinalizedAt: new Date() } }, + ); res.status(201).json({ ids: allNewIds, @@ -1274,7 +1293,7 @@ apiRoute(ctx.app, ctx.registry, { : undefined; // Persisted observation-criteria selection (issue #1156); deduped, evaluated - // by the pp-taxonomy post-processor. Absent/empty = no observations evaluated. + // by the pp-observations post-processor. Absent/empty = no observations evaluated. const persistedObservations: string[] | undefined = observationsProvided ? [...new Set(resolvedObservations)] : undefined; @@ -1316,6 +1335,9 @@ apiRoute(ctx.app, ctx.registry, { ...(persistedGates ? { gates: persistedGates } : {}), ...(persistedObservations ? { observations: persistedObservations } : {}), submissionId, + submissionExpectedRequestCount: count, + submissionRequestIndex: i, + submissionInitialRunId: runId, ...agentsMdFields, // Mint a distinct run id for the first attempt. Blob artifacts // are scoped under `{requestId}/runs/{runId}/...` so retries @@ -1327,6 +1349,20 @@ apiRoute(ctx.app, ctx.registry, { // Bulk insert all documents — scheduler will dispatch to queues await ctx.requestCollection.insertMany(newDocs); + await ensureNativeSubmission({ + collections: { submissions: ctx.submissionCollection }, + submissionId, + projectId, + requests: newDocs.map((doc) => ({ + requestId: doc._id, + runId: doc.run!._id, + })), + createdAt: newDocs[0]?.createdAt, + }); + await ctx.requestCollection.updateMany( + { _id: { $in: newIds }, submissionId }, + { $set: { submissionFinalizedAt: new Date() } }, + ); console.log(`Created ${count} ${mode} requests for ${workerType} (priority: ${requestedPriority ?? 0})`); @@ -1379,6 +1415,9 @@ apiRoute(ctx.app, ctx.registry, { ...(persistedGates ? { gates: persistedGates } : {}), ...(persistedObservations ? { observations: persistedObservations } : {}), submissionId, + submissionExpectedRequestCount: 1, + submissionRequestIndex: 0, + submissionInitialRunId: runId, ...agentsMdFields, // Mint a distinct run id for the first attempt. Blob artifacts // are scoped under `{requestId}/runs/{runId}/...` so retries @@ -1388,6 +1427,17 @@ apiRoute(ctx.app, ctx.registry, { // Store in MongoDB — scheduler will dispatch to queue await ctx.requestCollection.insertOne(requestDoc); + await ensureNativeSubmission({ + collections: { submissions: ctx.submissionCollection }, + submissionId, + projectId, + requests: [{ requestId, runId }], + createdAt: requestDoc.createdAt, + }); + await ctx.requestCollection.updateOne( + { _id: requestId, submissionId }, + { $set: { submissionFinalizedAt: new Date() } }, + ); console.log(`Created ${mode} request ${requestId} for ${workerType} (priority: ${requestedPriority ?? 0})`); @@ -2206,6 +2256,9 @@ apiRoute(ctx.app, ctx.registry, { ...(original.agentsMdPromptId ? { agentsMdPromptId: original.agentsMdPromptId } : {}), ...(original.agentsMdParentIds && original.agentsMdParentIds.length > 0 ? { agentsMdParentIds: original.agentsMdParentIds } : {}), submissionId, + submissionExpectedRequestCount: originalRuns.length * count, + submissionRequestIndex: newDocs.length, + submissionInitialRunId: runId, // Bulk re-submit creates a brand-new request — mint a distinct // run id for the first attempt so blob artifacts live under // `{requestId}/runs/{runId}/...` (independent of the original run). @@ -2219,6 +2272,20 @@ apiRoute(ctx.app, ctx.registry, { // Insert all new documents — scheduler will dispatch to queues if (newDocs.length > 0) { await ctx.requestCollection.insertMany(newDocs); + await ensureNativeSubmission({ + collections: { submissions: ctx.submissionCollection }, + submissionId, + projectId: newDocs[0].projectId, + requests: newDocs.map((doc) => ({ + requestId: doc._id, + runId: doc.run!._id, + })), + createdAt: newDocs[0].createdAt, + }); + await ctx.requestCollection.updateMany( + { _id: { $in: newIds }, submissionId }, + { $set: { submissionFinalizedAt: new Date() } }, + ); } console.log(`Bulk re-submitted ${newIds.length} runs from ${originalRuns.length} originals (count=${count})`); @@ -3129,6 +3196,24 @@ apiRoute(ctx.app, ctx.registry, { { _id: { $in: ids }, deletedAt: { $exists: false } }, ).toArray(); const requestMap = new Map(requests.map((r) => [r._id, r])); + const affectedSubmissionIds = [ + ...new Set( + requests.flatMap((request) => + request.submissionId ? [request.submissionId] : [], + ), + ), + ]; + if (affectedSubmissionIds.length > 0) { + await ctx.submissionCollection.updateMany( + { _id: { $in: affectedSubmissionIds } }, + { + $set: { + "run.reconcileAfter": new Date(Date.now() + 60_000), + updatedAt: new Date(), + }, + }, + ); + } const results: Array<{ requestId: string; runId?: string; attemptNumber?: number; error?: string }> = []; let retried = 0; @@ -3210,6 +3295,16 @@ apiRoute(ctx.app, ctx.registry, { retried++; } + if (affectedSubmissionIds.length > 0) { + await ctx.submissionCollection.updateMany( + { _id: { $in: affectedSubmissionIds } }, + { + $unset: { "run.reconcileAfter": "" }, + $set: { updatedAt: new Date() }, + }, + ); + } + res.status(201).json({ retried, skipped, results }); }, }); diff --git a/apps/api/src/routes/submissions.test.ts b/apps/api/src/routes/submissions.test.ts new file mode 100644 index 000000000..984bdd358 --- /dev/null +++ b/apps/api/src/routes/submissions.test.ts @@ -0,0 +1,253 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { OpenAPIRegistry } from "@asteasolutions/zod-to-openapi"; +import express from "express"; +import request from "supertest"; +import { describe, expect, it, vi } from "vitest"; +import type { Collection, Db } from "mongodb"; +import type { + RequestDocument, + ServiceHandlerDocument, + SubmissionDocument, + SubmissionRunHistoryDocument, +} from "shared"; +import { computeSubmissionVectorFingerprint } from "shared"; +import type { RouteContext } from "../route-context.js"; +import { registerSubmissionsRoutes } from "./submissions.js"; + +function queryResult(items: T[]) { + return { + sort: () => queryResult(items), + limit: () => queryResult(items), + toArray: async () => items, + }; +} + +function buildApp(input: { + requests: RequestDocument[]; + submission: SubmissionDocument; + history?: SubmissionRunHistoryDocument[]; +}) { + let submission = structuredClone(input.submission); + const services: ServiceHandlerDocument[] = [ + { + _id: "pp-atif", + type: "post-process-handler", + version: 1, + queue: "post-processor", + selector: "atif", + autoBackfill: true, + dependsOn: [], + }, + { + _id: "pp-observations", + type: "post-process-handler", + version: 1, + queue: "post-processor", + selector: "observations", + autoBackfill: true, + dependsOn: ["pp-atif"], + }, + ]; + const submissionCollection = { + findOne: vi.fn(async () => structuredClone(submission)), + find: vi.fn(() => queryResult([structuredClone(submission)])), + findOneAndUpdate: vi.fn(async ( + filter: Record, + update: { $set: Record }, + ) => { + if (filter["run._id"] !== submission.run._id) return null; + submission = { + ...submission, + updatedAt: update.$set.updatedAt as Date, + run: { + ...submission.run, + dependencyProjection: update.$set[ + "run.dependencyProjection" + ] as SubmissionDocument["run"]["dependencyProjection"], + blocking: update.$set[ + "run.blocking" + ] as SubmissionDocument["run"]["blocking"], + updatedAt: update.$set["run.updatedAt"] as Date, + }, + }; + return structuredClone(submission); + }), + } as unknown as Collection; + const db = { + collection: vi.fn(() => ({ + find: vi.fn(() => + queryResult(services.filter((service) => service.type === "post-process-handler")), + ), + findOne: vi.fn(async () => null), + })), + } as unknown as Db; + const ctx = { + app: express(), + registry: new OpenAPIRegistry(), + db, + requestCollection: { + find: vi.fn(() => queryResult(input.requests)), + } as unknown as Collection, + submissionCollection, + submissionRunsCollection: { + find: vi.fn(() => queryResult(input.history ?? [])), + findOne: vi.fn(async (filter: { _id: string }) => + (input.history ?? []).find((run) => run._id === filter._id) ?? null, + ), + } as unknown as Collection, + } as unknown as RouteContext; + ctx.app.use(express.json()); + registerSubmissionsRoutes(ctx); + ctx.app.use( + ( + error: Error, + _req: express.Request, + res: express.Response, + _next: express.NextFunction, + ) => res.status(500).json({ error: error.message }), + ); + return ctx.app; +} + +describe("submission routes", () => { + const requestDocument: RequestDocument = { + _id: "request-1", + projectId: "project-1", + scenario: { task: "Implement feature", criteria: ["criterion"] }, + workerType: "worker", + createdAt: new Date(), + priority: 0, + run: { + _id: "run-1", + attemptNumber: 1, + status: "done", + outcome: "failed", + handlerStatus: { + "pp-observations": { status: "done", version: 1 }, + }, + }, + }; + const submissionDocument: SubmissionDocument = { + _id: "submission-1", + submissionId: "submission-1", + projectId: "project-1", + requestIds: ["request-1"], + createdAt: new Date(), + run: { + _id: "submission-run-1", + attemptNumber: 1, + vectorFingerprint: computeSubmissionVectorFingerprint([ + { requestId: "request-1", runId: "run-1" }, + ]), + requests: [{ requestId: "request-1", runId: "run-1" }], + createdAt: new Date(), + dependencyProjection: [], + blocking: [], + }, + }; + + it("returns readiness and the exact request/run vector", async () => { + const response = await request( + buildApp({ + requests: [requestDocument], + submission: submissionDocument, + }), + ).get("/api/v1/submissions/submission-1"); + + expect(response.status, JSON.stringify(response.body)).toBe(200); + expect(response.body).toMatchObject({ + submissionId: "submission-1", + submissionRunId: "submission-run-1", + readiness: "ready", + requests: [{ requestId: "request-1", runId: "run-1", outcome: "failed" }], + dependencyProjection: [{ + requestId: "request-1", + runId: "run-1", + handlerId: "pp-observations", + state: "ready", + }], + }); + }); + + it("lists current and immutable archived attempts", async () => { + const historical: SubmissionRunHistoryDocument = { + ...submissionDocument.run, + _id: "submission-run-0", + attemptNumber: 1, + reconcileAfter: new Date(), + rollover: { + nextSubmissionRunId: "internal-next", + vectorFingerprint: "internal-fingerprint", + requests: [{ requestId: "request-1", runId: "run-1" }], + reason: "request-vector-changed", + claimedAt: new Date(), + }, + submissionId: "submission-1", + projectId: "project-1", + }; + const response = await request( + buildApp({ + requests: [requestDocument], + submission: submissionDocument, + history: [historical], + }), + ).get("/api/v1/submissions/submission-1/runs"); + + expect(response.status).toBe(200); + expect(response.body.runs.map((run: { _id: string }) => run._id)).toEqual([ + "submission-run-1", + "submission-run-0", + ]); + expect(response.body.runs[1]).not.toHaveProperty("reconcileAfter"); + expect(response.body.runs[1]).not.toHaveProperty("rollover"); + }); + + it("rejects a manual retry targeting a stale submission run", async () => { + const response = await request( + buildApp({ + requests: [requestDocument], + submission: submissionDocument, + }), + ) + .post("/api/v1/submissions/submission-1/retry") + .send({ submissionRunId: "stale-submission-run" }); + + expect(response.status).toBe(409); + expect(response.body).toMatchObject({ + error: "Submission attempt is stale", + submissionRunId: "submission-run-1", + }); + }); + + it("returns exact waiting request/run pairs when retry dependencies are not ready", async () => { + const waitingRequest: RequestDocument = { + ...requestDocument, + run: { + _id: "run-1", + attemptNumber: 1, + status: "done", + handlerStatus: {}, + }, + }; + const response = await request( + buildApp({ + requests: [waitingRequest], + submission: submissionDocument, + }), + ) + .post("/api/v1/submissions/submission-1/retry") + .send({ submissionRunId: "submission-run-1" }); + + expect(response.status).toBe(409); + expect(response.body.blocking).toEqual([ + { + requestId: "request-1", + runId: "run-1", + handlerId: "pp-observations", + reason: "dependency-not-ready", + }, + ]); + }); +}); diff --git a/apps/api/src/routes/submissions.ts b/apps/api/src/routes/submissions.ts new file mode 100644 index 000000000..201aae5bd --- /dev/null +++ b/apps/api/src/routes/submissions.ts @@ -0,0 +1,245 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { z } from "zod"; +import { + SubmissionCoordinator, + SubmissionListResponseSchema, + SubmissionResponseSchema, + SubmissionRetryResponseSchema, + SubmissionRunResponseSchema, + type ServiceHandlerDocument, + deriveSubmissionReadiness, +} from "shared"; +import type { SubmissionDocument } from "../route-context.js"; +import type { RouteContext } from "../route-context.js"; +import { apiRoute } from "../openapi/api-route.js"; +import { ProjectIdQuerySchema, getQueryProjectId } from "../utils/project-scope.js"; + +function coordinator(ctx: RouteContext): SubmissionCoordinator { + return new SubmissionCoordinator({ + requests: ctx.requestCollection, + submissions: ctx.submissionCollection, + submissionRuns: ctx.submissionRunsCollection, + services: ctx.db.collection("services"), + }); +} + +function toRunResponse( + run: T, +): Omit { + const { + reconcileAfter: _reconcileAfter, + rollover: _rollover, + ...response + } = run; + return response; +} + +function retryBlocking(submission: SubmissionDocument) { + if (submission.run.blocking.length > 0) return submission.run.blocking; + return submission.run.dependencyProjection + .filter((dependency) => dependency.state !== "ready") + .map((dependency) => ({ + requestId: dependency.requestId, + runId: dependency.runId, + handlerId: dependency.handlerId, + reason: "dependency-not-ready" as const, + ...(dependency.error ? { error: dependency.error } : {}), + })); +} + +async function toResponse(ctx: RouteContext, submission: SubmissionDocument) { + const requestDocs = await ctx.requestCollection + .find({ + _id: { $in: submission.requestIds }, + projectId: submission.projectId, + deletedAt: { $exists: false }, + }) + .toArray(); + const byId = new Map(requestDocs.map((request) => [request._id, request])); + const requests = submission.run.requests.flatMap((ref) => { + const request = byId.get(ref.requestId); + if (!request) return []; + return [{ + requestId: request._id, + runId: ref.runId, + task: request.scenario.task, + status: request.run?.status ?? "unknown", + ...(request.run?.outcome ? { outcome: request.run.outcome } : {}), + workerType: request.workerType, + ...(request.model ? { model: request.model } : {}), + ...(request.profileId ? { profileId: request.profileId } : {}), + }]; + }); + return { + submissionId: submission._id, + projectId: submission.projectId, + requestIds: submission.requestIds, + createdAt: submission.createdAt, + ...(submission.updatedAt ? { updatedAt: submission.updatedAt } : {}), + submissionRunId: submission.run._id, + attemptNumber: submission.run.attemptNumber, + readiness: deriveSubmissionReadiness(submission.run), + requests, + dependencyProjection: submission.run.dependencyProjection, + blocking: submission.run.blocking, + ...(submission.run.handlerStatus + ? { handlerProgress: submission.run.handlerStatus } + : {}), + ...(submission.run.report ? { report: submission.run.report } : {}), + ...(submission.run.supersededAt + ? { supersededAt: submission.run.supersededAt } + : {}), + ...(submission.run.supersededBySubmissionRunId + ? { supersededBySubmissionRunId: submission.run.supersededBySubmissionRunId } + : {}), + ...(submission.run.supersededReason + ? { supersededReason: submission.run.supersededReason } + : {}), + }; +} + +export function registerSubmissionsRoutes(ctx: RouteContext): void { + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/submissions", + tags: ["Submissions"], + summary: "List submissions", + query: ProjectIdQuerySchema.extend({ + limit: z.coerce.number().int().min(1).max(100).default(50), + }), + response: SubmissionListResponseSchema, + handler: async (req, res) => { + const projectId = getQueryProjectId(req); + const docs = await ctx.submissionCollection + .find({ projectId }) + .sort({ createdAt: -1, _id: -1 }) + .limit(req.query.limit) + .toArray(); + res.json({ + submissions: await Promise.all(docs.map((doc) => toResponse(ctx, doc))), + }); + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/submissions/:id", + tags: ["Submissions"], + summary: "Get a submission", + params: z.object({ id: z.string() }), + response: SubmissionResponseSchema, + errorResponses: { 404: { description: "Submission not found" } }, + handler: async (req, res) => { + const reconciled = await coordinator(ctx).reconcile(req.params.id); + if (!reconciled) { + res.status(404).json({ error: "Submission not found" }); + return; + } + res.json(await toResponse(ctx, reconciled.submission)); + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/submissions/:id/runs", + tags: ["Submissions"], + summary: "List current and historical submission attempts", + params: z.object({ id: z.string() }), + response: z.object({ runs: z.array(SubmissionRunResponseSchema) }), + errorResponses: { 404: { description: "Submission not found" } }, + handler: async (req, res) => { + const submission = await ctx.submissionCollection.findOne({ _id: req.params.id }); + if (!submission) { + res.status(404).json({ error: "Submission not found" }); + return; + } + const historical = await ctx.submissionRunsCollection + .find({ submissionId: req.params.id }) + .sort({ attemptNumber: -1 }) + .toArray(); + res.json({ + runs: [toRunResponse(submission.run), ...historical.map(toRunResponse)], + }); + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/submissions/:id/runs/:submissionRunId", + tags: ["Submissions"], + summary: "Get one submission attempt", + params: z.object({ id: z.string(), submissionRunId: z.string() }), + response: SubmissionRunResponseSchema, + errorResponses: { 404: { description: "Submission attempt not found" } }, + handler: async (req, res) => { + const submission = await ctx.submissionCollection.findOne({ _id: req.params.id }); + if (submission?.run._id === req.params.submissionRunId) { + res.json(toRunResponse(submission.run)); + return; + } + const historical = await ctx.submissionRunsCollection.findOne({ + _id: req.params.submissionRunId, + submissionId: req.params.id, + }); + if (!historical) { + res.status(404).json({ error: "Submission attempt not found" }); + return; + } + res.json(toRunResponse(historical)); + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "post", + path: "/api/v1/submissions/:id/retry", + tags: ["Submissions"], + summary: "Retry current submission processing", + params: z.object({ id: z.string() }), + body: z.object({ submissionRunId: z.string() }), + response: SubmissionRetryResponseSchema, + errorResponses: { + 404: { description: "Submission not found" }, + 409: { description: "Submission attempt is stale or dependencies are blocked" }, + }, + handler: async (req, res) => { + const current = await ctx.submissionCollection.findOne({ _id: req.params.id }); + if (!current) { + res.status(404).json({ error: "Submission not found" }); + return; + } + if (current.run._id !== req.body.submissionRunId) { + res.status(409).json({ + error: "Submission attempt is stale", + submissionRunId: current.run._id, + blocking: retryBlocking(current), + }); + return; + } + const result = await coordinator(ctx).retryCurrent( + req.params.id, + req.body.submissionRunId, + ); + if (!result) { + res.status(404).json({ error: "Submission not found" }); + return; + } + if (result.readiness === "blocked" || result.readiness === "waiting") { + res.status(409).json({ + error: "Submission dependencies are not ready", + submissionRunId: result.submission.run._id, + blocking: retryBlocking(result.submission), + }); + return; + } + res.status(201).json({ + submissionId: result.submission._id, + submissionRunId: result.submission.run._id, + attemptNumber: result.submission.run.attemptNumber, + readiness: result.readiness, + blocking: result.submission.run.blocking, + }); + }, + }); +} diff --git a/apps/api/src/test-helpers.ts b/apps/api/src/test-helpers.ts index 023c46b73..74447d6e7 100644 --- a/apps/api/src/test-helpers.ts +++ b/apps/api/src/test-helpers.ts @@ -49,11 +49,29 @@ export function createMockCollection(docs: T[] = insertedCount: 1, acknowledged: true, }), - updateOne: vi.fn().mockResolvedValue({ - matchedCount: 1, - modifiedCount: 1, - acknowledged: true, - }), + updateOne: vi.fn().mockImplementation( + async ( + filter: Record, + update: { $setOnInsert?: T }, + options?: { upsert?: boolean }, + ) => { + const existing = data.find((doc) => doc._id === filter._id); + if (!existing && options?.upsert && update.$setOnInsert) { + data.push({ ...update.$setOnInsert }); + return { + matchedCount: 0, + modifiedCount: 0, + upsertedCount: 1, + acknowledged: true, + }; + } + return { + matchedCount: 1, + modifiedCount: 1, + acknowledged: true, + }; + }, + ), updateMany: vi.fn().mockResolvedValue({ matchedCount: 1, modifiedCount: 1, @@ -204,6 +222,8 @@ export function createAllMockDependencies() { const db = createMockDb(); const collection = createMockCollection(); const runsCollection = createMockCollection(); + const submissionCollection = createMockCollection(); + const submissionRunsCollection = createMockCollection(); const criteriaCollection = createMockCollection(); const promptFeatureCollection = createMockCollection(); const reportCollection = createMockCollection(); @@ -232,6 +252,8 @@ export function createAllMockDependencies() { db, collection, runsCollection, + submissionCollection, + submissionRunsCollection, criteriaCollection, promptFeatureCollection, reportCollection, @@ -262,6 +284,8 @@ export function createAllMockDependencies() { db: ReturnType; collection: Collection; runsCollection: Collection; + submissionCollection: Collection; + submissionRunsCollection: Collection; criteriaCollection: Collection; promptFeatureCollection: Collection; reportCollection: Collection; diff --git a/apps/scheduler/src/index.ts b/apps/scheduler/src/index.ts index 101750a30..9fe0fbf79 100644 --- a/apps/scheduler/src/index.ts +++ b/apps/scheduler/src/index.ts @@ -8,10 +8,16 @@ import { DefaultAzureCredential } from "@azure/identity"; import { RequestScheduler } from "./request-scheduler.js"; import { PostProcessorDispatcher } from "./post-processor-dispatcher.js"; import { HandlerDispatcher } from "./handler-dispatcher.js"; +import { SubmissionDispatcher } from "./submission-dispatcher.js"; import { createHttpServer, type NotifyHandler } from "./notify-routes.js"; import { StuckRunReaper } from "./stuck-run-reaper.js"; import { RedisHeartbeatStore, type HeartbeatStore } from "shared"; -import type { CodingAgentDocument, RequestDocument } from "shared"; +import type { + CodingAgentDocument, + RequestDocument, + SubmissionDocument, + SubmissionRunHistoryDocument, +} from "shared"; import { initTelemetry, trackMetric, trackEvent, shutdownTelemetry } from "telemetry"; // ── Configuration ──────────────────────────────────────────────────── @@ -128,6 +134,9 @@ async function main(): Promise { const db = mongoClient.db(MONGO_DATABASE); const collection = db.collection(MONGO_COLLECTION); const agentCollection = db.collection("agents"); + const submissionCollection = db.collection("submissions"); + const submissionRunsCollection = + db.collection("submission-runs"); // Start the scheduler const scheduler = new RequestScheduler( @@ -191,6 +200,17 @@ async function main(): Promise { `[Scheduler] Handler dispatcher (DAG) started (poll=${ppPollIntervalMs}ms, topologyCacheTtl=${handlerCacheTtlMs}ms)`, ); + const submissionDispatcher = new SubmissionDispatcher( + collection, + submissionCollection, + submissionRunsCollection, + db, + createQueueClient, + ppPollIntervalMs, + ); + submissionDispatcher.start(); + console.log(`[Scheduler] Submission dispatcher started (poll=${ppPollIntervalMs}ms)`); + // Start the stuck-run reaper (authoritative backstop). Redis is non-fatal: // on misconfig or connection failure the reaper self-disables (its sweeps // skip when ping() fails) while the dispatch loop above keeps running. @@ -227,8 +247,25 @@ async function main(): Promise { ); } - // Wire notify handler to the HandlerDispatcher - const httpServer = createHttpServer(handlerDispatcher); + // Run notifications update the run DAG and immediately reconcile the parent + // submission. The polling loop remains the durable safety net. + const notifyHandler: NotifyHandler = { + onRunTerminal: async (requestId, runId) => { + await handlerDispatcher.onRunTerminal(requestId, runId); + await submissionDispatcher.onRequestChanged(requestId); + }, + onHandlerComplete: async (requestId, runId, handlerId, status) => { + await handlerDispatcher.onHandlerComplete( + requestId, + runId, + handlerId, + status, + ); + await submissionDispatcher.onRequestChanged(requestId); + }, + registerHandler: (doc) => handlerDispatcher.registerHandler(doc), + }; + const httpServer = createHttpServer(notifyHandler); httpServer.listen(HEALTH_PORT, () => { console.log(`[Scheduler] HTTP server listening on :${HEALTH_PORT}`); }); @@ -239,6 +276,7 @@ async function main(): Promise { await scheduler.stop(); await postProcessorDispatcher.stop(); await handlerDispatcher.stop(); + await submissionDispatcher.stop(); if (stuckRunReaper) await stuckRunReaper.stop(); if (heartbeatStore) await heartbeatStore.close(); httpServer.close(); diff --git a/apps/scheduler/src/notify-routes.ts b/apps/scheduler/src/notify-routes.ts index b63d13dd9..6810a90c0 100644 --- a/apps/scheduler/src/notify-routes.ts +++ b/apps/scheduler/src/notify-routes.ts @@ -2,7 +2,7 @@ // Licensed under the MIT License. import http from "node:http"; -import type { HandlerServiceDocument } from "shared"; +import type { ServiceHandlerDocument } from "shared"; /** * Maximum accepted request body size for the scheduler's HTTP endpoints. @@ -28,7 +28,7 @@ export interface NotifyHandler { handlerId: string, status: "done" | "failed", ): Promise; - registerHandler(doc: HandlerServiceDocument): Promise; + registerHandler(doc: ServiceHandlerDocument): Promise; } interface RunTerminalRequest { @@ -98,7 +98,7 @@ function parseHandlerCompleteRequest(body: unknown): HandlerCompleteRequest | nu }; } -function parseHandlerRegisterRequest(body: unknown): HandlerServiceDocument | null { +function parseHandlerRegisterRequest(body: unknown): ServiceHandlerDocument | null { if (!isRecord(body)) { return null; } @@ -115,7 +115,7 @@ function parseHandlerRegisterRequest(body: unknown): HandlerServiceDocument | nu if ( !isNonEmptyString(_id) || - type !== "post-process-handler" || + (type !== "post-process-handler" && type !== "submission-handler") || typeof version !== "number" || !Number.isFinite(version) || !isNonEmptyString(queue) || @@ -127,15 +127,23 @@ function parseHandlerRegisterRequest(body: unknown): HandlerServiceDocument | nu return null; } - return { + const common = { _id, - type, version, queue, selector, autoBackfill, dependsOn: dependsOn as string[], }; + if (type === "submission-handler") { + if (body.dependencyMode !== "all-runs") return null; + return { + ...common, + type, + dependencyMode: "all-runs", + }; + } + return { ...common, type }; } export function readJsonBody( @@ -262,7 +270,7 @@ export function createHttpServer( if (!parsedBody) { sendJson(res, 400, { error: - "Invalid handler registration: _id, type='post-process-handler', version (number), queue, selector, autoBackfill (boolean), and dependsOn (string[]) are required", + "Invalid handler registration: _id, supported type, version, queue, selector, autoBackfill, dependsOn, and submission dependencyMode when applicable are required", }); return; } diff --git a/apps/scheduler/src/submission-dispatcher.ts b/apps/scheduler/src/submission-dispatcher.ts new file mode 100644 index 000000000..cd2914853 --- /dev/null +++ b/apps/scheduler/src/submission-dispatcher.ts @@ -0,0 +1,112 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { QueueClient } from "@azure/storage-queue"; +import type { Collection, Db } from "mongodb"; +import { + SubmissionCoordinator, + type RequestDocument, + type SubmissionDocument, + type SubmissionHandlerServiceDocument, + type SubmissionQueueMessagePayload, + type SubmissionRunHistoryDocument, + type ServiceHandlerDocument, +} from "shared"; + +export class SubmissionDispatcher { + private interval: ReturnType | null = null; + private dispatching = false; + private scanAfterSubmissionId: string | null = null; + private readonly queueClients = new Map(); + private readonly coordinator: SubmissionCoordinator; + + constructor( + private readonly requests: Collection, + private readonly submissions: Collection, + submissionRuns: Collection, + db: Db, + private readonly createQueueClient: (queueName: string) => QueueClient, + private readonly pollIntervalMs = 30_000, + private readonly batchSize = 30, + ) { + this.coordinator = new SubmissionCoordinator( + { + requests, + submissions, + submissionRuns, + services: db.collection("services"), + }, + (service, message) => this.enqueue(service, message), + ); + } + + start(): void { + if (this.interval) return; + this.interval = setInterval(() => this.poll(), this.pollIntervalMs); + void this.poll(); + } + + async stop(): Promise { + if (this.interval) { + clearInterval(this.interval); + this.interval = null; + } + const deadline = Date.now() + 5_000; + while (this.dispatching && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 100)); + } + } + + async onRequestChanged(requestId: string): Promise { + const request = await this.requests.findOne( + { _id: requestId }, + { projection: { submissionId: 1 } }, + ); + if (request?.submissionId) { + await this.coordinator.reconcile(request.submissionId); + } + } + + private async poll(): Promise { + if (this.dispatching) return; + this.dispatching = true; + try { + await this.coordinator.recoverMissingNativeSubmissions(this.batchSize); + const candidates = await this.submissions + .find( + this.scanAfterSubmissionId + ? { _id: { $gt: this.scanAfterSubmissionId } } + : {}, + ) + .sort({ _id: 1 }) + .limit(this.batchSize) + .project({ _id: 1 }) + .toArray(); + this.scanAfterSubmissionId = + candidates.length === this.batchSize + ? candidates.at(-1)?._id ?? null + : null; + for (const candidate of candidates) { + await this.coordinator.reconcile(candidate._id); + } + } catch (error) { + console.error("[SubmissionDispatcher] Poll failed:", error); + } finally { + this.dispatching = false; + } + } + + private async enqueue( + service: SubmissionHandlerServiceDocument, + message: SubmissionQueueMessagePayload, + ): Promise { + let client = this.queueClients.get(service.queue); + if (!client) { + client = this.createQueueClient(service.queue); + await client.createIfNotExists(); + this.queueClients.set(service.queue, client); + } + const encoded = Buffer.from(JSON.stringify(message)).toString("base64"); + await client.sendMessage(encoded); + } +} diff --git a/docs/architecture/app-design.md b/docs/architecture/app-design.md index 8f1d67c01..9ff0c010f 100644 --- a/docs/architecture/app-design.md +++ b/docs/architecture/app-design.md @@ -470,6 +470,78 @@ Profile fan-out mode is also supported for comparative runs: 3. Each expanded request resolves configuration from its variation profile; unpinned specs resolve to `latestVersion` 4. Expanded requests persist `profileId` and `profileVersionId` on each `RequestDocument` for indexing and traceability +### Submission identity and processing attempts + +Every native submit creates a stable `SubmissionDocument`, including one-request +submissions. The API generates the submission id, every request id, and every +initial run id before writing anything. It inserts the requests first, then +idempotently finalizes the submission with: + +- immutable `projectId` and ordered `requestIds`; +- one embedded current `SubmissionRunState`; +- an exact immutable vector of `{ requestId, runId }` pairs for that attempt; +- a deterministic vector fingerprint, dependency projection, blocking details, + submission-level handler status, and report pointers. + +Requests carry `submissionExpectedRequestCount`, `submissionRequestIndex`, and +the immutable `submissionInitialRunId`, then receive `submissionFinalizedAt` +only after the parent document exists. The scheduler uses those markers to +recover the exact initial vector after an interrupted final write, even if a +request has since been retried. It never adopts historical/imported requests +that only have `submissionId`; those remain view-only and non-triggering. + +The current attempt lives only at `SubmissionDocument.run`. When any request's +current run id changes, reconciliation archives the previous attempt to the +immutable `submission-runs` collection and compare-and-swaps a fresh embedded +attempt containing every request's latest run id. The next attempt id is +deterministic for `(submissionId, attemptNumber, vectorFingerprint)`, so a +rollover first claims that successor on the exact current attempt. Only the +claim winner writes the immutable archive's supersession metadata, then swaps +the embedded current attempt. A restart between claim, archive, and swap resumes +the same claimed rollover instead of creating a dangling successor or parallel +current attempt. Claimed attempts reject worker claims/completions, and only the +latest embedded vector without a rollover claim is dispatchable. Individual +retries may therefore leave useful intermediate history. +Bulk request retry places a short, expiring reconciliation hold on each affected +submission while all replacement runs are installed, then releases it so the +completed batch produces one rollover. Polling clears expired holds and repairs +vector drift, missing projections, dropped notifications, and interrupted +archive/swap operations. + +`pp-readout` is a registered `submission-handler` with +`dependencyMode: all-runs` and `dependsOn: [pp-observations]`. Reconciliation +projects the authoritative `run.handlerStatus` for every exact request/run pair: + +- `pp-observations: done` is valid evidence even when the coding outcome failed + or an observation result has `passed: false`; +- a failed `pp-observations`, or one blocked by a failed DAG ancestor, blocks + readout; +- unrelated handler failures do not block; +- completed projections are naturally preserved for request/run pairs whose + run id did not change. + +Submission queue messages are attempt-scoped: +`{ type, submissionId, submissionRunId }`. PR B's worker must claim and complete +with filters on both `submissionId` and `run._id`, call +`isCurrentSubmissionRun()` before expensive work and final publication, and +discard a message when `claimSubmissionHandler()` returns `null`. Final writes +use `completeSubmissionHandler()`, whose compare-and-swap rejects stale +publication after supersession. + +The API exposes: + +- `GET /api/v1/submissions?projectId=...` +- `GET /api/v1/submissions/:id` +- `GET /api/v1/submissions/:id/runs` +- `GET /api/v1/submissions/:id/runs/:submissionRunId` +- `POST /api/v1/submissions/:id/retry` + +Responses expose readiness, exact blocking request/run pairs, handler progress, +supersession metadata, resolved request summaries, and the current report +pointer without queue names or scheduler claim fields. Portal submit flows +navigate to Submission Detail, and the CLI mirrors list/get/history/retry +capabilities under `scope submission`. + ## Runs List Query API `GET /api/v1/requests` powers the Portal **Runs** list and `scope run list`. All @@ -693,9 +765,9 @@ Criteria are reusable evaluation rules stored in the database and optionally def - **Criterion kind** — `kind: "gate" | "observation"` distinguishes criteria that gate the agent from record-only observations. Observation criteria can optionally set `taxonomyElementId` to one of the hard-coded observation dimensions and a `subject` (`run`|`iteration`) that selects whole-run vs per-iteration evaluation; run detail derives dimension grouping by joining each `observationResults[].criterionId` (per-turn or run-level) back to the criterion metadata. - **Kind** — `kind: "gate" | "observation"` (default `gate`). **Gate** criteria steer the coding agent through the gate DAG. **Observation** criteria (issue #1156) are record-only: they produce a boolean + evidence, never feed back to the agent, never pass/fail a gate, and never enter the gate DAG. Dependencies may only reference same-kind criteria. - **Taxonomy classification** — observation criteria carry an optional `taxonomyElementId` from a hard-coded enum of the three R&A Readout quality dimensions (`dimension:idiomatic-use`, `dimension:dependency-currency`, `dimension:configuration-correctness`). Classification is static/design-time, not per-run. -- **Evaluation subject** — `subject: "run" | "iteration"` decides the evaluation granularity of an observation. **`run`** (the default for observations) is evaluated **once** against the final snapshot + the merged trajectory across all iterations and stored at run level (`run.observationResults`); **`iteration`** is evaluated per turn and stored on each turn (`turn.observationResults`). Gate criteria are always `iteration` (the API rejects `run` for gates); `subject` is honored only by the `pp-taxonomy` pipeline. The AI authoring assistant auto-detects the subject and the user can override it. +- **Evaluation subject** — `subject: "run" | "iteration"` decides the evaluation granularity of an observation. **`run`** (the default for observations) is evaluated **once** against the final snapshot + the merged trajectory across all iterations and stored at run level (`run.observationResults`); **`iteration`** is evaluated per turn and stored on each turn (`turn.observationResults`). Gate criteria are always `iteration` (the API rejects `run` for gates); `subject` is honored only by the `pp-observations` pipeline. The AI authoring assistant auto-detects the subject and the user can override it. -Observation evaluation happens off the coding-agent critical path in the `pp-taxonomy` post-processor (see [post-processing](post-processing.md)). A run carries the chosen `observations: string[]` (observation-criteria ids); **iteration-subject** results are stored per turn as `turn.observationResults: CriterionResult[]`, while **run-subject** results are stored once at run level as `run.observationResults: CriterionResult[]` (both the same shape as gate `criteriaResults`). Dimension grouping is derived at read time by joining `criterionId` → criterion `taxonomyElementId`. +Observation evaluation happens off the coding-agent critical path in the `pp-observations` post-processor (see [post-processing](post-processing.md)). A run carries the chosen `observations: string[]` (observation-criteria ids); **iteration-subject** results are stored per turn as `turn.observationResults: CriterionResult[]`, while **run-subject** results are stored once at run level as `run.observationResults: CriterionResult[]` (both the same shape as gate `criteriaResults`). Dimension grouping is derived at read time by joining `criterionId` → criterion `taxonomyElementId`. See [`ENV_VARIABLES.md`](../../scope-mt-app/ENV_VARIABLES.md) for related configuration options. diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index a5c10eaf6..e6b179e56 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -79,7 +79,7 @@ flowchart TB | `workers/coder-acp-claude-code` | Claude Code agent via Agent Client Protocol (ACP) | | `workers/coder-acp-copilot` | GitHub Copilot agent via Agent Client Protocol (ACP) | | `workers/report-generator` | Copilot SDK worker that generates post-run markdown reports | -| `workers/post-processor` | DAG-based post-run handlers: `pp-atif` (trajectory ATIF) and `pp-taxonomy` (boolean observations via the judge, per-iteration or whole-run by criterion `subject`) | +| `workers/post-processor` | DAG-based post-run handlers: `pp-atif` (trajectory ATIF) and `pp-observations` (boolean observations via the judge, per-iteration or whole-run by criterion `subject`) | | `gateway` | AI Gateway — shared Rust TLS-intercepting proxy with plugin architecture (HAR capture, future: token refresh, rate limiting) | | `evaluations/static-prompts` | Developer-run static prompt quality and user-controlled prompt red-team suites | @@ -115,6 +115,14 @@ walkthrough, failures, cache isolation, and implemented/deferred boundary. 4. **Judge** — After the agent completes, the worker invokes the Judge to evaluate output against criteria. Results (pass/fail per criterion, scores) are persisted to CosmosDB. 5. **Snapshot** — Each iteration's workspace is snapshotted to Blob Storage for later inspection. +Every native submit also creates a stable submission that owns the complete +request set. Submission processing follows the requests' exact current run-id +vector: request retries archive and supersede the previous submission attempt, +run-level `pp-observations` completion feeds an all-runs dependency projection, +and only the embedded current submission attempt may dispatch `pp-readout`. +Portal and CLI users can inspect current/history attempts and retry current +submission processing without mutating request execution history. + > A run may execute as a sequence of **gates** (`Select → Build → Test → Run → > Deploy`), each with its own prompt, criteria subset, and iteration budget, > running stop-on-failure against the same workspace. Requests without an explicit diff --git a/website/src/openapi/scope-openapi.json b/website/src/openapi/scope-openapi.json index ca95e2002..eb0e8dfd8 100644 --- a/website/src/openapi/scope-openapi.json +++ b/website/src/openapi/scope-openapi.json @@ -1 +1 @@ -{"openapi":"3.1.0","info":{"title":"Scope API","version":"1.0.0","description":"REST API for the Scope platform — benchmarking AI coding agents"},"servers":[{"url":"/","description":"Current server"}],"components":{"securitySchemes":{"bearerAuth":{"type":"http","scheme":"bearer","bearerFormat":"JWT","description":"Use the unchanged IdP access token, without the Bearer prefix."}},"schemas":{"KeyResponse":{"type":"object","properties":{},"additionalProperties":{}},"KeyInput":{"type":"object","properties":{},"additionalProperties":{}},"ValidateKeyInput":{"type":"object","properties":{"token":{"type":"string"}},"required":["token"]},"AccountResponse":{"type":"object","properties":{},"additionalProperties":{}},"AccountInput":{"type":"object","properties":{},"additionalProperties":{}},"UserMeResponse":{"type":"object","properties":{"id":{"type":"string"},"role":{"type":"string"},"email":{"type":"string"},"displayName":{"type":"string"},"idp":{"type":"string"},"idpTenant":{"type":"string"}},"required":["id"]},"ProjectResponse":{"type":"object","properties":{"_id":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"creator":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"}},"required":["_id","name","createdAt"]},"CreateProjectInput":{"type":"object","properties":{"name":{"type":"string","minLength":1},"description":{"type":"string"},"creator":{"type":"string"}},"required":["name"]},"UpdateProjectInput":{"type":"object","properties":{"name":{"type":"string","minLength":1},"description":{"type":"string"}}},"RequestResponse":{"type":"object","properties":{"_id":{"type":"string"},"scenario":{"$ref":"#/components/schemas/Scenario"},"workerType":{"type":"string"},"model":{"type":"string"},"reasoningEffort":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"maxIterations":{"type":"number"},"personaInstructions":{"type":"string"},"persona":{"$ref":"#/components/schemas/Persona"},"deletedAt":{"type":["string","null"],"format":"date-time"},"taskPromptId":{"type":"string"},"agentsMdPromptId":{"type":"string"},"agentsMdParentIds":{"type":"array","items":{"type":"string"}},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"codebaseRevisionId":{"type":"string"},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceBinding"}},"extensions":{"type":"array","items":{"type":"string"}},"agentVersion":{"type":"string"},"profileId":{"type":"string"},"profileVersionId":{"type":"string"},"submissionId":{"type":"string"},"priority":{"type":"integer","default":0},"gates":{"type":"array","items":{"$ref":"#/components/schemas/GateConfig"}},"observations":{"type":"array","items":{"type":"string"}},"gateSummaries":{"type":"array","items":{"$ref":"#/components/schemas/GateRunSummary"}},"run":{"$ref":"#/components/schemas/RunState"},"projectId":{"type":"string"}},"required":["_id","scenario","workerType","createdAt","projectId"]},"Scenario":{"type":"object","properties":{"version":{"type":"string","enum":["v1","v2"]},"task":{"type":"string"},"criteria":{"type":"array","items":{"type":"string"}}},"required":["task","criteria"]},"Persona":{"type":"object","properties":{"personality":{"type":"string","enum":["demanding","friendly"]},"experience":{"type":"string","enum":["junior","senior"]},"verbosity":{"type":"string","enum":["brief","moderate"]},"type":{"type":"string","enum":["traditional","ai_assisted","vibe"]}},"required":["personality","experience","verbosity","type"]},"ResourceBinding":{"type":"object","properties":{"ref":{"type":"string"},"revisionId":{"type":"string"},"params":{"type":"object","additionalProperties":{"type":"string"}}},"required":["ref","revisionId","params"]},"GateConfig":{"type":"object","properties":{"gate":{"type":"string","enum":["select","build","test","run","deploy"]},"promptId":{"type":"string"},"promptText":{"type":"string"},"criteria":{"type":"array","items":{"type":"string"}},"maxIterations":{"type":"integer","minimum":1,"maximum":50}},"required":["gate","criteria"]},"GateRunSummary":{"type":"object","properties":{"gate":{"type":"string","enum":["select","build","test","run","deploy"]},"status":{"type":"string","enum":["passed","failed","skipped"]},"iterations":{"type":"integer","minimum":0}},"required":["gate","status","iterations"]},"RunState":{"type":"object","properties":{"_id":{"type":"string"},"attemptNumber":{"type":"integer","minimum":1},"status":{"type":"string","enum":["pending","queued","processing","paused","done"]},"queuedQueueName":{"type":"string"},"outcome":{"type":"string","enum":["succeeded","failed","finished"]},"result":{"type":"string"},"error":{"type":"string"},"logsUrl":{"type":"string"},"updatedAt":{"type":["string","null"],"format":"date-time"},"startedAt":{"type":["string","null"],"format":"date-time"},"finishedAt":{"type":["string","null"],"format":"date-time"},"durationMs":{"type":"number"},"turns":{"type":"array","items":{"$ref":"#/components/schemas/ConversationTurn"}},"workerVersion":{"type":"string"},"os":{"type":"object","properties":{"platform":{"type":"string"},"release":{"type":"string"},"arch":{"type":"string"}},"required":["platform","release","arch"]},"lastHeartbeatAt":{"type":["string","null"],"format":"date-time"},"worker":{"type":"object","properties":{"instanceId":{"type":"string"},"podName":{"type":"string"}},"required":["instanceId"]},"harUrl":{"type":"string"},"videoUrls":{"type":"array","items":{"type":"string"}},"setupVideoUrls":{"type":"array","items":{"type":"string"}},"tokenUsage":{"$ref":"#/components/schemas/TokenUsage"},"aiCallCount":{"type":"number"},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceRunOutcome"}},"mcpRegistered":{"type":"boolean"},"rawChatUrl":{"type":"string"},"rawChatFormat":{"type":"string"},"pausedAt":{"type":["string","null"],"format":"date-time"},"resumedAt":{"type":["string","null"],"format":"date-time"},"observationResults":{"type":"array","items":{"$ref":"#/components/schemas/CriterionResult"}}},"required":["_id","attemptNumber","status"]},"ConversationTurn":{"type":"object","properties":{"iteration":{"type":"number"},"gate":{"type":"string","enum":["select","build","test","run","deploy"]},"codingAgentResponse":{"type":"string"},"judgeFeedback":{"type":"string"},"snapshotUrl":{"type":"string"},"passed":{"type":"boolean"},"timestamp":{"type":["string","null"],"format":"date-time"},"criteriaResults":{"type":"array","items":{"$ref":"#/components/schemas/CriterionResult"}},"observationResults":{"type":"array","items":{"$ref":"#/components/schemas/CriterionResult"}},"harUrl":{"type":"string"},"videoUrls":{"type":"array","items":{"type":"string"}},"tokenUsage":{"$ref":"#/components/schemas/TokenUsage"},"startedAt":{"type":["string","null"],"format":"date-time"},"durationMs":{"type":"number"},"toolCalls":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"arguments":{"type":"object","additionalProperties":{}},"response":{"type":"string"},"timestamp":{"type":"string"}},"required":["id","name","arguments"]}},"toolCallsUrl":{"type":"string"},"toolCallCount":{"type":"number"},"aiCallCount":{"type":"number"},"rawChatUrl":{"type":"string"},"rawChatFormat":{"type":"string"},"chatResultUrl":{"type":"string"},"chatResultFormat":{"type":"string"}},"required":["iteration","judgeFeedback","snapshotUrl","passed","timestamp"]},"CriterionResult":{"type":"object","properties":{"criterionId":{"type":"string"},"passed":{"type":"boolean"},"feedback":{"type":"string"},"evaluated":{"type":"boolean"}},"required":["criterionId","passed","feedback","evaluated"]},"TokenUsage":{"type":"object","properties":{"promptTokens":{"type":"number"},"completionTokens":{"type":"number"},"totalTokens":{"type":"number"}},"required":["promptTokens","completionTokens","totalTokens"]},"ResourceRunOutcome":{"type":"object","properties":{"ref":{"type":"string"},"slug":{"type":"string"},"revisionId":{"type":"string"},"setupSucceeded":{"type":"boolean"},"published":{"type":"array","items":{"type":"string"}},"params":{"type":"object","additionalProperties":{"type":"string"}},"setupDurationMs":{"type":"number"},"error":{"type":"string"},"teardownRan":{"type":"boolean"}},"required":["ref","slug","revisionId","setupSucceeded","published"]},"ResourceBindingSpec":{"anyOf":[{"type":"string"},{"type":"object","properties":{"ref":{"type":"string"},"params":{"type":"object","additionalProperties":{"type":"string"}}},"required":["ref"]}]},"CreateRequestInput":{"type":"object","properties":{"scenario":{"$ref":"#/components/schemas/Scenario"},"model":{"type":"string"},"reasoningEffort":{"type":"string"},"maxIterations":{"type":"integer","minimum":1,"maximum":50},"personaInstructions":{"type":"string"},"persona":{"$ref":"#/components/schemas/Persona"},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"codebaseRevisionId":{"type":"string"},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceBindingSpec"}},"extensions":{"type":"array","items":{"type":"string"}},"profileId":{"type":"string"},"profileVariations":{"type":"array","items":{"type":"string"}},"priority":{"type":"integer"},"agentsMd":{"type":"string"},"agentsMdParentIds":{"type":"array","items":{"type":"string"}},"gates":{"type":"array","items":{"$ref":"#/components/schemas/GateConfig"}},"observations":{"type":"array","items":{"type":"string"}}},"required":["scenario"]},"RunFacetsResponse":{"type":"object","properties":{"total":{"type":"number"},"facets":{"type":"object","properties":{"workerType":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"status":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"outcome":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"model":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"os":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"priority":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"agentVersion":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"profileId":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}}},"required":["workerType","status","outcome","model","os","priority","agentVersion","profileId"]}},"required":["total","facets"]},"RunFacetBucket":{"type":"object","properties":{"value":{"type":"string"},"count":{"type":"number"}},"required":["value","count"]},"PaginatedRunsResponse":{"type":"object","properties":{"data":{"type":"array","items":{"$ref":"#/components/schemas/RequestResponse"}},"limit":{"type":"number"},"estimatedTotal":{"type":"number"},"cursors":{"$ref":"#/components/schemas/Cursors"}},"required":["data","limit","estimatedTotal","cursors"]},"Cursors":{"type":"object","properties":{"next":{"type":["string","null"]},"prev":{"type":["string","null"]}},"required":["next","prev"]},"PaginatedRunGroupsResponse":{"type":"object","properties":{"data":{"type":"array","items":{"$ref":"#/components/schemas/RunGroup"}},"limit":{"type":"number"},"estimatedTotal":{"type":"number"},"cursors":{"$ref":"#/components/schemas/Cursors"}},"required":["data","limit","estimatedTotal","cursors"]},"RunGroup":{"type":"object","properties":{"key":{"type":"string"},"label":{"type":"string"},"runIds":{"type":"array","items":{"type":"string"}},"aggregates":{"$ref":"#/components/schemas/GroupAggregates"},"uniform":{"$ref":"#/components/schemas/GroupUniformValues"}},"required":["key","label","runIds","aggregates","uniform"]},"GroupAggregates":{"type":"object","properties":{"count":{"type":"number"},"turns":{"$ref":"#/components/schemas/AggregateStats"},"duration":{"$ref":"#/components/schemas/AggregateStats"},"promptTokens":{"$ref":"#/components/schemas/AggregateStats"},"completionTokens":{"$ref":"#/components/schemas/AggregateStats"},"statusCounts":{"type":"object","additionalProperties":{"type":"number"}},"outcomeCounts":{"type":"object","additionalProperties":{"type":"number"}}},"required":["count","turns","duration","promptTokens","completionTokens","statusCounts","outcomeCounts"]},"AggregateStats":{"type":["object","null"],"properties":{"min":{"type":"number"},"max":{"type":"number"},"mean":{"type":"number"},"stdDev":{"type":"number"}},"required":["min","max","mean","stdDev"]},"GroupUniformValues":{"type":"object","properties":{"workerType":{"type":"string"},"agentVersion":{"type":"string"},"model":{"type":"string"},"platform":{"type":"string"},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"codebaseRevisionId":{"type":"string"},"extensions":{"type":"array","items":{"type":"string"}},"status":{"type":"string","enum":["pending","queued","processing","paused","done"]},"submissionId":{"type":"string"},"task":{"type":"string"}}},"BulkResubmitInput":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1},"count":{"type":"integer","minimum":1,"maximum":10,"default":1},"overrides":{"type":"object","properties":{"profileId":{"type":["string","null"]},"workerType":{"type":"string"},"agentVersion":{"type":"string"},"model":{"type":["string","null"]},"reasoningEffort":{"type":["string","null"]},"maxIterations":{"type":["number","null"]},"mcpServers":{"type":["array","null"],"items":{"type":"string"}},"skillRevisions":{"type":["array","null"],"items":{"type":"string"}},"extensions":{"type":["array","null"],"items":{"type":"string"}}}}},"required":["ids"]},"ReportResponse":{"type":"object","properties":{"_id":{"type":"string"},"requestId":{"type":"string"},"templateId":{"type":"string"},"reporter":{"$ref":"#/components/schemas/Reporter"},"content":{"type":"string"},"status":{"type":"string","enum":["pending","generating","completed","failed"]},"error":{"type":"string"},"insightReferences":{"type":"array","items":{"$ref":"#/components/schemas/InsightReference"}},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","requestId","status","createdAt","projectId"]},"Reporter":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"gitHash":{"type":"string"},"model":{"type":"string"},"agentId":{"type":"string"},"agentVersion":{"type":"string"}},"required":["id","name","gitHash","model","agentId","agentVersion"]},"InsightReference":{"type":"object","properties":{"insightId":{"type":"string"},"referencedAt":{"type":["string","null"],"format":"date-time"},"isNew":{"type":"boolean"}},"required":["insightId","referencedAt","isNew"]},"CreateCriteriaInput":{"type":"object","properties":{"id":{"type":"string","pattern":"^[a-z][a-z0-9_]*$"},"prompt":{"type":"string"},"dependsOn":{"type":"array","items":{"type":"string"}},"gates":{"type":"array","items":{"type":"string","enum":["select","build","test","run","deploy"]}},"kind":{"type":"string","enum":["gate","observation"],"default":"gate"},"taxonomyElementId":{"type":"string","enum":["dimension:idiomatic-use","dimension:dependency-currency","dimension:configuration-correctness"]},"subject":{"type":"string","enum":["run","iteration"]}},"required":["id","prompt"]},"CriteriaResponse":{"type":"object","properties":{"id":{"type":"string"},"prompt":{"type":"string"},"dependsOn":{"type":"array","items":{"type":"string"}},"gates":{"type":"array","items":{"type":"string","enum":["select","build","test","run","deploy"]}},"kind":{"type":"string","enum":["gate","observation"]},"taxonomyElementId":{"type":"string","enum":["dimension:idiomatic-use","dimension:dependency-currency","dimension:configuration-correctness"]},"subject":{"type":"string","enum":["run","iteration"]},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["id","prompt","createdAt","projectId"]},"CriteriaGraph":{"type":"object","properties":{"nodes":{"type":"array","items":{"$ref":"#/components/schemas/CriteriaGraphNode"}},"edges":{"type":"array","items":{"$ref":"#/components/schemas/CriteriaGraphEdge"}}},"required":["nodes","edges"]},"CriteriaGraphNode":{"type":"object","properties":{"id":{"type":"string"},"prompt":{"type":"string"},"dependsOn":{"type":"array","items":{"type":"string"}},"gates":{"type":"array","items":{"type":"string","enum":["select","build","test","run","deploy"]}},"kind":{"type":"string","enum":["gate","observation"]},"taxonomyElementId":{"type":"string","enum":["dimension:idiomatic-use","dimension:dependency-currency","dimension:configuration-correctness"]},"subject":{"type":"string","enum":["run","iteration"]}},"required":["id","prompt"]},"CriteriaGraphEdge":{"type":"object","properties":{"from":{"type":"string"},"to":{"type":"string"}},"required":["from","to"]},"UpdateCriteriaInput":{"type":"object","properties":{"prompt":{"type":"string"},"dependsOn":{"type":"array","items":{"type":"string"}},"gates":{"type":"array","items":{"type":"string","enum":["select","build","test","run","deploy"]}},"kind":{"type":"string","enum":["gate","observation"]},"taxonomyElementId":{"type":"string","enum":["dimension:idiomatic-use","dimension:dependency-currency","dimension:configuration-correctness"]},"subject":{"type":"string","enum":["run","iteration"]}}},"CreatePromptFeatureInput":{"type":"object","properties":{"id":{"type":"string","pattern":"^[a-z][a-z0-9_]*$"},"prompt":{"type":"string"},"type":{"$ref":"#/components/schemas/PromptType"}},"required":["id","prompt"]},"PromptType":{"type":"string","enum":["select","build","test","run","deploy","agents.md"]},"PromptFeatureResult":{"type":"object","properties":{"featureId":{"type":"string"},"detected":{"type":"boolean"},"evaluated":{"type":"boolean"}},"required":["featureId","detected","evaluated"]},"SuggestedPromptFeature":{"type":"object","properties":{"suggestedId":{"type":"string"},"behavior":{"type":"string"},"prompt":{"type":"string"}},"required":["suggestedId","behavior","prompt"]},"PromptFeatureResponse":{"type":"object","properties":{"id":{"type":"string"},"prompt":{"type":"string"},"type":{"$ref":"#/components/schemas/PromptType"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["id","prompt","createdAt","projectId"]},"UpdatePromptFeatureInput":{"type":"object","properties":{"prompt":{"type":"string"}}},"TaskPromptResponse":{"type":"object","properties":{"_id":{"type":"string"},"keyId":{"type":"string"},"type":{"$ref":"#/components/schemas/PromptType"},"text":{"type":"string"},"contentBlobUrl":{"type":"string"},"features":{"type":"array","items":{"$ref":"#/components/schemas/PromptFeatureResult"}},"featuresExtractedAt":{"type":["string","null"],"format":"date-time"},"createdAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","keyId","createdAt","projectId"]},"CreateTaskPromptInput":{"type":"object","properties":{"text":{"type":"string"},"type":{"$ref":"#/components/schemas/PromptType"}},"required":["text"]},"PatchTaskPromptFeatureInput":{"type":"object","properties":{"detected":{"type":"boolean"}},"required":["detected"]},"CreateReportInput":{"type":"object","properties":{"requestId":{"type":"string"},"templateId":{"type":"string"}},"required":["requestId"]},"BulkCreateReportsInput":{"type":"object","properties":{"requestIds":{"type":"array","items":{"type":"string"}},"templateId":{"type":"string"}},"required":["requestIds"]},"BulkReportStatusInput":{"type":"object","properties":{"reportIds":{"type":"array","items":{"type":"string"}}},"required":["reportIds"]},"BulkReportSummaryResponse":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/ReportSummary"}},"ReportSummary":{"type":"object","properties":{"total":{"type":"number"},"pending":{"type":"number"},"generating":{"type":"number"},"completed":{"type":"number"},"failed":{"type":"number"}},"required":["total","pending","generating","completed","failed"]},"BulkReportSummaryInput":{"type":"object","properties":{"requestIds":{"type":"array","items":{"type":"string"}}},"required":["requestIds"]},"TriggerReportsInput":{"type":"object","properties":{"requestId":{"type":"string"}},"required":["requestId"]},"BulkTriggerReportsInput":{"type":"object","properties":{"requestIds":{"type":"array","items":{"type":"string"}}},"required":["requestIds"]},"InsightResponse":{"type":"object","properties":{"_id":{"type":"string"},"title":{"type":"string"},"description":{"type":"string"},"category":{"type":"string"},"tags":{"type":"array","items":{"type":"string"}},"upvotes":{"type":"number"},"downvotes":{"type":"number"},"blocked":{"type":"boolean"},"referenceCount":{"type":"number"},"createdBy":{"type":"string","enum":["agent","user"]},"sourceReportId":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","title","description","upvotes","downvotes","blocked","referenceCount","createdBy","createdAt","projectId"]},"ProfileWithVersionResponse":{"type":"object","properties":{"_id":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"latestVersion":{"type":"number"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"version":{"$ref":"#/components/schemas/ProfileVersionResponse"},"projectId":{"type":"string"}},"required":["_id","name","latestVersion","createdAt","version","projectId"]},"ProfileVersionResponse":{"type":"object","properties":{"_id":{"type":"string"},"profileId":{"type":"string"},"version":{"type":"number"},"workerType":{"type":"string"},"model":{"type":"string"},"reasoningEffort":{"type":"string"},"agentVersion":{"type":"string"},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceBindingSpec"}},"extensions":{"type":"array","items":{"type":"string"}},"createdAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","profileId","version","workerType","model","createdAt","projectId"]},"CreateProfileInput":{"type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":128},"description":{"type":"string","maxLength":512},"workerType":{"type":"string"},"model":{"type":"string"},"reasoningEffort":{"type":"string"},"agentVersion":{"type":"string"},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceBindingSpec"}},"extensions":{"type":"array","items":{"type":"string"}}},"required":["name","workerType","model"]},"ProfileResponse":{"type":"object","properties":{"_id":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"latestVersion":{"type":"number"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","name","latestVersion","createdAt","projectId"]},"UpdateProfileIdentity":{"type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":128},"description":{"type":"string","maxLength":512}}},"ReportTemplateResponse":{"type":"object","properties":{"_id":{"type":"string"},"id":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"userPrompt":{"type":"string"},"systemPrompt":{"$ref":"#/components/schemas/ReportTemplateSystemPrompt"},"trigger":{"$ref":"#/components/schemas/ReportTrigger"},"model":{"type":"string"},"timeoutMs":{"type":"integer","exclusiveMinimum":0},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","id","name","userPrompt","createdAt","projectId"]},"ReportTemplateSystemPrompt":{"type":"object","properties":{"mode":{"type":"string","enum":["append","override"]},"content":{"type":"string"}},"required":["mode","content"]},"ReportTrigger":{"oneOf":[{"type":"object","properties":{"type":{"type":"string","enum":["always"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["criteria"]},"criteriaIds":{"type":"array","items":{"type":"string"}},"match":{"type":"string","enum":["any","all"]}},"required":["type","criteriaIds"]},{"type":"object","properties":{"type":{"type":"string","enum":["taskPrompt"]},"taskPromptIds":{"type":"array","items":{"type":"string"}}},"required":["type","taskPromptIds"]},{"type":"object","properties":{"type":{"type":"string","enum":["promptFeature"]},"featureIds":{"type":"array","items":{"type":"string"}},"match":{"type":"string","enum":["any","all"]}},"required":["type","featureIds"]}]},"CreateReportTemplateInput":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"userPrompt":{"type":"string"},"systemPrompt":{"$ref":"#/components/schemas/ReportTemplateSystemPrompt"},"trigger":{"$ref":"#/components/schemas/ReportTrigger"},"model":{"type":"string"},"timeoutMs":{"type":"integer","exclusiveMinimum":0}},"required":["id","name","userPrompt"]},"UpdateReportTemplateInput":{"type":"object","properties":{"name":{"type":"string"},"description":{"type":"string"},"userPrompt":{"type":"string"},"systemPrompt":{"allOf":[{"$ref":"#/components/schemas/ReportTemplateSystemPrompt"},{"type":["object","null"]}]},"trigger":{"$ref":"#/components/schemas/ReportTrigger"},"model":{"type":["string","null"]},"timeoutMs":{"type":["integer","null"],"exclusiveMinimum":0}}},"AgentResponse":{"type":"object","properties":{"_id":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"modelProvider":{"type":"string"},"supportedModels":{"type":"array","items":{"type":"string"}},"defaultModel":{"type":"string"},"available":{"type":"boolean"},"capabilities":{"$ref":"#/components/schemas/AgentCapabilities"},"versions":{"type":"array","items":{"$ref":"#/components/schemas/AgentVersion"}},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"}},"required":["_id","name","supportedModels","createdAt"]},"AgentCapabilities":{"type":"object","properties":{"supportsReasoningEffort":{"type":"boolean"},"supportsMcpServers":{"type":"boolean"},"supportsSkills":{"type":"boolean"},"supportsExtensions":{"type":"boolean"},"supportsResources":{"type":"boolean"}}},"AgentVersion":{"type":"object","properties":{"agentVersion":{"type":"string","minLength":1},"workerVersion":{"type":"string","minLength":1},"components":{"type":"object","additionalProperties":{"type":"string"}},"gitCommit":{"type":"string","minLength":1},"buildTime":{"type":"string","minLength":1},"imageTag":{"type":"string","minLength":1},"queueName":{"type":"string"},"status":{"type":"string","enum":["active","retired"]},"createdAt":{"type":["string","null"],"format":"date-time"}},"required":["agentVersion","workerVersion","components","gitCommit","buildTime","imageTag","status","createdAt"]},"CreateAgentInput":{"type":"object","properties":{"_id":{"type":"string","minLength":1},"name":{"type":"string","minLength":1},"description":{"type":"string"},"modelProvider":{"type":"string"},"supportedModels":{"type":"array","items":{"type":"string"}},"defaultModel":{"type":"string"},"available":{"type":"boolean"},"capabilities":{"$ref":"#/components/schemas/AgentCapabilities"}},"required":["_id","name"]},"UpdateAgentInput":{"type":"object","properties":{"name":{"type":"string"},"description":{"type":"string"},"modelProvider":{"type":"string"},"supportedModels":{"type":"array","items":{"type":"string"}},"defaultModel":{"type":"string"},"available":{"type":"boolean"},"capabilities":{"$ref":"#/components/schemas/AgentCapabilities"}}},"RegisterAgentVersionInput":{"type":"object","properties":{"agentVersion":{"type":"string","minLength":1},"workerVersion":{"type":"string","minLength":1},"components":{"type":"object","additionalProperties":{"type":"string"}},"gitCommit":{"type":"string","minLength":1},"buildTime":{"type":"string","minLength":1},"imageTag":{"type":"string","minLength":1},"queueName":{"type":"string","minLength":1}},"required":["agentVersion","workerVersion","components","gitCommit","buildTime","imageTag","queueName"]},"PatchAgentVersionInput":{"type":"object","properties":{"status":{"type":"string","enum":["active","retired"]}},"required":["status"]},"ModelResponse":{"type":"object","properties":{"_id":{"type":"string"},"modelId":{"type":"string"},"provider":{"type":"string"},"agentId":{"type":"string"},"firstSeenAt":{"type":["string","null"],"format":"date-time"},"lastSeenAt":{"type":["string","null"],"format":"date-time"},"disappearedAt":{"type":["string","null"],"format":"date-time"},"providerAvailableFrom":{"type":["string","null"],"format":"date-time"},"providerEndOfLife":{"type":["string","null"],"format":"date-time"},"metadata":{"type":"object","additionalProperties":{}},"capabilities":{"$ref":"#/components/schemas/ModelCapabilities"}},"required":["_id","modelId","provider","agentId","firstSeenAt","lastSeenAt"]},"ModelCapabilities":{"type":"object","properties":{"reasoningEffort":{"type":"array","items":{"type":"string"}},"toolCalls":{"type":"boolean"},"vision":{"type":"boolean"},"streaming":{"type":"boolean"},"adaptiveThinking":{"type":"boolean"},"maxThinkingBudget":{"type":"number"}}},"McpServerResponse":{"type":"object","properties":{"_id":{"type":"string"},"slug":{"type":"string"},"name":{"type":"string"},"type":{"type":"string","enum":["sse","http","stdio"]},"url":{"type":"string"},"command":{"type":"string"},"args":{"type":"array","items":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"headers":{"type":"array","items":{"$ref":"#/components/schemas/McpServerHeader"}},"sessionMode":{"type":"string","enum":["stateful","stateless"]},"version":{"type":"string"},"description":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","slug","name","type","createdAt","projectId"]},"McpServerHeader":{"type":"object","properties":{"name":{"type":"string"},"value":{"type":"string"}},"required":["name","value"]},"UpdateMcpServerInput":{"type":"object","properties":{"name":{"type":"string"},"type":{"type":"string","enum":["sse","http","stdio"]},"url":{"type":"string"},"command":{"type":"string"},"args":{"type":"array","items":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"headers":{"type":"array","items":{"$ref":"#/components/schemas/McpServerHeader"}},"sessionMode":{"type":"string","enum":["stateful","stateless"]},"version":{"type":"string"},"description":{"type":"string"}}},"SkillResponse":{"type":"object","properties":{"_id":{"type":"string"},"slug":{"type":"string"},"source":{"type":"string"},"skillName":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"origin":{"type":"string","enum":["skills-sh","manual"]},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","slug","source","skillName","name","origin","createdAt","projectId"]},"SkillSearchResult":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"source":{"type":"string"},"description":{"type":"string"},"internal":{"type":"boolean"},"installs":{"type":"number"}},"required":["id","name","source","internal"]},"SkillDiscoveryResult":{"type":"object","properties":{"skillName":{"type":"string"},"skillPath":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"existsInLibrary":{"type":"boolean"},"currentRevisionCommitSha":{"type":"string"},"latestUpstreamCommitSha":{"type":"string"},"updateAvailable":{"type":"boolean"},"lastImportedAt":{"type":"string"}},"required":["skillName","skillPath"]},"SkillRevisionResponse":{"type":"object","properties":{"_id":{"type":"string"},"ref":{"type":"string"},"source":{"type":"string"},"skillName":{"type":"string"},"skillPath":{"type":"string"},"commitHash":{"type":"string"},"commitTimestamp":{"type":["string","null"],"format":"date-time"},"name":{"type":"string"},"description":{"type":"string"},"license":{"type":"string"},"compatibility":{"type":"string"},"allowedTools":{"type":"string"},"metadata":{"type":"object","additionalProperties":{"type":"string"}},"content":{"type":"string"},"archiveUrl":{"type":"string"},"validationWarnings":{"type":"array","items":{"type":"string"}},"resolvedAt":{"type":["string","null"],"format":"date-time"},"createdAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","ref","source","skillName","skillPath","commitHash","commitTimestamp","name","description","content","archiveUrl","resolvedAt","createdAt","projectId"]},"CreateSkillInput":{"type":"object","properties":{"source":{"type":"string"},"skillName":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"origin":{"type":"string","enum":["skills-sh","manual"]}},"required":["source","skillName","name","origin"]},"CodebaseResponse":{"type":"object","properties":{"_id":{"type":"string"},"slug":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"sourceType":{"type":"string","enum":["git","archive"]},"source":{"type":"string"},"defaultBranch":{"type":"string"},"revisionCounter":{"type":"number"},"latestRevisionId":{"type":"string"},"creator":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","slug","name","sourceType","revisionCounter","createdAt","projectId"]},"UpdateCodebaseInput":{"type":"object","properties":{"name":{"type":"string","minLength":1},"description":{"type":"string"},"defaultBranch":{"type":"string"}}},"CodebaseRevisionResponse":{"type":"object","properties":{"_id":{"type":"string"},"codebaseId":{"type":"string"},"slug":{"type":"string"},"revisionNumber":{"type":"number"},"ref":{"type":"string"},"sourceType":{"type":"string","enum":["git","archive"]},"source":{"type":"string"},"requestedRef":{"type":"string"},"resolvedCommitSha":{"type":"string"},"commitTimestamp":{"type":["string","null"],"format":"date-time"},"originalFilename":{"type":"string"},"contentSha256":{"type":"string"},"archiveUrl":{"type":"string"},"sizeBytes":{"type":"number"},"fileCount":{"type":"number"},"creator":{"type":"string"},"resolvedAt":{"type":["string","null"],"format":"date-time"},"createdAt":{"type":["string","null"],"format":"date-time"},"deduplicated":{"type":"boolean"},"projectId":{"type":"string"}},"required":["_id","codebaseId","slug","revisionNumber","ref","sourceType","archiveUrl","resolvedAt","createdAt","projectId"]},"ResolveCodebaseRevisionInput":{"type":"object","properties":{"requestedRef":{"type":"string"},"creator":{"type":"string"}}},"ResourceResponse":{"type":"object","properties":{"_id":{"type":"string"},"slug":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"revisionCounter":{"type":"number"},"latestRevisionId":{"type":"string"},"latestRevisionNumber":{"type":"number"},"creator":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","slug","name","revisionCounter","createdAt","projectId"]},"CreateResourceInput":{"type":"object","properties":{"name":{"type":"string","minLength":1},"slug":{"type":"string","minLength":1},"description":{"type":"string"},"setup":{"$ref":"#/components/schemas/ResourceScript"},"teardown":{"$ref":"#/components/schemas/ResourceScript"},"exports":{"type":"array","items":{"type":"string","pattern":"^[A-Za-z_][A-Za-z0-9_]*$"}},"parameters":{"type":"array","items":{"$ref":"#/components/schemas/ResourceParameter"}},"creator":{"type":"string"}},"required":["name","setup"]},"ResourceScript":{"type":"object","properties":{"sh":{"type":"string","minLength":1}}},"ResourceParameter":{"type":"object","properties":{"name":{"type":"string","pattern":"^[A-Za-z_][A-Za-z0-9_]*$"},"description":{"type":"string"},"required":{"type":"boolean","default":false},"default":{"type":"string"},"example":{"type":"string"}},"required":["name"]},"UpdateResourceInput":{"type":"object","properties":{"name":{"type":"string","minLength":1},"description":{"type":"string"}}},"ResourceRevisionResponse":{"type":"object","properties":{"_id":{"type":"string"},"resourceId":{"type":"string"},"slug":{"type":"string"},"revisionNumber":{"type":"number"},"ref":{"type":"string"},"setup":{"$ref":"#/components/schemas/ResourceScript"},"teardown":{"$ref":"#/components/schemas/ResourceScript"},"exports":{"type":"array","items":{"type":"string"}},"parameters":{"type":"array","items":{"$ref":"#/components/schemas/ResourceParameter"}},"contentSha256":{"type":"string"},"creator":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"deduplicated":{"type":"boolean"},"projectId":{"type":"string"}},"required":["_id","resourceId","slug","revisionNumber","ref","setup","exports","contentSha256","createdAt","projectId"]},"CreateResourceRevisionInput":{"type":"object","properties":{"setup":{"$ref":"#/components/schemas/ResourceScript"},"teardown":{"$ref":"#/components/schemas/ResourceScript"},"exports":{"type":"array","items":{"type":"string","pattern":"^[A-Za-z_][A-Za-z0-9_]*$"}},"parameters":{"type":"array","items":{"$ref":"#/components/schemas/ResourceParameter"}},"creator":{"type":"string"}},"required":["setup"]},"ExtensionResponse":{"type":"object","properties":{"_id":{"type":"string"},"slug":{"type":"string"},"publisher":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"origin":{"type":"string","enum":["marketplace","manual"]},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","slug","publisher","name","origin","createdAt","projectId"]},"ExtensionSearchResult":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"publisher":{"type":"string"},"description":{"type":"string"},"internal":{"type":"boolean"},"version":{"type":"string"}},"required":["id","name","publisher","internal"]},"CreateExtensionInput":{"type":"object","properties":{"_id":{"type":"string","pattern":"^[a-zA-Z0-9-]+\\.[a-zA-Z0-9-]+$"},"publisher":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"origin":{"type":"string","enum":["marketplace","manual"]}},"required":["_id","publisher","name","origin"]},"UpdateExtensionInput":{"type":"object","properties":{"name":{"type":"string"},"description":{"type":"string"}}},"ExtensionVersionInfo":{"type":"object","properties":{"version":{"type":"string"},"preRelease":{"type":"boolean"},"lastUpdated":{"type":"string"}},"required":["version","preRelease","lastUpdated"]},"CreateInsightInput":{"type":"object","properties":{"title":{"type":"string"},"description":{"type":"string"},"category":{"type":"string"},"tags":{"type":"array","items":{"type":"string"}},"createdBy":{"type":"string","enum":["agent","user"]},"sourceReportId":{"type":"string"}},"required":["title","description"]},"UpdateInsightInput":{"type":"object","properties":{"title":{"type":"string"},"description":{"type":"string"},"category":{"type":"string"},"tags":{"type":"array","items":{"type":"string"}}}},"FeatureFlagResponse":{"type":"object","properties":{"key":{"type":"string"},"label":{"type":"string"},"enabled":{"type":"boolean"},"updatedAt":{"type":["string","null"],"format":"date-time"}},"required":["key","label","enabled","updatedAt"]},"UpdateFeatureFlagInput":{"type":"object","properties":{"enabled":{"type":"boolean"}},"required":["enabled"]}},"parameters":{}},"paths":{"/api/v1/keys/preview":{"post":{"tags":["Keys"],"summary":"Preview key","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyInput"}}}},"responses":{"200":{"description":"Key preview","content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyResponse"}}}}}}},"/api/v1/keys":{"post":{"tags":["Keys"],"summary":"Create key","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyInput"}}}},"responses":{"201":{"description":"Key created","content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyResponse"}}}}}},"get":{"tags":["Keys"],"summary":"List keys","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/KeyResponse"}}}}}}}},"/api/v1/keys/{id}":{"get":{"tags":["Keys"],"summary":"Get key","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyResponse"}}}}}},"put":{"tags":["Keys"],"summary":"Update key","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyInput"}}}},"responses":{"200":{"description":"Key updated","content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyResponse"}}}}}},"delete":{"tags":["Keys"],"summary":"Delete key","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Key deleted","content":{"application/json":{"schema":{"type":"object","properties":{"success":{"type":"boolean"}},"required":["success"]}}}}}}},"/api/v1/keys/{id}/validate":{"post":{"tags":["Keys"],"summary":"Validate key","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ValidateKeyInput"}}}},"responses":{"200":{"description":"Validation result","content":{"application/json":{"schema":{"type":"object","properties":{},"additionalProperties":{},"description":"Key validation result"}}}}}}},"/api/v1/accounts":{"post":{"tags":["Accounts"],"summary":"Create account","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/AccountInput"}}}},"responses":{"201":{"description":"Account created","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AccountResponse"}}}}}},"get":{"tags":["Accounts"],"summary":"List accounts","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/AccountResponse"}}}}}}}},"/api/v1/accounts/{id}":{"get":{"tags":["Accounts"],"summary":"Get account","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AccountResponse"}}}}}},"put":{"tags":["Accounts"],"summary":"Update account","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/AccountInput"}}}},"responses":{"200":{"description":"Account updated","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AccountResponse"}}}}}},"delete":{"tags":["Accounts"],"summary":"Delete account","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Account deleted","content":{"application/json":{"schema":{"type":"object","properties":{"success":{"type":"boolean"}},"required":["success"]}}}}}}},"/api/v1/users/me":{"get":{"tags":["Users"],"summary":"Get the authenticated user's identity","description":"Read the existing authenticated Scope identity without enrollment or profile writes. Responses must not be HTTP-cached.","security":[{"bearerAuth":[]}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/UserMeResponse"}}}},"401":{"description":"Not authenticated"},"403":{"description":"User is not enrolled or is disabled"},"503":{"description":"Authentication service unavailable"}}},"post":{"tags":["Users"],"summary":"Enroll the authenticated user","description":"JIT-enroll the authenticated user, refresh their profile and lastLoginAt, apply bootstrap-admin rules, and warm the access cache. Use this only after an explicit IdP login; do not prefetch, poll, or automatically retry transient failures.","security":[{"bearerAuth":[]}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/UserMeResponse"}}}},"401":{"description":"Not authenticated"},"403":{"description":"User is disabled"},"503":{"description":"Authentication service unavailable"}}}},"/health":{"get":{"tags":["Health"],"summary":"Liveness probe","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"status":{"type":"string"},"version":{"type":"string"}},"required":["status","version"]}}}}}}},"/ready":{"get":{"tags":["Health"],"summary":"Readiness probe","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"status":{"type":"string"},"migrations":{}},"required":["status"]}}}},"503":{"description":"Service is not ready"}}}},"/about":{"get":{"tags":["System"],"summary":"API metadata","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"name":{"type":"string"},"version":{"type":"string"},"buildTime":{"type":"string"},"environment":{"type":"string"},"description":{"type":"string"},"workers":{"type":"array","items":{"type":"string"}}},"required":["name","version","buildTime","environment","description","workers"]}}}}}}},"/api/v1/version":{"get":{"tags":["System"],"summary":"Version info","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"commit":{"type":"string"},"buildTime":{"type":"string"},"environment":{"type":"string"},"strictAgentCapabilities":{"type":"boolean"}},"required":["commit","buildTime","environment","strictAgentCapabilities"]}}}}}}},"/api/v1/projects":{"get":{"tags":["Projects"],"summary":"List all projects","parameters":[{"schema":{"type":"string","enum":["true","false"],"description":"When true, include soft-deleted projects in the result"},"required":false,"description":"When true, include soft-deleted projects in the result","name":"includeDeleted","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ProjectResponse"}}}}}}},"post":{"tags":["Projects"],"summary":"Create a project","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateProjectInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProjectResponse"}}}},"400":{"description":"Invalid input"}}}},"/api/v1/projects/{id}":{"get":{"tags":["Projects"],"summary":"Get a project","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProjectResponse"}}}},"404":{"description":"Project not found"}}},"patch":{"tags":["Projects"],"summary":"Update a project","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateProjectInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProjectResponse"}}}},"404":{"description":"Project not found"}}},"delete":{"tags":["Projects"],"summary":"Soft-delete a project","responses":{"204":{"description":"Success"},"404":{"description":"Project not found"}}}},"/api/v1/projects/{id}/restore":{"post":{"tags":["Projects"],"summary":"Restore a soft-deleted project","responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProjectResponse"}}}},"404":{"description":"Project not found"}}}},"/api/v1/requests":{"post":{"tags":["Requests"],"summary":"Submit request(s)","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"allOf":[{"$ref":"#/components/schemas/CreateRequestInput"},{"type":"object","properties":{"count":{"type":"number","minimum":1,"maximum":10,"default":1},"skills":{"type":"array","items":{"type":"string"}},"agentVersion":{"type":"string"},"codebase":{"type":"string"}}}]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"anyOf":[{"$ref":"#/components/schemas/RequestResponse"},{"type":"array","items":{"$ref":"#/components/schemas/RequestResponse"}}]}}}}}},"get":{"tags":["Requests"],"summary":"List requests","parameters":[{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"worker","in":"query"},{"schema":{"type":"string"},"required":false,"name":"taskPromptId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"criteria","in":"query"},{"schema":{"type":"string"},"required":false,"name":"submissionId","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"profileId","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"status","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"outcome","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"model","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"os","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"priority","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"agentVersion","in":"query"},{"schema":{"type":"string"},"required":false,"name":"search","in":"query"},{"schema":{"type":"string","enum":["task","submissionId","profile"]},"required":false,"name":"groupBy","in":"query"},{"schema":{"type":"integer","minimum":1,"maximum":100},"required":false,"name":"limit","in":"query"},{"schema":{"type":"string"},"required":false,"name":"after","in":"query"},{"schema":{"type":"string"},"required":false,"name":"before","in":"query"},{"schema":{"type":"string","enum":["true","false"]},"required":false,"name":"last","in":"query"},{"schema":{"type":"string","enum":["created","updated","priority","worker","status","id","duration","createdAt"]},"required":false,"name":"sortBy","in":"query"},{"schema":{"type":"string","enum":["asc","desc"]},"required":false,"name":"sortDir","in":"query"},{"schema":{"type":["string","null"],"format":"date-time"},"required":false,"name":"createdAfter","in":"query"},{"schema":{"type":["string","null"],"format":"date-time"},"required":false,"name":"createdBefore","in":"query"},{"schema":{"type":["integer","null"],"minimum":0},"required":false,"name":"turns","in":"query"},{"schema":{"type":"string","enum":["eq","gte","lte"]},"required":false,"name":"turnsOp","in":"query"},{"schema":{"type":["integer","null"],"minimum":0},"required":false,"name":"maxIterations","in":"query"},{"schema":{"type":"string","enum":["eq","gte","lte"]},"required":false,"name":"maxIterationsOp","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"anyOf":[{"$ref":"#/components/schemas/PaginatedRunsResponse"},{"$ref":"#/components/schemas/PaginatedRunGroupsResponse"}]}}}}}}},"/api/v1/requests/facets":{"get":{"tags":["Requests"],"summary":"List run filter facets","description":"Returns every distinct value and its full-dataset count per categorical filter dimension for the Runs list rail. Counts are absolute over all non-deleted runs **in the given project**: they intentionally ignore the active search, date range, iteration, and categorical selections so every selectable value stays visible with a stable count. Requires ?projectId=. Computed with parallel $group aggregations (Cosmos has no $facet) and cached per-project for a short TTL.","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/RunFacetsResponse"}}}}}}},"/api/v1/requests/{id}":{"get":{"tags":["Requests"],"summary":"Get request","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/RequestResponse"}}}},"404":{"description":"Not found"}}},"delete":{"tags":["Requests"],"summary":"Soft-delete request","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}},"404":{"description":"Not found"}}}},"/api/v1/analysis":{"get":{"tags":["Requests"],"summary":"Compute pass@k / success@T metrics","description":"Aggregates pass@k / success@T metrics over a single project's done runs. Requires ?projectId=.","parameters":[{"schema":{"type":"string"},"required":false,"name":"worker","in":"query"},{"schema":{"type":"string"},"required":false,"name":"taskPromptId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"criteria","in":"query"},{"schema":{"type":"string"},"required":false,"name":"features","in":"query"},{"schema":{"type":"string"},"required":false,"name":"submissionId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"k","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{},"additionalProperties":{},"description":"Analysis metrics"}}}}}}},"/api/v1/requests/bulk-resubmit":{"post":{"tags":["Requests"],"summary":"Bulk resubmit requests","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/BulkResubmitInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/RequestResponse"}}}}}}}},"/api/v1/requests/bulk":{"delete":{"tags":["Requests"],"summary":"Bulk soft-delete requests","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"}}},"required":["ids"]}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"deleted":{"type":"number"}},"required":["deleted"]}}}}}}},"/api/v1/requests/archive":{"post":{"tags":["Requests"],"summary":"Download batch archive of multiple runs","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1,"maxItems":100}},"required":["ids"]}}}},"responses":{"201":{"description":"Gzipped batch archive containing individual run archives"},"400":{"description":"Invalid input"},"404":{"description":"One or more runs not found"}}}},"/api/v1/runs/upload":{"post":{"tags":["Requests"],"summary":"Import run archive","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"201":{"description":"Success"}}}},"/api/v1/runs/upload-batch":{"post":{"tags":["Requests"],"summary":"Import batch run archive","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"207":{"description":"Multi-status: 201 if all runs imported, 400 if none imported, 207 if partial"},"400":{"description":"No archive uploaded, empty archive, or all runs failed"}}}},"/api/v1/requests/{id}/reports":{"get":{"tags":["Reports"],"summary":"Get reports for request","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ReportResponse"}}}}},"404":{"description":"Run not found"}}}},"/api/v1/requests/{id}/runs":{"get":{"tags":["Requests"],"summary":"List attempts for a request","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/RunState"}}}}},"404":{"description":"Request not found"}}}},"/api/v1/requests/{id}/runs/{runId}":{"get":{"tags":["Requests"],"summary":"Get a single attempt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/RunState"}}}},"404":{"description":"Request or run not found"}}}},"/api/v1/requests/bulk-retry":{"post":{"tags":["Requests"],"summary":"Bulk retry requests (start new attempts)","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1},"force":{"type":"boolean"}},"required":["ids"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"retried":{"type":"integer"},"skipped":{"type":"integer"},"results":{"type":"array","items":{"type":"object","properties":{"requestId":{"type":"string"},"runId":{"type":"string"},"attemptNumber":{"type":"integer"},"error":{"type":"string"}},"required":["requestId"]}}},"required":["retried","skipped","results"]}}}}}}},"/api/v1/requests/{id}/retry":{"post":{"tags":["Requests"],"summary":"Retry a request (start a new attempt)","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"force":{"type":"boolean"}}}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"requestId":{"type":"string"},"runId":{"type":"string"},"attemptNumber":{"type":"integer"}},"required":["requestId","runId","attemptNumber"]}}}},"404":{"description":"Request not found"},"409":{"description":"Conflict — request is not in a retryable state, or a concurrent retry won the race"},"422":{"description":"Cannot retry — current run not yet terminal"}}}},"/api/v1/requests/{id}/priority":{"post":{"tags":["Requests"],"summary":"Set priority on a single request","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"priority":{"type":"integer"}},"required":["priority"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"priority":{"type":"number"}},"required":["id","priority"]}}}}}}},"/api/v1/requests/bulk-priority":{"post":{"tags":["Requests"],"summary":"Set priority on multiple requests","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1,"maxItems":100},"priority":{"type":"integer"}},"required":["ids","priority"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"updated":{"type":"number"},"skipped":{"type":"number"}},"required":["updated","skipped"]}}}}}}},"/api/v1/requests/{id}/pause":{"post":{"tags":["Requests"],"summary":"Pause a single request","responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"status":{"type":"string"}},"required":["id","status"]}}}}}}},"/api/v1/requests/{id}/resume":{"post":{"tags":["Requests"],"summary":"Resume a paused request","responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"status":{"type":"string"}},"required":["id","status"]}}}}}}},"/api/v1/requests/bulk-pause":{"post":{"tags":["Requests"],"summary":"Pause multiple requests","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1,"maxItems":100}},"required":["ids"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"updated":{"type":"number"},"skipped":{"type":"number"}},"required":["updated","skipped"]}}}}}}},"/api/v1/requests/bulk-resume":{"post":{"tags":["Requests"],"summary":"Resume multiple paused requests","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1,"maxItems":100}},"required":["ids"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"updated":{"type":"number"},"skipped":{"type":"number"}},"required":["updated","skipped"]}}}}}}},"/api/v1/requests/{id}/cancel":{"post":{"tags":["Requests"],"summary":"Cancel a request (marks as done/failed and signals worker to exit)","responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"previousStatus":{"type":"string"},"status":{"type":"string"},"outcome":{"type":"string"}},"required":["id","previousStatus","status","outcome"]}}}}}}},"/api/v1/requests/bulk-cancel":{"post":{"tags":["Requests"],"summary":"Cancel multiple requests","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1,"maxItems":100}},"required":["ids"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"cancelled":{"type":"number"},"skipped":{"type":"number"},"results":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string"},"cancelled":{"type":"boolean"},"previousStatus":{"type":"string"},"error":{"type":"string"}},"required":["id","cancelled"]}}},"required":["cancelled","skipped","results"]}}}}}}},"/api/v1/requests/{id}/logs":{"get":{"tags":["Requests"],"summary":"Stream request logs (SSE)","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Server-sent event stream of log entries"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/logs":{"get":{"tags":["Requests"],"summary":"Stream logs for a specific attempt (SSE)","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"}],"responses":{"200":{"description":"Server-sent event stream of log entries"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/har":{"get":{"tags":["Requests"],"summary":"Download HAR file","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"HAR-format JSON file"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/har":{"get":{"tags":["Requests"],"summary":"Download HAR file for a specific attempt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"}],"responses":{"200":{"description":"HAR-format JSON file"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/atif":{"get":{"tags":["Requests"],"summary":"Download ATIF trajectory file for a specific iteration","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","description":"The iteration number (1-based)"},"required":true,"description":"The iteration number (1-based)","name":"iteration","in":"query"}],"responses":{"200":{"description":"ATIF v1.7 trajectory JSON file"},"400":{"description":"Missing or invalid iteration"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/atif":{"get":{"tags":["Requests"],"summary":"Download ATIF trajectory file for a specific attempt and iteration","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"},{"schema":{"type":"string","description":"The iteration number (1-based)"},"required":true,"description":"The iteration number (1-based)","name":"iteration","in":"query"}],"responses":{"200":{"description":"ATIF v1.7 trajectory JSON file"},"400":{"description":"Missing or invalid iteration"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/video":{"get":{"tags":["Requests"],"summary":"Download session recording","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"WebM video recording (supports Range requests)"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/video":{"get":{"tags":["Requests"],"summary":"Download session recording for a specific attempt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"}],"responses":{"200":{"description":"WebM video recording (supports Range requests)"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/tool-calls":{"get":{"tags":["Requests"],"summary":"Download per-iteration tool-calls JSONL","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"JSONL stream — one ToolCall per line"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/tool-calls":{"get":{"tags":["Requests"],"summary":"Download per-iteration tool-calls JSONL for a specific attempt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"}],"responses":{"200":{"description":"JSONL stream — one ToolCall per line"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/snapshots/{iteration}":{"get":{"tags":["Requests"],"summary":"Download iteration snapshot","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"iteration","in":"path"}],"responses":{"200":{"description":"Gzipped snapshot archive"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/snapshots/{iteration}":{"get":{"tags":["Requests"],"summary":"Download iteration snapshot for a specific attempt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"},{"schema":{"type":"string"},"required":true,"name":"iteration","in":"path"}],"responses":{"200":{"description":"Gzipped snapshot archive"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/archive":{"get":{"tags":["Requests"],"summary":"Download full run archive","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Gzipped run archive"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/archive":{"get":{"tags":["Requests"],"summary":"Download full run archive for a specific attempt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"}],"responses":{"200":{"description":"Gzipped run archive"},"404":{"description":"Not found"}}}},"/api/v1/criteria/generate-prompt":{"post":{"tags":["Criteria"],"summary":"Generate criterion prompt from behavior","parameters":[{"schema":{"type":"string","minLength":1,"description":"Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.","example":"00000000-0000-0000-0000-000000000000"},"required":false,"description":"Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"behavior":{"type":"string"},"currentId":{"type":"string"},"gates":{"type":"array","items":{"type":"string"}}},"required":["behavior"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"prompt":{"type":"string"},"suggestedId":{"type":"string"},"suggestedParents":{"type":"array","items":{"type":"string"}},"suggestedChildren":{"type":"array","items":{"type":"string"}}},"required":["prompt","suggestedId","suggestedParents","suggestedChildren"]}}}},"400":{"description":"Empty behavior string"},"503":{"description":"LLM not configured"}}}},"/api/v1/criteria/seed":{"post":{"tags":["Criteria"],"summary":"Seed criteria in bulk","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"criteria":{"type":"array","items":{"$ref":"#/components/schemas/CreateCriteriaInput"}}},"required":["criteria"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"seeded":{"type":"number"},"errors":{"type":"array","items":{"type":"string"}}},"required":["seeded","errors"]}}}},"400":{"description":"Seed batch would introduce a dependency cycle"}}}},"/api/v1/criteria":{"get":{"tags":["Criteria"],"summary":"List criteria","parameters":[{"schema":{"type":"string"},"required":false,"name":"q","in":"query"},{"schema":{"type":"string","description":"Comma-separated criterion IDs to include"},"required":false,"description":"Comma-separated criterion IDs to include","name":"ids","in":"query"},{"schema":{"type":"string","enum":["true","false"],"description":"When true and ids is set, also include dependency ancestors"},"required":false,"description":"When true and ids is set, also include dependency ancestors","name":"ancestors","in":"query"},{"schema":{"type":"string","enum":["gate","observation"],"description":"Filter by criterion kind"},"required":false,"description":"Filter by criterion kind","name":"kind","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/CriteriaResponse"}}}}}}},"post":{"tags":["Criteria"],"summary":"Create criterion","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateCriteriaInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CriteriaResponse"}}}},"409":{"description":"Criterion already exists"}}}},"/api/v1/criteria/mdp":{"get":{"tags":["Criteria"],"summary":"Compute MDP transitions","parameters":[{"schema":{"type":"string"},"required":false,"name":"criteria","in":"query"},{"schema":{"type":"string"},"required":false,"name":"features","in":"query"},{"schema":{"type":"string"},"required":false,"name":"since","in":"query"},{"schema":{"type":"string"},"required":false,"name":"worker","in":"query"},{"schema":{"type":"string"},"required":false,"name":"taskPromptId","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{},"additionalProperties":{}}}}}}}},"/api/v1/criteria/graph":{"get":{"tags":["Criteria"],"summary":"Get criteria DAG","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CriteriaGraph"}}}}}}},"/api/v1/criteria/{id}":{"get":{"tags":["Criteria"],"summary":"Get criterion","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CriteriaResponse"}}}},"404":{"description":"Criterion not found"}}},"put":{"tags":["Criteria"],"summary":"Update criterion","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateCriteriaInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CriteriaResponse"}}}},"400":{"description":"Invalid dependency reference or self-reference"},"404":{"description":"Criterion not found"}}},"delete":{"tags":["Criteria"],"summary":"Soft-delete criterion","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"deleted":{"type":"boolean"}},"required":["id","deleted"]}}}},"404":{"description":"Criterion not found"},"409":{"description":"Criterion has dependents"}}}},"/api/v1/prompt-features/generate-prompt":{"post":{"tags":["Prompt Features"],"summary":"Generate prompt feature from behavior","parameters":[{"schema":{"type":"string","minLength":1,"description":"Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.","example":"00000000-0000-0000-0000-000000000000"},"required":false,"description":"Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"behavior":{"type":"string"},"currentId":{"type":"string"}},"required":["behavior"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"prompt":{"type":"string"}},"required":["prompt"]}}}},"400":{"description":"Empty behavior string"},"503":{"description":"LLM not configured"}}}},"/api/v1/prompt-features/seed":{"post":{"tags":["Prompt Features"],"summary":"Seed prompt features in bulk","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"features":{"type":"array","items":{"$ref":"#/components/schemas/CreatePromptFeatureInput"}}},"required":["features"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"seeded":{"type":"number"},"errors":{"type":"array","items":{"type":"string"}}},"required":["seeded","errors"]}}}}}}},"/api/v1/prompt-features/extract-from-text":{"post":{"tags":["Prompt Features"],"summary":"Extract features from text","parameters":[{"schema":{"type":"string","minLength":1,"description":"Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.","example":"00000000-0000-0000-0000-000000000000"},"required":false,"description":"Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"text":{"type":"string"},"model":{"type":"string"},"type":{"type":"string","enum":["select","agents.md"]}},"required":["text"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"features":{"type":"array","items":{"$ref":"#/components/schemas/PromptFeatureResult"}},"suggestedFeatures":{"type":"array","items":{"$ref":"#/components/schemas/SuggestedPromptFeature"}},"cached":{"type":"boolean"}},"required":["features","cached"]}}}},"400":{"description":"Empty text string"},"503":{"description":"LLM not configured"}}}},"/api/v1/prompt-features":{"get":{"tags":["Prompt Features"],"summary":"List features","parameters":[{"schema":{"type":"string"},"required":false,"name":"q","in":"query"},{"schema":{"type":"string","enum":["select","agents.md"]},"required":false,"name":"type","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/PromptFeatureResponse"}}}}}}},"post":{"tags":["Prompt Features"],"summary":"Create feature","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreatePromptFeatureInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/PromptFeatureResponse"}}}},"400":{"description":"Invalid input"},"409":{"description":"Feature already exists"}}}},"/api/v1/prompt-features/{id}":{"get":{"tags":["Prompt Features"],"summary":"Get feature","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/PromptFeatureResponse"}}}},"404":{"description":"Feature not found"}}},"put":{"tags":["Prompt Features"],"summary":"Update feature","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdatePromptFeatureInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/PromptFeatureResponse"}}}},"400":{"description":"Invalid input"},"404":{"description":"Feature not found"}}},"delete":{"tags":["Prompt Features"],"summary":"Soft-delete feature","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"deleted":{"type":"boolean"}},"required":["id","deleted"]}}}},"404":{"description":"Feature not found"}}}},"/api/v1/task-prompts/generate":{"post":{"tags":["Task Prompts"],"summary":"Generate task prompts","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"description":{"type":"string"},"existingPrompt":{"type":"string"}}}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"tasks":{"type":"array","items":{"type":"string"}}},"required":["tasks"]}}}},"503":{"description":"LLM not configured"}}}},"/api/v1/task-prompts":{"get":{"tags":["Task Prompts"],"summary":"List task prompts","parameters":[{"schema":{"type":["number","null"]},"required":false,"name":"limit","in":"query"},{"schema":{"type":["number","null"]},"required":false,"name":"offset","in":"query"},{"schema":{"type":"string"},"required":false,"name":"search","in":"query"},{"schema":{"$ref":"#/components/schemas/PromptType"},"required":false,"name":"type","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"items":{"type":"array","items":{"$ref":"#/components/schemas/TaskPromptResponse"}},"total":{"type":"number"},"limit":{"type":"number"},"offset":{"type":"number"}},"required":["items","total","limit","offset"]}}}}}},"post":{"tags":["Task Prompts"],"summary":"Create or find task prompt","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateTaskPromptInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/TaskPromptResponse"}}}},"400":{"description":"Empty text string"}}}},"/api/v1/task-prompts/{id}":{"get":{"tags":["Task Prompts"],"summary":"Get task prompt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/TaskPromptResponse"}}}},"404":{"description":"Task prompt not found"}}},"delete":{"tags":["Task Prompts"],"summary":"Soft-delete task prompt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"deleted":{"type":"boolean"}},"required":["deleted"]}}}},"404":{"description":"Task prompt not found"}}}},"/api/v1/task-prompts/{id}/content":{"get":{"tags":["Task Prompts"],"summary":"Get resolved task prompt content","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"text":{"type":"string"}},"required":["id","text"]}}}},"404":{"description":"Task prompt not found"}}}},"/api/v1/task-prompts/{id}/extract-features":{"post":{"tags":["Task Prompts"],"summary":"Extract features from task prompt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":false,"name":"force","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"model":{"type":"string"}}}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"taskPromptId":{"type":"string"},"features":{"type":"array","items":{"$ref":"#/components/schemas/PromptFeatureResult"}},"featuresExtractedAt":{"type":["string","null"],"format":"date-time"},"suggestedFeatures":{"type":"array","items":{"$ref":"#/components/schemas/SuggestedPromptFeature"}},"cached":{"type":"boolean"}},"required":["taskPromptId","features","cached"]}}}},"404":{"description":"Task prompt not found"},"503":{"description":"LLM not configured"}}}},"/api/v1/task-prompts/{id}/features/{featureId}":{"patch":{"tags":["Task Prompts"],"summary":"Toggle feature flag on task prompt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"featureId","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/PatchTaskPromptFeatureInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/TaskPromptResponse"}}}},"400":{"description":"Invalid detected value"},"404":{"description":"Task prompt or feature not found"}}}},"/api/v1/reports":{"post":{"tags":["Reports"],"summary":"Create report","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateReportInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ReportResponse"}}}},"400":{"description":"Invalid input"},"404":{"description":"Run or template not found"}}},"get":{"tags":["Reports"],"summary":"List reports","parameters":[{"schema":{"type":"string"},"required":false,"name":"requestId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ReportResponse"}}}}}}}},"/api/v1/reports/bulk-create":{"post":{"tags":["Reports"],"summary":"Bulk create reports","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/BulkCreateReportsInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ReportResponse"}}}}},"400":{"description":"Invalid input"}}}},"/api/v1/reports/bulk-status":{"post":{"tags":["Reports"],"summary":"Bulk get report statuses","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/BulkReportStatusInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ReportResponse"}}}}},"400":{"description":"Invalid input"}}}},"/api/v1/reports/bulk-summary":{"post":{"tags":["Reports"],"summary":"Bulk get report summary per run","description":"Returns aggregated report status counts per run. Only the latest report per template is counted (re-triggers are deduplicated).","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/BulkReportSummaryInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/BulkReportSummaryResponse"}}}},"400":{"description":"Invalid input"}}}},"/api/v1/reports/{id}":{"get":{"tags":["Reports"],"summary":"Get report","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ReportResponse"}}}},"404":{"description":"Report not found"}}}},"/api/v1/reports/{id}/logs":{"get":{"tags":["Reports"],"summary":"Stream report logs (SSE)","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Server-sent event stream of log entries"},"404":{"description":"Report not found"}}}},"/api/v1/reports/trigger":{"post":{"tags":["Reports"],"summary":"Trigger reports","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/TriggerReportsInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"triggered":{"type":"number"},"reports":{"type":"array","items":{"$ref":"#/components/schemas/ReportResponse"}}},"required":["triggered","reports"]}}}},"400":{"description":"Invalid input"},"404":{"description":"Run not found"}}}},"/api/v1/reports/bulk-trigger":{"post":{"tags":["Reports"],"summary":"Bulk trigger reports","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/BulkTriggerReportsInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"type":"object","properties":{},"additionalProperties":{}}}}}},"400":{"description":"Invalid input"}}}},"/api/v1/reports/{id}/insights":{"get":{"tags":["Reports"],"summary":"Get insights for report","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"allOf":[{"$ref":"#/components/schemas/InsightResponse"},{"type":"object","properties":{"referencedAt":{"type":["string","null"],"format":"date-time"},"isNew":{"type":"boolean"}}}]}}}}},"404":{"description":"Report not found"}}},"post":{"tags":["Reports"],"summary":"Link insight to report","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"insightId":{"type":"string"},"isNew":{"type":"boolean"}},"required":["insightId"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"insightId":{"type":"string"},"referencedAt":{"type":["string","null"],"format":"date-time"},"isNew":{"type":"boolean"}},"required":["insightId","referencedAt","isNew"]}}}},"404":{"description":"Report or insight not found"},"409":{"description":"Insight already referenced by this report"}}}},"/api/v1/profiles":{"post":{"tags":["Profiles"],"summary":"Create a new profile","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateProfileInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProfileWithVersionResponse"}}}}}},"get":{"tags":["Profiles"],"summary":"List profiles","parameters":[{"schema":{"type":"string"},"required":false,"name":"workerType","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ProfileWithVersionResponse"}}}}}}}},"/api/v1/profiles/{profileId}":{"get":{"tags":["Profiles"],"summary":"Get profile with latest version","parameters":[{"schema":{"type":"string"},"required":true,"name":"profileId","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProfileWithVersionResponse"}}}}}},"post":{"tags":["Profiles"],"summary":"Create new profile version","parameters":[{"schema":{"type":"string"},"required":true,"name":"profileId","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"workerType":{"type":"string"},"model":{"type":"string"},"reasoningEffort":{"type":"string"},"agentVersion":{"type":"string"},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceBindingSpec"}},"extensions":{"type":"array","items":{"type":"string"}}},"required":["workerType","model"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProfileVersionResponse"}}}}}},"put":{"tags":["Profiles"],"summary":"Update profile identity","parameters":[{"schema":{"type":"string"},"required":true,"name":"profileId","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateProfileIdentity"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProfileResponse"}}}}}},"delete":{"tags":["Profiles"],"summary":"Delete profile","parameters":[{"schema":{"type":"string"},"required":true,"name":"profileId","in":"path"}],"responses":{"204":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}}}}},"/api/v1/profiles/{profileId}/versions":{"get":{"tags":["Profiles"],"summary":"List profile versions","parameters":[{"schema":{"type":"string"},"required":true,"name":"profileId","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ProfileVersionResponse"}}}}}}}},"/api/v1/profiles/{profileId}/versions/{version}":{"get":{"tags":["Profiles"],"summary":"Get profile version","parameters":[{"schema":{"type":"string"},"required":true,"name":"profileId","in":"path"},{"schema":{"type":["number","null"]},"required":false,"name":"version","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProfileVersionResponse"}}}}}}},"/api/v1/report-templates/default-system-prompt":{"get":{"tags":["Report Templates"],"summary":"Get default system prompt","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"content":{"type":"string"}},"required":["content"]}}}}}}},"/api/v1/report-templates/available-models":{"get":{"tags":["Report Templates"],"summary":"List models available for report generation","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"type":"object","properties":{"modelId":{"type":"string"}},"required":["modelId"]}}}}}}}},"/api/v1/report-templates":{"get":{"tags":["Report Templates"],"summary":"List report templates","parameters":[{"schema":{"type":"string"},"required":false,"name":"q","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ReportTemplateResponse"}}}}}}},"post":{"tags":["Report Templates"],"summary":"Create report template","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateReportTemplateInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ReportTemplateResponse"}}}},"409":{"description":"Report template already exists"}}}},"/api/v1/report-templates/{id}":{"get":{"tags":["Report Templates"],"summary":"Get report template","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ReportTemplateResponse"}}}},"404":{"description":"Report template not found"}}},"put":{"tags":["Report Templates"],"summary":"Update report template","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateReportTemplateInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ReportTemplateResponse"}}}},"404":{"description":"Report template not found"}}},"delete":{"tags":["Report Templates"],"summary":"Delete report template","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"204":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}},"404":{"description":"Report template not found"}}}},"/api/v1/agents":{"get":{"tags":["Agents"],"summary":"List agents","parameters":[{"schema":{"type":"string"},"required":false,"name":"modelProvider","in":"query"},{"schema":{"type":"string","enum":["true","false"]},"required":false,"name":"includeDeleted","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/AgentResponse"}}}}}}},"post":{"tags":["Agents"],"summary":"Create or update agent (upsert)","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateAgentInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AgentResponse"}}}},"400":{"description":"Validation error"},"409":{"description":"Restored agent versions conflict with an active queue owner"}}}},"/api/v1/agents/{id}":{"get":{"tags":["Agents"],"summary":"Get agent","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","enum":["true","false"]},"required":false,"name":"includeDeleted","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AgentResponse"}}}},"404":{"description":"Agent not found"}}},"put":{"tags":["Agents"],"summary":"Update agent","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateAgentInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AgentResponse"}}}},"400":{"description":"Validation error"},"404":{"description":"Agent not found"}}},"delete":{"tags":["Agents"],"summary":"Delete agent","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"204":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}},"404":{"description":"Agent not found"}}}},"/api/v1/agents/{id}/versions":{"get":{"tags":["Agents"],"summary":"List agent versions","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":false,"name":"status","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/AgentVersion"}}}}},"404":{"description":"Agent not found"}}},"post":{"tags":["Agents"],"summary":"Register agent version (upsert)","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/RegisterAgentVersionInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AgentVersion"}}}},"400":{"description":"Validation error"},"404":{"description":"Agent not found"},"409":{"description":"Queue is assigned to another agent"},"503":{"description":"Concurrent registration did not stabilize"}}}},"/api/v1/agents/{id}/versions/{agentVersion}":{"patch":{"tags":["Agents"],"summary":"Patch agent version","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"agentVersion","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/PatchAgentVersionInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AgentVersion"}}}},"400":{"description":"Invalid status value"},"404":{"description":"Agent or version not found"},"409":{"description":"Queue is assigned to another agent"},"503":{"description":"Concurrent activation did not stabilize"}}}},"/api/v1/models":{"get":{"tags":["Models"],"summary":"List models","parameters":[{"schema":{"type":"string"},"required":false,"name":"agentId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"provider","in":"query"},{"schema":{"type":"string","enum":["active","disappeared"]},"required":false,"name":"status","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ModelResponse"}}}}}}}},"/api/v1/models/{id}":{"get":{"tags":["Models"],"summary":"Get model","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ModelResponse"}}}}}}},"/api/v1/models/sync":{"post":{"tags":["Models"],"summary":"Sync models from provider","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"agentId":{"type":"string"},"provider":{"type":"string"},"models":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string"},"providerAvailableFrom":{"type":"string"},"providerEndOfLife":{"type":"string"},"metadata":{"type":"object","additionalProperties":{}},"capabilities":{"type":"object","properties":{"reasoningEffort":{"type":"array","items":{"type":"string"}},"toolCalls":{"type":"boolean"},"vision":{"type":"boolean"},"streaming":{"type":"boolean"},"adaptiveThinking":{"type":"boolean"},"maxThinkingBudget":{"type":"number"}}}},"required":["id"]}},"scannedAt":{"type":"string"}},"required":["agentId","provider","models","scannedAt"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"added":{"type":"array","items":{"type":"string"}},"removed":{"type":"array","items":{"type":"string"}},"unchanged":{"type":"array","items":{"type":"string"}}},"required":["added","removed","unchanged"]}}}}}}},"/api/v1/mcp/servers":{"get":{"tags":["MCP Servers"],"summary":"List MCP servers","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/McpServerResponse"}}}}}}},"post":{"tags":["MCP Servers"],"summary":"Create MCP server","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"_id":{"type":"string","pattern":"^[a-z0-9]([a-z0-9-]*[a-z0-9])?$"},"name":{"type":"string"},"type":{"type":"string","enum":["sse","http","stdio"]},"url":{"type":"string"},"command":{"type":"string"},"args":{"type":"array","items":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"headers":{"type":"array","items":{"$ref":"#/components/schemas/McpServerHeader"}},"sessionMode":{"type":"string","enum":["stateful","stateless"]},"version":{"type":"string"},"description":{"type":"string"}},"required":["_id","name","type"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/McpServerResponse"}}}}}}},"/api/v1/mcp/servers/{id}":{"get":{"tags":["MCP Servers"],"summary":"Get MCP server","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/McpServerResponse"}}}}}},"put":{"tags":["MCP Servers"],"summary":"Update MCP server","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateMcpServerInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/McpServerResponse"}}}}}},"delete":{"tags":["MCP Servers"],"summary":"Delete MCP server","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"204":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}}}}},"/api/v1/skills":{"get":{"tags":["Skills"],"summary":"List all skills","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/SkillResponse"}}}}}}},"post":{"tags":["Skills"],"summary":"Create or import a skill","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateSkillInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SkillResponse"}}}}}}},"/api/v1/skills/search":{"get":{"tags":["Skills"],"summary":"Search skills (internal + external)","parameters":[{"schema":{"type":"string"},"required":true,"name":"q","in":"query"},{"schema":{"type":"string"},"required":false,"name":"limit","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/SkillSearchResult"}}}}},"400":{"description":"Missing query parameter"}}}},"/api/v1/skills/search/external":{"get":{"tags":["Skills"],"summary":"Search external skills registry","parameters":[{"schema":{"type":"string"},"required":true,"name":"q","in":"query"},{"schema":{"type":"string"},"required":false,"name":"limit","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Optional project scope. When provided, external results are de-duplicated only against skills already installed in that project."},"required":false,"description":"Optional project scope. When provided, external results are de-duplicated only against skills already installed in that project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/SkillSearchResult"}}}}},"400":{"description":"Missing query parameter"}}}},"/api/v1/skills/discover":{"get":{"tags":["Skills"],"summary":"Discover skills in a GitHub repository","parameters":[{"schema":{"type":"string"},"required":true,"name":"source","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/SkillDiscoveryResult"}}}}},"400":{"description":"Missing or malformed source parameter"},"404":{"description":"Repository not found"},"502":{"description":"GitHub API error"}}}},"/api/v1/skills/{id}(*)/revisions":{"get":{"tags":["Skills"],"summary":"List skill revisions","parameters":[{"schema":{"type":"string"},"required":false,"name":"limit","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/SkillRevisionResponse"}}}}},"404":{"description":"Skill not found"}}}},"/api/v1/skills/{id}(*)":{"get":{"tags":["Skills"],"summary":"Get skill by slug","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SkillResponse"}}}},"404":{"description":"Skill not found"}}},"delete":{"tags":["Skills"],"summary":"Soft-delete a skill","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"204":{"description":"Success"},"404":{"description":"Skill not found"}}}},"/api/v1/skill-revisions/by-ref/{ref}(*)/archive":{"get":{"tags":["Skill Revisions"],"summary":"Download skill revision archive","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Binary tar.gz archive"},"404":{"description":"Skill revision or archive not found"},"500":{"description":"Blob storage error"}}}},"/api/v1/skill-revisions/by-ref/{ref}(*)":{"get":{"tags":["Skill Revisions"],"summary":"Get skill revision by ref","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SkillRevisionResponse"}}}},"404":{"description":"Skill revision not found"}}}},"/api/v1/skill-revisions/{id}":{"get":{"tags":["Skill Revisions"],"summary":"Get skill revision by ID","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SkillRevisionResponse"}}}},"404":{"description":"Skill revision not found"}}}},"/api/v1/skills/{id}(*)/resolve":{"post":{"tags":["Skills"],"summary":"Trigger skill resolution","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SkillRevisionResponse"}}}},"404":{"description":"Skill not found"},"500":{"description":"Blob storage error"}}}},"/api/v1/codebases":{"get":{"tags":["Codebases"],"summary":"List all codebases","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/CodebaseResponse"}}}}}}},"post":{"tags":["Codebases"],"summary":"Create a codebase (archive codebases require the archive file)","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"201":{"description":"Success"},"400":{"description":"Invalid input or missing archive"}}}},"/api/v1/codebases/{id}":{"get":{"tags":["Codebases"],"summary":"Get a codebase","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CodebaseResponse"}}}},"404":{"description":"Codebase not found"}}},"patch":{"tags":["Codebases"],"summary":"Update a codebase","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateCodebaseInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CodebaseResponse"}}}},"404":{"description":"Codebase not found"}}},"delete":{"tags":["Codebases"],"summary":"Delete a codebase","responses":{"204":{"description":"Success"},"404":{"description":"Codebase not found"}}}},"/api/v1/codebases/{id}/revisions":{"get":{"tags":["Codebases"],"summary":"List codebase revisions","parameters":[{"schema":{"type":"string"},"required":false,"name":"limit","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/CodebaseRevisionResponse"}}}}},"404":{"description":"Codebase not found"}}},"post":{"tags":["Codebases"],"summary":"Resolve a new git codebase revision","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ResolveCodebaseRevisionInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CodebaseRevisionResponse"}}}},"400":{"description":"Not a git codebase"},"404":{"description":"Codebase not found"},"502":{"description":"GitHub resolution failed"}}}},"/api/v1/codebases/{id}/upload":{"post":{"tags":["Codebases"],"summary":"Upload a codebase archive as a new revision","responses":{"201":{"description":"Success"},"400":{"description":"Missing archive or not an archive codebase"},"404":{"description":"Codebase not found"}}}},"/api/v1/codebase-revisions/{id}/archive":{"get":{"tags":["Codebase Revisions"],"summary":"Download codebase revision archive","responses":{"200":{"description":"Binary tar.gz archive"},"404":{"description":"Revision or archive not found"},"500":{"description":"Blob storage error"}}}},"/api/v1/codebase-revisions/{id}":{"get":{"tags":["Codebase Revisions"],"summary":"Get codebase revision by id","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CodebaseRevisionResponse"}}}},"404":{"description":"Revision not found"}}}},"/api/v1/resources":{"get":{"tags":["Resources"],"summary":"List all resources","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ResourceResponse"}}}}}}},"post":{"tags":["Resources"],"summary":"Create a resource and its first revision","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateResourceInput"}}}},"responses":{"201":{"description":"Success"},"400":{"description":"Invalid input"},"409":{"description":"Resource slug or revision ref already exists in this project"}}}},"/api/v1/resources/{id}":{"get":{"tags":["Resources"],"summary":"Get a resource","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ResourceResponse"}}}},"404":{"description":"Resource not found"}}},"patch":{"tags":["Resources"],"summary":"Update a resource","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateResourceInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ResourceResponse"}}}},"404":{"description":"Resource not found"}}},"delete":{"tags":["Resources"],"summary":"Delete a resource","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"204":{"description":"Success"},"404":{"description":"Resource not found"}}}},"/api/v1/resources/{id}/revisions":{"get":{"tags":["Resource Revisions"],"summary":"List resource revisions","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"limit","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ResourceRevisionResponse"}}}}},"404":{"description":"Resource not found"}}},"post":{"tags":["Resource Revisions"],"summary":"Create a resource revision","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateResourceRevisionInput"}}}},"responses":{"201":{"description":"Success"},"404":{"description":"Resource not found"},"409":{"description":"Resource revision ref already exists in this project"}}}},"/api/v1/resources/{id}/revisions/latest":{"get":{"tags":["Resource Revisions"],"summary":"Get the latest resource revision","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ResourceRevisionResponse"}}}},"404":{"description":"Revision not found"}}}},"/api/v1/resources/{id}/revisions/{revisionNumber}":{"get":{"tags":["Resource Revisions"],"summary":"Get a resource revision by resource and revision number","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"revisionNumber","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ResourceRevisionResponse"}}}},"404":{"description":"Revision not found"}}}},"/api/v1/resources/revisions/{id}":{"get":{"tags":["Resource Revisions"],"summary":"Get resource revision by id","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ResourceRevisionResponse"}}}},"404":{"description":"Revision not found"}}}},"/api/v1/extensions":{"get":{"tags":["Extensions"],"summary":"List all extensions","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ExtensionResponse"}}}}}}},"post":{"tags":["Extensions"],"summary":"Create or import an extension","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateExtensionInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ExtensionResponse"}}}}}}},"/api/v1/extensions/search":{"get":{"tags":["Extensions"],"summary":"Search extensions (internal + marketplace)","parameters":[{"schema":{"type":"string"},"required":true,"name":"q","in":"query"},{"schema":{"type":"string"},"required":false,"name":"limit","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ExtensionSearchResult"}}}}},"400":{"description":"Missing query parameter"}}}},"/api/v1/extensions/{id}":{"get":{"tags":["Extensions"],"summary":"Get extension by ID","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ExtensionResponse"}}}}}},"put":{"tags":["Extensions"],"summary":"Update extension","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateExtensionInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ExtensionResponse"}}}}}},"delete":{"tags":["Extensions"],"summary":"Delete extension","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"204":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}}}}},"/api/v1/extensions/{id}/versions":{"get":{"tags":["Extensions"],"summary":"List available versions for an extension","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":false,"name":"preRelease","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ExtensionVersionInfo"}}}}}}}},"/api/v1/insights":{"get":{"tags":["Insights"],"summary":"List insights","parameters":[{"schema":{"type":"string"},"required":false,"name":"q","in":"query"},{"schema":{"type":"string"},"required":false,"name":"blocked","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/InsightResponse"}}}}}}},"post":{"tags":["Insights"],"summary":"Create insight","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateInsightInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}}}}},"/api/v1/insights/search":{"get":{"tags":["Insights"],"summary":"Search insights","parameters":[{"schema":{"type":"string"},"required":true,"name":"q","in":"query"},{"schema":{"type":"string"},"required":false,"name":"blocked","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/InsightResponse"}}}}}}}},"/api/v1/insights/{id}":{"get":{"tags":["Insights"],"summary":"Get insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}},"404":{"description":"Insight not found"}}},"put":{"tags":["Insights"],"summary":"Update insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateInsightInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}},"404":{"description":"Insight not found"}}},"delete":{"tags":["Insights"],"summary":"Delete insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"204":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}},"404":{"description":"Insight not found"}}}},"/api/v1/insights/{id}/upvote":{"post":{"tags":["Insights"],"summary":"Upvote insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}},"404":{"description":"Insight not found"}}}},"/api/v1/insights/{id}/downvote":{"post":{"tags":["Insights"],"summary":"Downvote insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}},"404":{"description":"Insight not found"}}}},"/api/v1/insights/{id}/block":{"post":{"tags":["Insights"],"summary":"Block insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}},"404":{"description":"Insight not found"}}}},"/api/v1/insights/{id}/unblock":{"post":{"tags":["Insights"],"summary":"Unblock insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}},"404":{"description":"Insight not found"}}}},"/api/v1/insights/{id}/reports":{"get":{"tags":["Insights"],"summary":"Get reports referencing insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ReportResponse"}}}}},"404":{"description":"Insight not found"}}}},"/api/v1/feature-flags":{"get":{"tags":["Feature Flags"],"summary":"List feature flags","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/FeatureFlagResponse"}}}}}}}},"/api/v1/feature-flags/{key}":{"put":{"tags":["Feature Flags"],"summary":"Update feature flag","parameters":[{"schema":{"type":"string"},"required":true,"name":"key","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateFeatureFlagInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/FeatureFlagResponse"}}}}}}}},"webhooks":{}} \ No newline at end of file +{"openapi":"3.1.0","info":{"title":"Scope API","version":"1.0.0","description":"REST API for the Scope platform — benchmarking AI coding agents"},"servers":[{"url":"/","description":"Current server"}],"components":{"securitySchemes":{"bearerAuth":{"type":"http","scheme":"bearer","bearerFormat":"JWT","description":"Use the unchanged IdP access token, without the Bearer prefix."}},"schemas":{"KeyResponse":{"type":"object","properties":{},"additionalProperties":{}},"KeyInput":{"type":"object","properties":{},"additionalProperties":{}},"ValidateKeyInput":{"type":"object","properties":{"token":{"type":"string"}},"required":["token"]},"AccountResponse":{"type":"object","properties":{},"additionalProperties":{}},"AccountInput":{"type":"object","properties":{},"additionalProperties":{}},"UserMeResponse":{"type":"object","properties":{"id":{"type":"string"},"role":{"type":"string"},"email":{"type":"string"},"displayName":{"type":"string"},"idp":{"type":"string"},"idpTenant":{"type":"string"}},"required":["id"]},"ProjectResponse":{"type":"object","properties":{"_id":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"creator":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"}},"required":["_id","name","createdAt"]},"CreateProjectInput":{"type":"object","properties":{"name":{"type":"string","minLength":1},"description":{"type":"string"},"creator":{"type":"string"}},"required":["name"]},"UpdateProjectInput":{"type":"object","properties":{"name":{"type":"string","minLength":1},"description":{"type":"string"}}},"SubmissionListResponse":{"type":"object","properties":{"submissions":{"type":"array","items":{"$ref":"#/components/schemas/SubmissionResponse"}}},"required":["submissions"]},"SubmissionResponse":{"type":"object","properties":{"submissionId":{"type":"string"},"projectId":{"type":"string"},"requestIds":{"type":"array","items":{"type":"string"}},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"submissionRunId":{"type":"string"},"attemptNumber":{"type":"integer"},"readiness":{"type":"string","enum":["waiting","ready","blocked","processing","done","failed"]},"requests":{"type":"array","items":{"$ref":"#/components/schemas/SubmissionRequestSummary"}},"dependencyProjection":{"type":"array","items":{"$ref":"#/components/schemas/SubmissionDependencyProjection"}},"blocking":{"type":"array","items":{"$ref":"#/components/schemas/SubmissionBlockingDetail"}},"handlerProgress":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/SubmissionHandlerStatus"}},"report":{"$ref":"#/components/schemas/SubmissionReportPointer"},"supersededAt":{"type":["string","null"],"format":"date-time"},"supersededBySubmissionRunId":{"type":"string"},"supersededReason":{"type":"string","enum":["request-vector-changed","manual-retry"]}},"required":["submissionId","projectId","requestIds","createdAt","submissionRunId","attemptNumber","readiness","requests","dependencyProjection","blocking"]},"SubmissionRequestSummary":{"type":"object","properties":{"requestId":{"type":"string"},"runId":{"type":"string"},"task":{"type":"string"},"status":{"type":"string"},"outcome":{"type":"string"},"workerType":{"type":"string"},"model":{"type":"string"},"profileId":{"type":"string"}},"required":["requestId","runId","task","status","workerType"]},"SubmissionDependencyProjection":{"type":"object","properties":{"requestId":{"type":"string"},"runId":{"type":"string"},"handlerId":{"type":"string"},"state":{"type":"string","enum":["waiting","ready","failed","blocked"]},"version":{"type":"number"},"updatedAt":{"type":["string","null"],"format":"date-time"},"error":{"type":"string"}},"required":["requestId","runId","handlerId","state"]},"SubmissionBlockingDetail":{"type":"object","properties":{"requestId":{"type":"string"},"runId":{"type":"string"},"handlerId":{"type":"string"},"reason":{"type":"string","enum":["request-missing","run-changed","dependency-not-ready","dependency-failed","dependency-blocked"]},"error":{"type":"string"}},"required":["requestId","runId","handlerId","reason"]},"SubmissionHandlerStatus":{"type":"object","properties":{"status":{"type":"string","enum":["queued","processing","done","failed"]},"version":{"type":"number"},"updatedAt":{"type":["string","null"],"format":"date-time"},"error":{"type":"string"}},"required":["status"]},"SubmissionReportPointer":{"type":"object","properties":{"reportId":{"type":"string"},"url":{"type":"string"},"updatedAt":{"type":["string","null"],"format":"date-time"}}},"SubmissionRunResponse":{"type":"object","properties":{"_id":{"type":"string"},"attemptNumber":{"type":"integer","minimum":1},"vectorFingerprint":{"type":"string"},"requests":{"type":"array","items":{"$ref":"#/components/schemas/SubmissionRequestRef"}},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"startedAt":{"type":["string","null"],"format":"date-time"},"finishedAt":{"type":["string","null"],"format":"date-time"},"dependencyProjection":{"type":"array","items":{"$ref":"#/components/schemas/SubmissionDependencyProjection"},"default":[]},"handlerStatus":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/SubmissionHandlerStatus"}},"blocking":{"type":"array","items":{"$ref":"#/components/schemas/SubmissionBlockingDetail"},"default":[]},"report":{"$ref":"#/components/schemas/SubmissionReportPointer"},"supersededAt":{"type":["string","null"],"format":"date-time"},"supersededBySubmissionRunId":{"type":"string"},"supersededReason":{"type":"string","enum":["request-vector-changed","manual-retry"]},"submissionId":{"type":"string"},"projectId":{"type":"string"}},"required":["_id","attemptNumber","vectorFingerprint","requests","createdAt"]},"SubmissionRequestRef":{"type":"object","properties":{"requestId":{"type":"string"},"runId":{"type":"string"}},"required":["requestId","runId"]},"SubmissionRetryResponse":{"type":"object","properties":{"submissionId":{"type":"string"},"submissionRunId":{"type":"string"},"attemptNumber":{"type":"integer"},"readiness":{"type":"string","enum":["waiting","ready","blocked","processing","done","failed"]},"blocking":{"type":"array","items":{"$ref":"#/components/schemas/SubmissionBlockingDetail"}}},"required":["submissionId","submissionRunId","attemptNumber","readiness","blocking"]},"RequestResponse":{"type":"object","properties":{"_id":{"type":"string"},"scenario":{"$ref":"#/components/schemas/Scenario"},"workerType":{"type":"string"},"model":{"type":"string"},"reasoningEffort":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"maxIterations":{"type":"number"},"personaInstructions":{"type":"string"},"persona":{"$ref":"#/components/schemas/Persona"},"deletedAt":{"type":["string","null"],"format":"date-time"},"taskPromptId":{"type":"string"},"agentsMdPromptId":{"type":"string"},"agentsMdParentIds":{"type":"array","items":{"type":"string"}},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"codebaseRevisionId":{"type":"string"},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceBinding"}},"extensions":{"type":"array","items":{"type":"string"}},"agentVersion":{"type":"string"},"profileId":{"type":"string"},"profileVersionId":{"type":"string"},"submissionId":{"type":"string"},"submissionExpectedRequestCount":{"type":"integer","minimum":1},"submissionRequestIndex":{"type":"integer","minimum":0},"submissionInitialRunId":{"type":"string"},"submissionFinalizedAt":{"type":["string","null"],"format":"date-time"},"priority":{"type":"integer","default":0},"gates":{"type":"array","items":{"$ref":"#/components/schemas/GateConfig"}},"observations":{"type":"array","items":{"type":"string"}},"gateSummaries":{"type":"array","items":{"$ref":"#/components/schemas/GateRunSummary"}},"run":{"$ref":"#/components/schemas/RunState"},"projectId":{"type":"string"}},"required":["_id","scenario","workerType","createdAt","projectId"]},"Scenario":{"type":"object","properties":{"version":{"type":"string","enum":["v1","v2"]},"task":{"type":"string"},"criteria":{"type":"array","items":{"type":"string"}}},"required":["task","criteria"]},"Persona":{"type":"object","properties":{"personality":{"type":"string","enum":["demanding","friendly"]},"experience":{"type":"string","enum":["junior","senior"]},"verbosity":{"type":"string","enum":["brief","moderate"]},"type":{"type":"string","enum":["traditional","ai_assisted","vibe"]}},"required":["personality","experience","verbosity","type"]},"ResourceBinding":{"type":"object","properties":{"ref":{"type":"string"},"revisionId":{"type":"string"},"params":{"type":"object","additionalProperties":{"type":"string"}}},"required":["ref","revisionId","params"]},"GateConfig":{"type":"object","properties":{"gate":{"type":"string","enum":["select","build","test","run","deploy"]},"promptId":{"type":"string"},"promptText":{"type":"string"},"criteria":{"type":"array","items":{"type":"string"}},"maxIterations":{"type":"integer","minimum":1,"maximum":50}},"required":["gate","criteria"]},"GateRunSummary":{"type":"object","properties":{"gate":{"type":"string","enum":["select","build","test","run","deploy"]},"status":{"type":"string","enum":["passed","failed","skipped"]},"iterations":{"type":"integer","minimum":0}},"required":["gate","status","iterations"]},"RunState":{"type":"object","properties":{"_id":{"type":"string"},"attemptNumber":{"type":"integer","minimum":1},"status":{"type":"string","enum":["pending","queued","processing","paused","done"]},"queuedQueueName":{"type":"string"},"outcome":{"type":"string","enum":["succeeded","failed","finished"]},"result":{"type":"string"},"error":{"type":"string"},"logsUrl":{"type":"string"},"updatedAt":{"type":["string","null"],"format":"date-time"},"startedAt":{"type":["string","null"],"format":"date-time"},"finishedAt":{"type":["string","null"],"format":"date-time"},"durationMs":{"type":"number"},"turns":{"type":"array","items":{"$ref":"#/components/schemas/ConversationTurn"}},"workerVersion":{"type":"string"},"os":{"type":"object","properties":{"platform":{"type":"string"},"release":{"type":"string"},"arch":{"type":"string"}},"required":["platform","release","arch"]},"lastHeartbeatAt":{"type":["string","null"],"format":"date-time"},"worker":{"type":"object","properties":{"instanceId":{"type":"string"},"podName":{"type":"string"}},"required":["instanceId"]},"harUrl":{"type":"string"},"videoUrls":{"type":"array","items":{"type":"string"}},"setupVideoUrls":{"type":"array","items":{"type":"string"}},"tokenUsage":{"$ref":"#/components/schemas/TokenUsage"},"aiCallCount":{"type":"number"},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceRunOutcome"}},"mcpRegistered":{"type":"boolean"},"rawChatUrl":{"type":"string"},"rawChatFormat":{"type":"string"},"pausedAt":{"type":["string","null"],"format":"date-time"},"resumedAt":{"type":["string","null"],"format":"date-time"},"observationResults":{"type":"array","items":{"$ref":"#/components/schemas/CriterionResult"}}},"required":["_id","attemptNumber","status"]},"ConversationTurn":{"type":"object","properties":{"iteration":{"type":"number"},"gate":{"type":"string","enum":["select","build","test","run","deploy"]},"codingAgentResponse":{"type":"string"},"judgeFeedback":{"type":"string"},"snapshotUrl":{"type":"string"},"passed":{"type":"boolean"},"timestamp":{"type":["string","null"],"format":"date-time"},"criteriaResults":{"type":"array","items":{"$ref":"#/components/schemas/CriterionResult"}},"observationResults":{"type":"array","items":{"$ref":"#/components/schemas/CriterionResult"}},"harUrl":{"type":"string"},"videoUrls":{"type":"array","items":{"type":"string"}},"tokenUsage":{"$ref":"#/components/schemas/TokenUsage"},"startedAt":{"type":["string","null"],"format":"date-time"},"durationMs":{"type":"number"},"toolCalls":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"arguments":{"type":"object","additionalProperties":{}},"response":{"type":"string"},"timestamp":{"type":"string"}},"required":["id","name","arguments"]}},"toolCallsUrl":{"type":"string"},"toolCallCount":{"type":"number"},"aiCallCount":{"type":"number"},"rawChatUrl":{"type":"string"},"rawChatFormat":{"type":"string"},"chatResultUrl":{"type":"string"},"chatResultFormat":{"type":"string"}},"required":["iteration","judgeFeedback","snapshotUrl","passed","timestamp"]},"CriterionResult":{"type":"object","properties":{"criterionId":{"type":"string"},"passed":{"type":"boolean"},"feedback":{"type":"string"},"evaluated":{"type":"boolean"}},"required":["criterionId","passed","feedback","evaluated"]},"TokenUsage":{"type":"object","properties":{"promptTokens":{"type":"number"},"completionTokens":{"type":"number"},"totalTokens":{"type":"number"}},"required":["promptTokens","completionTokens","totalTokens"]},"ResourceRunOutcome":{"type":"object","properties":{"ref":{"type":"string"},"slug":{"type":"string"},"revisionId":{"type":"string"},"setupSucceeded":{"type":"boolean"},"published":{"type":"array","items":{"type":"string"}},"params":{"type":"object","additionalProperties":{"type":"string"}},"setupDurationMs":{"type":"number"},"error":{"type":"string"},"teardownRan":{"type":"boolean"}},"required":["ref","slug","revisionId","setupSucceeded","published"]},"ResourceBindingSpec":{"anyOf":[{"type":"string"},{"type":"object","properties":{"ref":{"type":"string"},"params":{"type":"object","additionalProperties":{"type":"string"}}},"required":["ref"]}]},"CreateRequestInput":{"type":"object","properties":{"scenario":{"$ref":"#/components/schemas/Scenario"},"model":{"type":"string"},"reasoningEffort":{"type":"string"},"maxIterations":{"type":"integer","minimum":1,"maximum":50},"personaInstructions":{"type":"string"},"persona":{"$ref":"#/components/schemas/Persona"},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"codebaseRevisionId":{"type":"string"},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceBindingSpec"}},"extensions":{"type":"array","items":{"type":"string"}},"profileId":{"type":"string"},"profileVariations":{"type":"array","items":{"type":"string"}},"priority":{"type":"integer"},"agentsMd":{"type":"string"},"agentsMdParentIds":{"type":"array","items":{"type":"string"}},"gates":{"type":"array","items":{"$ref":"#/components/schemas/GateConfig"}},"observations":{"type":"array","items":{"type":"string"}}},"required":["scenario"]},"RunFacetsResponse":{"type":"object","properties":{"total":{"type":"number"},"facets":{"type":"object","properties":{"workerType":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"status":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"outcome":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"model":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"os":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"priority":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"agentVersion":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}},"profileId":{"type":"array","items":{"$ref":"#/components/schemas/RunFacetBucket"}}},"required":["workerType","status","outcome","model","os","priority","agentVersion","profileId"]}},"required":["total","facets"]},"RunFacetBucket":{"type":"object","properties":{"value":{"type":"string"},"count":{"type":"number"}},"required":["value","count"]},"PaginatedRunsResponse":{"type":"object","properties":{"data":{"type":"array","items":{"$ref":"#/components/schemas/RequestResponse"}},"limit":{"type":"number"},"estimatedTotal":{"type":"number"},"cursors":{"$ref":"#/components/schemas/Cursors"}},"required":["data","limit","estimatedTotal","cursors"]},"Cursors":{"type":"object","properties":{"next":{"type":["string","null"]},"prev":{"type":["string","null"]}},"required":["next","prev"]},"PaginatedRunGroupsResponse":{"type":"object","properties":{"data":{"type":"array","items":{"$ref":"#/components/schemas/RunGroup"}},"limit":{"type":"number"},"estimatedTotal":{"type":"number"},"cursors":{"$ref":"#/components/schemas/Cursors"}},"required":["data","limit","estimatedTotal","cursors"]},"RunGroup":{"type":"object","properties":{"key":{"type":"string"},"label":{"type":"string"},"runIds":{"type":"array","items":{"type":"string"}},"aggregates":{"$ref":"#/components/schemas/GroupAggregates"},"uniform":{"$ref":"#/components/schemas/GroupUniformValues"}},"required":["key","label","runIds","aggregates","uniform"]},"GroupAggregates":{"type":"object","properties":{"count":{"type":"number"},"turns":{"$ref":"#/components/schemas/AggregateStats"},"duration":{"$ref":"#/components/schemas/AggregateStats"},"promptTokens":{"$ref":"#/components/schemas/AggregateStats"},"completionTokens":{"$ref":"#/components/schemas/AggregateStats"},"statusCounts":{"type":"object","additionalProperties":{"type":"number"}},"outcomeCounts":{"type":"object","additionalProperties":{"type":"number"}}},"required":["count","turns","duration","promptTokens","completionTokens","statusCounts","outcomeCounts"]},"AggregateStats":{"type":["object","null"],"properties":{"min":{"type":"number"},"max":{"type":"number"},"mean":{"type":"number"},"stdDev":{"type":"number"}},"required":["min","max","mean","stdDev"]},"GroupUniformValues":{"type":"object","properties":{"workerType":{"type":"string"},"agentVersion":{"type":"string"},"model":{"type":"string"},"platform":{"type":"string"},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"codebaseRevisionId":{"type":"string"},"extensions":{"type":"array","items":{"type":"string"}},"status":{"type":"string","enum":["pending","queued","processing","paused","done"]},"submissionId":{"type":"string"},"task":{"type":"string"}}},"BulkResubmitInput":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1},"count":{"type":"integer","minimum":1,"maximum":10,"default":1},"overrides":{"type":"object","properties":{"profileId":{"type":["string","null"]},"workerType":{"type":"string"},"agentVersion":{"type":"string"},"model":{"type":["string","null"]},"reasoningEffort":{"type":["string","null"]},"maxIterations":{"type":["number","null"]},"mcpServers":{"type":["array","null"],"items":{"type":"string"}},"skillRevisions":{"type":["array","null"],"items":{"type":"string"}},"extensions":{"type":["array","null"],"items":{"type":"string"}}}}},"required":["ids"]},"ReportResponse":{"type":"object","properties":{"_id":{"type":"string"},"requestId":{"type":"string"},"templateId":{"type":"string"},"reporter":{"$ref":"#/components/schemas/Reporter"},"content":{"type":"string"},"status":{"type":"string","enum":["pending","generating","completed","failed"]},"error":{"type":"string"},"insightReferences":{"type":"array","items":{"$ref":"#/components/schemas/InsightReference"}},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","requestId","status","createdAt","projectId"]},"Reporter":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"gitHash":{"type":"string"},"model":{"type":"string"},"agentId":{"type":"string"},"agentVersion":{"type":"string"}},"required":["id","name","gitHash","model","agentId","agentVersion"]},"InsightReference":{"type":"object","properties":{"insightId":{"type":"string"},"referencedAt":{"type":["string","null"],"format":"date-time"},"isNew":{"type":"boolean"}},"required":["insightId","referencedAt","isNew"]},"CreateCriteriaInput":{"type":"object","properties":{"id":{"type":"string","pattern":"^[a-z][a-z0-9_]*$"},"prompt":{"type":"string"},"dependsOn":{"type":"array","items":{"type":"string"}},"gates":{"type":"array","items":{"type":"string","enum":["select","build","test","run","deploy"]}},"kind":{"type":"string","enum":["gate","observation"],"default":"gate"},"taxonomyElementId":{"type":"string","enum":["dimension:idiomatic-use","dimension:dependency-currency","dimension:configuration-correctness"]},"subject":{"type":"string","enum":["run","iteration"]}},"required":["id","prompt"]},"CriteriaResponse":{"type":"object","properties":{"id":{"type":"string"},"prompt":{"type":"string"},"dependsOn":{"type":"array","items":{"type":"string"}},"gates":{"type":"array","items":{"type":"string","enum":["select","build","test","run","deploy"]}},"kind":{"type":"string","enum":["gate","observation"]},"taxonomyElementId":{"type":"string","enum":["dimension:idiomatic-use","dimension:dependency-currency","dimension:configuration-correctness"]},"subject":{"type":"string","enum":["run","iteration"]},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["id","prompt","createdAt","projectId"]},"CriteriaGraph":{"type":"object","properties":{"nodes":{"type":"array","items":{"$ref":"#/components/schemas/CriteriaGraphNode"}},"edges":{"type":"array","items":{"$ref":"#/components/schemas/CriteriaGraphEdge"}}},"required":["nodes","edges"]},"CriteriaGraphNode":{"type":"object","properties":{"id":{"type":"string"},"prompt":{"type":"string"},"dependsOn":{"type":"array","items":{"type":"string"}},"gates":{"type":"array","items":{"type":"string","enum":["select","build","test","run","deploy"]}},"kind":{"type":"string","enum":["gate","observation"]},"taxonomyElementId":{"type":"string","enum":["dimension:idiomatic-use","dimension:dependency-currency","dimension:configuration-correctness"]},"subject":{"type":"string","enum":["run","iteration"]}},"required":["id","prompt"]},"CriteriaGraphEdge":{"type":"object","properties":{"from":{"type":"string"},"to":{"type":"string"}},"required":["from","to"]},"UpdateCriteriaInput":{"type":"object","properties":{"prompt":{"type":"string"},"dependsOn":{"type":"array","items":{"type":"string"}},"gates":{"type":"array","items":{"type":"string","enum":["select","build","test","run","deploy"]}},"kind":{"type":"string","enum":["gate","observation"]},"taxonomyElementId":{"type":"string","enum":["dimension:idiomatic-use","dimension:dependency-currency","dimension:configuration-correctness"]},"subject":{"type":"string","enum":["run","iteration"]}}},"CreatePromptFeatureInput":{"type":"object","properties":{"id":{"type":"string","pattern":"^[a-z][a-z0-9_]*$"},"prompt":{"type":"string"},"type":{"$ref":"#/components/schemas/PromptType"}},"required":["id","prompt"]},"PromptType":{"type":"string","enum":["select","build","test","run","deploy","agents.md"]},"PromptFeatureResult":{"type":"object","properties":{"featureId":{"type":"string"},"detected":{"type":"boolean"},"evaluated":{"type":"boolean"}},"required":["featureId","detected","evaluated"]},"SuggestedPromptFeature":{"type":"object","properties":{"suggestedId":{"type":"string"},"behavior":{"type":"string"},"prompt":{"type":"string"}},"required":["suggestedId","behavior","prompt"]},"PromptFeatureResponse":{"type":"object","properties":{"id":{"type":"string"},"prompt":{"type":"string"},"type":{"$ref":"#/components/schemas/PromptType"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["id","prompt","createdAt","projectId"]},"UpdatePromptFeatureInput":{"type":"object","properties":{"prompt":{"type":"string"}}},"TaskPromptResponse":{"type":"object","properties":{"_id":{"type":"string"},"keyId":{"type":"string"},"type":{"$ref":"#/components/schemas/PromptType"},"text":{"type":"string"},"contentBlobUrl":{"type":"string"},"features":{"type":"array","items":{"$ref":"#/components/schemas/PromptFeatureResult"}},"featuresExtractedAt":{"type":["string","null"],"format":"date-time"},"createdAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","keyId","createdAt","projectId"]},"CreateTaskPromptInput":{"type":"object","properties":{"text":{"type":"string"},"type":{"$ref":"#/components/schemas/PromptType"}},"required":["text"]},"PatchTaskPromptFeatureInput":{"type":"object","properties":{"detected":{"type":"boolean"}},"required":["detected"]},"CreateReportInput":{"type":"object","properties":{"requestId":{"type":"string"},"templateId":{"type":"string"}},"required":["requestId"]},"BulkCreateReportsInput":{"type":"object","properties":{"requestIds":{"type":"array","items":{"type":"string"}},"templateId":{"type":"string"}},"required":["requestIds"]},"BulkReportStatusInput":{"type":"object","properties":{"reportIds":{"type":"array","items":{"type":"string"}}},"required":["reportIds"]},"BulkReportSummaryResponse":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/ReportSummary"}},"ReportSummary":{"type":"object","properties":{"total":{"type":"number"},"pending":{"type":"number"},"generating":{"type":"number"},"completed":{"type":"number"},"failed":{"type":"number"}},"required":["total","pending","generating","completed","failed"]},"BulkReportSummaryInput":{"type":"object","properties":{"requestIds":{"type":"array","items":{"type":"string"}}},"required":["requestIds"]},"TriggerReportsInput":{"type":"object","properties":{"requestId":{"type":"string"}},"required":["requestId"]},"BulkTriggerReportsInput":{"type":"object","properties":{"requestIds":{"type":"array","items":{"type":"string"}}},"required":["requestIds"]},"InsightResponse":{"type":"object","properties":{"_id":{"type":"string"},"title":{"type":"string"},"description":{"type":"string"},"category":{"type":"string"},"tags":{"type":"array","items":{"type":"string"}},"upvotes":{"type":"number"},"downvotes":{"type":"number"},"blocked":{"type":"boolean"},"referenceCount":{"type":"number"},"createdBy":{"type":"string","enum":["agent","user"]},"sourceReportId":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","title","description","upvotes","downvotes","blocked","referenceCount","createdBy","createdAt","projectId"]},"ProfileWithVersionResponse":{"type":"object","properties":{"_id":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"latestVersion":{"type":"number"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"version":{"$ref":"#/components/schemas/ProfileVersionResponse"},"projectId":{"type":"string"}},"required":["_id","name","latestVersion","createdAt","version","projectId"]},"ProfileVersionResponse":{"type":"object","properties":{"_id":{"type":"string"},"profileId":{"type":"string"},"version":{"type":"number"},"workerType":{"type":"string"},"model":{"type":"string"},"reasoningEffort":{"type":"string"},"agentVersion":{"type":"string"},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceBindingSpec"}},"extensions":{"type":"array","items":{"type":"string"}},"createdAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","profileId","version","workerType","model","createdAt","projectId"]},"CreateProfileInput":{"type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":128},"description":{"type":"string","maxLength":512},"workerType":{"type":"string"},"model":{"type":"string"},"reasoningEffort":{"type":"string"},"agentVersion":{"type":"string"},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceBindingSpec"}},"extensions":{"type":"array","items":{"type":"string"}}},"required":["name","workerType","model"]},"ProfileResponse":{"type":"object","properties":{"_id":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"latestVersion":{"type":"number"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","name","latestVersion","createdAt","projectId"]},"UpdateProfileIdentity":{"type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":128},"description":{"type":"string","maxLength":512}}},"ReportTemplateResponse":{"type":"object","properties":{"_id":{"type":"string"},"id":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"userPrompt":{"type":"string"},"systemPrompt":{"$ref":"#/components/schemas/ReportTemplateSystemPrompt"},"trigger":{"$ref":"#/components/schemas/ReportTrigger"},"model":{"type":"string"},"timeoutMs":{"type":"integer","exclusiveMinimum":0},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","id","name","userPrompt","createdAt","projectId"]},"ReportTemplateSystemPrompt":{"type":"object","properties":{"mode":{"type":"string","enum":["append","override"]},"content":{"type":"string"}},"required":["mode","content"]},"ReportTrigger":{"oneOf":[{"type":"object","properties":{"type":{"type":"string","enum":["always"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["criteria"]},"criteriaIds":{"type":"array","items":{"type":"string"}},"match":{"type":"string","enum":["any","all"]}},"required":["type","criteriaIds"]},{"type":"object","properties":{"type":{"type":"string","enum":["taskPrompt"]},"taskPromptIds":{"type":"array","items":{"type":"string"}}},"required":["type","taskPromptIds"]},{"type":"object","properties":{"type":{"type":"string","enum":["promptFeature"]},"featureIds":{"type":"array","items":{"type":"string"}},"match":{"type":"string","enum":["any","all"]}},"required":["type","featureIds"]}]},"CreateReportTemplateInput":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"userPrompt":{"type":"string"},"systemPrompt":{"$ref":"#/components/schemas/ReportTemplateSystemPrompt"},"trigger":{"$ref":"#/components/schemas/ReportTrigger"},"model":{"type":"string"},"timeoutMs":{"type":"integer","exclusiveMinimum":0}},"required":["id","name","userPrompt"]},"UpdateReportTemplateInput":{"type":"object","properties":{"name":{"type":"string"},"description":{"type":"string"},"userPrompt":{"type":"string"},"systemPrompt":{"allOf":[{"$ref":"#/components/schemas/ReportTemplateSystemPrompt"},{"type":["object","null"]}]},"trigger":{"$ref":"#/components/schemas/ReportTrigger"},"model":{"type":["string","null"]},"timeoutMs":{"type":["integer","null"],"exclusiveMinimum":0}}},"AgentResponse":{"type":"object","properties":{"_id":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"modelProvider":{"type":"string"},"supportedModels":{"type":"array","items":{"type":"string"}},"defaultModel":{"type":"string"},"available":{"type":"boolean"},"capabilities":{"$ref":"#/components/schemas/AgentCapabilities"},"versions":{"type":"array","items":{"$ref":"#/components/schemas/AgentVersion"}},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"}},"required":["_id","name","supportedModels","createdAt"]},"AgentCapabilities":{"type":"object","properties":{"supportsReasoningEffort":{"type":"boolean"},"supportsMcpServers":{"type":"boolean"},"supportsSkills":{"type":"boolean"},"supportsExtensions":{"type":"boolean"},"supportsResources":{"type":"boolean"}}},"AgentVersion":{"type":"object","properties":{"agentVersion":{"type":"string","minLength":1},"workerVersion":{"type":"string","minLength":1},"components":{"type":"object","additionalProperties":{"type":"string"}},"gitCommit":{"type":"string","minLength":1},"buildTime":{"type":"string","minLength":1},"imageTag":{"type":"string","minLength":1},"queueName":{"type":"string"},"status":{"type":"string","enum":["active","retired"]},"createdAt":{"type":["string","null"],"format":"date-time"}},"required":["agentVersion","workerVersion","components","gitCommit","buildTime","imageTag","status","createdAt"]},"CreateAgentInput":{"type":"object","properties":{"_id":{"type":"string","minLength":1},"name":{"type":"string","minLength":1},"description":{"type":"string"},"modelProvider":{"type":"string"},"supportedModels":{"type":"array","items":{"type":"string"}},"defaultModel":{"type":"string"},"available":{"type":"boolean"},"capabilities":{"$ref":"#/components/schemas/AgentCapabilities"}},"required":["_id","name"]},"UpdateAgentInput":{"type":"object","properties":{"name":{"type":"string"},"description":{"type":"string"},"modelProvider":{"type":"string"},"supportedModels":{"type":"array","items":{"type":"string"}},"defaultModel":{"type":"string"},"available":{"type":"boolean"},"capabilities":{"$ref":"#/components/schemas/AgentCapabilities"}}},"RegisterAgentVersionInput":{"type":"object","properties":{"agentVersion":{"type":"string","minLength":1},"workerVersion":{"type":"string","minLength":1},"components":{"type":"object","additionalProperties":{"type":"string"}},"gitCommit":{"type":"string","minLength":1},"buildTime":{"type":"string","minLength":1},"imageTag":{"type":"string","minLength":1},"queueName":{"type":"string","minLength":1}},"required":["agentVersion","workerVersion","components","gitCommit","buildTime","imageTag","queueName"]},"PatchAgentVersionInput":{"type":"object","properties":{"status":{"type":"string","enum":["active","retired"]}},"required":["status"]},"ModelResponse":{"type":"object","properties":{"_id":{"type":"string"},"modelId":{"type":"string"},"provider":{"type":"string"},"agentId":{"type":"string"},"firstSeenAt":{"type":["string","null"],"format":"date-time"},"lastSeenAt":{"type":["string","null"],"format":"date-time"},"disappearedAt":{"type":["string","null"],"format":"date-time"},"providerAvailableFrom":{"type":["string","null"],"format":"date-time"},"providerEndOfLife":{"type":["string","null"],"format":"date-time"},"metadata":{"type":"object","additionalProperties":{}},"capabilities":{"$ref":"#/components/schemas/ModelCapabilities"}},"required":["_id","modelId","provider","agentId","firstSeenAt","lastSeenAt"]},"ModelCapabilities":{"type":"object","properties":{"reasoningEffort":{"type":"array","items":{"type":"string"}},"toolCalls":{"type":"boolean"},"vision":{"type":"boolean"},"streaming":{"type":"boolean"},"adaptiveThinking":{"type":"boolean"},"maxThinkingBudget":{"type":"number"}}},"McpServerResponse":{"type":"object","properties":{"_id":{"type":"string"},"slug":{"type":"string"},"name":{"type":"string"},"type":{"type":"string","enum":["sse","http","stdio"]},"url":{"type":"string"},"command":{"type":"string"},"args":{"type":"array","items":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"headers":{"type":"array","items":{"$ref":"#/components/schemas/McpServerHeader"}},"sessionMode":{"type":"string","enum":["stateful","stateless"]},"version":{"type":"string"},"description":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","slug","name","type","createdAt","projectId"]},"McpServerHeader":{"type":"object","properties":{"name":{"type":"string"},"value":{"type":"string"}},"required":["name","value"]},"UpdateMcpServerInput":{"type":"object","properties":{"name":{"type":"string"},"type":{"type":"string","enum":["sse","http","stdio"]},"url":{"type":"string"},"command":{"type":"string"},"args":{"type":"array","items":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"headers":{"type":"array","items":{"$ref":"#/components/schemas/McpServerHeader"}},"sessionMode":{"type":"string","enum":["stateful","stateless"]},"version":{"type":"string"},"description":{"type":"string"}}},"SkillResponse":{"type":"object","properties":{"_id":{"type":"string"},"slug":{"type":"string"},"source":{"type":"string"},"skillName":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"origin":{"type":"string","enum":["skills-sh","manual"]},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","slug","source","skillName","name","origin","createdAt","projectId"]},"SkillSearchResult":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"source":{"type":"string"},"description":{"type":"string"},"internal":{"type":"boolean"},"installs":{"type":"number"}},"required":["id","name","source","internal"]},"SkillDiscoveryResult":{"type":"object","properties":{"skillName":{"type":"string"},"skillPath":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"existsInLibrary":{"type":"boolean"},"currentRevisionCommitSha":{"type":"string"},"latestUpstreamCommitSha":{"type":"string"},"updateAvailable":{"type":"boolean"},"lastImportedAt":{"type":"string"}},"required":["skillName","skillPath"]},"SkillRevisionResponse":{"type":"object","properties":{"_id":{"type":"string"},"ref":{"type":"string"},"source":{"type":"string"},"skillName":{"type":"string"},"skillPath":{"type":"string"},"commitHash":{"type":"string"},"commitTimestamp":{"type":["string","null"],"format":"date-time"},"name":{"type":"string"},"description":{"type":"string"},"license":{"type":"string"},"compatibility":{"type":"string"},"allowedTools":{"type":"string"},"metadata":{"type":"object","additionalProperties":{"type":"string"}},"content":{"type":"string"},"archiveUrl":{"type":"string"},"validationWarnings":{"type":"array","items":{"type":"string"}},"resolvedAt":{"type":["string","null"],"format":"date-time"},"createdAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","ref","source","skillName","skillPath","commitHash","commitTimestamp","name","description","content","archiveUrl","resolvedAt","createdAt","projectId"]},"CreateSkillInput":{"type":"object","properties":{"source":{"type":"string"},"skillName":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"origin":{"type":"string","enum":["skills-sh","manual"]}},"required":["source","skillName","name","origin"]},"CodebaseResponse":{"type":"object","properties":{"_id":{"type":"string"},"slug":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"sourceType":{"type":"string","enum":["git","archive"]},"source":{"type":"string"},"defaultBranch":{"type":"string"},"revisionCounter":{"type":"number"},"latestRevisionId":{"type":"string"},"creator":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","slug","name","sourceType","revisionCounter","createdAt","projectId"]},"UpdateCodebaseInput":{"type":"object","properties":{"name":{"type":"string","minLength":1},"description":{"type":"string"},"defaultBranch":{"type":"string"}}},"CodebaseRevisionResponse":{"type":"object","properties":{"_id":{"type":"string"},"codebaseId":{"type":"string"},"slug":{"type":"string"},"revisionNumber":{"type":"number"},"ref":{"type":"string"},"sourceType":{"type":"string","enum":["git","archive"]},"source":{"type":"string"},"requestedRef":{"type":"string"},"resolvedCommitSha":{"type":"string"},"commitTimestamp":{"type":["string","null"],"format":"date-time"},"originalFilename":{"type":"string"},"contentSha256":{"type":"string"},"archiveUrl":{"type":"string"},"sizeBytes":{"type":"number"},"fileCount":{"type":"number"},"creator":{"type":"string"},"resolvedAt":{"type":["string","null"],"format":"date-time"},"createdAt":{"type":["string","null"],"format":"date-time"},"deduplicated":{"type":"boolean"},"projectId":{"type":"string"}},"required":["_id","codebaseId","slug","revisionNumber","ref","sourceType","archiveUrl","resolvedAt","createdAt","projectId"]},"ResolveCodebaseRevisionInput":{"type":"object","properties":{"requestedRef":{"type":"string"},"creator":{"type":"string"}}},"ResourceResponse":{"type":"object","properties":{"_id":{"type":"string"},"slug":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"revisionCounter":{"type":"number"},"latestRevisionId":{"type":"string"},"latestRevisionNumber":{"type":"number"},"creator":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","slug","name","revisionCounter","createdAt","projectId"]},"CreateResourceInput":{"type":"object","properties":{"name":{"type":"string","minLength":1},"slug":{"type":"string","minLength":1},"description":{"type":"string"},"setup":{"$ref":"#/components/schemas/ResourceScript"},"teardown":{"$ref":"#/components/schemas/ResourceScript"},"exports":{"type":"array","items":{"type":"string","pattern":"^[A-Za-z_][A-Za-z0-9_]*$"}},"parameters":{"type":"array","items":{"$ref":"#/components/schemas/ResourceParameter"}},"creator":{"type":"string"}},"required":["name","setup"]},"ResourceScript":{"type":"object","properties":{"sh":{"type":"string","minLength":1}}},"ResourceParameter":{"type":"object","properties":{"name":{"type":"string","pattern":"^[A-Za-z_][A-Za-z0-9_]*$"},"description":{"type":"string"},"required":{"type":"boolean","default":false},"default":{"type":"string"},"example":{"type":"string"}},"required":["name"]},"UpdateResourceInput":{"type":"object","properties":{"name":{"type":"string","minLength":1},"description":{"type":"string"}}},"ResourceRevisionResponse":{"type":"object","properties":{"_id":{"type":"string"},"resourceId":{"type":"string"},"slug":{"type":"string"},"revisionNumber":{"type":"number"},"ref":{"type":"string"},"setup":{"$ref":"#/components/schemas/ResourceScript"},"teardown":{"$ref":"#/components/schemas/ResourceScript"},"exports":{"type":"array","items":{"type":"string"}},"parameters":{"type":"array","items":{"$ref":"#/components/schemas/ResourceParameter"}},"contentSha256":{"type":"string"},"creator":{"type":"string"},"createdAt":{"type":["string","null"],"format":"date-time"},"deduplicated":{"type":"boolean"},"projectId":{"type":"string"}},"required":["_id","resourceId","slug","revisionNumber","ref","setup","exports","contentSha256","createdAt","projectId"]},"CreateResourceRevisionInput":{"type":"object","properties":{"setup":{"$ref":"#/components/schemas/ResourceScript"},"teardown":{"$ref":"#/components/schemas/ResourceScript"},"exports":{"type":"array","items":{"type":"string","pattern":"^[A-Za-z_][A-Za-z0-9_]*$"}},"parameters":{"type":"array","items":{"$ref":"#/components/schemas/ResourceParameter"}},"creator":{"type":"string"}},"required":["setup"]},"ExtensionResponse":{"type":"object","properties":{"_id":{"type":"string"},"slug":{"type":"string"},"publisher":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"origin":{"type":"string","enum":["marketplace","manual"]},"createdAt":{"type":["string","null"],"format":"date-time"},"updatedAt":{"type":["string","null"],"format":"date-time"},"deletedAt":{"type":["string","null"],"format":"date-time"},"projectId":{"type":"string"}},"required":["_id","slug","publisher","name","origin","createdAt","projectId"]},"ExtensionSearchResult":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"publisher":{"type":"string"},"description":{"type":"string"},"internal":{"type":"boolean"},"version":{"type":"string"}},"required":["id","name","publisher","internal"]},"CreateExtensionInput":{"type":"object","properties":{"_id":{"type":"string","pattern":"^[a-zA-Z0-9-]+\\.[a-zA-Z0-9-]+$"},"publisher":{"type":"string"},"name":{"type":"string"},"description":{"type":"string"},"origin":{"type":"string","enum":["marketplace","manual"]}},"required":["_id","publisher","name","origin"]},"UpdateExtensionInput":{"type":"object","properties":{"name":{"type":"string"},"description":{"type":"string"}}},"ExtensionVersionInfo":{"type":"object","properties":{"version":{"type":"string"},"preRelease":{"type":"boolean"},"lastUpdated":{"type":"string"}},"required":["version","preRelease","lastUpdated"]},"CreateInsightInput":{"type":"object","properties":{"title":{"type":"string"},"description":{"type":"string"},"category":{"type":"string"},"tags":{"type":"array","items":{"type":"string"}},"createdBy":{"type":"string","enum":["agent","user"]},"sourceReportId":{"type":"string"}},"required":["title","description"]},"UpdateInsightInput":{"type":"object","properties":{"title":{"type":"string"},"description":{"type":"string"},"category":{"type":"string"},"tags":{"type":"array","items":{"type":"string"}}}},"FeatureFlagResponse":{"type":"object","properties":{"key":{"type":"string"},"label":{"type":"string"},"enabled":{"type":"boolean"},"updatedAt":{"type":["string","null"],"format":"date-time"}},"required":["key","label","enabled","updatedAt"]},"UpdateFeatureFlagInput":{"type":"object","properties":{"enabled":{"type":"boolean"}},"required":["enabled"]}},"parameters":{}},"paths":{"/api/v1/keys/preview":{"post":{"tags":["Keys"],"summary":"Preview key","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyInput"}}}},"responses":{"200":{"description":"Key preview","content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyResponse"}}}}}}},"/api/v1/keys":{"post":{"tags":["Keys"],"summary":"Create key","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyInput"}}}},"responses":{"201":{"description":"Key created","content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyResponse"}}}}}},"get":{"tags":["Keys"],"summary":"List keys","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/KeyResponse"}}}}}}}},"/api/v1/keys/{id}":{"get":{"tags":["Keys"],"summary":"Get key","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyResponse"}}}}}},"put":{"tags":["Keys"],"summary":"Update key","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyInput"}}}},"responses":{"200":{"description":"Key updated","content":{"application/json":{"schema":{"$ref":"#/components/schemas/KeyResponse"}}}}}},"delete":{"tags":["Keys"],"summary":"Delete key","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Key deleted","content":{"application/json":{"schema":{"type":"object","properties":{"success":{"type":"boolean"}},"required":["success"]}}}}}}},"/api/v1/keys/{id}/validate":{"post":{"tags":["Keys"],"summary":"Validate key","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ValidateKeyInput"}}}},"responses":{"200":{"description":"Validation result","content":{"application/json":{"schema":{"type":"object","properties":{},"additionalProperties":{},"description":"Key validation result"}}}}}}},"/api/v1/accounts":{"post":{"tags":["Accounts"],"summary":"Create account","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/AccountInput"}}}},"responses":{"201":{"description":"Account created","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AccountResponse"}}}}}},"get":{"tags":["Accounts"],"summary":"List accounts","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/AccountResponse"}}}}}}}},"/api/v1/accounts/{id}":{"get":{"tags":["Accounts"],"summary":"Get account","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AccountResponse"}}}}}},"put":{"tags":["Accounts"],"summary":"Update account","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/AccountInput"}}}},"responses":{"200":{"description":"Account updated","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AccountResponse"}}}}}},"delete":{"tags":["Accounts"],"summary":"Delete account","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Account deleted","content":{"application/json":{"schema":{"type":"object","properties":{"success":{"type":"boolean"}},"required":["success"]}}}}}}},"/api/v1/users/me":{"get":{"tags":["Users"],"summary":"Get the authenticated user's identity","description":"Read the existing authenticated Scope identity without enrollment or profile writes. Responses must not be HTTP-cached.","security":[{"bearerAuth":[]}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/UserMeResponse"}}}},"401":{"description":"Not authenticated"},"403":{"description":"User is not enrolled or is disabled"},"503":{"description":"Authentication service unavailable"}}},"post":{"tags":["Users"],"summary":"Enroll the authenticated user","description":"JIT-enroll the authenticated user, refresh their profile and lastLoginAt, apply bootstrap-admin rules, and warm the access cache. Use this only after an explicit IdP login; do not prefetch, poll, or automatically retry transient failures.","security":[{"bearerAuth":[]}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/UserMeResponse"}}}},"401":{"description":"Not authenticated"},"403":{"description":"User is disabled"},"503":{"description":"Authentication service unavailable"}}}},"/health":{"get":{"tags":["Health"],"summary":"Liveness probe","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"status":{"type":"string"},"version":{"type":"string"}},"required":["status","version"]}}}}}}},"/ready":{"get":{"tags":["Health"],"summary":"Readiness probe","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"status":{"type":"string"},"migrations":{}},"required":["status"]}}}},"503":{"description":"Service is not ready"}}}},"/about":{"get":{"tags":["System"],"summary":"API metadata","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"name":{"type":"string"},"version":{"type":"string"},"buildTime":{"type":"string"},"environment":{"type":"string"},"description":{"type":"string"},"workers":{"type":"array","items":{"type":"string"}}},"required":["name","version","buildTime","environment","description","workers"]}}}}}}},"/api/v1/version":{"get":{"tags":["System"],"summary":"Version info","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"commit":{"type":"string"},"buildTime":{"type":"string"},"environment":{"type":"string"},"strictAgentCapabilities":{"type":"boolean"}},"required":["commit","buildTime","environment","strictAgentCapabilities"]}}}}}}},"/api/v1/projects":{"get":{"tags":["Projects"],"summary":"List all projects","parameters":[{"schema":{"type":"string","enum":["true","false"],"description":"When true, include soft-deleted projects in the result"},"required":false,"description":"When true, include soft-deleted projects in the result","name":"includeDeleted","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ProjectResponse"}}}}}}},"post":{"tags":["Projects"],"summary":"Create a project","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateProjectInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProjectResponse"}}}},"400":{"description":"Invalid input"}}}},"/api/v1/projects/{id}":{"get":{"tags":["Projects"],"summary":"Get a project","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProjectResponse"}}}},"404":{"description":"Project not found"}}},"patch":{"tags":["Projects"],"summary":"Update a project","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateProjectInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProjectResponse"}}}},"404":{"description":"Project not found"}}},"delete":{"tags":["Projects"],"summary":"Soft-delete a project","responses":{"204":{"description":"Success"},"404":{"description":"Project not found"}}}},"/api/v1/projects/{id}/restore":{"post":{"tags":["Projects"],"summary":"Restore a soft-deleted project","responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProjectResponse"}}}},"404":{"description":"Project not found"}}}},"/api/v1/submissions":{"get":{"tags":["Submissions"],"summary":"List submissions","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"},{"schema":{"type":"integer","minimum":1,"maximum":100,"default":50},"required":false,"name":"limit","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SubmissionListResponse"}}}}}}},"/api/v1/submissions/{id}":{"get":{"tags":["Submissions"],"summary":"Get a submission","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SubmissionResponse"}}}},"404":{"description":"Submission not found"}}}},"/api/v1/submissions/{id}/runs":{"get":{"tags":["Submissions"],"summary":"List current and historical submission attempts","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"runs":{"type":"array","items":{"$ref":"#/components/schemas/SubmissionRunResponse"}}},"required":["runs"]}}}},"404":{"description":"Submission not found"}}}},"/api/v1/submissions/{id}/runs/{submissionRunId}":{"get":{"tags":["Submissions"],"summary":"Get one submission attempt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"submissionRunId","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SubmissionRunResponse"}}}},"404":{"description":"Submission attempt not found"}}}},"/api/v1/submissions/{id}/retry":{"post":{"tags":["Submissions"],"summary":"Retry current submission processing","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"submissionRunId":{"type":"string"}},"required":["submissionRunId"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SubmissionRetryResponse"}}}},"404":{"description":"Submission not found"},"409":{"description":"Submission attempt is stale or dependencies are blocked"}}}},"/api/v1/requests":{"post":{"tags":["Requests"],"summary":"Submit request(s)","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"allOf":[{"$ref":"#/components/schemas/CreateRequestInput"},{"type":"object","properties":{"count":{"type":"number","minimum":1,"maximum":10,"default":1},"skills":{"type":"array","items":{"type":"string"}},"agentVersion":{"type":"string"},"codebase":{"type":"string"}}}]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"anyOf":[{"$ref":"#/components/schemas/RequestResponse"},{"type":"array","items":{"$ref":"#/components/schemas/RequestResponse"}}]}}}}}},"get":{"tags":["Requests"],"summary":"List requests","parameters":[{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"worker","in":"query"},{"schema":{"type":"string"},"required":false,"name":"taskPromptId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"criteria","in":"query"},{"schema":{"type":"string"},"required":false,"name":"submissionId","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"profileId","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"status","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"outcome","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"model","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"os","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"priority","in":"query"},{"schema":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"required":false,"name":"agentVersion","in":"query"},{"schema":{"type":"string"},"required":false,"name":"search","in":"query"},{"schema":{"type":"string","enum":["task","submissionId","profile"]},"required":false,"name":"groupBy","in":"query"},{"schema":{"type":"integer","minimum":1,"maximum":100},"required":false,"name":"limit","in":"query"},{"schema":{"type":"string"},"required":false,"name":"after","in":"query"},{"schema":{"type":"string"},"required":false,"name":"before","in":"query"},{"schema":{"type":"string","enum":["true","false"]},"required":false,"name":"last","in":"query"},{"schema":{"type":"string","enum":["created","updated","priority","worker","status","id","duration","createdAt"]},"required":false,"name":"sortBy","in":"query"},{"schema":{"type":"string","enum":["asc","desc"]},"required":false,"name":"sortDir","in":"query"},{"schema":{"type":["string","null"],"format":"date-time"},"required":false,"name":"createdAfter","in":"query"},{"schema":{"type":["string","null"],"format":"date-time"},"required":false,"name":"createdBefore","in":"query"},{"schema":{"type":["integer","null"],"minimum":0},"required":false,"name":"turns","in":"query"},{"schema":{"type":"string","enum":["eq","gte","lte"]},"required":false,"name":"turnsOp","in":"query"},{"schema":{"type":["integer","null"],"minimum":0},"required":false,"name":"maxIterations","in":"query"},{"schema":{"type":"string","enum":["eq","gte","lte"]},"required":false,"name":"maxIterationsOp","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"anyOf":[{"$ref":"#/components/schemas/PaginatedRunsResponse"},{"$ref":"#/components/schemas/PaginatedRunGroupsResponse"}]}}}}}}},"/api/v1/requests/facets":{"get":{"tags":["Requests"],"summary":"List run filter facets","description":"Returns every distinct value and its full-dataset count per categorical filter dimension for the Runs list rail. Counts are absolute over all non-deleted runs **in the given project**: they intentionally ignore the active search, date range, iteration, and categorical selections so every selectable value stays visible with a stable count. Requires ?projectId=. Computed with parallel $group aggregations (Cosmos has no $facet) and cached per-project for a short TTL.","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/RunFacetsResponse"}}}}}}},"/api/v1/requests/{id}":{"get":{"tags":["Requests"],"summary":"Get request","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/RequestResponse"}}}},"404":{"description":"Not found"}}},"delete":{"tags":["Requests"],"summary":"Soft-delete request","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}},"404":{"description":"Not found"}}}},"/api/v1/analysis":{"get":{"tags":["Requests"],"summary":"Compute pass@k / success@T metrics","description":"Aggregates pass@k / success@T metrics over a single project's done runs. Requires ?projectId=.","parameters":[{"schema":{"type":"string"},"required":false,"name":"worker","in":"query"},{"schema":{"type":"string"},"required":false,"name":"taskPromptId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"criteria","in":"query"},{"schema":{"type":"string"},"required":false,"name":"features","in":"query"},{"schema":{"type":"string"},"required":false,"name":"submissionId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"k","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{},"additionalProperties":{},"description":"Analysis metrics"}}}}}}},"/api/v1/requests/bulk-resubmit":{"post":{"tags":["Requests"],"summary":"Bulk resubmit requests","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/BulkResubmitInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/RequestResponse"}}}}}}}},"/api/v1/requests/bulk":{"delete":{"tags":["Requests"],"summary":"Bulk soft-delete requests","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"}}},"required":["ids"]}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"deleted":{"type":"number"}},"required":["deleted"]}}}}}}},"/api/v1/requests/archive":{"post":{"tags":["Requests"],"summary":"Download batch archive of multiple runs","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1,"maxItems":100}},"required":["ids"]}}}},"responses":{"201":{"description":"Gzipped batch archive containing individual run archives"},"400":{"description":"Invalid input"},"404":{"description":"One or more runs not found"}}}},"/api/v1/runs/upload":{"post":{"tags":["Requests"],"summary":"Import run archive","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"201":{"description":"Success"}}}},"/api/v1/runs/upload-batch":{"post":{"tags":["Requests"],"summary":"Import batch run archive","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"207":{"description":"Multi-status: 201 if all runs imported, 400 if none imported, 207 if partial"},"400":{"description":"No archive uploaded, empty archive, or all runs failed"}}}},"/api/v1/requests/{id}/reports":{"get":{"tags":["Reports"],"summary":"Get reports for request","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ReportResponse"}}}}},"404":{"description":"Run not found"}}}},"/api/v1/requests/{id}/runs":{"get":{"tags":["Requests"],"summary":"List attempts for a request","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/RunState"}}}}},"404":{"description":"Request not found"}}}},"/api/v1/requests/{id}/runs/{runId}":{"get":{"tags":["Requests"],"summary":"Get a single attempt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/RunState"}}}},"404":{"description":"Request or run not found"}}}},"/api/v1/requests/bulk-retry":{"post":{"tags":["Requests"],"summary":"Bulk retry requests (start new attempts)","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1},"force":{"type":"boolean"}},"required":["ids"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"retried":{"type":"integer"},"skipped":{"type":"integer"},"results":{"type":"array","items":{"type":"object","properties":{"requestId":{"type":"string"},"runId":{"type":"string"},"attemptNumber":{"type":"integer"},"error":{"type":"string"}},"required":["requestId"]}}},"required":["retried","skipped","results"]}}}}}}},"/api/v1/requests/{id}/retry":{"post":{"tags":["Requests"],"summary":"Retry a request (start a new attempt)","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"force":{"type":"boolean"}}}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"requestId":{"type":"string"},"runId":{"type":"string"},"attemptNumber":{"type":"integer"}},"required":["requestId","runId","attemptNumber"]}}}},"404":{"description":"Request not found"},"409":{"description":"Conflict — request is not in a retryable state, or a concurrent retry won the race"},"422":{"description":"Cannot retry — current run not yet terminal"}}}},"/api/v1/requests/{id}/priority":{"post":{"tags":["Requests"],"summary":"Set priority on a single request","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"priority":{"type":"integer"}},"required":["priority"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"priority":{"type":"number"}},"required":["id","priority"]}}}}}}},"/api/v1/requests/bulk-priority":{"post":{"tags":["Requests"],"summary":"Set priority on multiple requests","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1,"maxItems":100},"priority":{"type":"integer"}},"required":["ids","priority"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"updated":{"type":"number"},"skipped":{"type":"number"}},"required":["updated","skipped"]}}}}}}},"/api/v1/requests/{id}/pause":{"post":{"tags":["Requests"],"summary":"Pause a single request","responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"status":{"type":"string"}},"required":["id","status"]}}}}}}},"/api/v1/requests/{id}/resume":{"post":{"tags":["Requests"],"summary":"Resume a paused request","responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"status":{"type":"string"}},"required":["id","status"]}}}}}}},"/api/v1/requests/bulk-pause":{"post":{"tags":["Requests"],"summary":"Pause multiple requests","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1,"maxItems":100}},"required":["ids"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"updated":{"type":"number"},"skipped":{"type":"number"}},"required":["updated","skipped"]}}}}}}},"/api/v1/requests/bulk-resume":{"post":{"tags":["Requests"],"summary":"Resume multiple paused requests","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1,"maxItems":100}},"required":["ids"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"updated":{"type":"number"},"skipped":{"type":"number"}},"required":["updated","skipped"]}}}}}}},"/api/v1/requests/{id}/cancel":{"post":{"tags":["Requests"],"summary":"Cancel a request (marks as done/failed and signals worker to exit)","responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"previousStatus":{"type":"string"},"status":{"type":"string"},"outcome":{"type":"string"}},"required":["id","previousStatus","status","outcome"]}}}}}}},"/api/v1/requests/bulk-cancel":{"post":{"tags":["Requests"],"summary":"Cancel multiple requests","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"},"minItems":1,"maxItems":100}},"required":["ids"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"cancelled":{"type":"number"},"skipped":{"type":"number"},"results":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string"},"cancelled":{"type":"boolean"},"previousStatus":{"type":"string"},"error":{"type":"string"}},"required":["id","cancelled"]}}},"required":["cancelled","skipped","results"]}}}}}}},"/api/v1/requests/{id}/logs":{"get":{"tags":["Requests"],"summary":"Stream request logs (SSE)","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Server-sent event stream of log entries"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/logs":{"get":{"tags":["Requests"],"summary":"Stream logs for a specific attempt (SSE)","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"}],"responses":{"200":{"description":"Server-sent event stream of log entries"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/har":{"get":{"tags":["Requests"],"summary":"Download HAR file","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"HAR-format JSON file"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/har":{"get":{"tags":["Requests"],"summary":"Download HAR file for a specific attempt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"}],"responses":{"200":{"description":"HAR-format JSON file"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/atif":{"get":{"tags":["Requests"],"summary":"Download ATIF trajectory file for a specific iteration","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","description":"The iteration number (1-based)"},"required":true,"description":"The iteration number (1-based)","name":"iteration","in":"query"}],"responses":{"200":{"description":"ATIF v1.7 trajectory JSON file"},"400":{"description":"Missing or invalid iteration"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/atif":{"get":{"tags":["Requests"],"summary":"Download ATIF trajectory file for a specific attempt and iteration","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"},{"schema":{"type":"string","description":"The iteration number (1-based)"},"required":true,"description":"The iteration number (1-based)","name":"iteration","in":"query"}],"responses":{"200":{"description":"ATIF v1.7 trajectory JSON file"},"400":{"description":"Missing or invalid iteration"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/video":{"get":{"tags":["Requests"],"summary":"Download session recording","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"WebM video recording (supports Range requests)"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/video":{"get":{"tags":["Requests"],"summary":"Download session recording for a specific attempt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"}],"responses":{"200":{"description":"WebM video recording (supports Range requests)"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/tool-calls":{"get":{"tags":["Requests"],"summary":"Download per-iteration tool-calls JSONL","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"JSONL stream — one ToolCall per line"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/tool-calls":{"get":{"tags":["Requests"],"summary":"Download per-iteration tool-calls JSONL for a specific attempt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"}],"responses":{"200":{"description":"JSONL stream — one ToolCall per line"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/snapshots/{iteration}":{"get":{"tags":["Requests"],"summary":"Download iteration snapshot","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"iteration","in":"path"}],"responses":{"200":{"description":"Gzipped snapshot archive"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/snapshots/{iteration}":{"get":{"tags":["Requests"],"summary":"Download iteration snapshot for a specific attempt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"},{"schema":{"type":"string"},"required":true,"name":"iteration","in":"path"}],"responses":{"200":{"description":"Gzipped snapshot archive"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/archive":{"get":{"tags":["Requests"],"summary":"Download full run archive","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Gzipped run archive"},"404":{"description":"Not found"}}}},"/api/v1/requests/{id}/runs/{runId}/archive":{"get":{"tags":["Requests"],"summary":"Download full run archive for a specific attempt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"runId","in":"path"}],"responses":{"200":{"description":"Gzipped run archive"},"404":{"description":"Not found"}}}},"/api/v1/criteria/generate-prompt":{"post":{"tags":["Criteria"],"summary":"Generate criterion prompt from behavior","parameters":[{"schema":{"type":"string","minLength":1,"description":"Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.","example":"00000000-0000-0000-0000-000000000000"},"required":false,"description":"Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"behavior":{"type":"string"},"currentId":{"type":"string"},"gates":{"type":"array","items":{"type":"string"}}},"required":["behavior"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"prompt":{"type":"string"},"suggestedId":{"type":"string"},"suggestedParents":{"type":"array","items":{"type":"string"}},"suggestedChildren":{"type":"array","items":{"type":"string"}}},"required":["prompt","suggestedId","suggestedParents","suggestedChildren"]}}}},"400":{"description":"Empty behavior string"},"503":{"description":"LLM not configured"}}}},"/api/v1/criteria/seed":{"post":{"tags":["Criteria"],"summary":"Seed criteria in bulk","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"criteria":{"type":"array","items":{"$ref":"#/components/schemas/CreateCriteriaInput"}}},"required":["criteria"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"seeded":{"type":"number"},"errors":{"type":"array","items":{"type":"string"}}},"required":["seeded","errors"]}}}},"400":{"description":"Seed batch would introduce a dependency cycle"}}}},"/api/v1/criteria":{"get":{"tags":["Criteria"],"summary":"List criteria","parameters":[{"schema":{"type":"string"},"required":false,"name":"q","in":"query"},{"schema":{"type":"string","description":"Comma-separated criterion IDs to include"},"required":false,"description":"Comma-separated criterion IDs to include","name":"ids","in":"query"},{"schema":{"type":"string","enum":["true","false"],"description":"When true and ids is set, also include dependency ancestors"},"required":false,"description":"When true and ids is set, also include dependency ancestors","name":"ancestors","in":"query"},{"schema":{"type":"string","enum":["gate","observation"],"description":"Filter by criterion kind"},"required":false,"description":"Filter by criterion kind","name":"kind","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/CriteriaResponse"}}}}}}},"post":{"tags":["Criteria"],"summary":"Create criterion","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateCriteriaInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CriteriaResponse"}}}},"409":{"description":"Criterion already exists"}}}},"/api/v1/criteria/mdp":{"get":{"tags":["Criteria"],"summary":"Compute MDP transitions","parameters":[{"schema":{"type":"string"},"required":false,"name":"criteria","in":"query"},{"schema":{"type":"string"},"required":false,"name":"features","in":"query"},{"schema":{"type":"string"},"required":false,"name":"since","in":"query"},{"schema":{"type":"string"},"required":false,"name":"worker","in":"query"},{"schema":{"type":"string"},"required":false,"name":"taskPromptId","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{},"additionalProperties":{}}}}}}}},"/api/v1/criteria/graph":{"get":{"tags":["Criteria"],"summary":"Get criteria DAG","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CriteriaGraph"}}}}}}},"/api/v1/criteria/{id}":{"get":{"tags":["Criteria"],"summary":"Get criterion","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CriteriaResponse"}}}},"404":{"description":"Criterion not found"}}},"put":{"tags":["Criteria"],"summary":"Update criterion","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateCriteriaInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CriteriaResponse"}}}},"400":{"description":"Invalid dependency reference or self-reference"},"404":{"description":"Criterion not found"}}},"delete":{"tags":["Criteria"],"summary":"Soft-delete criterion","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"deleted":{"type":"boolean"}},"required":["id","deleted"]}}}},"404":{"description":"Criterion not found"},"409":{"description":"Criterion has dependents"}}}},"/api/v1/prompt-features/generate-prompt":{"post":{"tags":["Prompt Features"],"summary":"Generate prompt feature from behavior","parameters":[{"schema":{"type":"string","minLength":1,"description":"Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.","example":"00000000-0000-0000-0000-000000000000"},"required":false,"description":"Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"behavior":{"type":"string"},"currentId":{"type":"string"}},"required":["behavior"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"prompt":{"type":"string"}},"required":["prompt"]}}}},"400":{"description":"Empty behavior string"},"503":{"description":"LLM not configured"}}}},"/api/v1/prompt-features/seed":{"post":{"tags":["Prompt Features"],"summary":"Seed prompt features in bulk","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"features":{"type":"array","items":{"$ref":"#/components/schemas/CreatePromptFeatureInput"}}},"required":["features"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"seeded":{"type":"number"},"errors":{"type":"array","items":{"type":"string"}}},"required":["seeded","errors"]}}}}}}},"/api/v1/prompt-features/extract-from-text":{"post":{"tags":["Prompt Features"],"summary":"Extract features from text","parameters":[{"schema":{"type":"string","minLength":1,"description":"Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.","example":"00000000-0000-0000-0000-000000000000"},"required":false,"description":"Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"text":{"type":"string"},"model":{"type":"string"},"type":{"type":"string","enum":["select","agents.md"]}},"required":["text"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"features":{"type":"array","items":{"$ref":"#/components/schemas/PromptFeatureResult"}},"suggestedFeatures":{"type":"array","items":{"$ref":"#/components/schemas/SuggestedPromptFeature"}},"cached":{"type":"boolean"}},"required":["features","cached"]}}}},"400":{"description":"Empty text string"},"503":{"description":"LLM not configured"}}}},"/api/v1/prompt-features":{"get":{"tags":["Prompt Features"],"summary":"List features","parameters":[{"schema":{"type":"string"},"required":false,"name":"q","in":"query"},{"schema":{"type":"string","enum":["select","agents.md"]},"required":false,"name":"type","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/PromptFeatureResponse"}}}}}}},"post":{"tags":["Prompt Features"],"summary":"Create feature","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreatePromptFeatureInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/PromptFeatureResponse"}}}},"400":{"description":"Invalid input"},"409":{"description":"Feature already exists"}}}},"/api/v1/prompt-features/{id}":{"get":{"tags":["Prompt Features"],"summary":"Get feature","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/PromptFeatureResponse"}}}},"404":{"description":"Feature not found"}}},"put":{"tags":["Prompt Features"],"summary":"Update feature","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdatePromptFeatureInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/PromptFeatureResponse"}}}},"400":{"description":"Invalid input"},"404":{"description":"Feature not found"}}},"delete":{"tags":["Prompt Features"],"summary":"Soft-delete feature","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"deleted":{"type":"boolean"}},"required":["id","deleted"]}}}},"404":{"description":"Feature not found"}}}},"/api/v1/task-prompts/generate":{"post":{"tags":["Task Prompts"],"summary":"Generate task prompts","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"description":{"type":"string"},"existingPrompt":{"type":"string"}}}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"tasks":{"type":"array","items":{"type":"string"}}},"required":["tasks"]}}}},"503":{"description":"LLM not configured"}}}},"/api/v1/task-prompts":{"get":{"tags":["Task Prompts"],"summary":"List task prompts","parameters":[{"schema":{"type":["number","null"]},"required":false,"name":"limit","in":"query"},{"schema":{"type":["number","null"]},"required":false,"name":"offset","in":"query"},{"schema":{"type":"string"},"required":false,"name":"search","in":"query"},{"schema":{"$ref":"#/components/schemas/PromptType"},"required":false,"name":"type","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"items":{"type":"array","items":{"$ref":"#/components/schemas/TaskPromptResponse"}},"total":{"type":"number"},"limit":{"type":"number"},"offset":{"type":"number"}},"required":["items","total","limit","offset"]}}}}}},"post":{"tags":["Task Prompts"],"summary":"Create or find task prompt","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateTaskPromptInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/TaskPromptResponse"}}}},"400":{"description":"Empty text string"}}}},"/api/v1/task-prompts/{id}":{"get":{"tags":["Task Prompts"],"summary":"Get task prompt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/TaskPromptResponse"}}}},"404":{"description":"Task prompt not found"}}},"delete":{"tags":["Task Prompts"],"summary":"Soft-delete task prompt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"deleted":{"type":"boolean"}},"required":["deleted"]}}}},"404":{"description":"Task prompt not found"}}}},"/api/v1/task-prompts/{id}/content":{"get":{"tags":["Task Prompts"],"summary":"Get resolved task prompt content","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"id":{"type":"string"},"text":{"type":"string"}},"required":["id","text"]}}}},"404":{"description":"Task prompt not found"}}}},"/api/v1/task-prompts/{id}/extract-features":{"post":{"tags":["Task Prompts"],"summary":"Extract features from task prompt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":false,"name":"force","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"model":{"type":"string"}}}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"taskPromptId":{"type":"string"},"features":{"type":"array","items":{"$ref":"#/components/schemas/PromptFeatureResult"}},"featuresExtractedAt":{"type":["string","null"],"format":"date-time"},"suggestedFeatures":{"type":"array","items":{"$ref":"#/components/schemas/SuggestedPromptFeature"}},"cached":{"type":"boolean"}},"required":["taskPromptId","features","cached"]}}}},"404":{"description":"Task prompt not found"},"503":{"description":"LLM not configured"}}}},"/api/v1/task-prompts/{id}/features/{featureId}":{"patch":{"tags":["Task Prompts"],"summary":"Toggle feature flag on task prompt","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"featureId","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/PatchTaskPromptFeatureInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/TaskPromptResponse"}}}},"400":{"description":"Invalid detected value"},"404":{"description":"Task prompt or feature not found"}}}},"/api/v1/reports":{"post":{"tags":["Reports"],"summary":"Create report","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateReportInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ReportResponse"}}}},"400":{"description":"Invalid input"},"404":{"description":"Run or template not found"}}},"get":{"tags":["Reports"],"summary":"List reports","parameters":[{"schema":{"type":"string"},"required":false,"name":"requestId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ReportResponse"}}}}}}}},"/api/v1/reports/bulk-create":{"post":{"tags":["Reports"],"summary":"Bulk create reports","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/BulkCreateReportsInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ReportResponse"}}}}},"400":{"description":"Invalid input"}}}},"/api/v1/reports/bulk-status":{"post":{"tags":["Reports"],"summary":"Bulk get report statuses","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/BulkReportStatusInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ReportResponse"}}}}},"400":{"description":"Invalid input"}}}},"/api/v1/reports/bulk-summary":{"post":{"tags":["Reports"],"summary":"Bulk get report summary per run","description":"Returns aggregated report status counts per run. Only the latest report per template is counted (re-triggers are deduplicated).","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/BulkReportSummaryInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/BulkReportSummaryResponse"}}}},"400":{"description":"Invalid input"}}}},"/api/v1/reports/{id}":{"get":{"tags":["Reports"],"summary":"Get report","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ReportResponse"}}}},"404":{"description":"Report not found"}}}},"/api/v1/reports/{id}/logs":{"get":{"tags":["Reports"],"summary":"Stream report logs (SSE)","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Server-sent event stream of log entries"},"404":{"description":"Report not found"}}}},"/api/v1/reports/trigger":{"post":{"tags":["Reports"],"summary":"Trigger reports","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/TriggerReportsInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"triggered":{"type":"number"},"reports":{"type":"array","items":{"$ref":"#/components/schemas/ReportResponse"}}},"required":["triggered","reports"]}}}},"400":{"description":"Invalid input"},"404":{"description":"Run not found"}}}},"/api/v1/reports/bulk-trigger":{"post":{"tags":["Reports"],"summary":"Bulk trigger reports","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/BulkTriggerReportsInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"type":"object","properties":{},"additionalProperties":{}}}}}},"400":{"description":"Invalid input"}}}},"/api/v1/reports/{id}/insights":{"get":{"tags":["Reports"],"summary":"Get insights for report","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"allOf":[{"$ref":"#/components/schemas/InsightResponse"},{"type":"object","properties":{"referencedAt":{"type":["string","null"],"format":"date-time"},"isNew":{"type":"boolean"}}}]}}}}},"404":{"description":"Report not found"}}},"post":{"tags":["Reports"],"summary":"Link insight to report","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"insightId":{"type":"string"},"isNew":{"type":"boolean"}},"required":["insightId"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"insightId":{"type":"string"},"referencedAt":{"type":["string","null"],"format":"date-time"},"isNew":{"type":"boolean"}},"required":["insightId","referencedAt","isNew"]}}}},"404":{"description":"Report or insight not found"},"409":{"description":"Insight already referenced by this report"}}}},"/api/v1/profiles":{"post":{"tags":["Profiles"],"summary":"Create a new profile","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateProfileInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProfileWithVersionResponse"}}}}}},"get":{"tags":["Profiles"],"summary":"List profiles","parameters":[{"schema":{"type":"string"},"required":false,"name":"workerType","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ProfileWithVersionResponse"}}}}}}}},"/api/v1/profiles/{profileId}":{"get":{"tags":["Profiles"],"summary":"Get profile with latest version","parameters":[{"schema":{"type":"string"},"required":true,"name":"profileId","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProfileWithVersionResponse"}}}}}},"post":{"tags":["Profiles"],"summary":"Create new profile version","parameters":[{"schema":{"type":"string"},"required":true,"name":"profileId","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"workerType":{"type":"string"},"model":{"type":"string"},"reasoningEffort":{"type":"string"},"agentVersion":{"type":"string"},"mcpServers":{"type":"array","items":{"type":"string"}},"skillRevisions":{"type":"array","items":{"type":"string"}},"resources":{"type":"array","items":{"$ref":"#/components/schemas/ResourceBindingSpec"}},"extensions":{"type":"array","items":{"type":"string"}}},"required":["workerType","model"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProfileVersionResponse"}}}}}},"put":{"tags":["Profiles"],"summary":"Update profile identity","parameters":[{"schema":{"type":"string"},"required":true,"name":"profileId","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateProfileIdentity"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProfileResponse"}}}}}},"delete":{"tags":["Profiles"],"summary":"Delete profile","parameters":[{"schema":{"type":"string"},"required":true,"name":"profileId","in":"path"}],"responses":{"204":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}}}}},"/api/v1/profiles/{profileId}/versions":{"get":{"tags":["Profiles"],"summary":"List profile versions","parameters":[{"schema":{"type":"string"},"required":true,"name":"profileId","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ProfileVersionResponse"}}}}}}}},"/api/v1/profiles/{profileId}/versions/{version}":{"get":{"tags":["Profiles"],"summary":"Get profile version","parameters":[{"schema":{"type":"string"},"required":true,"name":"profileId","in":"path"},{"schema":{"type":["number","null"]},"required":false,"name":"version","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProfileVersionResponse"}}}}}}},"/api/v1/report-templates/default-system-prompt":{"get":{"tags":["Report Templates"],"summary":"Get default system prompt","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"content":{"type":"string"}},"required":["content"]}}}}}}},"/api/v1/report-templates/available-models":{"get":{"tags":["Report Templates"],"summary":"List models available for report generation","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"type":"object","properties":{"modelId":{"type":"string"}},"required":["modelId"]}}}}}}}},"/api/v1/report-templates":{"get":{"tags":["Report Templates"],"summary":"List report templates","parameters":[{"schema":{"type":"string"},"required":false,"name":"q","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ReportTemplateResponse"}}}}}}},"post":{"tags":["Report Templates"],"summary":"Create report template","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateReportTemplateInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ReportTemplateResponse"}}}},"409":{"description":"Report template already exists"}}}},"/api/v1/report-templates/{id}":{"get":{"tags":["Report Templates"],"summary":"Get report template","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ReportTemplateResponse"}}}},"404":{"description":"Report template not found"}}},"put":{"tags":["Report Templates"],"summary":"Update report template","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateReportTemplateInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ReportTemplateResponse"}}}},"404":{"description":"Report template not found"}}},"delete":{"tags":["Report Templates"],"summary":"Delete report template","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"204":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}},"404":{"description":"Report template not found"}}}},"/api/v1/agents":{"get":{"tags":["Agents"],"summary":"List agents","parameters":[{"schema":{"type":"string"},"required":false,"name":"modelProvider","in":"query"},{"schema":{"type":"string","enum":["true","false"]},"required":false,"name":"includeDeleted","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/AgentResponse"}}}}}}},"post":{"tags":["Agents"],"summary":"Create or update agent (upsert)","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateAgentInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AgentResponse"}}}},"400":{"description":"Validation error"},"409":{"description":"Restored agent versions conflict with an active queue owner"}}}},"/api/v1/agents/{id}":{"get":{"tags":["Agents"],"summary":"Get agent","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","enum":["true","false"]},"required":false,"name":"includeDeleted","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AgentResponse"}}}},"404":{"description":"Agent not found"}}},"put":{"tags":["Agents"],"summary":"Update agent","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateAgentInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AgentResponse"}}}},"400":{"description":"Validation error"},"404":{"description":"Agent not found"}}},"delete":{"tags":["Agents"],"summary":"Delete agent","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"204":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}},"404":{"description":"Agent not found"}}}},"/api/v1/agents/{id}/versions":{"get":{"tags":["Agents"],"summary":"List agent versions","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":false,"name":"status","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/AgentVersion"}}}}},"404":{"description":"Agent not found"}}},"post":{"tags":["Agents"],"summary":"Register agent version (upsert)","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/RegisterAgentVersionInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AgentVersion"}}}},"400":{"description":"Validation error"},"404":{"description":"Agent not found"},"409":{"description":"Queue is assigned to another agent"},"503":{"description":"Concurrent registration did not stabilize"}}}},"/api/v1/agents/{id}/versions/{agentVersion}":{"patch":{"tags":["Agents"],"summary":"Patch agent version","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"agentVersion","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/PatchAgentVersionInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AgentVersion"}}}},"400":{"description":"Invalid status value"},"404":{"description":"Agent or version not found"},"409":{"description":"Queue is assigned to another agent"},"503":{"description":"Concurrent activation did not stabilize"}}}},"/api/v1/models":{"get":{"tags":["Models"],"summary":"List models","parameters":[{"schema":{"type":"string"},"required":false,"name":"agentId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"provider","in":"query"},{"schema":{"type":"string","enum":["active","disappeared"]},"required":false,"name":"status","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ModelResponse"}}}}}}}},"/api/v1/models/{id}":{"get":{"tags":["Models"],"summary":"Get model","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ModelResponse"}}}}}}},"/api/v1/models/sync":{"post":{"tags":["Models"],"summary":"Sync models from provider","requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"agentId":{"type":"string"},"provider":{"type":"string"},"models":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string"},"providerAvailableFrom":{"type":"string"},"providerEndOfLife":{"type":"string"},"metadata":{"type":"object","additionalProperties":{}},"capabilities":{"type":"object","properties":{"reasoningEffort":{"type":"array","items":{"type":"string"}},"toolCalls":{"type":"boolean"},"vision":{"type":"boolean"},"streaming":{"type":"boolean"},"adaptiveThinking":{"type":"boolean"},"maxThinkingBudget":{"type":"number"}}}},"required":["id"]}},"scannedAt":{"type":"string"}},"required":["agentId","provider","models","scannedAt"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"added":{"type":"array","items":{"type":"string"}},"removed":{"type":"array","items":{"type":"string"}},"unchanged":{"type":"array","items":{"type":"string"}}},"required":["added","removed","unchanged"]}}}}}}},"/api/v1/mcp/servers":{"get":{"tags":["MCP Servers"],"summary":"List MCP servers","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/McpServerResponse"}}}}}}},"post":{"tags":["MCP Servers"],"summary":"Create MCP server","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"_id":{"type":"string","pattern":"^[a-z0-9]([a-z0-9-]*[a-z0-9])?$"},"name":{"type":"string"},"type":{"type":"string","enum":["sse","http","stdio"]},"url":{"type":"string"},"command":{"type":"string"},"args":{"type":"array","items":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"headers":{"type":"array","items":{"$ref":"#/components/schemas/McpServerHeader"}},"sessionMode":{"type":"string","enum":["stateful","stateless"]},"version":{"type":"string"},"description":{"type":"string"}},"required":["_id","name","type"]}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/McpServerResponse"}}}}}}},"/api/v1/mcp/servers/{id}":{"get":{"tags":["MCP Servers"],"summary":"Get MCP server","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/McpServerResponse"}}}}}},"put":{"tags":["MCP Servers"],"summary":"Update MCP server","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateMcpServerInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/McpServerResponse"}}}}}},"delete":{"tags":["MCP Servers"],"summary":"Delete MCP server","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"204":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}}}}},"/api/v1/skills":{"get":{"tags":["Skills"],"summary":"List all skills","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/SkillResponse"}}}}}}},"post":{"tags":["Skills"],"summary":"Create or import a skill","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateSkillInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SkillResponse"}}}}}}},"/api/v1/skills/search":{"get":{"tags":["Skills"],"summary":"Search skills (internal + external)","parameters":[{"schema":{"type":"string"},"required":true,"name":"q","in":"query"},{"schema":{"type":"string"},"required":false,"name":"limit","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/SkillSearchResult"}}}}},"400":{"description":"Missing query parameter"}}}},"/api/v1/skills/search/external":{"get":{"tags":["Skills"],"summary":"Search external skills registry","parameters":[{"schema":{"type":"string"},"required":true,"name":"q","in":"query"},{"schema":{"type":"string"},"required":false,"name":"limit","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Optional project scope. When provided, external results are de-duplicated only against skills already installed in that project."},"required":false,"description":"Optional project scope. When provided, external results are de-duplicated only against skills already installed in that project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/SkillSearchResult"}}}}},"400":{"description":"Missing query parameter"}}}},"/api/v1/skills/discover":{"get":{"tags":["Skills"],"summary":"Discover skills in a GitHub repository","parameters":[{"schema":{"type":"string"},"required":true,"name":"source","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/SkillDiscoveryResult"}}}}},"400":{"description":"Missing or malformed source parameter"},"404":{"description":"Repository not found"},"502":{"description":"GitHub API error"}}}},"/api/v1/skills/{id}(*)/revisions":{"get":{"tags":["Skills"],"summary":"List skill revisions","parameters":[{"schema":{"type":"string"},"required":false,"name":"limit","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/SkillRevisionResponse"}}}}},"404":{"description":"Skill not found"}}}},"/api/v1/skills/{id}(*)":{"get":{"tags":["Skills"],"summary":"Get skill by slug","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SkillResponse"}}}},"404":{"description":"Skill not found"}}},"delete":{"tags":["Skills"],"summary":"Soft-delete a skill","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"204":{"description":"Success"},"404":{"description":"Skill not found"}}}},"/api/v1/skill-revisions/by-ref/{ref}(*)/archive":{"get":{"tags":["Skill Revisions"],"summary":"Download skill revision archive","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Binary tar.gz archive"},"404":{"description":"Skill revision or archive not found"},"500":{"description":"Blob storage error"}}}},"/api/v1/skill-revisions/by-ref/{ref}(*)":{"get":{"tags":["Skill Revisions"],"summary":"Get skill revision by ref","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SkillRevisionResponse"}}}},"404":{"description":"Skill revision not found"}}}},"/api/v1/skill-revisions/{id}":{"get":{"tags":["Skill Revisions"],"summary":"Get skill revision by ID","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SkillRevisionResponse"}}}},"404":{"description":"Skill revision not found"}}}},"/api/v1/skills/{id}(*)/resolve":{"post":{"tags":["Skills"],"summary":"Trigger skill resolution","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SkillRevisionResponse"}}}},"404":{"description":"Skill not found"},"500":{"description":"Blob storage error"}}}},"/api/v1/codebases":{"get":{"tags":["Codebases"],"summary":"List all codebases","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/CodebaseResponse"}}}}}}},"post":{"tags":["Codebases"],"summary":"Create a codebase (archive codebases require the archive file)","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"201":{"description":"Success"},"400":{"description":"Invalid input or missing archive"}}}},"/api/v1/codebases/{id}":{"get":{"tags":["Codebases"],"summary":"Get a codebase","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CodebaseResponse"}}}},"404":{"description":"Codebase not found"}}},"patch":{"tags":["Codebases"],"summary":"Update a codebase","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateCodebaseInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CodebaseResponse"}}}},"404":{"description":"Codebase not found"}}},"delete":{"tags":["Codebases"],"summary":"Delete a codebase","responses":{"204":{"description":"Success"},"404":{"description":"Codebase not found"}}}},"/api/v1/codebases/{id}/revisions":{"get":{"tags":["Codebases"],"summary":"List codebase revisions","parameters":[{"schema":{"type":"string"},"required":false,"name":"limit","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/CodebaseRevisionResponse"}}}}},"404":{"description":"Codebase not found"}}},"post":{"tags":["Codebases"],"summary":"Resolve a new git codebase revision","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ResolveCodebaseRevisionInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CodebaseRevisionResponse"}}}},"400":{"description":"Not a git codebase"},"404":{"description":"Codebase not found"},"502":{"description":"GitHub resolution failed"}}}},"/api/v1/codebases/{id}/upload":{"post":{"tags":["Codebases"],"summary":"Upload a codebase archive as a new revision","responses":{"201":{"description":"Success"},"400":{"description":"Missing archive or not an archive codebase"},"404":{"description":"Codebase not found"}}}},"/api/v1/codebase-revisions/{id}/archive":{"get":{"tags":["Codebase Revisions"],"summary":"Download codebase revision archive","responses":{"200":{"description":"Binary tar.gz archive"},"404":{"description":"Revision or archive not found"},"500":{"description":"Blob storage error"}}}},"/api/v1/codebase-revisions/{id}":{"get":{"tags":["Codebase Revisions"],"summary":"Get codebase revision by id","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/CodebaseRevisionResponse"}}}},"404":{"description":"Revision not found"}}}},"/api/v1/resources":{"get":{"tags":["Resources"],"summary":"List all resources","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ResourceResponse"}}}}}}},"post":{"tags":["Resources"],"summary":"Create a resource and its first revision","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateResourceInput"}}}},"responses":{"201":{"description":"Success"},"400":{"description":"Invalid input"},"409":{"description":"Resource slug or revision ref already exists in this project"}}}},"/api/v1/resources/{id}":{"get":{"tags":["Resources"],"summary":"Get a resource","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ResourceResponse"}}}},"404":{"description":"Resource not found"}}},"patch":{"tags":["Resources"],"summary":"Update a resource","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateResourceInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ResourceResponse"}}}},"404":{"description":"Resource not found"}}},"delete":{"tags":["Resources"],"summary":"Delete a resource","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"204":{"description":"Success"},"404":{"description":"Resource not found"}}}},"/api/v1/resources/{id}/revisions":{"get":{"tags":["Resource Revisions"],"summary":"List resource revisions","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"},{"schema":{"type":"string"},"required":false,"name":"limit","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ResourceRevisionResponse"}}}}},"404":{"description":"Resource not found"}}},"post":{"tags":["Resource Revisions"],"summary":"Create a resource revision","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateResourceRevisionInput"}}}},"responses":{"201":{"description":"Success"},"404":{"description":"Resource not found"},"409":{"description":"Resource revision ref already exists in this project"}}}},"/api/v1/resources/{id}/revisions/latest":{"get":{"tags":["Resource Revisions"],"summary":"Get the latest resource revision","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ResourceRevisionResponse"}}}},"404":{"description":"Revision not found"}}}},"/api/v1/resources/{id}/revisions/{revisionNumber}":{"get":{"tags":["Resource Revisions"],"summary":"Get a resource revision by resource and revision number","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":true,"name":"revisionNumber","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ResourceRevisionResponse"}}}},"404":{"description":"Revision not found"}}}},"/api/v1/resources/revisions/{id}":{"get":{"tags":["Resource Revisions"],"summary":"Get resource revision by id","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ResourceRevisionResponse"}}}},"404":{"description":"Revision not found"}}}},"/api/v1/extensions":{"get":{"tags":["Extensions"],"summary":"List all extensions","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ExtensionResponse"}}}}}}},"post":{"tags":["Extensions"],"summary":"Create or import an extension","parameters":[{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateExtensionInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ExtensionResponse"}}}}}}},"/api/v1/extensions/search":{"get":{"tags":["Extensions"],"summary":"Search extensions (internal + marketplace)","parameters":[{"schema":{"type":"string"},"required":true,"name":"q","in":"query"},{"schema":{"type":"string"},"required":false,"name":"limit","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ExtensionSearchResult"}}}}},"400":{"description":"Missing query parameter"}}}},"/api/v1/extensions/{id}":{"get":{"tags":["Extensions"],"summary":"Get extension by ID","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ExtensionResponse"}}}}}},"put":{"tags":["Extensions"],"summary":"Update extension","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateExtensionInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ExtensionResponse"}}}}}},"delete":{"tags":["Extensions"],"summary":"Delete extension","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"204":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}}}}},"/api/v1/extensions/{id}/versions":{"get":{"tags":["Extensions"],"summary":"List available versions for an extension","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"},{"schema":{"type":"string"},"required":false,"name":"preRelease","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ExtensionVersionInfo"}}}}}}}},"/api/v1/insights":{"get":{"tags":["Insights"],"summary":"List insights","parameters":[{"schema":{"type":"string"},"required":false,"name":"q","in":"query"},{"schema":{"type":"string"},"required":false,"name":"blocked","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/InsightResponse"}}}}}}},"post":{"tags":["Insights"],"summary":"Create insight","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateInsightInput"}}}},"responses":{"201":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}}}}},"/api/v1/insights/search":{"get":{"tags":["Insights"],"summary":"Search insights","parameters":[{"schema":{"type":"string"},"required":true,"name":"q","in":"query"},{"schema":{"type":"string"},"required":false,"name":"blocked","in":"query"},{"schema":{"type":"string","minLength":1,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","example":"00000000-0000-0000-0000-000000000000"},"required":true,"description":"Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.","name":"projectId","in":"query"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/InsightResponse"}}}}}}}},"/api/v1/insights/{id}":{"get":{"tags":["Insights"],"summary":"Get insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}},"404":{"description":"Insight not found"}}},"put":{"tags":["Insights"],"summary":"Update insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateInsightInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}},"404":{"description":"Insight not found"}}},"delete":{"tags":["Insights"],"summary":"Delete insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"204":{"description":"Success","content":{"application/json":{"schema":{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]}}}},"404":{"description":"Insight not found"}}}},"/api/v1/insights/{id}/upvote":{"post":{"tags":["Insights"],"summary":"Upvote insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}},"404":{"description":"Insight not found"}}}},"/api/v1/insights/{id}/downvote":{"post":{"tags":["Insights"],"summary":"Downvote insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}},"404":{"description":"Insight not found"}}}},"/api/v1/insights/{id}/block":{"post":{"tags":["Insights"],"summary":"Block insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}},"404":{"description":"Insight not found"}}}},"/api/v1/insights/{id}/unblock":{"post":{"tags":["Insights"],"summary":"Unblock insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/InsightResponse"}}}},"404":{"description":"Insight not found"}}}},"/api/v1/insights/{id}/reports":{"get":{"tags":["Insights"],"summary":"Get reports referencing insight","parameters":[{"schema":{"type":"string"},"required":true,"name":"id","in":"path"}],"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ReportResponse"}}}}},"404":{"description":"Insight not found"}}}},"/api/v1/feature-flags":{"get":{"tags":["Feature Flags"],"summary":"List feature flags","responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/FeatureFlagResponse"}}}}}}}},"/api/v1/feature-flags/{key}":{"put":{"tags":["Feature Flags"],"summary":"Update feature flag","parameters":[{"schema":{"type":"string"},"required":true,"name":"key","in":"path"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateFeatureFlagInput"}}}},"responses":{"200":{"description":"Success","content":{"application/json":{"schema":{"$ref":"#/components/schemas/FeatureFlagResponse"}}}}}}}},"webhooks":{}} \ No newline at end of file From 0220a1173fa76b4e96fac60df10437e00097b62a Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Tue, 13 Oct 2026 16:26:29 +0200 Subject: [PATCH 03/25] feat(submissions): add Portal and CLI workflows Navigate successful submits to a Submission Detail shell, link grouped runs and run detail back to their parent submission, surface readiness and blocking pairs, and add CLI list/get/history/retry parity with Storybook coverage. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 99f84f04-d3b0-40f1-a254-4dfbd21e4d87 --- apps/cli/src/commands/submission.ts | 206 ++++++++++++++++++ apps/cli/src/index.ts | 2 + apps/portal/src/App.tsx | 2 + .../SubmissionStatusCard.stories.tsx | 81 +++++++ .../components/SubmissionStatusCard.test.tsx | 69 ++++++ .../src/components/SubmissionStatusCard.tsx | 55 +++++ apps/portal/src/lib/api.ts | 25 ++- apps/portal/src/pages/RunDetail.tsx | 15 +- apps/portal/src/pages/RunsList.tsx | 52 +++-- apps/portal/src/pages/SubmissionDetail.tsx | 164 ++++++++++++++ apps/portal/src/pages/SubmitRun.tsx | 8 +- apps/portal/src/types.ts | 69 +++++- 12 files changed, 715 insertions(+), 33 deletions(-) create mode 100644 apps/cli/src/commands/submission.ts create mode 100644 apps/portal/src/components/SubmissionStatusCard.stories.tsx create mode 100644 apps/portal/src/components/SubmissionStatusCard.test.tsx create mode 100644 apps/portal/src/components/SubmissionStatusCard.tsx create mode 100644 apps/portal/src/pages/SubmissionDetail.tsx diff --git a/apps/cli/src/commands/submission.ts b/apps/cli/src/commands/submission.ts new file mode 100644 index 000000000..52b6c4b7a --- /dev/null +++ b/apps/cli/src/commands/submission.ts @@ -0,0 +1,206 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { Command } from "commander"; +import { configureHelp } from "../utils/helpFormatter.js"; +import { formatData, isMachineReadable } from "../utils/formatters.js"; +import type { DisplayField, OutputFormat } from "../utils/types.js"; +import { + getDefaultApiUrl, + withOutputOption, + withProjectOption, +} from "../utils/shared.js"; +import { requireProjectId } from "../utils/config.js"; +import { apiFetch } from "../utils/api-client.js"; +import { errorText, label, successText, value } from "../utils/style.js"; + +interface SubmissionSummary { + submissionId: string; + submissionRunId: string; + attemptNumber: number; + readiness: string; + createdAt: string; + requestIds: string[]; + requests: Array<{ + requestId: string; + runId: string; + task: string; + status: string; + outcome?: string; + }>; + blocking: Array<{ + requestId: string; + runId: string; + reason: string; + }>; +} + +async function responseJson(response: Response): Promise { + return response.json() as Promise; +} + +async function throwApiError(response: Response): Promise { + const body = await responseJson<{ error?: string }>(response); + throw new Error(body.error ?? `HTTP ${response.status}`); +} + +const fields: DisplayField[] = [ + { key: "submissionId", label: "Submission" }, + { key: "attemptNumber", label: "Attempt" }, + { key: "readiness", label: "Readiness" }, + { + key: "requestIds", + label: "Runs", + formatter: (row: Record) => + String((row.requestIds as string[] | undefined)?.length ?? 0), + }, + { key: "createdAt", label: "Created" }, +]; + +export function registerSubmissionCommands(program: Command): void { + const submission = program + .command("submission") + .description("Inspect and retry grouped run submissions") + .action(() => submission.help()); + configureHelp(submission); + + withProjectOption( + withOutputOption( + submission + .command("list") + .description("List submissions") + .option("-u, --url ", "API base URL", getDefaultApiUrl()), + ), + ).action(async (options) => { + try { + const projectId = requireProjectId(options.project); + const response = await apiFetch(options.url, "/submissions", { projectId }); + if (!response.ok) await throwApiError(response); + const body = await responseJson<{ submissions: SubmissionSummary[] }>(response); + console.log( + formatData(body.submissions, fields, (options.output ?? "table") as OutputFormat), + ); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exitCode = 1; + } + }); + + withOutputOption( + submission + .command("get") + .description("Get a submission and its current request/run vector") + .argument("", "Submission ID") + .option("-u, --url ", "API base URL", getDefaultApiUrl()), + ).action(async (id: string, options) => { + try { + const response = await apiFetch( + options.url, + `/submissions/${encodeURIComponent(id)}`, + ); + if (!response.ok) await throwApiError(response); + const doc = await responseJson(response); + const format = (options.output ?? "table") as OutputFormat; + if (isMachineReadable(format)) { + console.log(formatData([doc], fields, format)); + return; + } + console.log(`${label("Submission:")} ${value(doc.submissionId)}`); + console.log(`${label("Attempt:")} ${value(String(doc.attemptNumber))}`); + console.log(`${label("Readiness:")} ${value(doc.readiness)}`); + console.log(`${label("Submission run:")} ${value(doc.submissionRunId)}`); + for (const request of doc.requests) { + console.log( + ` ${value(request.requestId)} ${request.status}` + + `${request.outcome ? `/${request.outcome}` : ""} run=${request.runId}`, + ); + } + for (const blocking of doc.blocking) { + console.log( + ` blocked ${blocking.requestId}/${blocking.runId}: ${blocking.reason}`, + ); + } + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exitCode = 1; + } + }); + + withOutputOption( + submission + .command("runs") + .description("List current and archived submission attempts") + .argument("", "Submission ID") + .option("-u, --url ", "API base URL", getDefaultApiUrl()), + ).action(async (id: string, options) => { + try { + const response = await apiFetch( + options.url, + `/submissions/${encodeURIComponent(id)}/runs`, + ); + if (!response.ok) await throwApiError(response); + const body = await responseJson<{ + runs: Array<{ + _id: string; + attemptNumber: number; + createdAt: string; + requests: unknown[]; + supersededReason?: string; + }>; + }>(response); + const runFields: DisplayField[] = [ + { key: "_id", label: "Submission Run" }, + { key: "attemptNumber", label: "Attempt" }, + { + key: "requests", + label: "Runs", + formatter: (row: Record) => + String((row.requests as unknown[] | undefined)?.length ?? 0), + }, + { key: "supersededReason", label: "Superseded" }, + { key: "createdAt", label: "Created" }, + ]; + console.log( + formatData(body.runs, runFields, (options.output ?? "table") as OutputFormat), + ); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exitCode = 1; + } + }); + + submission + .command("retry") + .description("Retry the current submission processing attempt") + .argument("", "Submission ID") + .requiredOption("--submission-run ", "Current submission run ID") + .option("-u, --url ", "API base URL", getDefaultApiUrl()) + .action(async (id: string, options) => { + try { + const response = await apiFetch( + options.url, + `/submissions/${encodeURIComponent(id)}/retry`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ submissionRunId: options.submissionRun }), + }, + ); + if (!response.ok) await throwApiError(response); + const result = await responseJson<{ + submissionRunId: string; + attemptNumber: number; + readiness: string; + }>(response); + console.log( + successText( + `Submission retry started: attempt ${result.attemptNumber} (${result.submissionRunId}, ${result.readiness})`, + ), + ); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exitCode = 1; + } + }); +} + diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts index 399f12266..6d59d10a6 100644 --- a/apps/cli/src/index.ts +++ b/apps/cli/src/index.ts @@ -25,6 +25,7 @@ import { registerTaskPromptCommands } from "./commands/task-prompt.js"; import { registerProfileCommands } from "./commands/profile.js"; import { registerProjectCommands } from "./commands/project.js"; import { registerSecretCommands } from "./commands/secret.js"; +import { registerSubmissionCommands } from "./commands/submission.js"; import { registerUpdateCommand } from "./commands/update.js"; import { checkForUpdates } from "./utils/update-check.js"; import { errorText } from "./utils/style.js"; @@ -70,6 +71,7 @@ configureHelp(program); registerSecretCommands(program); registerProjectCommands(program); registerRunCommands(program); +registerSubmissionCommands(program); registerCriteriaCommands(program); registerPromptFeatureCommands(program); registerReportCommands(program); diff --git a/apps/portal/src/App.tsx b/apps/portal/src/App.tsx index 26d5f1d74..56e6a2d79 100644 --- a/apps/portal/src/App.tsx +++ b/apps/portal/src/App.tsx @@ -6,6 +6,7 @@ import { Layout } from "@/components/Layout"; import { RunsList } from "@/pages/RunsList"; import { RunDetail } from "@/pages/RunDetail"; import { SubmitRun } from "@/pages/SubmitRun"; +import { SubmissionDetail } from "@/pages/SubmissionDetail"; import { CriteriaList } from "@/pages/CriteriaList"; import { CriterionDetail } from "@/pages/CriterionDetail"; import { CriterionPreviewPanel } from "@/pages/CriterionPreviewPanel"; @@ -85,6 +86,7 @@ export function App() { } /> } /> + } /> }> } /> diff --git a/apps/portal/src/components/SubmissionStatusCard.stories.tsx b/apps/portal/src/components/SubmissionStatusCard.stories.tsx new file mode 100644 index 000000000..04d8a6365 --- /dev/null +++ b/apps/portal/src/components/SubmissionStatusCard.stories.tsx @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { Meta, StoryObj } from "@storybook/react-vite"; +import { SubmissionStatusCard } from "./SubmissionStatusCard"; +import type { Submission } from "@/types"; + +const submission: Submission = { + submissionId: "submission-123", + projectId: "project-123", + requestIds: ["request-1", "request-2"], + createdAt: new Date().toISOString(), + submissionRunId: "submission-run-1", + attemptNumber: 2, + readiness: "processing", + requests: [ + { + requestId: "request-1", + runId: "run-1", + task: "Implement the feature", + status: "done", + outcome: "succeeded", + workerType: "coder-acp-copilot", + }, + { + requestId: "request-2", + runId: "run-2", + task: "Implement the feature", + status: "done", + outcome: "failed", + workerType: "coder-acp-copilot", + }, + ], + dependencyProjection: [ + { + requestId: "request-1", + runId: "run-1", + handlerId: "pp-observations", + state: "ready", + }, + { + requestId: "request-2", + runId: "run-2", + handlerId: "pp-observations", + state: "ready", + }, + ], + blocking: [], + handlerProgress: { + "pp-readout": { status: "processing", version: 1 }, + }, +}; + +const meta = { + title: "Submissions/SubmissionStatusCard", + component: SubmissionStatusCard, + args: { submission }, +} satisfies Meta; + +export default meta; +type Story = StoryObj; + +export const Processing: Story = {}; + +export const Blocked: Story = { + args: { + submission: { + ...submission, + readiness: "blocked", + blocking: [ + { + requestId: "request-2", + runId: "run-2", + handlerId: "pp-observations", + reason: "dependency-failed", + }, + ], + }, + }, +}; + diff --git a/apps/portal/src/components/SubmissionStatusCard.test.tsx b/apps/portal/src/components/SubmissionStatusCard.test.tsx new file mode 100644 index 000000000..8ec046bb0 --- /dev/null +++ b/apps/portal/src/components/SubmissionStatusCard.test.tsx @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// @vitest-environment happy-dom +import { cleanup, render, screen } from "@testing-library/react"; +import { afterEach, describe, expect, it } from "vitest"; +import type { Submission } from "@/types"; +import { SubmissionStatusCard } from "./SubmissionStatusCard"; + +afterEach(cleanup); + +describe("SubmissionStatusCard", () => { + it("renders readiness and exact observation progress", () => { + const submission: Submission = { + submissionId: "submission-1", + projectId: "project-1", + requestIds: ["request-1", "request-2"], + createdAt: new Date().toISOString(), + submissionRunId: "submission-run-1", + attemptNumber: 3, + readiness: "blocked", + requests: [ + { + requestId: "request-1", + runId: "run-1", + task: "Task", + status: "done", + workerType: "worker", + }, + { + requestId: "request-2", + runId: "run-2", + task: "Task", + status: "done", + workerType: "worker", + }, + ], + dependencyProjection: [ + { + requestId: "request-1", + runId: "run-1", + handlerId: "pp-observations", + state: "ready", + }, + { + requestId: "request-2", + runId: "run-2", + handlerId: "pp-observations", + state: "failed", + }, + ], + blocking: [ + { + requestId: "request-2", + runId: "run-2", + handlerId: "pp-observations", + reason: "dependency-failed", + }, + ], + }; + + render(); + + expect(screen.getByText("blocked")).toBeDefined(); + expect(screen.getByText("3")).toBeDefined(); + expect(screen.getByText("1/2")).toBeDefined(); + }); +}); + diff --git a/apps/portal/src/components/SubmissionStatusCard.tsx b/apps/portal/src/components/SubmissionStatusCard.tsx new file mode 100644 index 000000000..6eca73210 --- /dev/null +++ b/apps/portal/src/components/SubmissionStatusCard.tsx @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { Badge } from "@/components/ui/badge"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import type { Submission } from "@/types"; + +const readinessVariant: Record< + Submission["readiness"], + "default" | "secondary" | "destructive" | "outline" +> = { + waiting: "secondary", + ready: "outline", + blocked: "destructive", + processing: "default", + done: "default", + failed: "destructive", +}; + +export function SubmissionStatusCard({ + submission, +}: { + submission: Submission; +}) { + const ready = submission.dependencyProjection.filter( + (dependency) => dependency.state === "ready", + ).length; + return ( + + + Processing + + {submission.readiness} + + + +
+
Attempt
+
{submission.attemptNumber}
+
+
+
Observations ready
+
+ {ready}/{submission.requests.length} +
+
+
+
Blocking runs
+
{submission.blocking.length}
+
+ + + ); +} + diff --git a/apps/portal/src/lib/api.ts b/apps/portal/src/lib/api.ts index 32e1d4a7d..1258ac38e 100644 --- a/apps/portal/src/lib/api.ts +++ b/apps/portal/src/lib/api.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import type { Run, RunState, CriteriaDocument, CriteriaGraphData, GeneratePromptResponse, AnalysisResponse, PromptFeatureDocument, Report, BulkReportStatus, BulkReportSummary, ReportTemplate, ReportTrigger, ReportTemplateSystemPrompt, KeyDocument, KeyValidationResult, CreateKeyRequest, UpdateKeyRequest, CodingAgent, AgentVersion, McpServerDocument, CreateMcpServerRequest, UpdateMcpServerRequest, BulkResubmitOverrides, Insight, InsightWithReference, TaskPrompt, TaskPromptFeatureExtractionResult, Model, FeatureFlag, SkillDocument, SkillSearchResult, SkillDiscoveryResult, SkillRevisionDocument, CodebaseDocument, CodebaseRevisionDocument, CodebaseSourceType, ResourceDocument, ResourceRevisionDocument, ResourceBindingSpec, CreateResourceBody, CreateResourceRevisionBody, ExtensionDocument, ExtensionSearchResult, ExtensionVersionInfo, MdpResponse, AccountDocument, CreateAccountRequest, UpdateAccountRequest, ProfileWithVersion, ProfileVersionDocument, ProfileDocument, RunGroup, RunFacetsResponse, CursorPaginatedResponse, IterationOp, GateConfig, GateId, PromptType, RunSortField, RunSortDir, CriterionKind, TaxonomyElementId, CriterionSubject } from "@/types"; +import type { Run, RunState, CriteriaDocument, CriteriaGraphData, GeneratePromptResponse, AnalysisResponse, PromptFeatureDocument, Report, BulkReportStatus, BulkReportSummary, ReportTemplate, ReportTrigger, ReportTemplateSystemPrompt, KeyDocument, KeyValidationResult, CreateKeyRequest, UpdateKeyRequest, CodingAgent, AgentVersion, McpServerDocument, CreateMcpServerRequest, UpdateMcpServerRequest, BulkResubmitOverrides, Insight, InsightWithReference, TaskPrompt, TaskPromptFeatureExtractionResult, Model, FeatureFlag, SkillDocument, SkillSearchResult, SkillDiscoveryResult, SkillRevisionDocument, CodebaseDocument, CodebaseRevisionDocument, CodebaseSourceType, ResourceDocument, ResourceRevisionDocument, ResourceBindingSpec, CreateResourceBody, CreateResourceRevisionBody, ExtensionDocument, ExtensionSearchResult, ExtensionVersionInfo, MdpResponse, AccountDocument, CreateAccountRequest, UpdateAccountRequest, ProfileWithVersion, ProfileVersionDocument, ProfileDocument, RunGroup, RunFacetsResponse, CursorPaginatedResponse, IterationOp, GateConfig, GateId, PromptType, RunSortField, RunSortDir, CriterionKind, TaxonomyElementId, CriterionSubject, Submission } from "@/types"; import type { Project, CreateProjectRequest, UpdateProjectRequest } from "@/types"; import { qs } from "./url"; @@ -231,7 +231,7 @@ export const api = { gates?: GateConfig[]; codebase?: string; codebaseRevisionId?: string; - }): Promise<(Run & { message: string }) | { ids: string[]; count: number; message: string }> => { + }): Promise<(Run & { message: string; submissionId: string }) | { ids: string[]; count: number; message: string; submissionId: string }> => { const { worker, ...payload } = body; const url = worker ? `/requests?worker=${encodeURIComponent(worker)}` : `/requests`; return request(url, { @@ -240,6 +240,27 @@ export const api = { }, { scoped: true }); }, + listSubmissions: (): Promise<{ submissions: Submission[] }> => + request("/submissions", undefined, { scoped: true }), + + getSubmission: (id: string): Promise => + request(`/submissions/${id}`), + + retrySubmission: ( + id: string, + submissionRunId: string, + ): Promise<{ + submissionId: string; + submissionRunId: string; + attemptNumber: number; + readiness: Submission["readiness"]; + blocking: Submission["blocking"]; + }> => + request(`/submissions/${id}/retry`, { + method: "POST", + body: JSON.stringify({ submissionRunId }), + }), + /** Soft-delete a run */ deleteRun: (id: string): Promise<{ id: string; deleted: boolean }> => { return request(`/requests/${id}`, { method: "DELETE" }); diff --git a/apps/portal/src/pages/RunDetail.tsx b/apps/portal/src/pages/RunDetail.tsx index 8cd9ab00e..9eaed350f 100644 --- a/apps/portal/src/pages/RunDetail.tsx +++ b/apps/portal/src/pages/RunDetail.tsx @@ -531,7 +531,7 @@ export function RunDetail() { (activeRun?.observationResults?.length ?? 0) > 0; const hasConfiguredObservations = (run?.observations?.length ?? 0) > 0; // Aggregate enrichment status across the whole post-processing DAG so the - // "Enriched" badge waits for pp-taxonomy instead of flipping after pp-atif. + // "Enriched" badge waits for pp-observations instead of flipping after pp-atif. const enrichmentStatus = useMemo( () => deriveEnrichmentStatus(activeRun, hasConfiguredObservations), [activeRun, hasConfiguredObservations], @@ -905,6 +905,19 @@ export function RunDetail() { /> )} + {run.submissionId && ( + + {run.submissionId.slice(0, 8)} + + )} + /> + )} {run.maxIterations && } {(() => { const totalDuration = activeRun?.turns?.reduce((sum, t) => sum + (t.durationMs ?? 0), 0); diff --git a/apps/portal/src/pages/RunsList.tsx b/apps/portal/src/pages/RunsList.tsx index 27b685df5..c7257fc97 100644 --- a/apps/portal/src/pages/RunsList.tsx +++ b/apps/portal/src/pages/RunsList.tsx @@ -1350,7 +1350,12 @@ export function RunsList() { hidden: columnVisibility.isHidden("submission"), cell: (r) => r.submissionId ? ( - + event.stopPropagation()} + > + + ) : ( — ), @@ -2666,31 +2671,36 @@ export function RunsList() { ? (profileNameById.get(groupKey) ?? formatId(groupKey)) : groupKey; return ( -
+ {groupBy === "submissionId" && groupKey !== "no-submission" && ( + + View submission + + )} {runs.length} run{runs.length !== 1 ? "s" : ""} - - {expanded ? "Click to collapse" : "Click to expand"} -
- + ); }, }} diff --git a/apps/portal/src/pages/SubmissionDetail.tsx b/apps/portal/src/pages/SubmissionDetail.tsx new file mode 100644 index 000000000..5dd5cab46 --- /dev/null +++ b/apps/portal/src/pages/SubmissionDetail.tsx @@ -0,0 +1,164 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { ArrowLeft, FileText, RotateCcw } from "lucide-react"; +import { Link, useParams } from "react-router-dom"; +import { toast } from "sonner"; +import { SubmissionStatusCard } from "@/components/SubmissionStatusCard"; +import { ShortId } from "@/components/ShortId"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { api } from "@/lib/api"; +import { formatDate } from "@/lib/utils"; + +export function SubmissionDetail() { + const { id } = useParams<{ id: string }>(); + const queryClient = useQueryClient(); + const submissionQuery = useQuery({ + queryKey: ["submission", id], + queryFn: () => api.getSubmission(id!), + enabled: Boolean(id), + refetchInterval: (query) => { + const readiness = query.state.data?.readiness; + return readiness === "done" || readiness === "failed" || readiness === "blocked" + ? false + : 5_000; + }, + }); + const retry = useMutation({ + mutationFn: () => + api.retrySubmission( + submissionQuery.data!.submissionId, + submissionQuery.data!.submissionRunId, + ), + onSuccess: () => { + toast.success("Submission processing restarted"); + void queryClient.invalidateQueries({ queryKey: ["submission", id] }); + }, + onError: (error) => { + toast.error(error instanceof Error ? error.message : "Retry failed"); + }, + }); + + if (submissionQuery.isLoading) { + return
Loading submission…
; + } + if (submissionQuery.isError || !submissionQuery.data) { + return ( +
+

Submission could not be loaded.

+
+ ); + } + + const submission = submissionQuery.data; + return ( +
+
+
+ +
+

Submission

+ +
+

+ Created {formatDate(submission.createdAt)} +

+
+ +
+ + + + {submission.blocking.length > 0 && ( + + + Blocking dependencies + + + {submission.blocking.map((blocking) => ( +
+ + {blocking.requestId.slice(0, 8)} + + {blocking.reason} + + {blocking.runId} + +
+ ))} + + + )} + + + + Runs + + + {submission.requests.map((request) => { + const dependency = submission.dependencyProjection.find( + (item) => + item.requestId === request.requestId && item.runId === request.runId, + ); + return ( + +
+
+ + {request.task} +
+
+ {request.requestId} · {request.runId} +
+
+
+ {request.status} + + {dependency?.state ?? "waiting"} + +
+ + ); + })} + + +
+ ); +} + diff --git a/apps/portal/src/pages/SubmitRun.tsx b/apps/portal/src/pages/SubmitRun.tsx index ec6b639ea..b6c51baeb 100644 --- a/apps/portal/src/pages/SubmitRun.tsx +++ b/apps/portal/src/pages/SubmitRun.tsx @@ -667,13 +667,7 @@ export function SubmitRun() { const submitMutation = useMutation({ mutationFn: api.submitRun, onSuccess: (data) => { - if ("ids" in data && data.ids.length > 1) { - navigate("/runs"); - } else if ("id" in data) { - navigate(`/runs/${data.id}`); - } else { - navigate("/runs"); - } + navigate(`/submissions/${data.submissionId}`); }, }); diff --git a/apps/portal/src/types.ts b/apps/portal/src/types.ts index 51a91eae0..1ee0d61f0 100644 --- a/apps/portal/src/types.ts +++ b/apps/portal/src/types.ts @@ -126,7 +126,7 @@ export interface OsInfo { /** * Per-handler post-processing status in the DAG (issue #1147). Keys in - * `RunState.handlerStatus` are handler IDs (e.g. "pp-atif", "pp-taxonomy"). + * `RunState.handlerStatus` are handler IDs (e.g. "pp-atif", "pp-observations"). */ export interface HandlerRunStatus { status: "queued" | "processing" | "done" | "failed"; @@ -173,7 +173,7 @@ export interface RunState { postProcessorStatus?: "queued" | "processing" | "done" | "failed"; /** * Per-handler post-processing status keyed by handler ID (e.g. "pp-atif", - * "pp-taxonomy"). Drives the aggregate "Enriched" badge so it only flips to + * "pp-observations"). Drives the aggregate "Enriched" badge so it only flips to * done once every handler in the DAG is terminal — not after the first * handler (pp-atif) completes. The legacy `postProcessorStatus` scalar only * tracks pp-atif and is kept for backward compatibility. @@ -225,6 +225,71 @@ export interface Run { gateSummaries?: GateRunSummary[]; } +export type SubmissionReadiness = + | "waiting" + | "ready" + | "blocked" + | "processing" + | "done" + | "failed"; + +export interface SubmissionDependencyProjection { + requestId: string; + runId: string; + handlerId: string; + state: "waiting" | "ready" | "failed" | "blocked"; + version?: number; + updatedAt?: string; + error?: string; +} + +export interface SubmissionBlockingDetail { + requestId: string; + runId: string; + handlerId: string; + reason: + | "request-missing" + | "run-changed" + | "dependency-failed" + | "dependency-blocked"; + error?: string; +} + +export interface SubmissionRequestSummary { + requestId: string; + runId: string; + task: string; + status: string; + outcome?: string; + workerType: string; + model?: string; + profileId?: string; +} + +export interface Submission { + submissionId: string; + projectId: string; + requestIds: string[]; + createdAt: string; + updatedAt?: string; + submissionRunId: string; + attemptNumber: number; + readiness: SubmissionReadiness; + requests: SubmissionRequestSummary[]; + dependencyProjection: SubmissionDependencyProjection[]; + blocking: SubmissionBlockingDetail[]; + handlerProgress?: Record< + string, + { + status: "queued" | "processing" | "done" | "failed"; + version?: number; + updatedAt?: string; + error?: string; + } + >; + report?: { reportId?: string; url?: string; updatedAt?: string }; +} + export interface CursorPaginatedResponse { data: T[]; limit: number; From a88da9788be4c0f3fcaf6c46fa71edfd5c797923 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Tue, 13 Oct 2026 16:26:42 +0200 Subject: [PATCH 04/25] style(submissions): normalize file endings Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 99f84f04-d3b0-40f1-a254-4dfbd21e4d87 --- apps/cli/src/commands/submission.ts | 2 -- apps/portal/src/components/SubmissionStatusCard.stories.tsx | 2 -- apps/portal/src/components/SubmissionStatusCard.test.tsx | 1 - apps/portal/src/components/SubmissionStatusCard.tsx | 1 - apps/portal/src/pages/SubmissionDetail.tsx | 2 -- .../db-migrations/src/034-create-submission-indexes.test.ts | 1 - 6 files changed, 9 deletions(-) diff --git a/apps/cli/src/commands/submission.ts b/apps/cli/src/commands/submission.ts index 52b6c4b7a..835e7f3ed 100644 --- a/apps/cli/src/commands/submission.ts +++ b/apps/cli/src/commands/submission.ts @@ -34,7 +34,6 @@ interface SubmissionSummary { reason: string; }>; } - async function responseJson(response: Response): Promise { return response.json() as Promise; } @@ -203,4 +202,3 @@ export function registerSubmissionCommands(program: Command): void { } }); } - diff --git a/apps/portal/src/components/SubmissionStatusCard.stories.tsx b/apps/portal/src/components/SubmissionStatusCard.stories.tsx index 04d8a6365..0c465e7e9 100644 --- a/apps/portal/src/components/SubmissionStatusCard.stories.tsx +++ b/apps/portal/src/components/SubmissionStatusCard.stories.tsx @@ -50,7 +50,6 @@ const submission: Submission = { "pp-readout": { status: "processing", version: 1 }, }, }; - const meta = { title: "Submissions/SubmissionStatusCard", component: SubmissionStatusCard, @@ -78,4 +77,3 @@ export const Blocked: Story = { }, }, }; - diff --git a/apps/portal/src/components/SubmissionStatusCard.test.tsx b/apps/portal/src/components/SubmissionStatusCard.test.tsx index 8ec046bb0..b735e762e 100644 --- a/apps/portal/src/components/SubmissionStatusCard.test.tsx +++ b/apps/portal/src/components/SubmissionStatusCard.test.tsx @@ -66,4 +66,3 @@ describe("SubmissionStatusCard", () => { expect(screen.getByText("1/2")).toBeDefined(); }); }); - diff --git a/apps/portal/src/components/SubmissionStatusCard.tsx b/apps/portal/src/components/SubmissionStatusCard.tsx index 6eca73210..f711bdd1c 100644 --- a/apps/portal/src/components/SubmissionStatusCard.tsx +++ b/apps/portal/src/components/SubmissionStatusCard.tsx @@ -52,4 +52,3 @@ export function SubmissionStatusCard({ ); } - diff --git a/apps/portal/src/pages/SubmissionDetail.tsx b/apps/portal/src/pages/SubmissionDetail.tsx index 5dd5cab46..a4a7b67bf 100644 --- a/apps/portal/src/pages/SubmissionDetail.tsx +++ b/apps/portal/src/pages/SubmissionDetail.tsx @@ -52,7 +52,6 @@ export function SubmissionDetail() { ); } - const submission = submissionQuery.data; return (
@@ -161,4 +160,3 @@ export function SubmissionDetail() {
); } - diff --git a/packages/db-migrations/src/034-create-submission-indexes.test.ts b/packages/db-migrations/src/034-create-submission-indexes.test.ts index 1fd59c8bd..efcd2017f 100644 --- a/packages/db-migrations/src/034-create-submission-indexes.test.ts +++ b/packages/db-migrations/src/034-create-submission-indexes.test.ts @@ -49,4 +49,3 @@ describe("migration 034: CreateSubmissionIndexes", () => { ); }); }); - From 57c76344b18e886986375cee8e815d562c577d82 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Thu, 15 Oct 2026 22:11:48 -0700 Subject: [PATCH 05/25] feat(shared): add canonical experiment storage Introduce the Experiment aggregate, bounded request payload schemas, deterministic Cosmos-safe chunk routing/layout, persistence CAS and repair primitives, focused tests, and migration indexes. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 8fe76b9e-1205-4910-b51b-e8fa51538125 --- docs/architecture/db-migrations.md | 1 + ...-create-experiment-storage-indexes.test.ts | 42 ++ .../034-create-experiment-storage-indexes.ts | 41 ++ .../db-migrations/src/required-migrations.ts | 2 +- packages/shared/package.json | 5 + .../src/experiments/experiment-store.test.ts | 193 +++++ .../src/experiments/experiment-store.ts | 683 ++++++++++++++++++ .../experiments/fixed-payload-store.test.ts | 130 ++++ .../src/experiments/fixed-payload-store.ts | 316 ++++++++ packages/shared/src/experiments/index.ts | 9 + .../shared/src/experiments/layout.test.ts | 112 +++ packages/shared/src/experiments/layout.ts | 123 ++++ .../shared/src/experiments/routing.test.ts | 34 + packages/shared/src/experiments/routing.ts | 51 ++ .../shared/src/experiments/schemas.test.ts | 80 ++ packages/shared/src/experiments/schemas.ts | 170 +++++ packages/shared/src/experiments/types.ts | 149 ++++ packages/shared/src/index.ts | 2 +- 18 files changed, 2141 insertions(+), 2 deletions(-) create mode 100644 packages/db-migrations/src/034-create-experiment-storage-indexes.test.ts create mode 100644 packages/db-migrations/src/migrations/034-create-experiment-storage-indexes.ts create mode 100644 packages/shared/src/experiments/experiment-store.test.ts create mode 100644 packages/shared/src/experiments/experiment-store.ts create mode 100644 packages/shared/src/experiments/fixed-payload-store.test.ts create mode 100644 packages/shared/src/experiments/fixed-payload-store.ts create mode 100644 packages/shared/src/experiments/index.ts create mode 100644 packages/shared/src/experiments/layout.test.ts create mode 100644 packages/shared/src/experiments/layout.ts create mode 100644 packages/shared/src/experiments/routing.test.ts create mode 100644 packages/shared/src/experiments/routing.ts create mode 100644 packages/shared/src/experiments/schemas.test.ts create mode 100644 packages/shared/src/experiments/schemas.ts create mode 100644 packages/shared/src/experiments/types.ts diff --git a/docs/architecture/db-migrations.md b/docs/architecture/db-migrations.md index d01bcc5aa..fc7cfd9a8 100644 --- a/docs/architecture/db-migrations.md +++ b/docs/architecture/db-migrations.md @@ -118,6 +118,7 @@ When you run a command: | `031-migrate-services-to-handler-dag` | Replaces the legacy single `post-processor` service row with per-handler service registrations and dependencies | | `032-add-handler-status-index` | Adds a handler-specific dispatch index for `pp-atif` polling | | `033-backfill-handler-status` | Backfills `run.handlerStatus.pp-atif` from deprecated `run.postProcessorStatus` / `run.postProcessorVersion` fields | +| `034-create-experiment-storage-indexes` | Adds the project paging and immutable execution-history indexes for canonical Experiment storage; request payload chunks remain point-read by deterministic `_id` only | ### Migration 029: users identity uniqueness diff --git a/packages/db-migrations/src/034-create-experiment-storage-indexes.test.ts b/packages/db-migrations/src/034-create-experiment-storage-indexes.test.ts new file mode 100644 index 000000000..c180b94ad --- /dev/null +++ b/packages/db-migrations/src/034-create-experiment-storage-indexes.test.ts @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { Db } from "mongodb"; +import { describe, expect, it, vi } from "vitest"; +import { CreateExperimentStorageIndexes } from "./migrations/034-create-experiment-storage-indexes.js"; + +function collection() { + return { createIndex: vi.fn(async () => "index") }; +} + +describe("migration 034: experiment storage indexes", () => { + it("creates only project paging and execution history indexes", async () => { + const experiments = collection(); + const executions = collection(); + const chunks = collection(); + const db = { + collection: vi.fn((name: string) => { + if (name === "experiments") return experiments; + if (name === "experiment-executions") return executions; + if (name === "experiment-chunks") return chunks; + throw new Error(`Unexpected collection ${name}`); + }), + } as unknown as Db; + + await new CreateExperimentStorageIndexes().up(db); + + expect(experiments.createIndex).toHaveBeenCalledWith( + { projectId: 1, _id: 1 }, + {}, + ); + expect(executions.createIndex).toHaveBeenCalledWith( + { projectId: 1 }, + {}, + ); + expect(executions.createIndex).toHaveBeenCalledWith( + { experimentId: 1, attemptNumber: -1 }, + {}, + ); + expect(chunks.createIndex).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/db-migrations/src/migrations/034-create-experiment-storage-indexes.ts b/packages/db-migrations/src/migrations/034-create-experiment-storage-indexes.ts new file mode 100644 index 000000000..aa643319e --- /dev/null +++ b/packages/db-migrations/src/migrations/034-create-experiment-storage-indexes.ts @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { Db } from "mongodb"; +import type { MigrationInterface } from "mongo-migrate-ts"; +import { ensureIndex } from "../cosmos-index-helpers.js"; + +const TAG = "034"; + +export class CreateExperimentStorageIndexes implements MigrationInterface { + async up(db: Db): Promise { + await ensureIndex( + db.collection("experiments"), + { projectId: 1, _id: 1 }, + {}, + "experiments", + TAG, + ); + await ensureIndex( + db.collection("experiment-executions"), + { projectId: 1 }, + {}, + "experiment-executions", + TAG, + ); + await ensureIndex( + db.collection("experiment-executions"), + { experimentId: 1, attemptNumber: -1 }, + {}, + "experiment-executions", + TAG, + ); + console.log(`[${TAG}] Experiment storage indexes complete`); + } + + async down(_db: Db): Promise { + console.log( + `[${TAG}-down] Skipping index changes - drop experiment indexes manually if needed`, + ); + } +} diff --git a/packages/db-migrations/src/required-migrations.ts b/packages/db-migrations/src/required-migrations.ts index bee754391..8d513270e 100644 --- a/packages/db-migrations/src/required-migrations.ts +++ b/packages/db-migrations/src/required-migrations.ts @@ -46,7 +46,7 @@ export const REQUIRED_MIGRATIONS: readonly string[] = [ "031-migrate-services-to-handler-dag.ts", "032-add-handler-status-index.ts", "033-backfill-handler-status.ts", - "034-create-submission-indexes.ts", + "034-create-experiment-storage-indexes.ts", "021-backfill-criteria-kind.ts", "022-backfill-criteria-subject.ts", ]; diff --git a/packages/shared/package.json b/packages/shared/package.json index db86b934f..155cb5643 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -44,6 +44,11 @@ "source": "./src/criteria/criteria-provider-factory.ts", "types": "./dist/criteria/criteria-provider-factory.d.ts", "import": "./dist/criteria/criteria-provider-factory.js" + }, + "./experiments": { + "source": "./src/experiments/index.ts", + "types": "./dist/experiments/index.d.ts", + "import": "./dist/experiments/index.js" } }, "scripts": { diff --git a/packages/shared/src/experiments/experiment-store.test.ts b/packages/shared/src/experiments/experiment-store.test.ts new file mode 100644 index 000000000..ef925baa0 --- /dev/null +++ b/packages/shared/src/experiments/experiment-store.test.ts @@ -0,0 +1,193 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { Collection, Document } from "mongodb"; +import { describe, expect, it } from "vitest"; +import { ExperimentStore } from "./experiment-store.js"; +import { + EXPERIMENT_STORAGE_PROFILE_VERSION, + type ExperimentDocument, + type ExperimentExecutionHistoryDocument, +} from "./types.js"; + +function getPath(source: object, path: string): unknown { + let value: unknown = source; + for (const segment of path.split(".")) { + if (typeof value !== "object" || value === null) return undefined; + value = (value as Record)[segment]; + } + return value; +} + +function setPath(target: object, path: string, value: unknown): void { + const segments = path.split("."); + let current = target as Record; + for (const segment of segments.slice(0, -1)) { + const next = current[segment]; + if (typeof next !== "object" || next === null) { + current[segment] = {}; + } + current = current[segment] as Record; + } + current[segments.at(-1)!] = value; +} + +function matches(document: object, filter: Document): boolean { + return Object.entries(filter).every(([path, expected]) => { + const actual = getPath(document, path); + if ( + typeof expected === "object" && + expected !== null && + "$exists" in expected + ) { + return ( + (actual !== undefined) === + Boolean((expected as { $exists: unknown }).$exists) + ); + } + return actual === expected; + }); +} + +function atomicCollection(initial: ExperimentDocument) { + let document = structuredClone(initial); + return { + current: () => structuredClone(document), + async findOne(filter: Document) { + return matches(document, filter) ? structuredClone(document) : null; + }, + async findOneAndUpdate( + filter: Document, + update: { + $set?: Record; + $inc?: Record; + }, + ) { + if (!matches(document, filter)) return null; + const next = structuredClone(document); + for (const [path, value] of Object.entries(update.$set ?? {})) { + setPath(next, path, value); + } + for (const [path, increment] of Object.entries(update.$inc ?? {})) { + const value = getPath(next, path); + if (typeof value !== "number") throw new Error(`Non-number ${path}`); + setPath(next, path, value + increment); + } + document = next; + return structuredClone(document); + }, + }; +} + +function experiment(): ExperimentDocument { + const now = new Date("2026-01-01T00:00:00.000Z"); + return { + _id: "experiment-1", + experimentId: "experiment-1", + projectId: "project-1", + metadata: { name: "Experiment" }, + storageProfileVersion: EXPERIMENT_STORAGE_PROFILE_VERSION, + stateVersion: 2, + revision: { + experimentRevisionId: "revision-1", + revisionNumber: 1, + memberCount: 1, + membershipFingerprint: "membership", + definitionFingerprint: "definition", + createdAt: now, + }, + execution: { + experimentExecutionId: "execution-1", + attemptNumber: 1, + version: 1, + requests: { + storage: "inline", + entryCount: 1, + payloadFingerprint: "payload", + entries: [ + { + requestId: "request-1", + runId: "run-1", + dependencies: [], + blockers: [], + }, + ], + }, + vectorFingerprint: "vector", + handlerSummary: { + report: { status: "queued", version: 1, updatedAt: now }, + }, + blocking: [], + createdAt: now, + updatedAt: now, + }, + createdAt: now, + updatedAt: now, + finalizedAt: now, + }; +} + +describe("ExperimentStore CAS contracts", () => { + it("allows exactly one winner in a writer-freezer race", async () => { + const experiments = atomicCollection(experiment()); + const store = new ExperimentStore({ + experiments: + experiments as unknown as Collection, + executions: + {} as Collection, + chunks: {} as Collection, + }); + + const [writer, freezer] = await Promise.all([ + store.claimHandler({ + projectId: "project-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-1", + expectedExecutionVersion: 1, + handlerId: "report", + handlerVersion: 1, + }), + store.freezeCurrent({ + projectId: "project-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-1", + expectedExecutionVersion: 1, + }), + ]); + + expect([writer, freezer].filter(Boolean)).toHaveLength(1); + expect(experiments.current().execution.version).toBe(2); + expect(experiments.current().stateVersion).toBe(3); + }); + + it("requires project, execution identity, and version on claims", async () => { + const experiments = atomicCollection(experiment()); + const store = new ExperimentStore({ + experiments: + experiments as unknown as Collection, + executions: + {} as Collection, + chunks: {} as Collection, + }); + await expect( + store.claimHandler({ + projectId: "wrong-project", + experimentId: "experiment-1", + experimentExecutionId: "execution-1", + expectedExecutionVersion: 1, + handlerId: "report", + handlerVersion: 1, + }), + ).resolves.toBeNull(); + await expect( + store.claimHandler({ + projectId: "project-1", + experimentId: "experiment-1", + experimentExecutionId: "wrong-execution", + expectedExecutionVersion: 1, + handlerId: "report", + handlerVersion: 1, + }), + ).resolves.toBeNull(); + }); +}); diff --git a/packages/shared/src/experiments/experiment-store.ts b/packages/shared/src/experiments/experiment-store.ts new file mode 100644 index 000000000..43b13ca7f --- /dev/null +++ b/packages/shared/src/experiments/experiment-store.ts @@ -0,0 +1,683 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { Collection, Document } from "mongodb"; +import { FixedPayloadStore } from "./fixed-payload-store.js"; +import { + ExperimentDocumentSchema, + ExperimentExecutionHistoryDocumentSchema, + ExperimentExecutionSchema, + ExperimentMetadataSchema, + ExperimentRequestStateSchema, + ExperimentRevisionSchema, +} from "./schemas.js"; +import { fingerprintExperimentPayload } from "./routing.js"; +import { + EXPERIMENT_STORAGE_PROFILE_VERSION, + type ExperimentDocument, + type ExperimentExecution, + type ExperimentExecutionHistoryDocument, + type ExperimentHandlerStatus, + type ExperimentMetadata, + type ExperimentPage, + type ExperimentRequestState, + type ExperimentRevision, + type FixedPayload, +} from "./types.js"; + +export interface ExperimentCollections { + experiments: Collection; + executions: Collection; + chunks: Collection; +} + +export interface CreateExperimentInput { + experimentId: string; + projectId: string; + metadata: ExperimentMetadata; + revision: ExperimentRevision; + experimentExecutionId: string; + requests: readonly ExperimentRequestState[]; + handlerSummary?: ExperimentExecution["handlerSummary"]; + blocking?: ExperimentExecution["blocking"]; + now?: Date; +} + +export interface RolloverExperimentInput { + projectId: string; + experimentId: string; + expectedExperimentExecutionId: string; + expectedExecutionVersion: number; + nextExperimentExecutionId: string; + requests: readonly ExperimentRequestState[]; + reason: "rollover" | "retry"; + now?: Date; +} + +function parseExperiment(value: unknown): ExperimentDocument { + return ExperimentDocumentSchema.parse(value) as ExperimentDocument; +} + +function parseExecution(value: unknown): ExperimentExecution { + return ExperimentExecutionSchema.parse(value) as ExperimentExecution; +} + +function parseHistory(value: unknown): ExperimentExecutionHistoryDocument { + return ExperimentExecutionHistoryDocumentSchema.parse( + value, + ) as ExperimentExecutionHistoryDocument; +} + +function assertHandlerId(handlerId: string): void { + if ( + handlerId.length === 0 || + handlerId.length > 200 || + handlerId.includes(".") || + handlerId.startsWith("$") + ) { + throw new Error(`Invalid experiment handler id '${handlerId}'`); + } +} + +function comparable(value: unknown): string { + return JSON.stringify(value, (_key, item: unknown) => + item instanceof Date ? item.toISOString() : item, + ); +} + +export class ExperimentStore { + private readonly payloads: FixedPayloadStore; + + constructor(private readonly collections: ExperimentCollections) { + this.payloads = new FixedPayloadStore(collections.chunks); + } + + async createPending(input: CreateExperimentInput): Promise { + const now = input.now ?? new Date(); + const metadata = ExperimentMetadataSchema.parse( + input.metadata, + ) as ExperimentMetadata; + const revision = ExperimentRevisionSchema.parse( + input.revision, + ) as ExperimentRevision; + const requests = input.requests.map((request) => + ExperimentRequestStateSchema.parse(request), + ) as ExperimentRequestState[]; + if (revision.memberCount !== requests.length) { + throw new Error( + `Revision memberCount ${revision.memberCount} does not match request count ${requests.length}`, + ); + } + + const identity = { + projectId: input.projectId, + experimentId: input.experimentId, + ownerId: input.experimentExecutionId, + generation: 1, + }; + const shell = (payload: FixedPayload): object => ({ + _id: input.experimentId, + experimentId: input.experimentId, + projectId: input.projectId, + metadata, + storageProfileVersion: EXPERIMENT_STORAGE_PROFILE_VERSION, + stateVersion: 1, + revision, + execution: { + experimentExecutionId: input.experimentExecutionId, + attemptNumber: 1, + version: 1, + requests: payload, + vectorFingerprint: fingerprintExperimentPayload(requests), + handlerSummary: input.handlerSummary ?? {}, + blocking: input.blocking ?? [], + createdAt: now, + updatedAt: now, + }, + createdAt: now, + updatedAt: now, + }); + const payload = await this.payloads.prepareRequests({ + identity, + entries: requests, + inlineDocument: shell, + now, + }); + const document = parseExperiment(shell(payload)); + await this.collections.experiments.updateOne( + { _id: document._id } as Document, + { $setOnInsert: document } as Document, + { upsert: true }, + ); + const raw: unknown = await this.collections.experiments.findOne({ + _id: document._id, + } as Document); + if (!raw) { + throw new Error(`Experiment ${document.experimentId} was not persisted`); + } + const persisted = parseExperiment(raw); + if ( + persisted.projectId !== document.projectId || + persisted.execution.experimentExecutionId !== + document.execution.experimentExecutionId || + persisted.execution.vectorFingerprint !== + document.execution.vectorFingerprint || + persisted.revision.experimentRevisionId !== + document.revision.experimentRevisionId || + persisted.revision.revisionNumber !== document.revision.revisionNumber || + persisted.revision.membershipFingerprint !== + document.revision.membershipFingerprint || + persisted.revision.definitionFingerprint !== + document.revision.definitionFingerprint || + persisted.storageProfileVersion !== EXPERIMENT_STORAGE_PROFILE_VERSION + ) { + throw new Error( + `Experiment ${document.experimentId} already exists with different immutable identity`, + ); + } + return persisted; + } + + async finalizeCreation(input: { + projectId: string; + experimentId: string; + expectedExperimentExecutionId: string; + expectedStateVersion: number; + now?: Date; + }): Promise { + const current = await this.getRaw(input.projectId, input.experimentId); + if ( + !current || + current.execution.experimentExecutionId !== + input.expectedExperimentExecutionId + ) { + return null; + } + if (current.finalizedAt) return current; + if (current.stateVersion !== input.expectedStateVersion) return null; + await this.payloads.verifyCompleteSet( + this.executionIdentity(current), + current.execution.requests, + ); + const now = input.now ?? new Date(); + const updated = await this.collections.experiments.findOneAndUpdate( + { + _id: input.experimentId, + projectId: input.projectId, + stateVersion: input.expectedStateVersion, + "execution.experimentExecutionId": + input.expectedExperimentExecutionId, + finalizedAt: { $exists: false }, + } as Document, + { + $set: { finalizedAt: now, updatedAt: now }, + $inc: { stateVersion: 1 }, + } as Document, + { returnDocument: "after" }, + ); + if (updated) return parseExperiment(updated); + const raced = await this.getRaw(input.projectId, input.experimentId); + return raced?.finalizedAt ? raced : null; + } + + async get( + projectId: string, + experimentId: string, + ): Promise { + const raw: unknown = await this.collections.experiments.findOne({ + _id: experimentId, + projectId, + finalizedAt: { $exists: true }, + } as Document); + return raw ? parseExperiment(raw) : null; + } + + async listPage(input: { + projectId: string; + limit?: number; + afterExperimentId?: string; + }): Promise { + const limit = Math.min(Math.max(input.limit ?? 50, 1), 100); + const filter: Document = { + projectId: input.projectId, + finalizedAt: { $exists: true }, + ...(input.afterExperimentId + ? { _id: { $gt: input.afterExperimentId } } + : {}), + }; + const raw = await this.collections.experiments + .find(filter) + .sort({ _id: 1 }) + .limit(limit + 1) + .toArray(); + const parsed = raw.map(parseExperiment); + const hasMore = parsed.length > limit; + const experiments = parsed.slice(0, limit); + return { + experiments, + ...(hasMore + ? { nextCursor: experiments.at(-1)?.experimentId } + : {}), + }; + } + + async getExecution(input: { + projectId: string; + experimentId: string; + experimentExecutionId: string; + }): Promise { + const experiment = await this.get(input.projectId, input.experimentId); + if ( + experiment?.execution.experimentExecutionId === + input.experimentExecutionId + ) { + return experiment.execution; + } + const raw: unknown = await this.collections.executions.findOne({ + _id: input.experimentExecutionId, + projectId: input.projectId, + experimentId: input.experimentId, + } as Document); + return raw ? parseHistory(raw) : null; + } + + async listExecutionHistory(input: { + projectId: string; + experimentId: string; + limit?: number; + }): Promise { + const limit = Math.min(Math.max(input.limit ?? 50, 1), 100); + const raw = await this.collections.executions + .find({ + projectId: input.projectId, + experimentId: input.experimentId, + } as Document) + .sort({ attemptNumber: -1 }) + .limit(limit) + .toArray(); + return raw.map(parseHistory); + } + + async claimHandler(input: { + projectId: string; + experimentId: string; + experimentExecutionId: string; + expectedExecutionVersion: number; + handlerId: string; + handlerVersion: number; + now?: Date; + }): Promise { + assertHandlerId(input.handlerId); + const now = input.now ?? new Date(); + const path = `execution.handlerSummary.${input.handlerId}`; + const updated = await this.collections.experiments.findOneAndUpdate( + { + _id: input.experimentId, + projectId: input.projectId, + finalizedAt: { $exists: true }, + "execution.experimentExecutionId": + input.experimentExecutionId, + "execution.version": input.expectedExecutionVersion, + "execution.frozenAt": { $exists: false }, + "execution.rollover": { $exists: false }, + [`${path}.status`]: "queued", + [`${path}.version`]: input.handlerVersion, + } as Document, + { + $set: { + [path]: { + status: "processing", + version: input.handlerVersion, + updatedAt: now, + }, + "execution.startedAt": now, + "execution.updatedAt": now, + updatedAt: now, + }, + $inc: { "execution.version": 1, stateVersion: 1 }, + } as Document, + { returnDocument: "after" }, + ); + return updated ? parseExperiment(updated) : null; + } + + async completeHandler(input: { + projectId: string; + experimentId: string; + experimentExecutionId: string; + expectedExecutionVersion: number; + handlerId: string; + handlerVersion: number; + status: Extract; + error?: string; + now?: Date; + }): Promise { + assertHandlerId(input.handlerId); + const now = input.now ?? new Date(); + const path = `execution.handlerSummary.${input.handlerId}`; + const updated = await this.collections.experiments.findOneAndUpdate( + { + _id: input.experimentId, + projectId: input.projectId, + "execution.experimentExecutionId": + input.experimentExecutionId, + "execution.version": input.expectedExecutionVersion, + "execution.rollover": { $exists: false }, + "execution.frozenAt": { $exists: false }, + [`${path}.status`]: "processing", + [`${path}.version`]: input.handlerVersion, + } as Document, + { + $set: { + [path]: { + status: input.status, + version: input.handlerVersion, + updatedAt: now, + ...(input.error ? { error: input.error } : {}), + }, + "execution.updatedAt": now, + "execution.finishedAt": now, + updatedAt: now, + }, + $inc: { "execution.version": 1, stateVersion: 1 }, + } as Document, + { returnDocument: "after" }, + ); + return updated ? parseExperiment(updated) : null; + } + + async freezeCurrent(input: { + projectId: string; + experimentId: string; + experimentExecutionId: string; + expectedExecutionVersion: number; + now?: Date; + }): Promise { + const now = input.now ?? new Date(); + const updated = await this.collections.experiments.findOneAndUpdate( + { + _id: input.experimentId, + projectId: input.projectId, + "execution.experimentExecutionId": + input.experimentExecutionId, + "execution.version": input.expectedExecutionVersion, + "execution.frozenAt": { $exists: false }, + "execution.rollover": { $exists: false }, + } as Document, + { + $set: { + "execution.frozenAt": now, + "execution.updatedAt": now, + updatedAt: now, + }, + $inc: { "execution.version": 1, stateVersion: 1 }, + } as Document, + { returnDocument: "after" }, + ); + return updated ? parseExperiment(updated) : null; + } + + async archiveCurrent(input: { + projectId: string; + experimentId: string; + experimentExecutionId: string; + now?: Date; + }): Promise { + const experiment = await this.getRaw(input.projectId, input.experimentId); + if ( + !experiment || + experiment.execution.experimentExecutionId !== + input.experimentExecutionId || + !experiment.execution.frozenAt + ) { + return null; + } + return this.archive(experiment, input.now ?? new Date()); + } + + async rollover(input: RolloverExperimentInput): Promise { + const experiment = await this.getRaw(input.projectId, input.experimentId); + if ( + !experiment || + experiment.execution.experimentExecutionId !== + input.expectedExperimentExecutionId || + experiment.execution.version !== input.expectedExecutionVersion || + !experiment.execution.frozenAt + ) { + return null; + } + const now = input.now ?? new Date(); + const requests = input.requests.map((request) => + ExperimentRequestStateSchema.parse(request), + ) as ExperimentRequestState[]; + const generation = experiment.execution.attemptNumber + 1; + const payload = await this.payloads.prepareRequests({ + identity: { + projectId: input.projectId, + experimentId: input.experimentId, + ownerId: input.nextExperimentExecutionId, + generation, + }, + entries: requests, + inlineDocument: (candidate) => ({ + ...experiment, + execution: this.nextExecution( + experiment.execution, + input.nextExperimentExecutionId, + candidate, + requests, + now, + ), + }), + now, + }); + const nextExecution = this.nextExecution( + experiment.execution, + input.nextExperimentExecutionId, + payload, + requests, + now, + ); + const claimed = await this.collections.experiments.findOneAndUpdate( + { + _id: input.experimentId, + projectId: input.projectId, + "execution.experimentExecutionId": + input.expectedExperimentExecutionId, + "execution.version": input.expectedExecutionVersion, + "execution.frozenAt": { $exists: true }, + "execution.rollover": { $exists: false }, + } as Document, + { + $set: { + "execution.rollover": { + nextExecution, + reason: input.reason, + claimedAt: now, + }, + "execution.updatedAt": now, + updatedAt: now, + }, + $inc: { "execution.version": 1, stateVersion: 1 }, + } as Document, + { returnDocument: "after" }, + ); + if (!claimed) return null; + return this.repairRollover(input.projectId, input.experimentId); + } + + async retryCurrent(input: Omit): Promise { + const experiment = await this.getRaw(input.projectId, input.experimentId); + if ( + !experiment || + experiment.execution.experimentExecutionId !== + input.expectedExperimentExecutionId + ) { + return null; + } + const requests = await this.payloads.readRequests( + this.executionIdentity(experiment), + experiment.execution.requests, + ); + return this.rollover({ ...input, requests, reason: "retry" }); + } + + async discardStale(input: { + projectId: string; + experimentId: string; + experimentExecutionId: string; + expectedExecutionVersion: number; + reason: string; + now?: Date; + }): Promise { + const now = input.now ?? new Date(); + const updated = await this.collections.experiments.findOneAndUpdate( + { + _id: input.experimentId, + projectId: input.projectId, + "execution.experimentExecutionId": + input.experimentExecutionId, + "execution.version": input.expectedExecutionVersion, + "execution.rollover": { $exists: false }, + "execution.discardedAt": { $exists: false }, + } as Document, + { + $set: { + "execution.discardedAt": now, + "execution.discardReason": input.reason.slice(0, 2_048), + "execution.updatedAt": now, + updatedAt: now, + }, + $inc: { "execution.version": 1, stateVersion: 1 }, + } as Document, + { returnDocument: "after" }, + ); + return updated ? parseExperiment(updated) : null; + } + + async repairRollover( + projectId: string, + experimentId: string, + ): Promise { + const experiment = await this.getRaw(projectId, experimentId); + const rollover = experiment?.execution.rollover; + if (!experiment || !rollover) return experiment; + const next = parseExecution(rollover.nextExecution); + await this.payloads.verifyCompleteSet( + { + projectId, + experimentId, + ownerId: next.experimentExecutionId, + generation: next.attemptNumber, + }, + next.requests, + ); + await this.archive(experiment, rollover.claimedAt); + const now = new Date(); + const updated = await this.collections.experiments.findOneAndUpdate( + { + _id: experimentId, + projectId, + "execution.experimentExecutionId": + experiment.execution.experimentExecutionId, + "execution.rollover.nextExecution.experimentExecutionId": + next.experimentExecutionId, + } as Document, + { + $set: { execution: next, updatedAt: now }, + $inc: { stateVersion: 1 }, + } as Document, + { returnDocument: "after" }, + ); + if (updated) return parseExperiment(updated); + return this.getRaw(projectId, experimentId); + } + + private async getRaw( + projectId: string, + experimentId: string, + ): Promise { + const raw: unknown = await this.collections.experiments.findOne({ + _id: experimentId, + projectId, + } as Document); + return raw ? parseExperiment(raw) : null; + } + + private executionIdentity(experiment: ExperimentDocument) { + return { + projectId: experiment.projectId, + experimentId: experiment.experimentId, + ownerId: experiment.execution.experimentExecutionId, + generation: experiment.execution.attemptNumber, + }; + } + + private nextExecution( + current: ExperimentExecution, + experimentExecutionId: string, + requests: FixedPayload, + entries: ExperimentRequestState[], + now: Date, + ): ExperimentExecution { + return { + experimentExecutionId, + attemptNumber: current.attemptNumber + 1, + version: 1, + requests, + vectorFingerprint: fingerprintExperimentPayload(entries), + handlerSummary: {}, + blocking: [], + createdAt: now, + updatedAt: now, + }; + } + + private async archive( + experiment: ExperimentDocument, + archivedAt: Date, + ): Promise { + const rollover = experiment.execution.rollover; + const { + rollover: _rollover, + ...execution + } = experiment.execution; + const history = parseHistory({ + ...execution, + _id: execution.experimentExecutionId, + experimentId: experiment.experimentId, + projectId: experiment.projectId, + storageProfileVersion: experiment.storageProfileVersion, + revision: experiment.revision, + archivedAt, + ...(rollover + ? { + supersededAt: rollover.claimedAt, + supersededByExperimentExecutionId: + rollover.nextExecution.experimentExecutionId, + supersededReason: rollover.reason, + } + : {}), + }); + await this.collections.executions.updateOne( + { _id: history._id } as Document, + { $setOnInsert: history } as Document, + { upsert: true }, + ); + const raw: unknown = await this.collections.executions.findOne({ + _id: history._id, + } as Document); + if (!raw) { + throw new Error( + `Experiment execution history ${history._id} was not persisted`, + ); + } + const persisted = parseHistory(raw); + if (comparable(persisted) !== comparable(history)) { + throw new Error( + `Experiment execution history ${history._id} is immutable and does not match`, + ); + } + return persisted; + } +} diff --git a/packages/shared/src/experiments/fixed-payload-store.test.ts b/packages/shared/src/experiments/fixed-payload-store.test.ts new file mode 100644 index 000000000..9339dbecb --- /dev/null +++ b/packages/shared/src/experiments/fixed-payload-store.test.ts @@ -0,0 +1,130 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { Collection, Document } from "mongodb"; +import { describe, expect, it } from "vitest"; +import { FixedPayloadStore } from "./fixed-payload-store.js"; +import type { ExperimentRequestState } from "./types.js"; + +function request(index: number): ExperimentRequestState { + return { + requestId: `request-${index}`, + runId: `run-${index}`, + dependencies: [], + blockers: [], + }; +} + +function chunkCollection(failAt?: number) { + const documents = new Map(); + let writes = 0; + return { + documents, + async updateOne( + filter: Document, + update: { $setOnInsert: Document }, + ) { + writes += 1; + if (writes === failAt) throw new Error("injected write failure"); + const id = String(filter._id); + if (!documents.has(id)) documents.set(id, structuredClone(update.$setOnInsert)); + return { matchedCount: 1, modifiedCount: 1 }; + }, + async findOne(filter: Document) { + return documents.get(String(filter._id)) ?? null; + }, + async deleteMany(filter: { _id: { $in: string[] } }) { + let deletedCount = 0; + for (const id of filter._id.$in) { + if (documents.delete(id)) deletedCount += 1; + } + return { deletedCount }; + }, + }; +} + +describe("FixedPayloadStore", () => { + const identity = { + projectId: "project-1", + experimentId: "experiment-1", + ownerId: "execution-1", + generation: 1, + }; + + it("has identical logical reads for inline and chunked payloads", async () => { + const entries = Array.from({ length: 60 }, (_, index) => request(index)); + const collection = chunkCollection(); + const store = new FixedPayloadStore( + collection as unknown as Collection, + ); + const inline = await store.prepareRequests({ + identity, + entries, + inlineDocument: (payload) => ({ payload }), + }); + expect(inline.storage).toBe("inline"); + expect(await store.readRequests(identity, inline)).toEqual(entries); + + const chunked = await store.prepareRequests({ + identity: { ...identity, ownerId: "execution-2", generation: 2 }, + entries, + inlineDocument: (payload) => ({ + payload, + padding: "x".repeat(256 * 1024), + }), + }); + expect(chunked.storage).toBe("chunked"); + expect( + await store.readRequests( + { ...identity, ownerId: "execution-2", generation: 2 }, + chunked, + ), + ).toEqual(entries); + }); + + it("fails before returning a publishable descriptor on a partial write", async () => { + const collection = chunkCollection(2); + const store = new FixedPayloadStore( + collection as unknown as Collection, + ); + await expect( + store.prepareRequests({ + identity, + entries: Array.from({ length: 100 }, (_, index) => ({ + ...request(index), + dependencies: [ + { + handlerId: "obs", + state: "ready", + error: "x".repeat(1_800), + }, + ], + })), + inlineDocument: (payload) => ({ + payload, + padding: "x".repeat(256 * 1024), + }), + }), + ).rejects.toThrow("injected write failure"); + expect(collection.documents.size).toBe(1); + }); + + it("rejects deterministic-id identity mismatches", async () => { + const collection = chunkCollection(); + const store = new FixedPayloadStore( + collection as unknown as Collection, + ); + const payload = await store.prepareRequests({ + identity, + entries: [request(1)], + inlineDocument: (value) => ({ + value, + padding: "x".repeat(256 * 1024), + }), + }); + expect(payload.storage).toBe("chunked"); + await expect( + store.readRequests({ ...identity, projectId: "other-project" }, payload), + ).rejects.toThrow(/identity mismatch/); + }); +}); diff --git a/packages/shared/src/experiments/fixed-payload-store.ts b/packages/shared/src/experiments/fixed-payload-store.ts new file mode 100644 index 000000000..9d7b91340 --- /dev/null +++ b/packages/shared/src/experiments/fixed-payload-store.ts @@ -0,0 +1,316 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { Collection, Document } from "mongodb"; +import { z } from "zod"; +import { selectFixedPayloadLayout } from "./layout.js"; +import { + ExperimentRequestStateSchema, +} from "./schemas.js"; +import { + computeExperimentChunkId, + fingerprintExperimentPayload, +} from "./routing.js"; +import { + EXPERIMENT_REQUEST_BUCKET_ASSIGNMENT_VERSION, + EXPERIMENT_STORAGE_PROFILE_VERSION, + type ExperimentRequestState, + type FixedPayload, +} from "./types.js"; + +interface ExperimentChunkDocument { + _id: string; + projectId: string; + experimentId: string; + kind: "requests"; + ownerId: string; + generation: number; + chunkIndex: number; + bucketCount: number; + storageProfileVersion: typeof EXPERIMENT_STORAGE_PROFILE_VERSION; + assignmentVersion: typeof EXPERIMENT_REQUEST_BUCKET_ASSIGNMENT_VERSION; + payloadFingerprint: string; + entries: Array<{ position: number; state: ExperimentRequestState }>; + createdAt: Date; +} + +const ExperimentChunkDocumentSchema = z + .object({ + _id: z.string().min(1), + projectId: z.string().min(1), + experimentId: z.string().min(1), + kind: z.literal("requests"), + ownerId: z.string().min(1), + generation: z.number().int().positive(), + chunkIndex: z.number().int().nonnegative(), + bucketCount: z.number().int().positive(), + storageProfileVersion: z.literal(EXPERIMENT_STORAGE_PROFILE_VERSION), + assignmentVersion: z.literal( + EXPERIMENT_REQUEST_BUCKET_ASSIGNMENT_VERSION, + ), + payloadFingerprint: z.string().min(1), + entries: z.array( + z + .object({ + position: z.number().int().nonnegative(), + state: ExperimentRequestStateSchema, + }) + .strict(), + ), + createdAt: z.coerce.date(), + }) + .strict(); + +export interface FixedPayloadIdentity { + projectId: string; + experimentId: string; + ownerId: string; + generation: number; +} + +function sameChunk( + left: ExperimentChunkDocument, + right: ExperimentChunkDocument, +): boolean { + return ( + left._id === right._id && + left.projectId === right.projectId && + left.experimentId === right.experimentId && + left.kind === right.kind && + left.ownerId === right.ownerId && + left.generation === right.generation && + left.chunkIndex === right.chunkIndex && + left.bucketCount === right.bucketCount && + left.storageProfileVersion === right.storageProfileVersion && + left.assignmentVersion === right.assignmentVersion && + left.payloadFingerprint === right.payloadFingerprint && + fingerprintExperimentPayload(left.entries) === + fingerprintExperimentPayload(right.entries) + ); +} + +export class FixedPayloadStore { + constructor( + private readonly chunks: Collection, + ) {} + + async prepareRequests(input: { + identity: FixedPayloadIdentity; + entries: readonly ExperimentRequestState[]; + inlineDocument: (payload: FixedPayload) => object; + now?: Date; + }): Promise> { + if (input.identity.generation < 1) { + throw new Error("Fixed payload generation must be positive"); + } + const entries = input.entries.map((entry) => + ExperimentRequestStateSchema.parse(entry), + ); + const payloadFingerprint = fingerprintExperimentPayload(entries); + const positionByRequestId = new Map( + entries.map((entry, position) => [entry.requestId, position]), + ); + const layout = selectFixedPayloadLayout({ + entries, + inlineDocument: (inlineEntries) => + input.inlineDocument({ + storage: "inline", + entryCount: inlineEntries.length, + payloadFingerprint, + entries: inlineEntries, + }), + chunkDocument: (chunkIndex, bucketCount, bucketEntries) => + this.chunkDocument({ + identity: input.identity, + chunkIndex, + bucketCount, + entries: bucketEntries.map((state) => ({ + position: positionByRequestId.get(state.requestId)!, + state, + })), + payloadFingerprint, + createdAt: input.now ?? new Date(0), + }), + }); + if (layout.storage === "inline") { + return { + storage: "inline", + entryCount: entries.length, + payloadFingerprint, + entries, + }; + } + + const createdAt = input.now ?? new Date(); + const documents = layout.buckets.map((bucketEntries, chunkIndex) => + this.chunkDocument({ + identity: input.identity, + chunkIndex, + bucketCount: layout.bucketCount, + entries: bucketEntries.map((state) => ({ + position: positionByRequestId.get(state.requestId)!, + state, + })), + payloadFingerprint, + createdAt, + }), + ); + for (const document of documents) { + await this.chunks.updateOne( + { _id: document._id } as Document, + { $setOnInsert: document } as Document, + { upsert: true }, + ); + } + await this.assertDocuments(documents); + return { + storage: "chunked", + entryCount: entries.length, + payloadFingerprint, + assignmentVersion: EXPERIMENT_REQUEST_BUCKET_ASSIGNMENT_VERSION, + bucketCount: layout.bucketCount, + chunkIds: documents.map((document) => document._id), + }; + } + + async readRequests( + identity: FixedPayloadIdentity, + payload: FixedPayload, + ): Promise { + if (payload.storage === "inline") { + const entries = payload.entries.map((entry) => + ExperimentRequestStateSchema.parse(entry), + ); + this.assertPayloadFingerprint(payload, entries); + return entries; + } + const documents: ExperimentChunkDocument[] = []; + for (const chunkId of payload.chunkIds) { + const raw: unknown = await this.chunks.findOne({ _id: chunkId } as Document); + if (!raw) throw new Error(`Missing experiment chunk ${chunkId}`); + const document = ExperimentChunkDocumentSchema.parse( + raw, + ) as ExperimentChunkDocument; + this.assertIdentity(document, identity, payload); + documents.push(document); + } + documents.sort((left, right) => left.chunkIndex - right.chunkIndex); + const positioned = documents + .flatMap((document) => document.entries) + .sort((left, right) => left.position - right.position); + if ( + positioned.some((entry, index) => entry.position !== index) + ) { + throw new Error("Experiment payload positions are incomplete or duplicated"); + } + const entries = positioned.map((entry) => entry.state); + if (entries.length !== payload.entryCount) { + throw new Error( + `Experiment payload expected ${payload.entryCount} entries but loaded ${entries.length}`, + ); + } + this.assertPayloadFingerprint(payload, entries); + return entries; + } + + async verifyCompleteSet( + identity: FixedPayloadIdentity, + payload: FixedPayload, + ): Promise { + await this.readRequests(identity, payload); + } + + async discardUnpublished( + payload: FixedPayload, + ): Promise { + if (payload.storage === "inline") return 0; + const result = await this.chunks.deleteMany({ + _id: { $in: payload.chunkIds }, + } as Document); + return result.deletedCount; + } + + private chunkDocument(input: { + identity: FixedPayloadIdentity; + chunkIndex: number; + bucketCount: number; + entries: Array<{ position: number; state: ExperimentRequestState }>; + payloadFingerprint: string; + createdAt: Date; + }): ExperimentChunkDocument { + return { + _id: computeExperimentChunkId({ + ...input.identity, + kind: "requests", + chunkIndex: input.chunkIndex, + }), + ...input.identity, + kind: "requests", + chunkIndex: input.chunkIndex, + bucketCount: input.bucketCount, + storageProfileVersion: EXPERIMENT_STORAGE_PROFILE_VERSION, + assignmentVersion: EXPERIMENT_REQUEST_BUCKET_ASSIGNMENT_VERSION, + payloadFingerprint: input.payloadFingerprint, + entries: input.entries, + createdAt: input.createdAt, + }; + } + + private async assertDocuments( + expected: ExperimentChunkDocument[], + ): Promise { + for (const expectedDocument of expected) { + const raw: unknown = await this.chunks.findOne({ + _id: expectedDocument._id, + } as Document); + if (!raw) { + throw new Error( + `Experiment chunk ${expectedDocument._id} was not persisted`, + ); + } + const actual = ExperimentChunkDocumentSchema.parse( + raw, + ) as ExperimentChunkDocument; + if (!sameChunk(actual, expectedDocument)) { + throw new Error( + `Experiment chunk ${expectedDocument._id} identity or content mismatch`, + ); + } + } + } + + private assertIdentity( + document: ExperimentChunkDocument, + identity: FixedPayloadIdentity, + payload: Extract, { storage: "chunked" }>, + ): void { + const expectedId = computeExperimentChunkId({ + ...identity, + kind: "requests", + chunkIndex: document.chunkIndex, + }); + if ( + document._id !== expectedId || + document.projectId !== identity.projectId || + document.experimentId !== identity.experimentId || + document.ownerId !== identity.ownerId || + document.generation !== identity.generation || + document.bucketCount !== payload.bucketCount || + document.assignmentVersion !== payload.assignmentVersion || + document.payloadFingerprint !== payload.payloadFingerprint + ) { + throw new Error(`Experiment chunk ${document._id} identity mismatch`); + } + } + + private assertPayloadFingerprint( + payload: FixedPayload, + entries: ExperimentRequestState[], + ): void { + if ( + fingerprintExperimentPayload(entries) !== payload.payloadFingerprint + ) { + throw new Error("Experiment payload fingerprint mismatch"); + } + } +} diff --git a/packages/shared/src/experiments/index.ts b/packages/shared/src/experiments/index.ts new file mode 100644 index 000000000..acdbd6b87 --- /dev/null +++ b/packages/shared/src/experiments/index.ts @@ -0,0 +1,9 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +export * from "./types.js"; +export * from "./schemas.js"; +export * from "./routing.js"; +export * from "./layout.js"; +export * from "./fixed-payload-store.js"; +export * from "./experiment-store.js"; diff --git a/packages/shared/src/experiments/layout.test.ts b/packages/shared/src/experiments/layout.test.ts new file mode 100644 index 000000000..41db38862 --- /dev/null +++ b/packages/shared/src/experiments/layout.test.ts @@ -0,0 +1,112 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, expect, it } from "vitest"; +import { + EXPERIMENT_STORAGE_PROFILE, + bsonSize, + selectFixedPayloadLayout, +} from "./layout.js"; +import type { ExperimentRequestState } from "./types.js"; + +function request(index: number, error = ""): ExperimentRequestState { + return { + requestId: `request-${index}`, + runId: `run-${index}`, + dependencies: [ + { + handlerId: "observations", + state: "ready", + ...(error ? { error } : {}), + }, + ], + blockers: [], + }; +} + +describe("experiment fixed payload layout", () => { + it("chooses inline at the exact target and chunks one byte above it", () => { + const entries = [request(1)]; + const base = bsonSize({ entries, padding: "" }); + const exactPadding = "x".repeat( + EXPERIMENT_STORAGE_PROFILE.targetDocumentBytes - base, + ); + const exact = selectFixedPayloadLayout({ + entries, + inlineDocument: (value) => ({ entries: value, padding: exactPadding }), + chunkDocument: (chunkIndex, bucketCount, value) => ({ + chunkIndex, + bucketCount, + entries: value, + }), + }); + expect(exact.bsonBytes[0]).toBe( + EXPERIMENT_STORAGE_PROFILE.targetDocumentBytes, + ); + expect(exact.storage).toBe("inline"); + + const adjacent = selectFixedPayloadLayout({ + entries, + inlineDocument: (value) => ({ + entries: value, + padding: `${exactPadding}x`, + }), + chunkDocument: (chunkIndex, bucketCount, value) => ({ + chunkIndex, + bucketCount, + entries: value, + }), + }); + expect(adjacent.storage).toBe("chunked"); + expect(adjacent.bucketCount).toBe(1); + }); + + it("selects the first safe bucket count after recomputing distribution", () => { + const entries = Array.from({ length: 120 }, (_, index) => + request(index, "x".repeat(3_000)), + ); + const layout = selectFixedPayloadLayout({ + entries, + inlineDocument: (value) => ({ + entries: value, + padding: "x".repeat(EXPERIMENT_STORAGE_PROFILE.targetDocumentBytes), + }), + chunkDocument: (chunkIndex, bucketCount, value) => ({ + chunkIndex, + bucketCount, + entries: value, + }), + }); + expect(layout.storage).toBe("chunked"); + // Three is the count-only lower bound; the hash distribution overloads + // one bucket, so four is the first count that satisfies the BSON budget. + expect(layout.bucketCount).toBe(4); + expect(layout.bsonBytes.every((size) => size < 256 * 1024)).toBe(true); + }); + + it("rejects oversized entries and duplicate request ids", () => { + const oversized = request(1, "x".repeat(4_096)); + expect(() => + selectFixedPayloadLayout({ + entries: [oversized], + inlineDocument: (entries) => ({ entries }), + chunkDocument: (chunkIndex, bucketCount, entries) => ({ + chunkIndex, + bucketCount, + entries, + }), + }), + ).toThrow(/maximum/); + expect(() => + selectFixedPayloadLayout({ + entries: [request(1), request(1)], + inlineDocument: (entries) => ({ entries }), + chunkDocument: (chunkIndex, bucketCount, entries) => ({ + chunkIndex, + bucketCount, + entries, + }), + }), + ).toThrow(/Duplicate/); + }); +}); diff --git a/packages/shared/src/experiments/layout.ts b/packages/shared/src/experiments/layout.ts new file mode 100644 index 000000000..c2ce92b07 --- /dev/null +++ b/packages/shared/src/experiments/layout.ts @@ -0,0 +1,123 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { BSON } from "mongodb"; +import { assignExperimentRequestBucket } from "./routing.js"; +import type { ExperimentRequestState } from "./types.js"; + +export const EXPERIMENT_STORAGE_PROFILE = Object.freeze({ + maxDocumentBytes: 2 * 1024 * 1024, + targetDocumentBytes: 256 * 1024, + maxRequestEntryBytes: 4 * 1024, + metadataReserveBytes: (256 * 1024) / 16, + loadFactor: 0.8, + maxBucketSearch: 65_536, +}); + +export interface FixedPayloadLayout { + storage: "inline" | "chunked"; + bucketCount: number; + buckets: ExperimentRequestState[][]; + bsonBytes: number[]; +} + +export function bsonSize(value: object): number { + return BSON.calculateObjectSize(value); +} + +function payloadBudgetBytes(): number { + return Math.floor( + (EXPERIMENT_STORAGE_PROFILE.targetDocumentBytes - + EXPERIMENT_STORAGE_PROFILE.metadataReserveBytes) * + EXPERIMENT_STORAGE_PROFILE.loadFactor, + ); +} + +export function selectFixedPayloadLayout(input: { + entries: readonly ExperimentRequestState[]; + inlineDocument: (entries: ExperimentRequestState[]) => object; + chunkDocument: ( + chunkIndex: number, + bucketCount: number, + entries: ExperimentRequestState[], + ) => object; +}): FixedPayloadLayout { + const entries = input.entries.map((entry) => structuredClone(entry)); + const ids = new Set(); + for (const entry of entries) { + if (ids.has(entry.requestId)) { + throw new Error(`Duplicate experiment requestId '${entry.requestId}'`); + } + ids.add(entry.requestId); + const bytes = bsonSize({ + position: Number.MAX_SAFE_INTEGER, + state: entry, + }); + if (bytes > EXPERIMENT_STORAGE_PROFILE.maxRequestEntryBytes) { + throw new Error( + `Experiment request ${entry.requestId} is ${bytes} bytes; maximum is ${EXPERIMENT_STORAGE_PROFILE.maxRequestEntryBytes}`, + ); + } + } + + const inlineBytes = bsonSize(input.inlineDocument(entries)); + if ( + inlineBytes <= EXPERIMENT_STORAGE_PROFILE.targetDocumentBytes && + inlineBytes <= EXPERIMENT_STORAGE_PROFILE.maxDocumentBytes + ) { + return { + storage: "inline", + bucketCount: 0, + buckets: [entries], + bsonBytes: [inlineBytes], + }; + } + if (entries.length === 0) { + throw new Error("An empty fixed payload must be stored inline"); + } + + const budget = payloadBudgetBytes(); + const maxEntriesPerBucket = Math.max( + 1, + Math.floor( + budget / EXPERIMENT_STORAGE_PROFILE.maxRequestEntryBytes, + ), + ); + const firstBucketCount = Math.ceil(entries.length / maxEntriesPerBucket); + const lastBucketCount = Math.min( + entries.length, + EXPERIMENT_STORAGE_PROFILE.maxBucketSearch, + ); + + for ( + let bucketCount = firstBucketCount; + bucketCount <= lastBucketCount; + bucketCount += 1 + ) { + const buckets = Array.from( + { length: bucketCount }, + () => [] as ExperimentRequestState[], + ); + for (const entry of entries) { + buckets[ + assignExperimentRequestBucket(entry.requestId, bucketCount) + ]!.push(entry); + } + const bsonBytes = buckets.map((bucket, chunkIndex) => + bsonSize(input.chunkDocument(chunkIndex, bucketCount, bucket)), + ); + if ( + buckets.every((bucket) => bucket.length <= maxEntriesPerBucket) && + bsonBytes.every( + (bytes) => + bytes <= budget && + bytes <= EXPERIMENT_STORAGE_PROFILE.maxDocumentBytes, + ) + ) { + return { storage: "chunked", bucketCount, buckets, bsonBytes }; + } + } + throw new Error( + `No safe experiment request layout found within ${EXPERIMENT_STORAGE_PROFILE.maxBucketSearch} buckets`, + ); +} diff --git a/packages/shared/src/experiments/routing.test.ts b/packages/shared/src/experiments/routing.test.ts new file mode 100644 index 000000000..4c696f0f1 --- /dev/null +++ b/packages/shared/src/experiments/routing.test.ts @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, expect, it } from "vitest"; +import { + assignExperimentRequestBucket, + computeExperimentChunkId, +} from "./routing.js"; + +describe("experiment routing", () => { + it("matches the sha256-mod-v1 golden assignment", () => { + expect(assignExperimentRequestBucket("request-123", 17)).toBe(15); + expect(assignExperimentRequestBucket("request-123", 17)).toBe(15); + }); + + it("matches the deterministic canonical tuple chunk id", () => { + expect( + computeExperimentChunkId({ + projectId: "p", + experimentId: "e", + kind: "requests", + ownerId: "x", + generation: 2, + chunkIndex: 3, + }), + ).toBe("ec_v1_nFlNA4koLtvQvUwoexRibIpfKx89eMnNfsFCPFXfIkU"); + }); + + it("rejects invalid bucket counts", () => { + expect(() => assignExperimentRequestBucket("request-1", 0)).toThrow( + /positive safe integer/, + ); + }); +}); diff --git a/packages/shared/src/experiments/routing.ts b/packages/shared/src/experiments/routing.ts new file mode 100644 index 000000000..bd9afd8fe --- /dev/null +++ b/packages/shared/src/experiments/routing.ts @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { createHash } from "node:crypto"; + +const REQUEST_BUCKET_HASH_PREFIX = "experiment-request-bucket:v1\0"; + +function sha256(value: string): Buffer { + return createHash("sha256").update(value, "utf8").digest(); +} + +export function assignExperimentRequestBucket( + requestId: string, + bucketCount: number, +): number { + if (!Number.isSafeInteger(bucketCount) || bucketCount <= 0) { + throw new Error("bucketCount must be a positive safe integer"); + } + const digest = sha256(`${REQUEST_BUCKET_HASH_PREFIX}${requestId}`); + let unsignedBigEndian = 0n; + for (const byte of digest) { + unsignedBigEndian = (unsignedBigEndian << 8n) + BigInt(byte); + } + return Number(unsignedBigEndian % BigInt(bucketCount)); +} + +export function computeExperimentChunkId(input: { + projectId: string; + experimentId: string; + kind: string; + ownerId: string; + generation: number; + chunkIndex: number; +}): string { + const canonical = JSON.stringify([ + "sha256-v1", + input.projectId, + input.experimentId, + input.kind, + input.ownerId, + input.generation, + input.chunkIndex, + ]); + return `ec_v1_${sha256(canonical).toString("base64url")}`; +} + +export function fingerprintExperimentPayload(value: unknown): string { + return createHash("sha256") + .update(JSON.stringify(value), "utf8") + .digest("base64url"); +} diff --git a/packages/shared/src/experiments/schemas.test.ts b/packages/shared/src/experiments/schemas.test.ts new file mode 100644 index 000000000..9a4b6fb2d --- /dev/null +++ b/packages/shared/src/experiments/schemas.test.ts @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, expect, it } from "vitest"; +import { + ExperimentDocumentSchema, + ExperimentRequestStateSchema, +} from "./schemas.js"; + +describe("experiment schemas", () => { + it("parses the bounded request leaf and rejects recursive or unknown data", () => { + expect( + ExperimentRequestStateSchema.parse({ + requestId: "request-1", + runId: "run-1", + dependencies: [{ handlerId: "obs", state: "ready" }], + blockers: [], + }), + ).toMatchObject({ requestId: "request-1", runId: "run-1" }); + expect(() => + ExperimentRequestStateSchema.parse({ + requestId: "request-1", + runId: "run-1", + dependencies: [], + blockers: [], + requests: [], + }), + ).toThrow(); + expect(() => + ExperimentRequestStateSchema.parse({ + requestId: "request-1", + runId: "run-1", + dependencies: [], + blockers: Array.from({ length: 33 }, () => ({ + handlerId: "obs", + reason: "dependency-blocked", + })), + }), + ).toThrow(); + }); + + it("rejects an unknown storage profile at the document boundary", () => { + expect(() => + ExperimentDocumentSchema.parse({ + _id: "experiment-1", + experimentId: "experiment-1", + projectId: "project-1", + metadata: { name: "Experiment" }, + storageProfileVersion: "future-profile", + stateVersion: 1, + revision: { + experimentRevisionId: "revision-1", + revisionNumber: 1, + memberCount: 0, + membershipFingerprint: "m", + definitionFingerprint: "d", + createdAt: new Date(), + }, + execution: { + experimentExecutionId: "execution-1", + attemptNumber: 1, + version: 1, + requests: { + storage: "inline", + entryCount: 0, + payloadFingerprint: "f", + entries: [], + }, + vectorFingerprint: "v", + handlerSummary: {}, + blocking: [], + createdAt: new Date(), + updatedAt: new Date(), + }, + createdAt: new Date(), + updatedAt: new Date(), + }), + ).toThrow(); + }); +}); diff --git a/packages/shared/src/experiments/schemas.ts b/packages/shared/src/experiments/schemas.ts new file mode 100644 index 000000000..8491cf493 --- /dev/null +++ b/packages/shared/src/experiments/schemas.ts @@ -0,0 +1,170 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { z } from "zod"; +import { + EXPERIMENT_REQUEST_BUCKET_ASSIGNMENT_VERSION, + EXPERIMENT_STORAGE_PROFILE_VERSION, +} from "./types.js"; + +const IdSchema = z.string().trim().min(1).max(200); +const ErrorSchema = z.string().max(2_048); + +export const ExperimentDependencyProjectionSchema = z + .object({ + handlerId: IdSchema, + state: z.enum(["waiting", "ready", "failed", "blocked"]), + version: z.number().int().nonnegative().optional(), + updatedAt: z.coerce.date().optional(), + error: ErrorSchema.optional(), + }) + .strict(); + +export const ExperimentBlockerSchema = z + .object({ + handlerId: IdSchema, + reason: z.enum([ + "request-missing", + "run-changed", + "dependency-not-ready", + "dependency-failed", + "dependency-blocked", + ]), + error: ErrorSchema.optional(), + }) + .strict(); + +export const ExperimentRequestStateSchema = z + .object({ + requestId: IdSchema, + runId: IdSchema, + dependencies: z.array(ExperimentDependencyProjectionSchema).max(32), + blockers: z.array(ExperimentBlockerSchema).max(32), + }) + .strict(); + +export const ExperimentMetadataSchema = z + .object({ + name: z.string().trim().min(1).max(256), + description: z.string().max(4_096).optional(), + tags: z.array(z.string().trim().min(1).max(64)).max(32).optional(), + createdBy: IdSchema.optional(), + }) + .strict(); + +export const ExperimentRevisionSchema = z + .object({ + experimentRevisionId: IdSchema, + revisionNumber: z.number().int().min(1), + memberCount: z.number().int().nonnegative(), + membershipFingerprint: IdSchema, + definitionFingerprint: IdSchema, + createdAt: z.coerce.date(), + }) + .strict(); + +export const ExperimentHandlerSummarySchema = z + .object({ + status: z.enum(["queued", "processing", "done", "failed"]), + version: z.number().int().nonnegative(), + updatedAt: z.coerce.date(), + error: ErrorSchema.optional(), + }) + .strict(); + +export const ExperimentFixedRequestPayloadSchema = z.discriminatedUnion( + "storage", + [ + z + .object({ + storage: z.literal("inline"), + entryCount: z.number().int().nonnegative(), + payloadFingerprint: IdSchema, + entries: z.array(ExperimentRequestStateSchema), + }) + .strict() + .refine((value) => value.entryCount === value.entries.length, { + message: "Inline entryCount must equal entries.length", + }), + z + .object({ + storage: z.literal("chunked"), + entryCount: z.number().int().positive(), + payloadFingerprint: IdSchema, + assignmentVersion: z.literal( + EXPERIMENT_REQUEST_BUCKET_ASSIGNMENT_VERSION, + ), + bucketCount: z.number().int().positive(), + chunkIds: z.array(IdSchema).min(1), + }) + .strict() + .refine((value) => value.bucketCount === value.chunkIds.length, { + message: "Chunked bucketCount must equal chunkIds.length", + }), + ], +); + +const ExperimentExecutionBaseSchema = z + .object({ + experimentExecutionId: IdSchema, + attemptNumber: z.number().int().min(1), + version: z.number().int().min(1), + requests: ExperimentFixedRequestPayloadSchema, + vectorFingerprint: IdSchema, + handlerSummary: z + .record(IdSchema, ExperimentHandlerSummarySchema) + .refine((value) => Object.keys(value).length <= 64, { + message: "handlerSummary is limited to 64 handlers", + }), + blocking: z.array(ExperimentBlockerSchema).max(32), + createdAt: z.coerce.date(), + updatedAt: z.coerce.date(), + startedAt: z.coerce.date().optional(), + finishedAt: z.coerce.date().optional(), + frozenAt: z.coerce.date().optional(), + discardedAt: z.coerce.date().optional(), + discardReason: z.string().max(2_048).optional(), + supersededAt: z.coerce.date().optional(), + supersededByExperimentExecutionId: IdSchema.optional(), + supersededReason: z.enum(["rollover", "retry"]).optional(), + }) + .strict(); + +export const ExperimentExecutionSchema: z.ZodType = ExperimentExecutionBaseSchema + .extend({ + rollover: z + .object({ + nextExecution: ExperimentExecutionBaseSchema, + reason: z.enum(["rollover", "retry"]), + claimedAt: z.coerce.date(), + }) + .strict() + .optional(), + }) + .strict(); + +export const ExperimentDocumentSchema = z + .object({ + _id: IdSchema, + experimentId: IdSchema, + projectId: IdSchema, + metadata: ExperimentMetadataSchema, + storageProfileVersion: z.literal(EXPERIMENT_STORAGE_PROFILE_VERSION), + stateVersion: z.number().int().min(1), + revision: ExperimentRevisionSchema, + execution: ExperimentExecutionSchema, + createdAt: z.coerce.date(), + updatedAt: z.coerce.date(), + finalizedAt: z.coerce.date().optional(), + }) + .strict(); + +export const ExperimentExecutionHistoryDocumentSchema = + ExperimentExecutionBaseSchema.extend({ + _id: IdSchema, + experimentId: IdSchema, + projectId: IdSchema, + storageProfileVersion: z.literal(EXPERIMENT_STORAGE_PROFILE_VERSION), + revision: ExperimentRevisionSchema, + archivedAt: z.coerce.date(), + }).strict(); diff --git a/packages/shared/src/experiments/types.ts b/packages/shared/src/experiments/types.ts new file mode 100644 index 000000000..e31b2f328 --- /dev/null +++ b/packages/shared/src/experiments/types.ts @@ -0,0 +1,149 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +export const EXPERIMENT_STORAGE_PROFILE_VERSION = "cosmos-mongo-2mb-v1" as const; +export const EXPERIMENT_REQUEST_BUCKET_ASSIGNMENT_VERSION = + "sha256-mod-v1" as const; + +export type ExperimentStorageProfileVersion = + typeof EXPERIMENT_STORAGE_PROFILE_VERSION; + +export interface InlineFixedPayload { + storage: "inline"; + entryCount: number; + payloadFingerprint: string; + entries: T[]; +} + +export interface ChunkedFixedPayload { + storage: "chunked"; + entryCount: number; + payloadFingerprint: string; + assignmentVersion: typeof EXPERIMENT_REQUEST_BUCKET_ASSIGNMENT_VERSION; + bucketCount: number; + chunkIds: string[]; +} + +export type FixedPayload = + | InlineFixedPayload + | ChunkedFixedPayload; + +export type ExperimentDependencyState = + | "waiting" + | "ready" + | "failed" + | "blocked"; + +export interface ExperimentDependencyProjection { + handlerId: string; + state: ExperimentDependencyState; + version?: number; + updatedAt?: Date; + error?: string; +} + +export type ExperimentBlockerReason = + | "request-missing" + | "run-changed" + | "dependency-not-ready" + | "dependency-failed" + | "dependency-blocked"; + +export interface ExperimentBlocker { + handlerId: string; + reason: ExperimentBlockerReason; + error?: string; +} + +/** Bounded, nonrecursive leaf stored in an experiment execution request vector. */ +export interface ExperimentRequestState { + requestId: string; + runId: string; + dependencies: ExperimentDependencyProjection[]; + blockers: ExperimentBlocker[]; +} + +export interface ExperimentMetadata { + name: string; + description?: string; + tags?: string[]; + createdBy?: string; +} + +export interface ExperimentRevision { + experimentRevisionId: string; + revisionNumber: number; + memberCount: number; + membershipFingerprint: string; + definitionFingerprint: string; + createdAt: Date; +} + +export type ExperimentHandlerStatus = + | "queued" + | "processing" + | "done" + | "failed"; + +export interface ExperimentHandlerSummary { + status: ExperimentHandlerStatus; + version: number; + updatedAt: Date; + error?: string; +} + +export interface ExperimentRolloverClaim { + nextExecution: ExperimentExecution; + reason: "rollover" | "retry"; + claimedAt: Date; +} + +export interface ExperimentExecution { + experimentExecutionId: string; + attemptNumber: number; + version: number; + requests: FixedPayload; + vectorFingerprint: string; + handlerSummary: Record; + blocking: ExperimentBlocker[]; + createdAt: Date; + updatedAt: Date; + startedAt?: Date; + finishedAt?: Date; + frozenAt?: Date; + discardedAt?: Date; + discardReason?: string; + supersededAt?: Date; + supersededByExperimentExecutionId?: string; + supersededReason?: "rollover" | "retry"; + rollover?: ExperimentRolloverClaim; +} + +export interface ExperimentDocument { + _id: string; + experimentId: string; + projectId: string; + metadata: ExperimentMetadata; + storageProfileVersion: ExperimentStorageProfileVersion; + stateVersion: number; + revision: ExperimentRevision; + execution: ExperimentExecution; + createdAt: Date; + updatedAt: Date; + finalizedAt?: Date; +} + +export interface ExperimentExecutionHistoryDocument + extends Omit { + _id: string; + experimentId: string; + projectId: string; + storageProfileVersion: ExperimentStorageProfileVersion; + revision: ExperimentRevision; + archivedAt: Date; +} + +export interface ExperimentPage { + experiments: ExperimentDocument[]; + nextCursor?: string; +} diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index fae65d02f..94bfa7c85 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -20,7 +20,7 @@ export * from "./codebases/index.js"; export * from "./resources/index.js"; export * from "./extensions/index.js"; export * from "./projects/index.js"; -export * from "./submissions/index.js"; +export * from "./experiments/index.js"; export * from "./agent-version.js"; export * from "./resolve-agent-version.js"; export * from "./resolve-agent-target.js"; From e25164129a9cae1fb3a2e9507876bc2d48e419a0 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Thu, 15 Oct 2026 22:42:56 -0700 Subject: [PATCH 06/25] refactor(experiments): remove submission model Replace imported Submission persistence surfaces with Experiment correlation and handler types. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 92d03055-789e-48b7-9a44-6de5b357f486 --- .../src/034-create-submission-indexes.test.ts | 51 -- .../034-create-submission-indexes.ts | 67 -- packages/shared/src/schemas/index.ts | 1 - packages/shared/src/schemas/request.ts | 12 +- packages/shared/src/schemas/submission.ts | 182 ----- .../shared/src/scripts/register-handler.ts | 10 +- packages/shared/src/submissions/index.ts | 5 - .../submission-coordinator.test.ts | 693 ------------------ .../src/submissions/submission-coordinator.ts | 573 --------------- .../src/submissions/submission-state.test.ts | 155 ---- .../src/submissions/submission-state.ts | 184 ----- packages/shared/src/types/types.ts | 131 +--- 12 files changed, 28 insertions(+), 2036 deletions(-) delete mode 100644 packages/db-migrations/src/034-create-submission-indexes.test.ts delete mode 100644 packages/db-migrations/src/migrations/034-create-submission-indexes.ts delete mode 100644 packages/shared/src/schemas/submission.ts delete mode 100644 packages/shared/src/submissions/index.ts delete mode 100644 packages/shared/src/submissions/submission-coordinator.test.ts delete mode 100644 packages/shared/src/submissions/submission-coordinator.ts delete mode 100644 packages/shared/src/submissions/submission-state.test.ts delete mode 100644 packages/shared/src/submissions/submission-state.ts diff --git a/packages/db-migrations/src/034-create-submission-indexes.test.ts b/packages/db-migrations/src/034-create-submission-indexes.test.ts deleted file mode 100644 index efcd2017f..000000000 --- a/packages/db-migrations/src/034-create-submission-indexes.test.ts +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -import type { Db } from "mongodb"; -import { describe, expect, it, vi } from "vitest"; -import { CreateSubmissionIndexes } from "./migrations/034-create-submission-indexes.js"; - -function collection() { - return { - createIndex: vi.fn(async () => "index"), - }; -} - -describe("migration 034: CreateSubmissionIndexes", () => { - it("creates submission, history, dispatch, and recovery indexes", async () => { - const submissions = collection(); - const submissionRuns = collection(); - const requests = collection(); - const db = { - collection: vi.fn((name: string) => { - if (name === "submissions") return submissions; - if (name === "submission-runs") return submissionRuns; - if (name === "requests") return requests; - throw new Error(`Unexpected collection ${name}`); - }), - } as unknown as Db; - - await new CreateSubmissionIndexes().up(db); - - expect(submissions.createIndex).toHaveBeenCalledWith( - { projectId: 1, createdAt: -1, _id: -1 }, - {}, - ); - expect(submissions.createIndex).toHaveBeenCalledWith( - { "run.handlerStatus.pp-readout.status": 1 }, - {}, - ); - expect(submissionRuns.createIndex).toHaveBeenCalledWith( - { submissionId: 1, attemptNumber: -1 }, - {}, - ); - expect(requests.createIndex).toHaveBeenCalledWith( - { submissionFinalizedAt: 1 }, - {}, - ); - expect(requests.createIndex).toHaveBeenCalledWith( - { submissionId: 1, projectId: 1, submissionRequestIndex: 1, _id: 1 }, - {}, - ); - }); -}); diff --git a/packages/db-migrations/src/migrations/034-create-submission-indexes.ts b/packages/db-migrations/src/migrations/034-create-submission-indexes.ts deleted file mode 100644 index 8ceb836e9..000000000 --- a/packages/db-migrations/src/migrations/034-create-submission-indexes.ts +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -import type { Db } from "mongodb"; -import type { MigrationInterface } from "mongo-migrate-ts"; -import { ensureIndex } from "../cosmos-index-helpers.js"; - -const TAG = "034"; - -export class CreateSubmissionIndexes implements MigrationInterface { - async up(db: Db): Promise { - const submissions = db.collection("submissions"); - await ensureIndex( - submissions, - { projectId: 1, createdAt: -1, _id: -1 }, - {}, - "submissions", - TAG, - ); - await ensureIndex( - submissions, - { "run.handlerStatus.pp-readout.status": 1 }, - {}, - "submissions", - TAG, - ); - - const submissionRuns = db.collection("submission-runs"); - await ensureIndex( - submissionRuns, - { submissionId: 1 }, - {}, - "submission-runs", - TAG, - ); - await ensureIndex( - submissionRuns, - { submissionId: 1, attemptNumber: -1 }, - {}, - "submission-runs", - TAG, - ); - - const requests = db.collection("requests"); - await ensureIndex( - requests, - { submissionFinalizedAt: 1 }, - {}, - "requests", - TAG, - ); - await ensureIndex( - requests, - { submissionId: 1, projectId: 1, submissionRequestIndex: 1, _id: 1 }, - {}, - "requests", - TAG, - ); - console.log(`[${TAG}] Submission indexes complete`); - } - - async down(_db: Db): Promise { - console.log( - `[${TAG}-down] Skipping index changes - drop submission indexes manually if needed`, - ); - } -} diff --git a/packages/shared/src/schemas/index.ts b/packages/shared/src/schemas/index.ts index 52a7bed3f..cb1df8b4c 100644 --- a/packages/shared/src/schemas/index.ts +++ b/packages/shared/src/schemas/index.ts @@ -22,4 +22,3 @@ export * from "./account.js"; export * from "./feature-flag.js"; export * from "./profile.js"; export * from "./project.js"; -export * from "./submission.js"; diff --git a/packages/shared/src/schemas/request.ts b/packages/shared/src/schemas/request.ts index 5aefb2466..ac7766f2d 100644 --- a/packages/shared/src/schemas/request.ts +++ b/packages/shared/src/schemas/request.ts @@ -180,11 +180,13 @@ export const RequestResponseSchema = z agentVersion: z.string().optional(), profileId: z.string().optional(), profileVersionId: z.string().optional(), - submissionId: z.string().optional(), - submissionExpectedRequestCount: z.number().int().min(1).optional(), - submissionRequestIndex: z.number().int().min(0).optional(), - submissionInitialRunId: z.string().optional(), - submissionFinalizedAt: z.coerce.date().optional(), + experimentId: z.string().optional(), + experimentRevisionId: z.string().optional(), + experimentExecutionId: z.string().optional(), + experimentExpectedRequestCount: z.number().int().min(1).optional(), + experimentRequestIndex: z.number().int().min(0).optional(), + experimentInitialRunId: z.string().optional(), + experimentFinalizedAt: z.coerce.date().optional(), priority: z.number().int().default(0), gates: z.array(GateConfigSchema).optional(), observations: z.array(z.string()).optional(), diff --git a/packages/shared/src/schemas/submission.ts b/packages/shared/src/schemas/submission.ts deleted file mode 100644 index 3558ae603..000000000 --- a/packages/shared/src/schemas/submission.ts +++ /dev/null @@ -1,182 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -import { z } from "zod"; -import { extendZodWithOpenApi } from "@asteasolutions/zod-to-openapi"; - -extendZodWithOpenApi(z); - -export const SubmissionRequestRefSchema = z - .object({ - requestId: z.string(), - runId: z.string(), - }) - .openapi("SubmissionRequestRef"); - -export const SubmissionDependencyStateSchema = z.enum([ - "waiting", - "ready", - "failed", - "blocked", -]); - -export const SubmissionDependencyProjectionSchema = z - .object({ - requestId: z.string(), - runId: z.string(), - handlerId: z.string(), - state: SubmissionDependencyStateSchema, - version: z.number().optional(), - updatedAt: z.coerce.date().optional(), - error: z.string().optional(), - }) - .openapi("SubmissionDependencyProjection"); - -export const SubmissionBlockingDetailSchema = z - .object({ - requestId: z.string(), - runId: z.string(), - handlerId: z.string(), - reason: z.enum([ - "request-missing", - "run-changed", - "dependency-not-ready", - "dependency-failed", - "dependency-blocked", - ]), - error: z.string().optional(), - }) - .openapi("SubmissionBlockingDetail"); - -export const SubmissionHandlerStatusSchema = z - .object({ - status: z.enum(["queued", "processing", "done", "failed"]), - version: z.number().optional(), - updatedAt: z.coerce.date().optional(), - error: z.string().optional(), - }) - .openapi("SubmissionHandlerStatus"); - -export const SubmissionReportPointerSchema = z - .object({ - reportId: z.string().optional(), - url: z.string().optional(), - updatedAt: z.coerce.date().optional(), - }) - .openapi("SubmissionReportPointer"); - -export const SubmissionRolloverClaimSchema = z.object({ - nextSubmissionRunId: z.string(), - vectorFingerprint: z.string(), - requests: z.array(SubmissionRequestRefSchema), - reason: z.enum(["request-vector-changed", "manual-retry"]), - claimedAt: z.coerce.date(), -}); - -export const SubmissionRunStateSchema = z - .object({ - _id: z.string(), - attemptNumber: z.number().int().min(1), - vectorFingerprint: z.string(), - requests: z.array(SubmissionRequestRefSchema), - createdAt: z.coerce.date(), - updatedAt: z.coerce.date().optional(), - startedAt: z.coerce.date().optional(), - finishedAt: z.coerce.date().optional(), - dependencyProjection: z.array(SubmissionDependencyProjectionSchema).default([]), - handlerStatus: z.record(z.string(), SubmissionHandlerStatusSchema).optional(), - blocking: z.array(SubmissionBlockingDetailSchema).default([]), - report: SubmissionReportPointerSchema.optional(), - reconcileAfter: z.coerce.date().optional(), - rollover: SubmissionRolloverClaimSchema.optional(), - supersededAt: z.coerce.date().optional(), - supersededBySubmissionRunId: z.string().optional(), - supersededReason: z.enum(["request-vector-changed", "manual-retry"]).optional(), - }) - .openapi("SubmissionRunState"); - -export const SubmissionDocumentSchema = z - .object({ - _id: z.string(), - submissionId: z.string(), - projectId: z.string(), - requestIds: z.array(z.string()).min(1), - createdAt: z.coerce.date(), - updatedAt: z.coerce.date().optional(), - run: SubmissionRunStateSchema, - }) - .openapi("SubmissionDocument"); - -export const SubmissionRunHistoryDocumentSchema = SubmissionRunStateSchema.extend({ - submissionId: z.string(), - projectId: z.string(), -}).openapi("SubmissionRunHistoryDocument"); - -export const SubmissionRunResponseSchema = SubmissionRunStateSchema.omit({ - reconcileAfter: true, - rollover: true, -}) - .extend({ - submissionId: z.string().optional(), - projectId: z.string().optional(), - }) - .openapi("SubmissionRunResponse"); - -export const SubmissionReadinessSchema = z.enum([ - "waiting", - "ready", - "blocked", - "processing", - "done", - "failed", -]); - -export const SubmissionRequestSummarySchema = z - .object({ - requestId: z.string(), - runId: z.string(), - task: z.string(), - status: z.string(), - outcome: z.string().optional(), - workerType: z.string(), - model: z.string().optional(), - profileId: z.string().optional(), - }) - .openapi("SubmissionRequestSummary"); - -export const SubmissionResponseSchema = z - .object({ - submissionId: z.string(), - projectId: z.string(), - requestIds: z.array(z.string()), - createdAt: z.coerce.date(), - updatedAt: z.coerce.date().optional(), - submissionRunId: z.string(), - attemptNumber: z.number().int(), - readiness: SubmissionReadinessSchema, - requests: z.array(SubmissionRequestSummarySchema), - dependencyProjection: z.array(SubmissionDependencyProjectionSchema), - blocking: z.array(SubmissionBlockingDetailSchema), - handlerProgress: z.record(z.string(), SubmissionHandlerStatusSchema).optional(), - report: SubmissionReportPointerSchema.optional(), - supersededAt: z.coerce.date().optional(), - supersededBySubmissionRunId: z.string().optional(), - supersededReason: z.enum(["request-vector-changed", "manual-retry"]).optional(), - }) - .openapi("SubmissionResponse"); - -export const SubmissionListResponseSchema = z - .object({ - submissions: z.array(SubmissionResponseSchema), - }) - .openapi("SubmissionListResponse"); - -export const SubmissionRetryResponseSchema = z - .object({ - submissionId: z.string(), - submissionRunId: z.string(), - attemptNumber: z.number().int(), - readiness: SubmissionReadinessSchema, - blocking: z.array(SubmissionBlockingDetailSchema), - }) - .openapi("SubmissionRetryResponse"); diff --git a/packages/shared/src/scripts/register-handler.ts b/packages/shared/src/scripts/register-handler.ts index 730b860f6..cc06045df 100644 --- a/packages/shared/src/scripts/register-handler.ts +++ b/packages/shared/src/scripts/register-handler.ts @@ -42,9 +42,9 @@ export function parseHandlerDocument(raw: unknown): ServiceHandlerDocument { if (!isNonEmptyString(doc._id)) { throw new Error("handler.yaml: '_id' is required"); } - if (doc.type !== "post-process-handler" && doc.type !== "submission-handler") { + if (doc.type !== "post-process-handler" && doc.type !== "experiment-handler") { throw new Error( - "handler.yaml: 'type' must be 'post-process-handler' or 'submission-handler'", + "handler.yaml: 'type' must be 'post-process-handler' or 'experiment-handler'", ); } if (typeof doc.version !== "number" || !Number.isFinite(doc.version)) { @@ -72,15 +72,15 @@ export function parseHandlerDocument(raw: unknown): ServiceHandlerDocument { autoBackfill: doc.autoBackfill, dependsOn: dependsOn as string[], }; - if (doc.type === "submission-handler") { + if (doc.type === "experiment-handler") { if (doc.dependencyMode !== "all-runs") { throw new Error( - "handler.yaml: submission handlers require dependencyMode: 'all-runs'", + "handler.yaml: experiment handlers require dependencyMode: 'all-runs'", ); } return { ...common, - type: "submission-handler", + type: "experiment-handler", dependencyMode: "all-runs", }; } diff --git a/packages/shared/src/submissions/index.ts b/packages/shared/src/submissions/index.ts deleted file mode 100644 index 5aa35026a..000000000 --- a/packages/shared/src/submissions/index.ts +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -export * from "./submission-state.js"; -export * from "./submission-coordinator.js"; diff --git a/packages/shared/src/submissions/submission-coordinator.test.ts b/packages/shared/src/submissions/submission-coordinator.test.ts deleted file mode 100644 index d44b4c12e..000000000 --- a/packages/shared/src/submissions/submission-coordinator.test.ts +++ /dev/null @@ -1,693 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -import { describe, expect, it, vi } from "vitest"; -import type { Collection } from "mongodb"; -import type { - RequestDocument, - ServiceHandlerDocument, - SubmissionDocument, - SubmissionRunHistoryDocument, -} from "../types/types.js"; -import { - SubmissionCoordinator, - claimSubmissionHandler, - completeSubmissionHandler, - ensureNativeSubmission, -} from "./submission-coordinator.js"; - -function queryResult(items: T[]) { - return { - project: () => queryResult(items), - sort: () => queryResult(items), - limit: () => queryResult(items), - toArray: async () => items, - }; -} - -describe("submission coordinator persistence", () => { - it("creates a native submission idempotently with the exact initial vector", async () => { - let stored: SubmissionDocument | null = null; - const updateOne = vi.fn(async ( - _filter: unknown, - update: { $setOnInsert: SubmissionDocument }, - ) => { - stored ??= update.$setOnInsert; - return { matchedCount: stored ? 1 : 0, modifiedCount: 0 }; - }); - const findOne = vi.fn(async () => stored); - const submissions = { - updateOne, - findOne, - } as unknown as Collection; - - const first = await ensureNativeSubmission({ - collections: { submissions }, - submissionId: "submission-1", - projectId: "project-1", - requests: [ - { requestId: "request-1", runId: "run-1" }, - { requestId: "request-2", runId: "run-2" }, - ], - createdAt: new Date("2026-01-01T00:00:00Z"), - }); - const second = await ensureNativeSubmission({ - collections: { submissions }, - submissionId: "submission-1", - projectId: "project-1", - requests: [ - { requestId: "request-1", runId: "run-1" }, - { requestId: "request-2", runId: "run-2" }, - ], - createdAt: new Date("2026-01-01T00:00:00Z"), - }); - - expect(first).toEqual(second); - expect(first.requestIds).toEqual(["request-1", "request-2"]); - expect(first.run.requests).toEqual([ - { requestId: "request-1", runId: "run-1" }, - { requestId: "request-2", runId: "run-2" }, - ]); - expect(first.run._id).toBe(second.run._id); - expect(updateOne).toHaveBeenCalledTimes(2); - }); - - it("rejects an idempotency key reused with a different immutable vector", async () => { - const stored: SubmissionDocument = { - _id: "submission-1", - submissionId: "submission-1", - projectId: "project-1", - requestIds: ["request-1"], - createdAt: new Date(), - run: { - _id: "submission-run-1", - attemptNumber: 1, - vectorFingerprint: "fingerprint", - requests: [{ requestId: "request-1", runId: "run-1" }], - createdAt: new Date(), - dependencyProjection: [], - blocking: [], - }, - }; - const submissions = { - updateOne: vi.fn(async () => ({ matchedCount: 1, modifiedCount: 0 })), - findOne: vi.fn(async () => stored), - } as unknown as Collection; - - await expect( - ensureNativeSubmission({ - collections: { submissions }, - submissionId: "submission-1", - projectId: "project-1", - requests: [{ requestId: "request-1", runId: "different-run" }], - }), - ).rejects.toThrow("different immutable request vector"); - }); - - it("scopes claims and completions to the exact embedded submission run", async () => { - const findOneAndUpdate = vi.fn(async (_filter: unknown, _update: unknown) => null); - const updateOne = vi.fn(async (_filter: unknown, _update: unknown) => ({ - modifiedCount: 0, - })); - const submissions = { - findOneAndUpdate, - updateOne, - } as unknown as Collection; - - expect( - await claimSubmissionHandler({ - submissions, - submissionId: "submission-1", - submissionRunId: "stale-run", - handlerId: "pp-readout", - version: 1, - }), - ).toBeNull(); - expect(findOneAndUpdate.mock.calls[0]?.[0]).toMatchObject({ - _id: "submission-1", - "run._id": "stale-run", - "run.rollover": { $exists: false }, - }); - - expect( - await completeSubmissionHandler({ - submissions, - submissionId: "submission-1", - submissionRunId: "stale-run", - handlerId: "pp-readout", - status: "done", - version: 1, - }), - ).toBe(false); - expect(updateOne.mock.calls[0]?.[0]).toMatchObject({ - _id: "submission-1", - "run._id": "stale-run", - "run.rollover": { $exists: false }, - "run.handlerStatus.pp-readout.status": "processing", - }); - }); - - it("defers reconciliation while a bulk retry replaces its request vector", async () => { - const submission: SubmissionDocument = { - _id: "submission-1", - submissionId: "submission-1", - projectId: "project-1", - requestIds: ["request-1"], - createdAt: new Date(), - run: { - _id: "submission-run-1", - attemptNumber: 1, - vectorFingerprint: "old-vector", - requests: [{ requestId: "request-1", runId: "run-1" }], - createdAt: new Date(), - reconcileAfter: new Date(Date.now() + 60_000), - dependencyProjection: [], - blocking: [], - }, - }; - const requestsFind = vi.fn(); - const archiveInsert = vi.fn(); - const coordinator = new SubmissionCoordinator({ - requests: { - find: requestsFind, - } as unknown as Collection, - submissions: { - findOne: vi.fn(async () => submission), - } as unknown as Collection, - submissionRuns: { - insertOne: archiveInsert, - } as unknown as Collection, - services: {} as Collection, - }); - - const result = await coordinator.reconcile("submission-1"); - - expect(result?.rolled).toBe(false); - expect(requestsFind).not.toHaveBeenCalled(); - expect(archiveInsert).not.toHaveBeenCalled(); - }); - - it("resumes an interrupted drift rollover without double-rolling a manual retry", async () => { - const requestDocs: RequestDocument[] = [ - { - _id: "request-1", - projectId: "project-1", - scenario: { task: "Task", criteria: ["criterion"] }, - workerType: "worker", - createdAt: new Date(), - priority: 0, - submissionId: "submission-1", - run: { - _id: "run-2", - attemptNumber: 2, - status: "done", - outcome: "failed", - handlerStatus: { - "pp-observations": { status: "done", version: 2 }, - unrelated: { status: "failed" }, - }, - }, - }, - ]; - let submission: SubmissionDocument = { - _id: "submission-1", - submissionId: "submission-1", - projectId: "project-1", - requestIds: ["request-1"], - createdAt: new Date(), - run: { - _id: "submission-run-1", - attemptNumber: 1, - vectorFingerprint: "old-vector", - requests: [{ requestId: "request-1", runId: "run-1" }], - createdAt: new Date(), - dependencyProjection: [], - blocking: [], - }, - }; - const history: SubmissionRunHistoryDocument[] = []; - const services: ServiceHandlerDocument[] = [ - { - _id: "pp-atif", - type: "post-process-handler", - version: 1, - queue: "post-processor", - selector: "atif", - autoBackfill: true, - dependsOn: [], - }, - { - _id: "pp-observations", - type: "post-process-handler", - version: 2, - queue: "post-processor", - selector: "observations", - autoBackfill: true, - dependsOn: ["pp-atif"], - }, - { - _id: "pp-readout", - type: "submission-handler", - version: 1, - queue: "readout", - selector: "readout", - autoBackfill: false, - dependsOn: ["pp-observations"], - dependencyMode: "all-runs", - }, - ]; - const submissions = { - findOne: vi.fn(async () => structuredClone(submission)), - updateOne: vi.fn(async () => ({ matchedCount: 1, modifiedCount: 1 })), - findOneAndUpdate: vi.fn(async ( - filter: Record, - update: { $set: Record }, - ) => { - if (filter["run._id"] && filter["run._id"] !== submission.run._id) return null; - if (update.$set["run.rollover"]) { - submission = { - ...submission, - updatedAt: update.$set.updatedAt as Date, - run: { - ...submission.run, - rollover: update.$set[ - "run.rollover" - ] as SubmissionDocument["run"]["rollover"], - updatedAt: update.$set["run.updatedAt"] as Date, - }, - }; - } else if (update.$set.run) { - rollAttempts++; - if (rollAttempts === 1) return null; - submission = { - ...submission, - run: update.$set.run as SubmissionDocument["run"], - updatedAt: update.$set.updatedAt as Date, - }; - } else { - submission = { - ...submission, - updatedAt: update.$set.updatedAt as Date, - run: { - ...submission.run, - dependencyProjection: update.$set[ - "run.dependencyProjection" - ] as SubmissionDocument["run"]["dependencyProjection"], - blocking: update.$set[ - "run.blocking" - ] as SubmissionDocument["run"]["blocking"], - updatedAt: update.$set["run.updatedAt"] as Date, - }, - }; - } - return structuredClone(submission); - }), - } as unknown as Collection; - let rollAttempts = 0; - const enqueue = vi.fn(async ( - _service: ServiceHandlerDocument, - _message: { submissionId: string; submissionRunId: string; type: string }, - ) => undefined); - const archiveUpdate = vi.fn(async ( - _filter: unknown, - update: { $setOnInsert: SubmissionRunHistoryDocument }, - ) => { - if (!history.some((item) => item._id === update.$setOnInsert._id)) { - history.push(update.$setOnInsert); - } - return { matchedCount: 1, modifiedCount: 1 }; - }); - const coordinator = new SubmissionCoordinator( - { - requests: { - find: vi.fn(() => queryResult(requestDocs)), - } as unknown as Collection, - submissions, - submissionRuns: { - updateOne: archiveUpdate, - } as unknown as Collection, - services: { - find: vi.fn(() => - queryResult(services.filter((service) => service.type === "post-process-handler")), - ), - findOne: vi.fn(async () => - services.find((service) => service._id === "pp-readout") ?? null, - ), - } as unknown as Collection, - }, - enqueue, - ); - - const result = await coordinator.retryCurrent( - "submission-1", - "submission-run-1", - ); - - expect(result?.rolled).toBe(true); - expect(result?.submission.run.attemptNumber).toBe(2); - expect(result?.submission.run.requests).toEqual([ - { requestId: "request-1", runId: "run-2" }, - ]); - expect(result?.submission.run.dependencyProjection[0]).toMatchObject({ - runId: "run-2", - state: "ready", - version: 2, - }); - expect(history).toHaveLength(1); - expect(archiveUpdate).toHaveBeenCalledTimes(2); - expect(history[0]).toMatchObject({ - _id: "submission-run-1", - supersededBySubmissionRunId: result?.submission.run._id, - supersededReason: "request-vector-changed", - }); - expect(enqueue).toHaveBeenCalledTimes(1); - expect(enqueue.mock.calls[0]?.[1]).toMatchObject({ - submissionId: "submission-1", - submissionRunId: result?.submission.run._id, - }); - }); - - it("links immutable history only to CAS-winning successors under concurrent vector drift", async () => { - const request = (runId: string, attemptNumber: number): RequestDocument => ({ - _id: "request-1", - projectId: "project-1", - scenario: { task: "Task", criteria: ["criterion"] }, - workerType: "worker", - createdAt: new Date(), - priority: 0, - submissionId: "submission-1", - run: { - _id: runId, - attemptNumber, - status: "done", - }, - }); - let submission: SubmissionDocument = { - _id: "submission-1", - submissionId: "submission-1", - projectId: "project-1", - requestIds: ["request-1"], - createdAt: new Date(), - run: { - _id: "submission-run-1", - attemptNumber: 1, - vectorFingerprint: "initial", - requests: [{ requestId: "request-1", runId: "run-1" }], - createdAt: new Date(), - dependencyProjection: [], - blocking: [], - }, - }; - const history = new Map(); - const submissions = { - findOne: vi.fn(async () => structuredClone(submission)), - updateOne: vi.fn(async () => ({ matchedCount: 1, modifiedCount: 1 })), - findOneAndUpdate: vi.fn(async ( - filter: Record, - update: { $set: Record }, - ) => { - if (filter["run._id"] !== submission.run._id) return null; - if (update.$set["run.rollover"]) { - if (submission.run.rollover) return null; - submission = { - ...submission, - run: { - ...submission.run, - rollover: update.$set[ - "run.rollover" - ] as SubmissionDocument["run"]["rollover"], - }, - }; - } else if (update.$set.run) { - if ( - filter["run.rollover.nextSubmissionRunId"] !== - submission.run.rollover?.nextSubmissionRunId - ) { - return null; - } - submission = { - ...submission, - run: update.$set.run as SubmissionDocument["run"], - }; - } else { - submission = { - ...submission, - run: { - ...submission.run, - dependencyProjection: update.$set[ - "run.dependencyProjection" - ] as SubmissionDocument["run"]["dependencyProjection"], - blocking: update.$set[ - "run.blocking" - ] as SubmissionDocument["run"]["blocking"], - }, - }; - } - return structuredClone(submission); - }), - } as unknown as Collection; - const submissionRuns = { - updateOne: vi.fn(async ( - filter: { _id: string }, - update: { $setOnInsert: SubmissionRunHistoryDocument }, - ) => { - if (!history.has(filter._id)) { - history.set(filter._id, structuredClone(update.$setOnInsert)); - } - return { matchedCount: 1, modifiedCount: 1 }; - }), - } as unknown as Collection; - const services = { - find: vi.fn(() => queryResult([])), - findOne: vi.fn(async () => null), - } as unknown as Collection; - let firstCoordinatorReads = 0; - const coordinatorA = new SubmissionCoordinator({ - requests: { - find: vi.fn(() => - queryResult([ - firstCoordinatorReads++ === 0 - ? request("run-2", 2) - : request("run-3", 3), - ]), - ), - } as unknown as Collection, - submissions, - submissionRuns, - services, - }); - const coordinatorB = new SubmissionCoordinator({ - requests: { - find: vi.fn(() => queryResult([request("run-3", 3)])), - } as unknown as Collection, - submissions, - submissionRuns, - services, - }); - - await Promise.all([ - coordinatorA.reconcile("submission-1"), - coordinatorB.reconcile("submission-1"), - ]); - - expect(submission.run.requests).toEqual([ - { requestId: "request-1", runId: "run-3" }, - ]); - expect(submission.run.rollover).toBeUndefined(); - expect(history.size).toBe(2); - const knownRunIds = new Set([...history.keys(), submission.run._id]); - for (const archived of history.values()) { - expect(knownRunIds.has(archived.supersededBySubmissionRunId!)).toBe(true); - } - }); - - it("repairs an interrupted claimed rollover on the next polling reconciliation", async () => { - const requestDocument: RequestDocument = { - _id: "request-1", - projectId: "project-1", - scenario: { task: "Task", criteria: ["criterion"] }, - workerType: "worker", - createdAt: new Date(), - priority: 0, - submissionId: "submission-1", - run: { - _id: "run-2", - attemptNumber: 2, - status: "done", - }, - }; - let submission: SubmissionDocument = { - _id: "submission-1", - submissionId: "submission-1", - projectId: "project-1", - requestIds: ["request-1"], - createdAt: new Date(), - run: { - _id: "submission-run-1", - attemptNumber: 1, - vectorFingerprint: "initial", - requests: [{ requestId: "request-1", runId: "run-1" }], - createdAt: new Date(), - dependencyProjection: [], - blocking: [], - }, - }; - const history = new Map(); - let interruptReplacement = true; - const submissions = { - findOne: vi.fn(async () => structuredClone(submission)), - updateOne: vi.fn(async () => ({ matchedCount: 1, modifiedCount: 1 })), - findOneAndUpdate: vi.fn(async ( - filter: Record, - update: { $set: Record }, - ) => { - if (filter["run._id"] !== submission.run._id) return null; - if (update.$set["run.rollover"]) { - if (submission.run.rollover) return null; - submission = { - ...submission, - run: { - ...submission.run, - rollover: update.$set[ - "run.rollover" - ] as SubmissionDocument["run"]["rollover"], - }, - }; - } else if (update.$set.run) { - if (interruptReplacement) { - interruptReplacement = false; - throw new Error("simulated process interruption"); - } - submission = { - ...submission, - run: update.$set.run as SubmissionDocument["run"], - }; - } else { - submission = { - ...submission, - run: { - ...submission.run, - dependencyProjection: update.$set[ - "run.dependencyProjection" - ] as SubmissionDocument["run"]["dependencyProjection"], - blocking: update.$set[ - "run.blocking" - ] as SubmissionDocument["run"]["blocking"], - }, - }; - } - return structuredClone(submission); - }), - } as unknown as Collection; - const submissionRuns = { - updateOne: vi.fn(async ( - filter: { _id: string }, - update: { $setOnInsert: SubmissionRunHistoryDocument }, - ) => { - if (!history.has(filter._id)) { - history.set(filter._id, structuredClone(update.$setOnInsert)); - } - return { matchedCount: 1, modifiedCount: 1 }; - }), - } as unknown as Collection; - const collections = { - requests: { - find: vi.fn(() => queryResult([requestDocument])), - } as unknown as Collection, - submissions, - submissionRuns, - services: { - find: vi.fn(() => queryResult([])), - findOne: vi.fn(async () => null), - } as unknown as Collection, - }; - - await expect( - new SubmissionCoordinator(collections).reconcile("submission-1"), - ).rejects.toThrow("simulated process interruption"); - expect(submission.run.rollover).toBeDefined(); - expect(history.size).toBe(1); - - const repaired = await new SubmissionCoordinator(collections).reconcile( - "submission-1", - ); - - expect(repaired?.submission.run.attemptNumber).toBe(2); - expect(repaired?.submission.run.requests).toEqual([ - { requestId: "request-1", runId: "run-2" }, - ]); - expect(repaired?.submission.run.rollover).toBeUndefined(); - expect(history.get("submission-run-1")?.supersededBySubmissionRunId).toBe( - repaired?.submission.run._id, - ); - }); - - it("recovers an interrupted final submission write only for complete native request groups", async () => { - const requests: RequestDocument[] = [ - { - _id: "request-1", - projectId: "project-1", - scenario: { task: "Task", criteria: ["criterion"] }, - workerType: "worker", - createdAt: new Date(), - priority: 0, - submissionId: "submission-1", - submissionExpectedRequestCount: 2, - submissionRequestIndex: 0, - submissionInitialRunId: "run-1", - run: { _id: "retried-run", attemptNumber: 2, status: "pending" }, - }, - { - _id: "request-2", - projectId: "project-1", - scenario: { task: "Task", criteria: ["criterion"] }, - workerType: "worker", - createdAt: new Date(), - priority: 0, - submissionId: "submission-1", - submissionExpectedRequestCount: 2, - submissionRequestIndex: 1, - submissionInitialRunId: "run-2", - run: { _id: "run-2", attemptNumber: 1, status: "pending" }, - }, - ]; - let stored: SubmissionDocument | null = null; - const updateMany = vi.fn(async () => ({ modifiedCount: 2 })); - const coordinator = new SubmissionCoordinator({ - requests: { - find: vi.fn(() => queryResult(requests)), - updateMany, - } as unknown as Collection, - submissions: { - findOne: vi.fn(async () => stored), - updateOne: vi.fn(async ( - _filter: unknown, - update: { $setOnInsert: SubmissionDocument }, - ) => { - stored ??= update.$setOnInsert; - return { upsertedCount: 1 }; - }), - } as unknown as Collection, - submissionRuns: {} as Collection, - services: {} as Collection, - }); - - expect(await coordinator.recoverMissingNativeSubmissions()).toBe(1); - expect(stored).not.toBeNull(); - const recovered = stored as SubmissionDocument | null; - expect(recovered?.requestIds).toEqual(["request-1", "request-2"]); - expect(recovered?.run.requests).toEqual([ - { requestId: "request-1", runId: "run-1" }, - { requestId: "request-2", runId: "run-2" }, - ]); - expect(await coordinator.recoverMissingNativeSubmissions()).toBe(1); - expect(updateMany).toHaveBeenCalledTimes(2); - expect(updateMany).toHaveBeenCalledWith( - expect.objectContaining({ - submissionId: "submission-1", - submissionFinalizedAt: { $exists: false }, - }), - expect.objectContaining({ $set: { submissionFinalizedAt: expect.any(Date) } }), - ); - }); -}); diff --git a/packages/shared/src/submissions/submission-coordinator.ts b/packages/shared/src/submissions/submission-coordinator.ts deleted file mode 100644 index 5720a744a..000000000 --- a/packages/shared/src/submissions/submission-coordinator.ts +++ /dev/null @@ -1,573 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -import { v5 as uuidv5 } from "uuid"; -import type { Collection, Document } from "mongodb"; -import type { - HandlerServiceDocument, - RequestDocument, - SubmissionDocument, - SubmissionHandlerServiceDocument, - SubmissionQueueMessagePayload, - SubmissionReportPointer, - SubmissionRunHistoryDocument, - SubmissionRunState, - ServiceHandlerDocument, -} from "../types/types.js"; -import { - OBSERVATIONS_HANDLER_ID, - READOUT_HANDLER_ID, - computeSubmissionVectorFingerprint, - deriveSubmissionReadiness, - projectSubmissionDependency, -} from "./submission-state.js"; - -const SUBMISSION_RUN_NAMESPACE = "6b0f8a6a-5cda-4cb4-bcc4-79bb0ac7f54d"; - -export interface SubmissionCollections { - requests: Collection; - submissions: Collection; - submissionRuns: Collection; - services: Collection; -} - -export type EnqueueSubmission = ( - service: SubmissionHandlerServiceDocument, - message: SubmissionQueueMessagePayload, -) => Promise; - -export interface ReconcileSubmissionResult { - submission: SubmissionDocument; - readiness: ReturnType; - rolled: boolean; - dispatched: boolean; -} - -function submissionRunId( - submissionId: string, - attemptNumber: number, - vectorFingerprint: string, -): string { - return uuidv5( - `${submissionId}:${attemptNumber}:${vectorFingerprint}`, - SUBMISSION_RUN_NAMESPACE, - ); -} - -function initialSubmissionRun( - submissionId: string, - requests: Array<{ requestId: string; runId: string }>, - createdAt: Date, -): SubmissionRunState { - const vectorFingerprint = computeSubmissionVectorFingerprint(requests); - return { - _id: submissionRunId(submissionId, 1, vectorFingerprint), - attemptNumber: 1, - vectorFingerprint, - requests, - createdAt, - dependencyProjection: [], - blocking: [], - }; -} - -export async function ensureNativeSubmission(input: { - collections: Pick; - submissionId: string; - projectId: string; - requests: Array<{ requestId: string; runId: string }>; - createdAt?: Date; -}): Promise { - const requestIds = input.requests.map((request) => request.requestId); - const createdAt = input.createdAt ?? new Date(); - const doc: SubmissionDocument = { - _id: input.submissionId, - submissionId: input.submissionId, - projectId: input.projectId, - requestIds, - createdAt, - run: initialSubmissionRun(input.submissionId, input.requests, createdAt), - }; - await input.collections.submissions.updateOne( - { _id: input.submissionId } as Document, - { $setOnInsert: doc } as Document, - { upsert: true }, - ); - const persisted = await input.collections.submissions.findOne({ - _id: input.submissionId, - } as Document); - if (!persisted) { - throw new Error(`Submission ${input.submissionId} was not persisted`); - } - const sameProject = persisted.projectId === input.projectId; - const sameRequests = - persisted.requestIds.length === requestIds.length && - persisted.requestIds.every((requestId, index) => requestId === requestIds[index]); - const sameInitialVector = - persisted.run.attemptNumber > 1 || - (persisted.run.requests.length === input.requests.length && - persisted.run.requests.every((request, index) => { - const expected = input.requests[index]; - return ( - request.requestId === expected?.requestId && - request.runId === expected.runId - ); - })); - if (!sameProject || !sameRequests || !sameInitialVector) { - throw new Error( - `Submission ${input.submissionId} already exists with a different immutable request vector`, - ); - } - return persisted; -} - -async function loadPostProcessHandlers( - services: Collection, -): Promise { - const documents = await services - .find({ type: "post-process-handler" }) - .toArray(); - return documents.filter( - (document): document is HandlerServiceDocument => - document.type === "post-process-handler", - ); -} - -async function loadReadoutService( - services: Collection, -): Promise { - const document = await services.findOne({ - _id: READOUT_HANDLER_ID, - type: "submission-handler", - }); - return document?.type === "submission-handler" ? document : null; -} - -async function archiveAndRoll(input: { - collections: SubmissionCollections; - submission: SubmissionDocument; - requests: Array<{ requestId: string; runId: string }>; - reason: "request-vector-changed" | "manual-retry"; -}): Promise { - let claimed = input.submission; - if (!claimed.run.rollover) { - const claimedAt = new Date(); - const nextAttemptNumber = claimed.run.attemptNumber + 1; - const vectorFingerprint = computeSubmissionVectorFingerprint(input.requests); - const nextSubmissionRunId = submissionRunId( - claimed._id, - nextAttemptNumber, - vectorFingerprint, - ); - const claimedDocument = await input.collections.submissions.findOneAndUpdate( - { - _id: claimed._id, - "run._id": claimed.run._id, - "run.vectorFingerprint": claimed.run.vectorFingerprint, - "run.rollover": { $exists: false }, - } as Document, - { - $set: { - "run.rollover": { - nextSubmissionRunId, - vectorFingerprint, - requests: input.requests, - reason: input.reason, - claimedAt, - }, - "run.updatedAt": claimedAt, - updatedAt: claimedAt, - }, - } as Document, - { returnDocument: "after" }, - ); - if (!claimedDocument) return null; - claimed = claimedDocument; - } - const rollover = claimed.run.rollover; - if (!rollover) return null; - const { rollover: _rollover, reconcileAfter: _reconcileAfter, ...archivedRun } = - claimed.run; - const archived: SubmissionRunHistoryDocument = { - ...archivedRun, - submissionId: claimed._id, - projectId: claimed.projectId, - supersededAt: rollover.claimedAt, - supersededBySubmissionRunId: rollover.nextSubmissionRunId, - supersededReason: rollover.reason, - }; - await input.collections.submissionRuns.updateOne( - { _id: archived._id } as Document, - { $setOnInsert: archived } as Document, - { upsert: true }, - ); - - const nextRun: SubmissionRunState = { - _id: rollover.nextSubmissionRunId, - attemptNumber: claimed.run.attemptNumber + 1, - vectorFingerprint: rollover.vectorFingerprint, - requests: rollover.requests, - createdAt: rollover.claimedAt, - dependencyProjection: [], - blocking: [], - }; - const updated = await input.collections.submissions.findOneAndUpdate( - { - _id: claimed._id, - "run._id": claimed.run._id, - "run.rollover.nextSubmissionRunId": rollover.nextSubmissionRunId, - } as Document, - { - $set: { - run: nextRun, - updatedAt: rollover.claimedAt, - }, - } as Document, - { returnDocument: "after" }, - ); - return updated; -} - -export class SubmissionCoordinator { - private recoveryAfterRequestId: string | null = null; - - constructor( - private readonly collections: SubmissionCollections, - private readonly enqueue?: EnqueueSubmission, - ) {} - - async recoverMissingNativeSubmissions(limit = 30): Promise { - const filter: Document = { - submissionId: { $exists: true }, - submissionExpectedRequestCount: { $exists: true }, - submissionFinalizedAt: { $exists: false }, - deletedAt: { $exists: false }, - ...(this.recoveryAfterRequestId - ? { _id: { $gt: this.recoveryAfterRequestId } } - : {}), - }; - const candidates = await this.collections.requests - .find(filter) - .project({ - _id: 1, - projectId: 1, - submissionId: 1, - submissionExpectedRequestCount: 1, - submissionRequestIndex: 1, - submissionInitialRunId: 1, - "run._id": 1, - createdAt: 1, - }) - .sort({ _id: 1 }) - .limit(limit) - .toArray(); - this.recoveryAfterRequestId = - candidates.length === limit ? candidates.at(-1)?._id ?? null : null; - let recovered = 0; - const visited = new Set(); - for (const candidate of candidates) { - const submissionId = candidate.submissionId; - const expected = candidate.submissionExpectedRequestCount; - if (!submissionId || !expected || visited.has(submissionId)) continue; - visited.add(submissionId); - const requests = await this.collections.requests - .find({ - submissionId, - projectId: candidate.projectId, - submissionExpectedRequestCount: expected, - deletedAt: { $exists: false }, - } as Document) - .sort({ submissionRequestIndex: 1, _id: 1 }) - .toArray(); - if ( - requests.length !== expected || - requests.some( - (request, index) => - request.submissionRequestIndex !== index || - !request.submissionInitialRunId, - ) - ) { - continue; - } - await ensureNativeSubmission({ - collections: this.collections, - submissionId, - projectId: candidate.projectId, - requests: requests.map((request) => ({ - requestId: request._id, - runId: request.submissionInitialRunId!, - })), - createdAt: requests[0]?.createdAt, - }); - await this.collections.requests.updateMany( - { - submissionId, - projectId: candidate.projectId, - submissionExpectedRequestCount: expected, - submissionFinalizedAt: { $exists: false }, - } as Document, - { $set: { submissionFinalizedAt: new Date() } } as Document, - ); - recovered++; - } - return recovered; - } - - async reconcile(submissionId: string): Promise { - let rolled = false; - for (let attempt = 0; attempt < 4; attempt++) { - const submission = await this.collections.submissions.findOne({ - _id: submissionId, - } as Document); - if (!submission) return null; - if ( - submission.run.reconcileAfter && - submission.run.reconcileAfter.getTime() > Date.now() - ) { - return { - submission, - readiness: deriveSubmissionReadiness(submission.run), - rolled, - dispatched: false, - }; - } - if (submission.run.reconcileAfter) { - await this.collections.submissions.updateOne( - { _id: submissionId, "run._id": submission.run._id } as Document, - { - $unset: { "run.reconcileAfter": "" }, - $set: { updatedAt: new Date() }, - } as Document, - ); - delete submission.run.reconcileAfter; - } - if (submission.run.rollover) { - const updated = await archiveAndRoll({ - collections: this.collections, - submission, - requests: submission.run.rollover.requests, - reason: submission.run.rollover.reason, - }); - rolled = true; - if (!updated) continue; - continue; - } - - const requestDocs = await this.collections.requests - .find({ - _id: { $in: submission.requestIds }, - projectId: submission.projectId, - deletedAt: { $exists: false }, - } as Document) - .toArray(); - const byId = new Map(requestDocs.map((request) => [request._id, request])); - const latestVector = submission.requestIds.flatMap((requestId) => { - const runId = byId.get(requestId)?.run?._id; - return runId ? [{ requestId, runId }] : []; - }); - const latestFingerprint = - latestVector.length === submission.requestIds.length - ? computeSubmissionVectorFingerprint(latestVector) - : submission.run.vectorFingerprint; - - if ( - latestVector.length === submission.requestIds.length && - latestFingerprint !== submission.run.vectorFingerprint - ) { - const updated = await archiveAndRoll({ - collections: this.collections, - submission, - requests: latestVector, - reason: "request-vector-changed", - }); - rolled = true; - if (!updated) continue; - continue; - } - - const handlers = await loadPostProcessHandlers(this.collections.services); - const projected = submission.run.requests.map((ref) => - projectSubmissionDependency({ - requestId: ref.requestId, - expectedRunId: ref.runId, - run: byId.get(ref.requestId)?.run, - handlers, - handlerId: OBSERVATIONS_HANDLER_ID, - }), - ); - const dependencyProjection = projected.map((result) => result.projection); - const blocking = projected.flatMap((result) => - result.blocking ? [result.blocking] : [], - ); - const now = new Date(); - const refreshed = await this.collections.submissions.findOneAndUpdate( - { _id: submissionId, "run._id": submission.run._id } as Document, - { - $set: { - "run.dependencyProjection": dependencyProjection, - "run.blocking": blocking, - "run.updatedAt": now, - updatedAt: now, - }, - } as Document, - { returnDocument: "after" }, - ); - if (!refreshed) continue; - - let dispatched = false; - const readiness = deriveSubmissionReadiness(refreshed.run); - if (readiness === "ready" && this.enqueue) { - const service = await loadReadoutService(this.collections.services); - const existing = refreshed.run.handlerStatus?.[READOUT_HANDLER_ID]; - const eligible = - service && - (!existing || - (service.autoBackfill && - existing.status === "done" && - (existing.version ?? 0) < service.version)); - if (service && eligible) { - await this.enqueue(service, { - type: service.selector, - submissionId, - submissionRunId: refreshed.run._id, - }); - const queuedAt = new Date(); - const marked = await this.collections.submissions.updateOne( - { - _id: submissionId, - "run._id": refreshed.run._id, - [`run.handlerStatus.${READOUT_HANDLER_ID}.status`]: { - $nin: ["queued", "processing", "failed"], - }, - } as Document, - { - $set: { - [`run.handlerStatus.${READOUT_HANDLER_ID}`]: { - status: "queued", - version: service.version, - updatedAt: queuedAt, - }, - "run.updatedAt": queuedAt, - updatedAt: queuedAt, - }, - } as Document, - ); - dispatched = marked.modifiedCount === 1; - } - } - const finalSubmission = dispatched - ? await this.collections.submissions.findOne({ _id: submissionId } as Document) - : refreshed; - if (!finalSubmission) return null; - return { - submission: finalSubmission, - readiness: deriveSubmissionReadiness(finalSubmission.run), - rolled, - dispatched, - }; - } - throw new Error(`Submission ${submissionId} could not be reconciled after concurrent updates`); - } - - async retryCurrent( - submissionId: string, - expectedSubmissionRunId: string, - ): Promise { - const reconciled = await this.reconcile(submissionId); - if (!reconciled) return null; - if (reconciled.submission.run._id !== expectedSubmissionRunId) { - return reconciled; - } - if (reconciled.readiness === "blocked" || reconciled.readiness === "waiting") { - return reconciled; - } - const rolled = await archiveAndRoll({ - collections: this.collections, - submission: reconciled.submission, - requests: reconciled.submission.run.requests, - reason: "manual-retry", - }); - if (!rolled) return this.reconcile(submissionId); - return this.reconcile(submissionId); - } -} - -export async function claimSubmissionHandler(input: { - submissions: Collection; - submissionId: string; - submissionRunId: string; - handlerId: string; - version: number; -}): Promise { - const now = new Date(); - return input.submissions.findOneAndUpdate( - { - _id: input.submissionId, - "run._id": input.submissionRunId, - "run.rollover": { $exists: false }, - [`run.handlerStatus.${input.handlerId}.status`]: { $in: ["queued"] }, - } as Document, - { - $set: { - [`run.handlerStatus.${input.handlerId}`]: { - status: "processing", - version: input.version, - updatedAt: now, - }, - "run.startedAt": now, - "run.updatedAt": now, - updatedAt: now, - }, - } as Document, - { returnDocument: "after" }, - ); -} - -export async function isCurrentSubmissionRun(input: { - submissions: Collection; - submissionId: string; - submissionRunId: string; -}): Promise { - return Boolean( - await input.submissions.findOne({ - _id: input.submissionId, - "run._id": input.submissionRunId, - "run.rollover": { $exists: false }, - } as Document), - ); -} - -export async function completeSubmissionHandler(input: { - submissions: Collection; - submissionId: string; - submissionRunId: string; - handlerId: string; - status: "done" | "failed"; - version: number; - error?: string; - report?: SubmissionReportPointer; -}): Promise { - const now = new Date(); - const set: Record = { - [`run.handlerStatus.${input.handlerId}`]: { - status: input.status, - version: input.version, - updatedAt: now, - ...(input.error ? { error: input.error } : {}), - }, - "run.updatedAt": now, - "run.finishedAt": now, - updatedAt: now, - }; - if (input.report) set["run.report"] = input.report; - const result = await input.submissions.updateOne( - { - _id: input.submissionId, - "run._id": input.submissionRunId, - "run.rollover": { $exists: false }, - [`run.handlerStatus.${input.handlerId}.status`]: "processing", - } as Document, - { $set: set } as Document, - ); - return result.modifiedCount === 1; -} diff --git a/packages/shared/src/submissions/submission-state.test.ts b/packages/shared/src/submissions/submission-state.test.ts deleted file mode 100644 index 854558c68..000000000 --- a/packages/shared/src/submissions/submission-state.test.ts +++ /dev/null @@ -1,155 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -import { describe, expect, it } from "vitest"; -import type { HandlerServiceDocument, RunState, SubmissionRunState } from "../types/types.js"; -import { - OBSERVATIONS_HANDLER_ID, - READOUT_HANDLER_ID, - computeSubmissionVectorFingerprint, - deriveSubmissionReadiness, - evaluateNamedHandlerState, - isHandlerDagDrained, - projectSubmissionDependency, -} from "./submission-state.js"; - -const handlers: HandlerServiceDocument[] = [ - { - _id: "pp-atif", - type: "post-process-handler", - version: 1, - queue: "post-processor", - selector: "atif", - autoBackfill: true, - dependsOn: [], - }, - { - _id: OBSERVATIONS_HANDLER_ID, - type: "post-process-handler", - version: 1, - queue: "post-processor", - selector: "observations", - autoBackfill: true, - dependsOn: ["pp-atif"], - }, -]; - -function run(handlerStatus: RunState["handlerStatus"]): RunState { - return { - _id: "run-1", - attemptNumber: 1, - status: "done", - outcome: "failed", - handlerStatus, - observationResults: [ - { criterionId: "observation-1", passed: false, feedback: "evidence", evaluated: true }, - ], - }; -} - -describe("submission state helpers", () => { - it("fingerprints vectors deterministically independent of request order", () => { - expect( - computeSubmissionVectorFingerprint([ - { requestId: "b", runId: "2" }, - { requestId: "a", runId: "1" }, - ]), - ).toBe( - computeSubmissionVectorFingerprint([ - { requestId: "a", runId: "1" }, - { requestId: "b", runId: "2" }, - ]), - ); - }); - - it("treats completed observations as ready even for failed coding outcomes and passed:false evidence", () => { - const result = projectSubmissionDependency({ - requestId: "request-1", - expectedRunId: "run-1", - run: run({ - [OBSERVATIONS_HANDLER_ID]: { status: "done", version: 1 }, - unrelated: { status: "failed", error: "ignored" }, - }), - handlers, - }); - - expect(result).toEqual({ - projection: { - requestId: "request-1", - runId: "run-1", - handlerId: OBSERVATIONS_HANDLER_ID, - state: "ready", - version: 1, - }, - drifted: false, - }); - }); - - it("blocks when observations fail or are blocked by a failed dependency", () => { - expect( - evaluateNamedHandlerState( - OBSERVATIONS_HANDLER_ID, - handlers, - { [OBSERVATIONS_HANDLER_ID]: { status: "failed" } }, - ), - ).toBe("failed"); - expect( - evaluateNamedHandlerState( - OBSERVATIONS_HANDLER_ID, - handlers, - { "pp-atif": { status: "failed" } }, - ), - ).toBe("blocked"); - }); - - it("reuses named-handler DAG semantics for existing report triggering", () => { - expect( - isHandlerDagDrained(handlers, { - "pp-atif": { status: "failed" }, - }), - ).toBe(true); - expect( - isHandlerDagDrained(handlers, { - "pp-atif": { status: "processing" }, - }), - ).toBe(false); - }); - - it("detects exact run drift", () => { - const result = projectSubmissionDependency({ - requestId: "request-1", - expectedRunId: "old-run", - run: run({ [OBSERVATIONS_HANDLER_ID]: { status: "done" } }), - handlers, - }); - expect(result.drifted).toBe(true); - expect(result.blocking?.reason).toBe("run-changed"); - }); - - it("derives submission readiness from dependencies and readout progress", () => { - const base: SubmissionRunState = { - _id: "submission-run-1", - attemptNumber: 1, - vectorFingerprint: "fingerprint", - requests: [{ requestId: "request-1", runId: "run-1" }], - createdAt: new Date(), - dependencyProjection: [{ - requestId: "request-1", - runId: "run-1", - handlerId: OBSERVATIONS_HANDLER_ID, - state: "ready", - }], - blocking: [], - }; - - expect(deriveSubmissionReadiness(base)).toBe("ready"); - expect(deriveSubmissionReadiness({ - ...base, - handlerStatus: { [READOUT_HANDLER_ID]: { status: "processing" } }, - })).toBe("processing"); - expect(deriveSubmissionReadiness({ - ...base, - handlerStatus: { [READOUT_HANDLER_ID]: { status: "done" } }, - })).toBe("done"); - }); -}); diff --git a/packages/shared/src/submissions/submission-state.ts b/packages/shared/src/submissions/submission-state.ts deleted file mode 100644 index cbaf44452..000000000 --- a/packages/shared/src/submissions/submission-state.ts +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -import type { - HandlerRunStatus, - HandlerServiceDocument, - RunState, - SubmissionBlockingDetail, - SubmissionDependencyProjection, - SubmissionHandlerState, - SubmissionReadiness, - SubmissionRequestRef, - SubmissionRunState, -} from "../types/types.js"; -import { DependencyGraph } from "../graph/dependency-graph.js"; - -export const OBSERVATIONS_HANDLER_ID = "pp-observations"; -export const READOUT_HANDLER_ID = "pp-readout"; - -export function computeSubmissionVectorFingerprint( - requests: readonly SubmissionRequestRef[], -): string { - const canonical = [...requests] - .sort((left, right) => left.requestId.localeCompare(right.requestId)) - .map(({ requestId, runId }) => `${requestId}:${runId}`) - .join("\n"); - let hash = 0xcbf29ce484222325n; - for (let index = 0; index < canonical.length; index++) { - hash ^= BigInt(canonical.charCodeAt(index)); - hash = BigInt.asUintN(64, hash * 0x100000001b3n); - } - return hash.toString(16).padStart(16, "0"); -} - -export function evaluateNamedHandlerState( - handlerId: string, - handlers: readonly HandlerServiceDocument[], - handlerStatus: Record | undefined, -): SubmissionHandlerState { - const status = handlerStatus?.[handlerId]; - if (status?.status === "done") return "ready"; - if (status?.status === "failed") return "failed"; - - const graph = new DependencyGraph( - handlers.map((handler) => ({ - id: handler._id, - dependsOn: handler.dependsOn, - })), - ); - - if (!handlers.some((handler) => handler._id === handlerId)) { - return "waiting"; - } - - for (const ancestorId of graph.getAncestors(handlerId)) { - if (handlerStatus?.[ancestorId]?.status === "failed") return "blocked"; - } - return "waiting"; -} - -function isTerminalHandlerState( - handlerId: string, - graph: DependencyGraph, - handlerStatus: Record | undefined, -): boolean { - const status = handlerStatus?.[handlerId]?.status; - if (status === "done" || status === "failed") return true; - if (status !== undefined) return false; - for (const ancestorId of graph.getAncestors(handlerId)) { - if (handlerStatus?.[ancestorId]?.status === "failed") return true; - } - return false; -} - -export function isHandlerDagDrained( - handlers: readonly HandlerServiceDocument[], - handlerStatus: Record | undefined, -): boolean { - const graph = new DependencyGraph( - handlers.map((handler) => ({ - id: handler._id, - dependsOn: handler.dependsOn, - })), - ); - return handlers.every((handler) => - isTerminalHandlerState(handler._id, graph, handlerStatus), - ); -} - -export function projectSubmissionDependency(input: { - requestId: string; - expectedRunId: string; - run?: RunState; - handlers: readonly HandlerServiceDocument[]; - handlerId?: string; -}): { - projection: SubmissionDependencyProjection; - blocking?: SubmissionBlockingDetail; - drifted: boolean; -} { - const handlerId = input.handlerId ?? OBSERVATIONS_HANDLER_ID; - if (!input.run) { - return { - projection: { - requestId: input.requestId, - runId: input.expectedRunId, - handlerId, - state: "blocked", - }, - blocking: { - requestId: input.requestId, - runId: input.expectedRunId, - handlerId, - reason: "request-missing", - }, - drifted: false, - }; - } - if (input.run._id !== input.expectedRunId) { - return { - projection: { - requestId: input.requestId, - runId: input.expectedRunId, - handlerId, - state: "blocked", - }, - blocking: { - requestId: input.requestId, - runId: input.expectedRunId, - handlerId, - reason: "run-changed", - }, - drifted: true, - }; - } - - const source = input.run.handlerStatus?.[handlerId]; - const state = evaluateNamedHandlerState( - handlerId, - input.handlers, - input.run.handlerStatus, - ); - const projection: SubmissionDependencyProjection = { - requestId: input.requestId, - runId: input.expectedRunId, - handlerId, - state, - ...(source?.version !== undefined ? { version: source.version } : {}), - ...(source?.updatedAt ? { updatedAt: source.updatedAt } : {}), - ...(source?.error ? { error: source.error } : {}), - }; - if (state !== "failed" && state !== "blocked") { - return { projection, drifted: false }; - } - return { - projection, - blocking: { - requestId: input.requestId, - runId: input.expectedRunId, - handlerId, - reason: state === "failed" ? "dependency-failed" : "dependency-blocked", - ...(source?.error ? { error: source.error } : {}), - }, - drifted: false, - }; -} - -export function deriveSubmissionReadiness(run: SubmissionRunState): SubmissionReadiness { - const readout = run.handlerStatus?.[READOUT_HANDLER_ID]; - if (readout?.status === "done") return "done"; - if (readout?.status === "failed") return "failed"; - if (readout?.status === "queued" || readout?.status === "processing") { - return "processing"; - } - if (run.blocking.length > 0) return "blocked"; - if ( - run.requests.length > 0 && - run.dependencyProjection.length === run.requests.length && - run.dependencyProjection.every((dependency) => dependency.state === "ready") - ) { - return "ready"; - } - return "waiting"; -} diff --git a/packages/shared/src/types/types.ts b/packages/shared/src/types/types.ts index 5f8315307..9108cb690 100644 --- a/packages/shared/src/types/types.ts +++ b/packages/shared/src/types/types.ts @@ -384,19 +384,17 @@ export interface RequestDocument { agentVersion?: string; // Agent software version prefix (e.g. "copilot-0.0.415") — FK → AgentVersion.agentVersion profileId?: string; // FK → ProfileDocument._id (the profile lineage) profileVersionId?: string; // FK → ProfileVersionDocument._id (exact version used) - submissionId?: string; // FK → SubmissionDocument._id - /** Expected native request count for interrupted SubmissionDocument finalization. - * Historical/imported requests omit it and remain view-only if their parent - * submission document is missing. */ - submissionExpectedRequestCount?: number; - /** Stable position within the native submission's immutable request vector. */ - submissionRequestIndex?: number; - /** Run id minted for the submission's first attempt. It remains unchanged - * across request retries so recovery can rebuild the exact initial vector. */ - submissionInitialRunId?: string; - /** Set after the parent SubmissionDocument has been idempotently finalized. - * Missing on interrupted native writes so the scheduler can recover them. */ - submissionFinalizedAt?: Date; + experimentId?: string; + experimentRevisionId?: string; + experimentExecutionId?: string; + /** Expected request count for interrupted Experiment finalization. */ + experimentExpectedRequestCount?: number; + /** Stable position within the Experiment's immutable request vector. */ + experimentRequestIndex?: number; + /** Initial run id retained so recovery can rebuild the exact first vector. */ + experimentInitialRunId?: string; + /** Set only after the parent Experiment has been published. */ + experimentFinalizedAt?: Date; /** Scheduling priority. Higher = processed first. Default: 0. */ priority: number; /** @@ -579,9 +577,9 @@ export interface HandlerServiceDocument { dependsOn: string[]; } -export interface SubmissionHandlerServiceDocument { +export interface ExperimentHandlerServiceDocument { _id: string; - type: "submission-handler"; + type: "experiment-handler"; version: number; queue: string; selector: string; @@ -592,104 +590,7 @@ export interface SubmissionHandlerServiceDocument { export type ServiceHandlerDocument = | HandlerServiceDocument - | SubmissionHandlerServiceDocument; - -export interface SubmissionRequestRef { - requestId: string; - runId: string; -} - -export type SubmissionHandlerState = "waiting" | "ready" | "failed" | "blocked"; - -export interface SubmissionDependencyProjection { - requestId: string; - runId: string; - handlerId: string; - state: SubmissionHandlerState; - version?: number; - updatedAt?: Date; - error?: string; -} - -export interface SubmissionBlockingDetail { - requestId: string; - runId: string; - handlerId: string; - reason: - | "request-missing" - | "run-changed" - | "dependency-not-ready" - | "dependency-failed" - | "dependency-blocked"; - error?: string; -} - -export interface SubmissionReportPointer { - reportId?: string; - url?: string; - updatedAt?: Date; -} - -export interface SubmissionRolloverClaim { - nextSubmissionRunId: string; - vectorFingerprint: string; - requests: SubmissionRequestRef[]; - reason: "request-vector-changed" | "manual-retry"; - claimedAt: Date; -} - -export interface SubmissionRunState { - _id: string; - attemptNumber: number; - vectorFingerprint: string; - requests: SubmissionRequestRef[]; - createdAt: Date; - updatedAt?: Date; - startedAt?: Date; - finishedAt?: Date; - dependencyProjection: SubmissionDependencyProjection[]; - handlerStatus?: Record; - blocking: SubmissionBlockingDetail[]; - report?: SubmissionReportPointer; - /** Internal scheduler hold used by bulk request retry so one completed - * replacement batch produces one vector rollover. Expiry self-heals crashes. */ - reconcileAfter?: Date; - /** Internal resumable rollover claim. Its CAS winner exclusively determines - * the immutable archive's successor metadata. */ - rollover?: SubmissionRolloverClaim; - supersededAt?: Date; - supersededBySubmissionRunId?: string; - supersededReason?: "request-vector-changed" | "manual-retry"; -} - -export interface SubmissionDocument { - _id: string; - submissionId: string; - projectId: string; - requestIds: string[]; - createdAt: Date; - updatedAt?: Date; - run: SubmissionRunState; -} - -export interface SubmissionRunHistoryDocument extends SubmissionRunState { - submissionId: string; - projectId: string; -} - -export type SubmissionReadiness = - | "waiting" - | "ready" - | "blocked" - | "processing" - | "done" - | "failed"; - -export interface SubmissionQueueMessagePayload { - type: string; - submissionId: string; - submissionRunId: string; -} + | ExperimentHandlerServiceDocument; /** * Field names whose values move from `RequestDocument` (top-level, legacy @@ -824,7 +725,7 @@ export interface QueueProcessorConfig extends BaseQueueProcessorConfig { // --- Runs grouping types (shared between API and portal) --- -export type GroupByKey = "none" | "task" | "submissionId" | "profile"; +export type GroupByKey = "none" | "task" | "experimentId" | "profile"; export interface AggregateStats { min: number; @@ -842,7 +743,7 @@ export interface GroupUniformValues { skillRevisions?: string[]; extensions?: string[]; status?: "pending" | "processing" | "done"; - submissionId?: string; + experimentId?: string; task?: string; } From ed1ca2c6bdc53ac38834b8bfed865a343f2385a6 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Thu, 15 Oct 2026 22:39:12 -0700 Subject: [PATCH 07/25] feat(experiments): add canonical API runtime Replace legacy Submission API and scheduler surfaces with project-scoped Experiment creation, execution history, retry, exact queue claims, recovery, and OpenAPI contracts. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b05e74ff-44da-4e78-9f05-0e256009e7ee --- .../openapi-snapshot.test.ts.snap | 1344 ++++++++++++++--- apps/api/src/endpoints.test.ts | 80 +- apps/api/src/experiment-openapi.test.ts | 68 + apps/api/src/grouping.test.ts | 10 +- apps/api/src/grouping.ts | 14 +- apps/api/src/index.ts | 33 +- apps/api/src/openapi/openapi.test.ts | 2 +- apps/api/src/route-context.ts | 12 +- apps/api/src/routes/experiments.test.ts | 247 +++ apps/api/src/routes/experiments.ts | 691 +++++++++ apps/api/src/routes/profiles.ts | 2 +- apps/api/src/routes/requests/archive.test.ts | 2 - apps/api/src/routes/requests/index.ts | 171 +-- apps/api/src/routes/submissions.test.ts | 253 ---- apps/api/src/routes/submissions.ts | 245 --- apps/api/src/test-helpers.ts | 15 +- apps/api/src/utils/agent-helpers.ts | 2 +- apps/api/src/utils/codebase-helpers.ts | 2 +- .../src/experiment-coordinator.test.ts | 365 +++++ apps/scheduler/src/experiment-coordinator.ts | 527 +++++++ apps/scheduler/src/experiment-dispatcher.ts | 169 +++ apps/scheduler/src/handler-dispatcher.ts | 28 +- apps/scheduler/src/index.ts | 39 +- apps/scheduler/src/notify-routes.test.ts | 188 +++ apps/scheduler/src/notify-routes.ts | 122 +- apps/scheduler/src/submission-dispatcher.ts | 112 -- packages/shared/src/cursor.test.ts | 6 +- .../src/experiments/experiment-store.test.ts | 60 +- .../src/experiments/experiment-store.ts | 6 + packages/shared/src/experiments/schemas.ts | 3 + packages/shared/src/experiments/types.ts | 3 + packages/shared/src/schemas/request.ts | 21 +- packages/shared/src/types/types.ts | 22 +- 33 files changed, 3798 insertions(+), 1066 deletions(-) create mode 100644 apps/api/src/experiment-openapi.test.ts create mode 100644 apps/api/src/routes/experiments.test.ts create mode 100644 apps/api/src/routes/experiments.ts delete mode 100644 apps/api/src/routes/submissions.test.ts delete mode 100644 apps/api/src/routes/submissions.ts create mode 100644 apps/scheduler/src/experiment-coordinator.test.ts create mode 100644 apps/scheduler/src/experiment-coordinator.ts create mode 100644 apps/scheduler/src/experiment-dispatcher.ts delete mode 100644 apps/scheduler/src/submission-dispatcher.ts diff --git a/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap b/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap index 317154c30..c1d80e316 100644 --- a/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap +++ b/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap @@ -721,6 +721,158 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` ], "type": "object", }, + "CreateExperimentInput": { + "properties": { + "metadata": { + "properties": { + "description": { + "maxLength": 4096, + "type": "string", + }, + "name": { + "maxLength": 256, + "minLength": 1, + "type": "string", + }, + "tags": { + "items": { + "maxLength": 64, + "minLength": 1, + "type": "string", + }, + "maxItems": 32, + "type": "array", + }, + }, + "required": [ + "name", + ], + "type": "object", + }, + "mode": { + "enum": [ + "compose", + "empty", + ], + "type": "string", + }, + "requests": { + "default": [], + "items": { + "properties": { + "agentsMd": { + "type": "string", + }, + "agentsMdParentIds": { + "items": { + "type": "string", + }, + "type": "array", + }, + "codebaseRevisionId": { + "type": "string", + }, + "extensions": { + "items": { + "type": "string", + }, + "type": "array", + }, + "gates": { + "items": { + "$ref": "#/components/schemas/GateConfig", + }, + "type": "array", + }, + "maxIterations": { + "maximum": 50, + "minimum": 1, + "type": "integer", + }, + "mcpServers": { + "items": { + "type": "string", + }, + "type": "array", + }, + "model": { + "type": "string", + }, + "observations": { + "items": { + "type": "string", + }, + "type": "array", + }, + "persona": { + "$ref": "#/components/schemas/Persona", + }, + "personaInstructions": { + "type": "string", + }, + "priority": { + "type": "integer", + }, + "profileId": { + "type": "string", + }, + "reasoningEffort": { + "type": "string", + }, + "resources": { + "items": { + "$ref": "#/components/schemas/ResourceBindingSpec", + }, + "type": "array", + }, + "scenario": { + "$ref": "#/components/schemas/Scenario", + }, + "skillRevisions": { + "items": { + "type": "string", + }, + "type": "array", + }, + "workerType": { + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "scenario", + "workerType", + ], + "type": "object", + }, + "maxItems": 250000, + "type": "array", + }, + }, + "required": [ + "mode", + "metadata", + ], + "type": "object", + }, + "CreateExperimentResponse": { + "properties": { + "experiment": { + "$ref": "#/components/schemas/ExperimentResponse", + }, + "requestIds": { + "items": { + "type": "string", + }, + "type": "array", + }, + }, + "required": [ + "experiment", + "requestIds", + ], + "type": "object", + }, "CreateExtensionInput": { "properties": { "_id": { @@ -986,6 +1138,8 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "type": "string", }, "profileVariations": { + "deprecated": true, + "description": "Deprecated for composition. Use POST /api/v1/experiments and its requests array.", "items": { "type": "string", }, @@ -1349,11 +1503,55 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` ], "type": "object", }, - "ExtensionResponse": { + "ExperimentExecutionListResponse": { "properties": { - "_id": { + "data": { + "items": { + "$ref": "#/components/schemas/ExperimentExecutionResponse", + }, + "type": "array", + }, + "nextCursor": { "type": "string", }, + }, + "required": [ + "data", + ], + "type": "object", + }, + "ExperimentExecutionResponse": { + "properties": { + "attemptNumber": { + "type": "integer", + }, + "blockers": { + "items": { + "properties": { + "error": { + "type": "string", + }, + "handlerId": { + "type": "string", + }, + "reason": { + "type": "string", + }, + "requestId": { + "type": "string", + }, + "runId": { + "type": "string", + }, + }, + "required": [ + "handlerId", + "reason", + ], + "type": "object", + }, + "type": "array", + }, "createdAt": { "format": "date-time", "type": [ @@ -1361,33 +1559,91 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "null", ], }, - "deletedAt": { + "discardReason": { + "type": "string", + }, + "discardedAt": { "format": "date-time", "type": [ "string", "null", ], }, - "description": { + "experimentExecutionId": { "type": "string", }, - "name": { - "type": "string", + "finishedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], }, - "origin": { + "progress": { + "additionalProperties": { + "properties": { + "error": { + "type": "string", + }, + "status": { + "enum": [ + "queued", + "processing", + "done", + "failed", + ], + "type": "string", + }, + "updatedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + }, + "required": [ + "status", + "updatedAt", + ], + "type": "object", + }, + "type": "object", + }, + "readiness": { "enum": [ - "marketplace", - "manual", + "empty", + "waiting", + "blocked", + "ready", + "processing", + "done", + "failed", ], "type": "string", }, - "projectId": { - "type": "string", + "startedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], }, - "publisher": { + "supersededAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "supersededByExperimentExecutionId": { "type": "string", }, - "slug": { + "supersededReason": { + "enum": [ + "rollover", + "retry", + ], "type": "string", }, "updatedAt": { @@ -1399,83 +1655,396 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, }, "required": [ - "_id", - "slug", - "publisher", - "name", - "origin", + "experimentExecutionId", + "attemptNumber", + "readiness", + "blockers", + "progress", "createdAt", - "projectId", - ], - "type": "object", - }, - "ExtensionSearchResult": { - "properties": { - "description": { - "type": "string", - }, - "id": { - "type": "string", - }, - "internal": { - "type": "boolean", - }, - "name": { - "type": "string", - }, - "publisher": { - "type": "string", - }, - "version": { - "type": "string", - }, - }, - "required": [ - "id", - "name", - "publisher", - "internal", + "updatedAt", ], "type": "object", }, - "ExtensionVersionInfo": { + "ExperimentListResponse": { "properties": { - "lastUpdated": { - "type": "string", - }, - "preRelease": { - "type": "boolean", + "data": { + "items": { + "$ref": "#/components/schemas/ExperimentResponse", + }, + "type": "array", }, - "version": { + "nextCursor": { "type": "string", }, }, "required": [ - "version", - "preRelease", - "lastUpdated", + "data", ], "type": "object", }, - "FeatureFlagResponse": { + "ExperimentResponse": { "properties": { - "enabled": { - "type": "boolean", - }, - "key": { - "type": "string", - }, - "label": { - "type": "string", - }, - "updatedAt": { + "createdAt": { "format": "date-time", "type": [ "string", "null", ], }, - }, + "currentExecution": { + "properties": { + "attemptNumber": { + "type": "integer", + }, + "blockers": { + "items": { + "properties": { + "error": { + "type": "string", + }, + "handlerId": { + "type": "string", + }, + "reason": { + "type": "string", + }, + "requestId": { + "type": "string", + }, + "runId": { + "type": "string", + }, + }, + "required": [ + "handlerId", + "reason", + ], + "type": "object", + }, + "type": "array", + }, + "createdAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "discardReason": { + "type": "string", + }, + "discardedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "experimentExecutionId": { + "type": "string", + }, + "finishedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "progress": { + "additionalProperties": { + "properties": { + "error": { + "type": "string", + }, + "status": { + "enum": [ + "queued", + "processing", + "done", + "failed", + ], + "type": "string", + }, + "updatedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + }, + "required": [ + "status", + "updatedAt", + ], + "type": "object", + }, + "type": "object", + }, + "readiness": { + "enum": [ + "empty", + "waiting", + "blocked", + "ready", + "processing", + "done", + "failed", + ], + "type": "string", + }, + "report": { + "properties": { + "reportId": { + "type": "string", + }, + "updatedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "url": { + "type": "string", + }, + }, + "type": "object", + }, + "startedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "updatedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + }, + "required": [ + "experimentExecutionId", + "attemptNumber", + "readiness", + "blockers", + "progress", + "createdAt", + "updatedAt", + ], + "type": "object", + }, + "experimentId": { + "type": "string", + }, + "metadata": { + "properties": { + "createdBy": { + "type": "string", + }, + "description": { + "maxLength": 4096, + "type": "string", + }, + "name": { + "maxLength": 256, + "minLength": 1, + "type": "string", + }, + "tags": { + "items": { + "maxLength": 64, + "minLength": 1, + "type": "string", + }, + "maxItems": 32, + "type": "array", + }, + }, + "required": [ + "name", + ], + "type": "object", + }, + "projectId": { + "type": "string", + }, + "revision": { + "properties": { + "createdAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "experimentRevisionId": { + "type": "string", + }, + "memberCount": { + "type": "integer", + }, + "revisionNumber": { + "type": "integer", + }, + }, + "required": [ + "experimentRevisionId", + "revisionNumber", + "memberCount", + "createdAt", + ], + "type": "object", + }, + "updatedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + }, + "required": [ + "experimentId", + "projectId", + "metadata", + "revision", + "currentExecution", + "createdAt", + "updatedAt", + ], + "type": "object", + }, + "ExtensionResponse": { + "properties": { + "_id": { + "type": "string", + }, + "createdAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "deletedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "description": { + "type": "string", + }, + "name": { + "type": "string", + }, + "origin": { + "enum": [ + "marketplace", + "manual", + ], + "type": "string", + }, + "projectId": { + "type": "string", + }, + "publisher": { + "type": "string", + }, + "slug": { + "type": "string", + }, + "updatedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + }, + "required": [ + "_id", + "slug", + "publisher", + "name", + "origin", + "createdAt", + "projectId", + ], + "type": "object", + }, + "ExtensionSearchResult": { + "properties": { + "description": { + "type": "string", + }, + "id": { + "type": "string", + }, + "internal": { + "type": "boolean", + }, + "name": { + "type": "string", + }, + "publisher": { + "type": "string", + }, + "version": { + "type": "string", + }, + }, + "required": [ + "id", + "name", + "publisher", + "internal", + ], + "type": "object", + }, + "ExtensionVersionInfo": { + "properties": { + "lastUpdated": { + "type": "string", + }, + "preRelease": { + "type": "boolean", + }, + "version": { + "type": "string", + }, + }, + "required": [ + "version", + "preRelease", + "lastUpdated", + ], + "type": "object", + }, + "FeatureFlagResponse": { + "properties": { + "enabled": { + "type": "boolean", + }, + "key": { + "type": "string", + }, + "label": { + "type": "string", + }, + "updatedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + }, "required": [ "key", "label", @@ -1601,6 +2170,9 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "codebaseRevisionId": { "type": "string", }, + "experimentId": { + "type": "string", + }, "extensions": { "items": { "type": "string", @@ -1635,9 +2207,6 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` ], "type": "string", }, - "submissionId": { - "type": "string", - }, "task": { "type": "string", }, @@ -2741,6 +3310,27 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "null", ], }, + "experimentExpectedRequestCount": { + "minimum": 0, + "type": "integer", + }, + "experimentFinalizedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "experimentId": { + "type": "string", + }, + "experimentInitialRunId": { + "type": "string", + }, + "experimentRequestIndex": { + "minimum": 0, + "type": "integer", + }, "extensions": { "items": { "type": "string", @@ -2817,9 +3407,6 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, "type": "array", }, - "submissionId": { - "type": "string", - }, "taskPromptId": { "type": "string", }, @@ -3109,6 +3696,66 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, "type": "object", }, + "RetryExperimentResponse": { + "properties": { + "attemptNumber": { + "type": "integer", + }, + "blockers": { + "items": { + "properties": { + "error": { + "type": "string", + }, + "handlerId": { + "type": "string", + }, + "reason": { + "type": "string", + }, + "requestId": { + "type": "string", + }, + "runId": { + "type": "string", + }, + }, + "required": [ + "handlerId", + "reason", + ], + "type": "object", + }, + "type": "array", + }, + "experimentExecutionId": { + "type": "string", + }, + "experimentId": { + "type": "string", + }, + "readiness": { + "enum": [ + "empty", + "waiting", + "blocked", + "ready", + "processing", + "done", + "failed", + ], + "type": "string", + }, + }, + "required": [ + "experimentId", + "experimentExecutionId", + "attemptNumber", + "readiness", + "blockers", + ], + "type": "object", + }, "RunFacetBucket": { "properties": { "count": { @@ -4690,7 +5337,7 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, { "in": "query", - "name": "submissionId", + "name": "experimentId", "required": false, "schema": { "type": "string", @@ -5198,10 +5845,252 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, }, "required": [ - "prompt", - "suggestedId", - "suggestedParents", - "suggestedChildren", + "prompt", + "suggestedId", + "suggestedParents", + "suggestedChildren", + ], + "type": "object", + }, + }, + }, + "description": "Success", + }, + "400": { + "description": "Empty behavior string", + }, + "503": { + "description": "LLM not configured", + }, + }, + "summary": "Generate criterion prompt from behavior", + "tags": [ + "Criteria", + ], + }, + }, + "/api/v1/criteria/graph": { + "get": { + "parameters": [ + { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "in": "query", + "name": "projectId", + "required": true, + "schema": { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, + "type": "string", + }, + }, + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CriteriaGraph", + }, + }, + }, + "description": "Success", + }, + }, + "summary": "Get criteria DAG", + "tags": [ + "Criteria", + ], + }, + }, + "/api/v1/criteria/mdp": { + "get": { + "parameters": [ + { + "in": "query", + "name": "criteria", + "required": false, + "schema": { + "type": "string", + }, + }, + { + "in": "query", + "name": "features", + "required": false, + "schema": { + "type": "string", + }, + }, + { + "in": "query", + "name": "since", + "required": false, + "schema": { + "type": "string", + }, + }, + { + "in": "query", + "name": "worker", + "required": false, + "schema": { + "type": "string", + }, + }, + { + "in": "query", + "name": "taskPromptId", + "required": false, + "schema": { + "type": "string", + }, + }, + { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "in": "query", + "name": "projectId", + "required": true, + "schema": { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, + "type": "string", + }, + }, + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": {}, + "properties": {}, + "type": "object", + }, + }, + }, + "description": "Success", + }, + }, + "summary": "Compute MDP transitions", + "tags": [ + "Criteria", + ], + }, + }, + "/api/v1/criteria/seed": { + "post": { + "parameters": [ + { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "in": "query", + "name": "projectId", + "required": true, + "schema": { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, + "type": "string", + }, + }, + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "criteria": { + "items": { + "$ref": "#/components/schemas/CreateCriteriaInput", + }, + "type": "array", + }, + }, + "required": [ + "criteria", + ], + "type": "object", + }, + }, + }, + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "properties": { + "errors": { + "items": { + "type": "string", + }, + "type": "array", + }, + "seeded": { + "type": "number", + }, + }, + "required": [ + "seeded", + "errors", + ], + "type": "object", + }, + }, + }, + "description": "Success", + }, + "400": { + "description": "Seed batch would introduce a dependency cycle", + }, + }, + "summary": "Seed criteria in bulk", + "tags": [ + "Criteria", + ], + }, + }, + "/api/v1/criteria/{id}": { + "delete": { + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "type": "string", + }, + }, + { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "in": "query", + "name": "projectId", + "required": true, + "schema": { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, + "type": "string", + }, + }, + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "deleted": { + "type": "boolean", + }, + "id": { + "type": "string", + }, + }, + "required": [ + "id", + "deleted", ], "type": "object", }, @@ -5209,22 +6098,28 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, "description": "Success", }, - "400": { - "description": "Empty behavior string", + "404": { + "description": "Criterion not found", }, - "503": { - "description": "LLM not configured", + "409": { + "description": "Criterion has dependents", }, }, - "summary": "Generate criterion prompt from behavior", + "summary": "Soft-delete criterion", "tags": [ "Criteria", ], }, - }, - "/api/v1/criteria/graph": { "get": { "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "type": "string", + }, + }, { "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", "in": "query", @@ -5243,70 +6138,108 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/CriteriaGraph", + "$ref": "#/components/schemas/CriteriaResponse", }, }, }, "description": "Success", }, + "404": { + "description": "Criterion not found", + }, }, - "summary": "Get criteria DAG", + "summary": "Get criterion", "tags": [ "Criteria", ], }, - }, - "/api/v1/criteria/mdp": { - "get": { + "put": { "parameters": [ { - "in": "query", - "name": "criteria", - "required": false, + "in": "path", + "name": "id", + "required": true, "schema": { "type": "string", }, }, { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", "in": "query", - "name": "features", - "required": false, + "name": "projectId", + "required": true, "schema": { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, "type": "string", }, }, - { - "in": "query", - "name": "since", - "required": false, - "schema": { - "type": "string", + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateCriteriaInput", + }, }, }, + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CriteriaResponse", + }, + }, + }, + "description": "Success", + }, + "400": { + "description": "Invalid dependency reference or self-reference", + }, + "404": { + "description": "Criterion not found", + }, + }, + "summary": "Update criterion", + "tags": [ + "Criteria", + ], + }, + }, + "/api/v1/experiments": { + "get": { + "parameters": [ { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", "in": "query", - "name": "worker", - "required": false, + "name": "projectId", + "required": true, "schema": { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, "type": "string", }, }, { "in": "query", - "name": "taskPromptId", + "name": "limit", "required": false, "schema": { - "type": "string", + "default": 50, + "maximum": 100, + "minimum": 1, + "type": "integer", }, }, { - "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", "in": "query", - "name": "projectId", - "required": true, + "name": "after", + "required": false, "schema": { - "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", - "example": "00000000-0000-0000-0000-000000000000", "minLength": 1, "type": "string", }, @@ -5317,23 +6250,20 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "content": { "application/json": { "schema": { - "additionalProperties": {}, - "properties": {}, - "type": "object", + "$ref": "#/components/schemas/ExperimentListResponse", }, }, }, "description": "Success", }, }, - "summary": "Compute MDP transitions", + "summary": "List experiments", "tags": [ - "Criteria", + "Experiments", ], }, - }, - "/api/v1/criteria/seed": { "post": { + "description": "Canonical composition endpoint. Request and run IDs are minted before insertion; the exact revision/execution is then finalized idempotently.", "parameters": [ { "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", @@ -5352,18 +6282,7 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "content": { "application/json": { "schema": { - "properties": { - "criteria": { - "items": { - "$ref": "#/components/schemas/CreateCriteriaInput", - }, - "type": "array", - }, - }, - "required": [ - "criteria", - ], - "type": "object", + "$ref": "#/components/schemas/CreateExperimentInput", }, }, }, @@ -5373,45 +6292,31 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "content": { "application/json": { "schema": { - "properties": { - "errors": { - "items": { - "type": "string", - }, - "type": "array", - }, - "seeded": { - "type": "number", - }, - }, - "required": [ - "seeded", - "errors", - ], - "type": "object", + "$ref": "#/components/schemas/CreateExperimentResponse", }, }, }, "description": "Success", }, - "400": { - "description": "Seed batch would introduce a dependency cycle", + "409": { + "description": "Experiment identity conflict", }, }, - "summary": "Seed criteria in bulk", + "summary": "Create an experiment and zero or more new requests", "tags": [ - "Criteria", + "Experiments", ], }, }, - "/api/v1/criteria/{id}": { - "delete": { + "/api/v1/experiments/{id}": { + "get": { "parameters": [ { "in": "path", "name": "id", "required": true, "schema": { + "minLength": 1, "type": "string", }, }, @@ -5433,36 +6338,23 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "content": { "application/json": { "schema": { - "properties": { - "deleted": { - "type": "boolean", - }, - "id": { - "type": "string", - }, - }, - "required": [ - "id", - "deleted", - ], - "type": "object", + "$ref": "#/components/schemas/ExperimentResponse", }, }, }, "description": "Success", }, "404": { - "description": "Criterion not found", - }, - "409": { - "description": "Criterion has dependents", + "description": "Experiment not found", }, }, - "summary": "Soft-delete criterion", + "summary": "Get an experiment", "tags": [ - "Criteria", + "Experiments", ], }, + }, + "/api/v1/experiments/{id}/executions": { "get": { "parameters": [ { @@ -5470,17 +6362,69 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "name": "id", "required": true, "schema": { + "minLength": 1, "type": "string", }, }, { - "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", "in": "query", - "name": "projectId", + "name": "limit", + "required": false, + "schema": { + "default": 50, + "maximum": 100, + "minimum": 1, + "type": "integer", + }, + }, + { + "in": "query", + "name": "after", + "required": false, + "schema": { + "minLength": 1, + "type": "string", + }, + }, + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ExperimentExecutionListResponse", + }, + }, + }, + "description": "Success", + }, + "404": { + "description": "Experiment not found", + }, + }, + "summary": "List current and historical experiment executions", + "tags": [ + "Experiments", + ], + }, + }, + "/api/v1/experiments/{id}/executions/{experimentExecutionId}": { + "get": { + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "minLength": 1, + "type": "string", + }, + }, + { + "in": "path", + "name": "experimentExecutionId", "required": true, "schema": { - "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", - "example": "00000000-0000-0000-0000-000000000000", "minLength": 1, "type": "string", }, @@ -5491,28 +6435,31 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/CriteriaResponse", + "$ref": "#/components/schemas/ExperimentExecutionResponse", }, }, }, "description": "Success", }, "404": { - "description": "Criterion not found", + "description": "Experiment execution not found", }, }, - "summary": "Get criterion", + "summary": "Get one exact experiment execution", "tags": [ - "Criteria", + "Experiments", ], }, - "put": { + }, + "/api/v1/experiments/{id}/retry": { + "post": { "parameters": [ { "in": "path", "name": "id", "required": true, "schema": { + "minLength": 1, "type": "string", }, }, @@ -5533,32 +6480,41 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/UpdateCriteriaInput", + "properties": { + "experimentExecutionId": { + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "experimentExecutionId", + ], + "type": "object", }, }, }, }, "responses": { - "200": { + "201": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/CriteriaResponse", + "$ref": "#/components/schemas/RetryExperimentResponse", }, }, }, "description": "Success", }, - "400": { - "description": "Invalid dependency reference or self-reference", - }, "404": { - "description": "Criterion not found", + "description": "Experiment not found", + }, + "409": { + "description": "Execution is stale or cannot be retried", }, }, - "summary": "Update criterion", + "summary": "Retry current experiment processing", "tags": [ - "Criteria", + "Experiments", ], }, }, @@ -8828,7 +9784,7 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, { "in": "query", - "name": "submissionId", + "name": "experimentId", "required": false, "schema": { "type": "string", @@ -8975,7 +9931,7 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "schema": { "enum": [ "task", - "submissionId", + "experimentId", "profile", ], "type": "string", @@ -9194,6 +10150,8 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, "count": { "default": 1, + "deprecated": true, + "description": "Deprecated for composition. Use POST /api/v1/experiments and its requests array.", "maximum": 10, "minimum": 1, "type": "number", diff --git a/apps/api/src/endpoints.test.ts b/apps/api/src/endpoints.test.ts index c105562c7..893288464 100644 --- a/apps/api/src/endpoints.test.ts +++ b/apps/api/src/endpoints.test.ts @@ -766,19 +766,19 @@ describe("API Endpoints", () => { expect(res.body.data[0]).toHaveProperty("uniform"); }); - it("returns grouped results when groupBy=submissionId", async () => { + it("returns grouped results when groupBy=experimentId", async () => { const groupedDocs = [ - { key: "sub-1", label: "sub-1", aggregates: { count: 2, turns: null, duration: null, promptTokens: null, completionTokens: null }, uniform: {} }, + { key: "experiment-1", label: "experiment-1", aggregates: { count: 2, turns: null, duration: null, promptTokens: null, completionTokens: null }, uniform: {} }, ]; (mocks.collection.aggregate as any) - .mockReturnValueOnce({ toArray: vi.fn().mockResolvedValue([{ _id: "sub-1" }]) }) + .mockReturnValueOnce({ toArray: vi.fn().mockResolvedValue([{ _id: "experiment-1" }]) }) .mockReturnValueOnce({ toArray: vi.fn().mockResolvedValue(groupedDocs) }) .mockReturnValueOnce({ toArray: vi.fn().mockResolvedValue([]) }) .mockReturnValueOnce({ toArray: vi.fn().mockResolvedValue([]) }); - const res = await request(testServer()).get(`/api/v1/requests?groupBy=submissionId&projectId=${TEST_PROJECT_ID}`); + const res = await request(testServer()).get(`/api/v1/requests?groupBy=experimentId&projectId=${TEST_PROJECT_ID}`); expect(res.status).toBe(200); - expect(res.body.data[0]).toHaveProperty("key", "sub-1"); + expect(res.body.data[0]).toHaveProperty("key", "experiment-1"); }); it("calls aggregate pipeline when groupBy is provided", async () => { @@ -1955,15 +1955,14 @@ describe("API Endpoints", () => { expect(res.status).toBe(201); const doc = (mocks.collection.insertOne as any).mock.calls[0][0]; expect(doc).toHaveProperty("reasoningEffort", "high"); - expect(doc).toHaveProperty("submissionExpectedRequestCount", 1); - const submissionInsert = (mocks.submissionCollection.updateOne as any).mock.calls[0][1].$setOnInsert; - expect(submissionInsert.requestIds).toEqual([doc._id]); - expect(submissionInsert.run.requests).toEqual([ - { requestId: doc._id, runId: doc.run._id }, - ]); + expect(doc).not.toHaveProperty("experimentId"); + expect(mocks.experimentCollection.updateOne).not.toHaveBeenCalled(); + expect(res.headers).not.toHaveProperty("deprecation"); + expect(res.headers).not.toHaveProperty("link"); + expect(res.headers).not.toHaveProperty("sunset"); }); - it("creates one complete submission vector for count fan-out", async () => { + it("keeps count fan-out independent from experiments", async () => { const res = await request(testServer()) .post(`/api/v1/requests?worker=coder-acp-copilot&projectId=${TEST_PROJECT_ID}`) .send({ @@ -1974,25 +1973,13 @@ describe("API Endpoints", () => { expect(res.status).toBe(201); const docs = (mocks.collection.insertMany as any).mock.calls[0][0]; expect(docs).toHaveLength(3); - expect(docs.every((doc: { submissionExpectedRequestCount: number }) => - doc.submissionExpectedRequestCount === 3)).toBe(true); - expect( - docs.map((doc: { submissionRequestIndex: number }) => doc.submissionRequestIndex), - ).toEqual([0, 1, 2]); - expect( - docs.every( - (doc: { submissionInitialRunId: string; run: { _id: string } }) => - doc.submissionInitialRunId === doc.run._id, - ), - ).toBe(true); - const submissionInsert = (mocks.submissionCollection.updateOne as any).mock.calls[0][1].$setOnInsert; - expect(submissionInsert.requestIds).toEqual(docs.map((doc: { _id: string }) => doc._id)); - expect(submissionInsert.run.requests).toEqual( - docs.map((doc: { _id: string; run: { _id: string } }) => ({ - requestId: doc._id, - runId: doc.run._id, - })), + expect(docs.every((doc: object) => !("experimentId" in doc))).toBe(true); + expect(mocks.experimentCollection.updateOne).not.toHaveBeenCalled(); + expect(res.headers.deprecation).toBe("true"); + expect(res.headers.link).toBe( + '; rel="successor-version"; title="POST /api/v1/experiments"', ); + expect(res.headers).not.toHaveProperty("sunset"); }); it("profile reasoningEffort takes precedence over client-provided effort", async () => { @@ -2297,7 +2284,7 @@ describe("API Endpoints", () => { new Set(["base-profile", "var-profile"]), ); for (const doc of docs) { - expect(doc.submissionExpectedRequestCount).toBe(2); + expect(doc.experimentId).toBeUndefined(); expect(Array.isArray(doc.gates)).toBe(true); const selectGate = doc.gates.find((g: { gate: string }) => g.gate === "select"); const buildGate = doc.gates.find((g: { gate: string }) => g.gate === "build"); @@ -2308,10 +2295,7 @@ describe("API Endpoints", () => { expect(buildGate.promptId).toBe("tp-build"); expect(buildGate.promptText).toBeUndefined(); } - const submissionInsert = (mocks.submissionCollection.updateOne as any).mock.calls[0][1].$setOnInsert; - expect(submissionInsert.requestIds).toEqual( - docs.map((doc: { _id: string }) => doc._id), - ); + expect(mocks.experimentCollection.updateOne).not.toHaveBeenCalled(); // Response surfaces the configured gate count. expect(res.body.gates).toBe(2); }); @@ -2469,9 +2453,8 @@ describe("API Endpoints", () => { expect(insertCall[0]).toHaveProperty("taskPromptId", "tp-123"); expect(insertCall[0]).toHaveProperty("model", "gpt-4o"); expect(insertCall[0]).toHaveProperty("maxIterations", 5); - expect(insertCall[0]).toHaveProperty("submissionExpectedRequestCount", 1); - const submissionInsert = (mocks.submissionCollection.updateOne as any).mock.calls[0][1].$setOnInsert; - expect(submissionInsert.requestIds).toEqual([insertCall[0]._id]); + expect(insertCall[0]).not.toHaveProperty("experimentId"); + expect(mocks.experimentCollection.updateOne).not.toHaveBeenCalled(); }); it("applies profile override: uses profile fields for worker, model, mcpServers, skillRevisions, extensions", async () => { @@ -2848,13 +2831,13 @@ describe("API Endpoints", () => { it("retries eligible requests and skips non-terminal ones", async () => { const doc1 = { _id: "req-1", - submissionId: "submission-1", + experimentId: "experiment-1", workerType: "coder-acp-copilot", run: { _id: "run-1", attemptNumber: 1, status: "done", outcome: "failed" }, }; const doc2 = { _id: "req-2", - submissionId: "submission-1", + experimentId: "experiment-1", workerType: "coder-acp-copilot", run: { _id: "run-2", attemptNumber: 1, status: "processing" }, }; @@ -2880,22 +2863,7 @@ describe("API Endpoints", () => { $set: expect.objectContaining({ agentVersion: "1.0.0" }), }), ); - expect(mocks.submissionCollection.updateMany).toHaveBeenNthCalledWith( - 1, - { _id: { $in: ["submission-1"] } }, - expect.objectContaining({ - $set: expect.objectContaining({ - "run.reconcileAfter": expect.any(Date), - }), - }), - ); - expect(mocks.submissionCollection.updateMany).toHaveBeenNthCalledWith( - 2, - { _id: { $in: ["submission-1"] } }, - expect.objectContaining({ - $unset: { "run.reconcileAfter": "" }, - }), - ); + expect(mocks.experimentCollection.updateMany).not.toHaveBeenCalled(); }); it("returns 400 when ids array is empty", async () => { diff --git a/apps/api/src/experiment-openapi.test.ts b/apps/api/src/experiment-openapi.test.ts new file mode 100644 index 000000000..fd5288a78 --- /dev/null +++ b/apps/api/src/experiment-openapi.test.ts @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { beforeAll, describe, expect, it, vi } from "vitest"; +import type { OpenAPIObject } from "openapi3-ts/oas31"; + +vi.mock("db-migrations/check-migrations", () => ({ + checkMigrations: vi.fn().mockResolvedValue({ + ready: true, + applied: [], + pending: [], + }), +})); +vi.mock("./llm.js", () => ({ + isLlmAvailable: vi.fn().mockReturnValue(false), + generateCriteriaPrompt: vi.fn(), +})); +vi.mock("./prompt-feature-llm.js", () => ({ + isLlmAvailable: vi.fn().mockReturnValue(false), + generatePromptFeaturePrompt: vi.fn(), + extractPromptFeatures: vi.fn(), +})); +vi.mock("./task-prompt-llm.js", () => ({ + isTaskPromptLlmAvailable: vi.fn().mockReturnValue(false), + generateTaskPrompt: vi.fn(), +})); + +let document: OpenAPIObject; + +beforeAll(async () => { + await import("./index.js"); + const { generateOpenAPIDocument } = await import("./openapi/index.js"); + document = generateOpenAPIDocument(); +}); + +describe("Experiment OpenAPI contract", () => { + it("publishes canonical experiment operations", () => { + expect(document.paths).toHaveProperty("/api/v1/experiments"); + expect(document.paths).toHaveProperty("/api/v1/experiments/{id}"); + expect(document.paths).toHaveProperty( + "/api/v1/experiments/{id}/executions", + ); + expect(document.paths).toHaveProperty( + "/api/v1/experiments/{id}/executions/{experimentExecutionId}", + ); + expect(document.paths).toHaveProperty("/api/v1/experiments/{id}/retry"); + expect( + Object.keys(document.paths ?? {}).filter((path) => + path.includes("submission"), + ), + ).toEqual([]); + }); + + it("keeps POST /requests active while deprecating only composition properties", () => { + const operation = document.paths?.["/api/v1/requests"]?.post; + expect(operation?.deprecated).not.toBe(true); + const requestSchema = document.components?.schemas?.CreateRequestInput; + const operationJson = JSON.stringify(operation); + const requestSchemaJson = JSON.stringify(requestSchema); + + expect(operationJson).toContain('"count"'); + expect(operationJson).toContain('"deprecated":true'); + expect(requestSchemaJson).toContain('"profileVariations"'); + expect(requestSchemaJson).toContain('"deprecated":true'); + expect(operationJson).toContain("POST /api/v1/experiments"); + expect(requestSchemaJson).toContain("POST /api/v1/experiments"); + }); +}); diff --git a/apps/api/src/grouping.test.ts b/apps/api/src/grouping.test.ts index 56a834ce3..746498c58 100644 --- a/apps/api/src/grouping.test.ts +++ b/apps/api/src/grouping.test.ts @@ -14,8 +14,8 @@ describe("buildGroupingPipeline", () => { expect(pipeline[3]).toHaveProperty("$sort"); }); - it("returns 4 pipeline stages for submissionId grouping", () => { - const pipeline = buildGroupingPipeline("submissionId"); + it("returns 4 pipeline stages for experimentId grouping", () => { + const pipeline = buildGroupingPipeline("experimentId"); expect(pipeline).toHaveLength(4); }); @@ -32,8 +32,8 @@ describe("buildGroupingPipeline", () => { expect(addFields._groupKey).toBe("$taskPromptId"); }); - it("groups by submissionId for submissionId grouping", () => { - const pipeline = buildGroupingPipeline("submissionId"); + it("groups by experimentId for experimentId grouping", () => { + const pipeline = buildGroupingPipeline("experimentId"); const addFields = pipeline[0].$addFields as Record; expect(addFields._groupKey).toBeDefined(); }); @@ -74,7 +74,7 @@ describe("buildGroupingPipeline", () => { expect(group._agentVersions).toBeDefined(); expect(group._platforms).toBeDefined(); expect(group._statuses).toBeDefined(); - expect(group._submissionIds).toBeDefined(); + expect(group._experimentIds).toBeDefined(); expect(group._tasks).toBeDefined(); expect(group._mcpKeys).toBeDefined(); expect(group._skillKeys).toBeDefined(); diff --git a/apps/api/src/grouping.ts b/apps/api/src/grouping.ts index 3cc34cb18..d3ae0f60b 100644 --- a/apps/api/src/grouping.ts +++ b/apps/api/src/grouping.ts @@ -12,17 +12,17 @@ import type { GroupByKey, RunGroup, AggregateStats, GroupUniformValues } from "s * The pipeline is appended after a $match stage with the caller's filter. */ export function buildGroupingPipeline( - groupBy: "task" | "submissionId" | "profile", + groupBy: "task" | "experimentId" | "profile", ): Document[] { const groupField = - groupBy === "task" ? "$taskPromptId" : groupBy === "profile" ? "$profileId" : "$submissionId"; + groupBy === "task" ? "$taskPromptId" : groupBy === "profile" ? "$profileId" : "$experimentId"; const fallbackGroupField = - groupBy === "profile" ? { $literal: "no-profile" } : { $literal: "no-submission" }; + groupBy === "profile" ? { $literal: "no-profile" } : { $literal: "no-experiment" }; // `task` groups purely by the always-present `taskPromptId` — it is // materialized on every run-creation path (findOrCreate → taskPrompt._id) and // backfilled onto legacy docs by migration 001, so the group key is always a // real id the member-fetch filter (`{ taskPromptId: key }`) can match. Only - // `profile`/`submission` fall back to a fixed literal sentinel for runs that + // `profile`/`experiment` fall back to a fixed literal sentinel for runs that // genuinely lack the field. const groupKeyExpr = groupBy === "task" ? groupField : { $ifNull: [groupField, fallbackGroupField] }; @@ -131,7 +131,7 @@ export function buildGroupingPipeline( _agentVersions: { $addToSet: { $ifNull: ["$agentVersion", ""] } }, _platforms: { $addToSet: { $ifNull: ["$run.os.platform", ""] } }, _statuses: { $addToSet: "$run.status" }, - _submissionIds: { $addToSet: { $ifNull: ["$submissionId", ""] } }, + _experimentIds: { $addToSet: { $ifNull: ["$experimentId", ""] } }, _tasks: { $addToSet: { $ifNull: ["$scenario.task", ""] } }, _mcpKeys: { $addToSet: "$_mcpKey" }, _skillKeys: { $addToSet: "$_skillKey" }, @@ -160,7 +160,7 @@ export function buildGroupingPipeline( label: { $switch: { branches: [ - { case: { $eq: ["$_id", "no-submission"] }, then: "No submission ID" }, + { case: { $eq: ["$_id", "no-experiment"] }, then: "No experiment ID" }, { case: { $eq: ["$_id", "no-profile"] }, then: "No profile" }, ], default: { $ifNull: ["$_firstTask", "$_id"] }, @@ -183,7 +183,7 @@ export function buildGroupingPipeline( agentVersion: { $cond: { if: { $and: [{ $eq: [{ $size: "$_agentVersions" }, 1] }, { $ne: [{ $arrayElemAt: ["$_agentVersions", 0] }, ""] }] }, then: { $arrayElemAt: ["$_agentVersions", 0] }, else: "$$REMOVE" } }, platform: { $cond: { if: { $and: [{ $eq: [{ $size: "$_platforms" }, 1] }, { $ne: [{ $arrayElemAt: ["$_platforms", 0] }, ""] }] }, then: { $arrayElemAt: ["$_platforms", 0] }, else: "$$REMOVE" } }, status: { $cond: { if: { $eq: [{ $size: "$_statuses" }, 1] }, then: { $arrayElemAt: ["$_statuses", 0] }, else: "$$REMOVE" } }, - submissionId: { $cond: { if: { $and: [{ $eq: [{ $size: "$_submissionIds" }, 1] }, { $ne: [{ $arrayElemAt: ["$_submissionIds", 0] }, ""] }] }, then: { $arrayElemAt: ["$_submissionIds", 0] }, else: "$$REMOVE" } }, + experimentId: { $cond: { if: { $and: [{ $eq: [{ $size: "$_experimentIds" }, 1] }, { $ne: [{ $arrayElemAt: ["$_experimentIds", 0] }, ""] }] }, then: { $arrayElemAt: ["$_experimentIds", 0] }, else: "$$REMOVE" } }, task: { $cond: { if: { $and: [{ $eq: [{ $size: "$_tasks" }, 1] }, { $ne: [{ $arrayElemAt: ["$_tasks", 0] }, ""] }] }, then: { $arrayElemAt: ["$_tasks", 0] }, else: "$$REMOVE" } }, mcpServers: { $cond: { if: { $and: [{ $eq: [{ $size: "$_mcpKeys" }, 1] }, { $ne: [{ $arrayElemAt: ["$_mcpKeys", 0] }, ""] }] }, then: "$_firstMcpServers", else: "$$REMOVE" } }, skillRevisions: { $cond: { if: { $and: [{ $eq: [{ $size: "$_skillKeys" }, 1] }, { $ne: [{ $arrayElemAt: ["$_skillKeys", 0] }, ""] }] }, then: "$_firstSkillRevisions", else: "$$REMOVE" } }, diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 7f95ab2f5..d9229ca51 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -10,6 +10,7 @@ import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; import dotenv from "dotenv"; import { TaskPromptStore, SkillRevisionStore, SkillResolver, CodebaseStore, CodebaseRevisionStore, CodebaseResolver, ResourceStore, ResourceRevisionStore, ResourceResolver, McpSecretClient, McpSecretUnavailableError, BlobStorage, RedisHeartbeatStore, ProjectStore, loadAuthConfigFromEnv } from "shared"; +import type { ExperimentDocument, ExperimentExecutionHistoryDocument } from "shared"; import { initTelemetry } from "telemetry"; import type { TaskPromptDocument, SkillDocument, SkillRevisionDocument, CodebaseDocument, CodebaseRevisionDocument, ResourceDocument, ResourceRevisionDocument, ProfileDocument, ProfileVersionDocument, ProjectDocument, HeartbeatStore, AuthProvider, ProfileEnricher, UserDocument } from "shared"; import { UserStore } from "./auth/user-store.js"; @@ -47,7 +48,7 @@ import { registerInsightsRoutes } from "./routes/insights.js"; import { registerSecretsRoutes } from "./routes/secrets.js"; import { registerProfilesRoutes } from "./routes/profiles.js"; import { registerProjectsRoutes } from "./routes/projects.js"; -import { registerSubmissionsRoutes } from "./routes/submissions.js"; +import { registerExperimentsRoutes } from "./routes/experiments.js"; import { ProjectScopeError } from "./utils/project-scope.js"; import { registerUsersRoutes } from "./routes/users.js"; import { shutdownPortalCopilotClientPool } from "./copilot-client-pool.js"; @@ -60,8 +61,6 @@ import type { InsightDocument, RequestDocument, RunHistoryDocument, - SubmissionDocument, - SubmissionRunHistoryDocument, CodingAgentDocument, ModelDocument, McpServerDocument, @@ -100,8 +99,9 @@ let mongoClient: MongoClient; let db: Db; let collection: Collection; let runsCol: Collection; -let submissionCollection: Collection; -let submissionRunsCollection: Collection; +let experimentCollection: Collection; +let experimentExecutionsCollection: Collection; +let experimentChunksCollection: Collection; let projectCollection: Collection; let projectStore: ProjectStore; let criteriaCollection: Collection; @@ -149,8 +149,10 @@ async function initializeClients(): Promise { db = mongoClient.db(mongoDatabase); collection = db.collection(mongoCollection); runsCol = db.collection("runs"); - submissionCollection = db.collection("submissions"); - submissionRunsCollection = db.collection("submission-runs"); + experimentCollection = db.collection("experiments"); + experimentExecutionsCollection = + db.collection("experiment-executions"); + experimentChunksCollection = db.collection("experiment-chunks"); projectCollection = db.collection("projects"); projectStore = new ProjectStore(projectCollection); criteriaCollection = db.collection("criteria"); @@ -284,8 +286,9 @@ const routeCtx: RouteContext = { get db() { return db; }, get requestCollection() { return collection; }, get runsCollection() { return runsCol; }, - get submissionCollection() { return submissionCollection; }, - get submissionRunsCollection() { return submissionRunsCollection; }, + get experimentCollection() { return experimentCollection; }, + get experimentExecutionsCollection() { return experimentExecutionsCollection; }, + get experimentChunksCollection() { return experimentChunksCollection; }, get projectCollection() { return projectCollection; }, get criteriaCollection() { return criteriaCollection; }, get promptFeatureCollection() { return promptFeatureCollection; }, @@ -347,7 +350,7 @@ app.use(createUserAccessMiddleware(() => userAccessResolver)); registerSecretsRoutes(routeCtx); registerSystemRoutes(routeCtx); registerProjectsRoutes(routeCtx); -registerSubmissionsRoutes(routeCtx); +registerExperimentsRoutes(routeCtx); registerRequestsRoutes(routeCtx); registerRequestsCancelRoutes(routeCtx); registerRequestsLogsRoutes(routeCtx); @@ -434,8 +437,9 @@ export interface TestDependencies { db?: Db; collection?: Collection; runsCollection?: Collection; - submissionCollection?: Collection; - submissionRunsCollection?: Collection; + experimentCollection?: Collection; + experimentExecutionsCollection?: Collection; + experimentChunksCollection?: Collection; criteriaCollection?: Collection; promptFeatureCollection?: Collection; reportCollection?: Collection; @@ -477,8 +481,9 @@ export function _injectTestDependencies(deps: TestDependencies): void { if (deps.db) db = deps.db; if (deps.collection) collection = deps.collection; if (deps.runsCollection) runsCol = deps.runsCollection; - if (deps.submissionCollection) submissionCollection = deps.submissionCollection; - if (deps.submissionRunsCollection) submissionRunsCollection = deps.submissionRunsCollection; + if (deps.experimentCollection) experimentCollection = deps.experimentCollection; + if (deps.experimentExecutionsCollection) experimentExecutionsCollection = deps.experimentExecutionsCollection; + if (deps.experimentChunksCollection) experimentChunksCollection = deps.experimentChunksCollection; if (deps.criteriaCollection) criteriaCollection = deps.criteriaCollection; if (deps.promptFeatureCollection) promptFeatureCollection = deps.promptFeatureCollection; if (deps.reportCollection) reportCollection = deps.reportCollection; diff --git a/apps/api/src/openapi/openapi.test.ts b/apps/api/src/openapi/openapi.test.ts index 87426de43..6dbfa9152 100644 --- a/apps/api/src/openapi/openapi.test.ts +++ b/apps/api/src/openapi/openapi.test.ts @@ -466,7 +466,7 @@ apiRoute(testApp, registry, { apiRoute(testApp, registry, { method: "get", path: "/api/v1/analysis", tags: ["Requests"], summary: "Compute pass@k / success@T metrics", - query: z.object({ worker: z.string().optional(), taskPromptId: z.string().optional(), criteria: z.string().optional(), submissionId: z.string().optional(), k: z.string().optional() }), + query: z.object({ worker: z.string().optional(), taskPromptId: z.string().optional(), criteria: z.string().optional(), experimentId: z.string().optional(), k: z.string().optional() }), response: z.object({}).passthrough().describe("Analysis metrics"), handler: noop, }); apiRoute(testApp, registry, { diff --git a/apps/api/src/route-context.ts b/apps/api/src/route-context.ts index b675e8b9f..2037c04f0 100644 --- a/apps/api/src/route-context.ts +++ b/apps/api/src/route-context.ts @@ -52,8 +52,8 @@ import type { ReportTemplateResponseSchema, RequestResponseSchema, RunHistoryDocumentSchema, - SubmissionDocumentSchema, - SubmissionRunHistoryDocumentSchema, + ExperimentDocument, + ExperimentExecutionHistoryDocument, } from "shared"; // ─── Document types (inferred from Zod schemas) ───────────────────────────── @@ -76,9 +76,6 @@ export type ReportTrigger = z.infer; export type ReportTemplateDocument = Omit, "_id">; export type RequestDocument = z.infer; export type RunHistoryDocument = z.infer; -export type SubmissionDocument = z.infer; -export type SubmissionRunHistoryDocument = z.infer; - export type WorkerType = string; // ─── RouteContext ──────────────────────────────────────────────────────────── @@ -96,8 +93,9 @@ export interface RouteContext { db: Db; requestCollection: Collection; runsCollection: Collection; - submissionCollection: Collection; - submissionRunsCollection: Collection; + experimentCollection: Collection; + experimentExecutionsCollection: Collection; + experimentChunksCollection: Collection; projectCollection: Collection; criteriaCollection: Collection; promptFeatureCollection: Collection; diff --git a/apps/api/src/routes/experiments.test.ts b/apps/api/src/routes/experiments.test.ts new file mode 100644 index 000000000..677e1633f --- /dev/null +++ b/apps/api/src/routes/experiments.test.ts @@ -0,0 +1,247 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import express from "express"; +import request from "supertest"; +import { + OpenAPIRegistry, + OpenApiGeneratorV31, + extendZodWithOpenApi, +} from "@asteasolutions/zod-to-openapi"; +import { describe, expect, it, vi } from "vitest"; +import { z } from "zod"; +import type { RouteContext } from "../route-context.js"; +import type { ExperimentDocument } from "shared"; +import { registerExperimentsRoutes } from "./experiments.js"; + +extendZodWithOpenApi(z); + +function harness(overrides: Partial = {}) { + const app = express(); + app.use(express.json()); + const registry = new OpenAPIRegistry(); + registerExperimentsRoutes({ + app, + registry, + ...overrides, + } as unknown as RouteContext); + return { app, registry }; +} + +function emptyExperimentCollection() { + let document: ExperimentDocument | undefined; + return { + updateOne: vi.fn( + async ( + _filter: object, + update: { $setOnInsert?: ExperimentDocument }, + ) => { + document ??= structuredClone(update.$setOnInsert); + return { acknowledged: true, matchedCount: document ? 1 : 0 }; + }, + ), + findOne: vi.fn(async () => + document ? structuredClone(document) : null, + ), + findOneAndUpdate: vi.fn( + async ( + _filter: object, + update: { + $set?: Partial; + $inc?: { stateVersion?: number }; + }, + ) => { + if (!document) return null; + document = { + ...document, + ...update.$set, + stateVersion: + document.stateVersion + (update.$inc?.stateVersion ?? 0), + }; + return structuredClone(document); + }, + ), + }; +} + +function generateDocument(registry: OpenAPIRegistry) { + return new OpenApiGeneratorV31(registry.definitions).generateDocument({ + openapi: "3.1.0", + info: { title: "Experiment contract", version: "1.0.0" }, + }); +} + +describe("Experiment API contract", () => { + it("requires project scope on root routes", async () => { + const { app } = harness(); + + const create = await request(app) + .post("/api/v1/experiments") + .send({ + mode: "empty", + metadata: { name: "Empty experiment" }, + requests: [], + }); + const list = await request(app).get("/api/v1/experiments"); + const get = await request(app).get("/api/v1/experiments/experiment-1"); + const retry = await request(app) + .post("/api/v1/experiments/experiment-1/retry") + .send({ experimentExecutionId: "execution-1" }); + + expect(create.status).toBe(400); + expect(list.status).toBe(400); + expect(get.status).toBe(400); + expect(retry.status).toBe(400); + }); + + it("enforces explicit empty and compose modes", async () => { + const { app } = harness(); + + const emptyWithMembers = await request(app) + .post("/api/v1/experiments?projectId=project-1") + .send({ + mode: "empty", + metadata: { name: "Invalid empty experiment" }, + requests: [{ scenario: { task: "task" }, workerType: "worker" }], + }); + const composeWithoutMembers = await request(app) + .post("/api/v1/experiments?projectId=project-1") + .send({ + mode: "compose", + metadata: { name: "Invalid composed experiment" }, + requests: [], + }); + + expect(emptyWithMembers.status).toBe(400); + expect(composeWithoutMembers.status).toBe(400); + }); + + it("creates an empty, visible, non-dispatchable experiment", async () => { + const experimentCollection = emptyExperimentCollection(); + const { app } = harness({ + experimentCollection, + experimentExecutionsCollection: {}, + experimentChunksCollection: {}, + requestCollection: { + insertMany: vi.fn(), + updateMany: vi.fn(), + }, + } as unknown as Partial); + + const response = await request(app) + .post("/api/v1/experiments?projectId=project-1") + .send({ + mode: "empty", + metadata: { name: "Empty experiment" }, + requests: [], + }); + + expect(response.status).toBe(201); + expect(response.body.requestIds).toEqual([]); + expect(response.body.experiment).toMatchObject({ + projectId: "project-1", + metadata: { name: "Empty experiment" }, + revision: { memberCount: 0 }, + currentExecution: { + readiness: "empty", + blockers: [], + progress: {}, + }, + }); + expect(JSON.stringify(response.body)).not.toMatch( + /stateVersion|claimToken|vectorFingerprint|storageProfile|rollover/, + ); + }); + + it("inserts composed requests before publishing their exact revision", async () => { + const experimentCollection = emptyExperimentCollection(); + const insertMany = vi.fn().mockResolvedValue({ + acknowledged: true, + insertedCount: 1, + }); + const updateMany = vi.fn().mockResolvedValue({ + acknowledged: true, + matchedCount: 1, + modifiedCount: 1, + }); + const { app } = harness({ + experimentCollection, + experimentExecutionsCollection: {}, + experimentChunksCollection: {}, + requestCollection: { insertMany, updateMany }, + taskPromptStore: { + findOrCreate: vi.fn().mockResolvedValue({ _id: "prompt-1" }), + }, + blobStorage: { + getLogsBlobUrl: vi.fn( + (path: string) => `https://storage.invalid/${path}`, + ), + }, + } as unknown as Partial); + + const response = await request(app) + .post("/api/v1/experiments?projectId=project-1") + .send({ + mode: "compose", + metadata: { name: "One request" }, + requests: [ + { + scenario: { task: "Build it", criteria: ["works"] }, + workerType: "coder", + }, + ], + }); + + expect(response.status).toBe(201); + expect(response.body.requestIds).toHaveLength(1); + const inserted = insertMany.mock.calls[0]?.[0]?.[0]; + expect(inserted).toMatchObject({ + projectId: "project-1", + experimentId: response.body.experiment.experimentId, + experimentExpectedRequestCount: 1, + experimentRequestIndex: 0, + experimentInitialRunId: expect.any(String), + run: { + _id: expect.any(String), + status: "pending", + }, + }); + expect(updateMany).toHaveBeenCalledWith( + expect.objectContaining({ + projectId: "project-1", + experimentId: response.body.experiment.experimentId, + }), + { $set: { experimentFinalizedAt: expect.any(Date) } }, + ); + }); + + it("publishes only the canonical routes and public execution fields", () => { + const { registry } = harness(); + const document = generateDocument(registry); + + expect(Object.keys(document.paths ?? {}).sort()).toEqual([ + "/api/v1/experiments", + "/api/v1/experiments/{id}", + "/api/v1/experiments/{id}/executions", + "/api/v1/experiments/{id}/executions/{experimentExecutionId}", + "/api/v1/experiments/{id}/retry", + ]); + expect(document.paths).not.toHaveProperty("/api/v1/submissions"); + + const executionSchema = document.components?.schemas + ?.ExperimentExecutionResponse; + expect(executionSchema).toMatchObject({ + type: "object", + properties: { + experimentExecutionId: { type: "string" }, + readiness: expect.any(Object), + blockers: expect.any(Object), + progress: expect.any(Object), + supersededByExperimentExecutionId: { type: "string" }, + }, + }); + expect(JSON.stringify(executionSchema)).not.toMatch( + /claimToken|stateVersion|vectorFingerprint|storageProfile/, + ); + }); +}); diff --git a/apps/api/src/routes/experiments.ts b/apps/api/src/routes/experiments.ts new file mode 100644 index 000000000..824b1442b --- /dev/null +++ b/apps/api/src/routes/experiments.ts @@ -0,0 +1,691 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { v4 as uuidv4 } from "uuid"; +import { z } from "zod"; +import type { Document } from "mongodb"; +import { + CreateRequestInputSchema, + ExperimentStore, + fingerprintExperimentPayload, + type ExperimentDocument, + type ExperimentExecution, + type ExperimentExecutionHistoryDocument, + type ExperimentMetadata, + type ExperimentRequestState, +} from "shared"; +import type { RequestDocument, RouteContext } from "../route-context.js"; +import { apiRoute } from "../openapi/api-route.js"; +import { + ProjectIdQuerySchema, + getQueryProjectId, +} from "../utils/project-scope.js"; + +const CursorSchema = z.string().min(1).optional(); +const PageQuerySchema = ProjectIdQuerySchema.extend({ + limit: z.coerce.number().int().min(1).max(100).default(50), + after: CursorSchema, +}); + +const ExperimentMetadataInputSchema = z.object({ + name: z.string().trim().min(1).max(256), + description: z.string().max(4_096).optional(), + tags: z.array(z.string().trim().min(1).max(64)).max(32).optional(), +}); + +const ExperimentRequestInputSchema = CreateRequestInputSchema.omit({ + profileVariations: true, +}).extend({ + workerType: z.string().trim().min(1), +}); + +const CreateExperimentInputSchema = z + .object({ + mode: z.enum(["compose", "empty"]), + metadata: ExperimentMetadataInputSchema, + requests: z.array(ExperimentRequestInputSchema).max(250_000).default([]), + }) + .superRefine((value, ctx) => { + if (value.mode === "empty" && value.requests.length !== 0) { + ctx.addIssue({ + code: "custom", + path: ["requests"], + message: "empty mode requires an empty requests array", + }); + } + if (value.mode === "compose" && value.requests.length === 0) { + ctx.addIssue({ + code: "custom", + path: ["requests"], + message: "compose mode requires at least one request", + }); + } + }) + .openapi("CreateExperimentInput"); + +const HandlerProgressSchema = z.record( + z.string(), + z.object({ + status: z.enum(["queued", "processing", "done", "failed"]), + updatedAt: z.coerce.date(), + error: z.string().optional(), + }), +); + +const ReadinessSchema = z.enum([ + "empty", + "waiting", + "blocked", + "ready", + "processing", + "done", + "failed", +]); + +const ExperimentResponseSchema = z + .object({ + experimentId: z.string(), + projectId: z.string(), + metadata: ExperimentMetadataInputSchema.extend({ + createdBy: z.string().optional(), + }), + revision: z.object({ + experimentRevisionId: z.string(), + revisionNumber: z.number().int(), + memberCount: z.number().int(), + createdAt: z.coerce.date(), + }), + currentExecution: z.object({ + experimentExecutionId: z.string(), + attemptNumber: z.number().int(), + readiness: ReadinessSchema, + blockers: z.array( + z.object({ + requestId: z.string().optional(), + runId: z.string().optional(), + handlerId: z.string(), + reason: z.string(), + error: z.string().optional(), + }), + ), + progress: HandlerProgressSchema, + createdAt: z.coerce.date(), + updatedAt: z.coerce.date(), + startedAt: z.coerce.date().optional(), + finishedAt: z.coerce.date().optional(), + discardedAt: z.coerce.date().optional(), + discardReason: z.string().optional(), + report: z + .object({ + reportId: z.string().optional(), + url: z.string().optional(), + updatedAt: z.coerce.date().optional(), + }) + .optional(), + }), + createdAt: z.coerce.date(), + updatedAt: z.coerce.date(), + }) + .openapi("ExperimentResponse"); + +const ExperimentListResponseSchema = z + .object({ + data: z.array(ExperimentResponseSchema), + nextCursor: z.string().optional(), + }) + .openapi("ExperimentListResponse"); + +const ExecutionResponseSchema = z + .object({ + experimentExecutionId: z.string(), + attemptNumber: z.number().int(), + readiness: ReadinessSchema, + blockers: z.array(z.object({ + requestId: z.string().optional(), + runId: z.string().optional(), + handlerId: z.string(), + reason: z.string(), + error: z.string().optional(), + })), + progress: HandlerProgressSchema, + createdAt: z.coerce.date(), + updatedAt: z.coerce.date(), + startedAt: z.coerce.date().optional(), + finishedAt: z.coerce.date().optional(), + discardedAt: z.coerce.date().optional(), + discardReason: z.string().optional(), + supersededAt: z.coerce.date().optional(), + supersededByExperimentExecutionId: z.string().optional(), + supersededReason: z.enum(["rollover", "retry"]).optional(), + }) + .openapi("ExperimentExecutionResponse"); + +const ExecutionListResponseSchema = z + .object({ + data: z.array(ExecutionResponseSchema), + nextCursor: z.string().optional(), + }) + .openapi("ExperimentExecutionListResponse"); + +const CreateExperimentResponseSchema = z + .object({ + experiment: ExperimentResponseSchema, + requestIds: z.array(z.string()), + }) + .openapi("CreateExperimentResponse"); + +const RetryExperimentResponseSchema = z + .object({ + experimentId: z.string(), + experimentExecutionId: z.string(), + attemptNumber: z.number().int(), + readiness: ReadinessSchema, + blockers: z.array(z.object({ + requestId: z.string().optional(), + runId: z.string().optional(), + handlerId: z.string(), + reason: z.string(), + error: z.string().optional(), + })), + }) + .openapi("RetryExperimentResponse"); + +type Readiness = z.infer; + +function store(ctx: RouteContext): ExperimentStore { + return new ExperimentStore({ + experiments: ctx.experimentCollection, + executions: ctx.experimentExecutionsCollection, + chunks: ctx.experimentChunksCollection, + }); +} + +function readiness(execution: ExperimentExecution): Readiness { + if (execution.requests.entryCount === 0) return "empty"; + if (execution.discardedAt) return "failed"; + if (execution.blocking.some((blocker) => blocker.reason === "dependency-failed")) { + return "failed"; + } + if ( + execution.blocking.length > 0 && + execution.blocking.every( + (blocker) => blocker.reason === "dependency-not-ready", + ) + ) { + return "waiting"; + } + if (execution.blocking.length > 0) return "blocked"; + const statuses = Object.values(execution.handlerSummary); + if (statuses.some((status) => status.status === "failed")) return "failed"; + if (statuses.some((status) => status.status === "processing" || status.status === "queued")) { + return "processing"; + } + if (statuses.length > 0 && statuses.every((status) => status.status === "done")) { + return "done"; + } + return "ready"; +} + +function executionResponse(execution: ExperimentExecution) { + return { + experimentExecutionId: execution.experimentExecutionId, + attemptNumber: execution.attemptNumber, + readiness: readiness(execution), + blockers: execution.blocking, + progress: Object.fromEntries( + Object.entries(execution.handlerSummary).map(([handlerId, status]) => [ + handlerId, + { + status: status.status, + updatedAt: status.updatedAt, + ...(status.error ? { error: status.error } : {}), + }, + ]), + ), + createdAt: execution.createdAt, + updatedAt: execution.updatedAt, + ...(execution.startedAt ? { startedAt: execution.startedAt } : {}), + ...(execution.finishedAt ? { finishedAt: execution.finishedAt } : {}), + ...(execution.discardedAt ? { discardedAt: execution.discardedAt } : {}), + ...(execution.discardReason ? { discardReason: execution.discardReason } : {}), + ...(execution.supersededAt ? { supersededAt: execution.supersededAt } : {}), + ...(execution.supersededByExperimentExecutionId + ? { + supersededByExperimentExecutionId: + execution.supersededByExperimentExecutionId, + } + : {}), + ...(execution.supersededReason + ? { supersededReason: execution.supersededReason } + : {}), + }; +} + +function experimentResponse(experiment: ExperimentDocument) { + return { + experimentId: experiment.experimentId, + projectId: experiment.projectId, + metadata: experiment.metadata, + revision: { + experimentRevisionId: experiment.revision.experimentRevisionId, + revisionNumber: experiment.revision.revisionNumber, + memberCount: experiment.revision.memberCount, + createdAt: experiment.revision.createdAt, + }, + currentExecution: executionResponse(experiment.execution), + createdAt: experiment.createdAt, + updatedAt: experiment.updatedAt, + }; +} + +interface ExecutionCursor { + attemptNumber: number; + experimentExecutionId: string; +} + +function encodeExecutionCursor(cursor: ExecutionCursor): string { + return Buffer.from(JSON.stringify({ v: 1, ...cursor }), "utf8").toString( + "base64url", + ); +} + +function decodeExecutionCursor( + cursor: string | undefined, +): ExecutionCursor | undefined { + if (!cursor) return undefined; + try { + const parsed: unknown = JSON.parse( + Buffer.from(cursor, "base64url").toString("utf8"), + ); + const result = z + .object({ + v: z.literal(1), + attemptNumber: z.number().int().positive(), + experimentExecutionId: z.string().min(1), + }) + .strict() + .safeParse(parsed); + if (result.success) { + return { + attemptNumber: result.data.attemptNumber, + experimentExecutionId: result.data.experimentExecutionId, + }; + } + } catch { + // Converted to the route's standard 400 below. + } + throw new Error("Invalid experiment cursor"); +} + +function createRequestDocument(input: { + requestId: string; + runId: string; + experimentId: string; + expectedCount: number; + index: number; + projectId: string; + body: z.infer; + taskPromptId: string; + logsUrl: string; + now: Date; +}): RequestDocument { + const { body } = input; + return { + _id: input.requestId, + projectId: input.projectId, + scenario: body.scenario, + workerType: body.workerType, + createdAt: input.now, + taskPromptId: input.taskPromptId, + priority: body.priority ?? 0, + ...(body.model ? { model: body.model } : {}), + ...(body.reasoningEffort ? { reasoningEffort: body.reasoningEffort } : {}), + ...(body.maxIterations ? { maxIterations: body.maxIterations } : {}), + ...(body.personaInstructions + ? { personaInstructions: body.personaInstructions } + : {}), + ...(body.persona ? { persona: body.persona } : {}), + ...(body.mcpServers ? { mcpServers: body.mcpServers } : {}), + ...(body.skillRevisions ? { skillRevisions: body.skillRevisions } : {}), + ...(body.codebaseRevisionId + ? { codebaseRevisionId: body.codebaseRevisionId } + : {}), + ...(body.extensions ? { extensions: body.extensions } : {}), + ...(body.profileId ? { profileId: body.profileId } : {}), + ...(body.gates ? { gates: body.gates } : {}), + ...(body.observations ? { observations: body.observations } : {}), + experimentId: input.experimentId, + experimentExpectedRequestCount: input.expectedCount, + experimentRequestIndex: input.index, + experimentInitialRunId: input.runId, + run: { + _id: input.runId, + attemptNumber: 1, + status: "pending", + logsUrl: input.logsUrl, + }, + }; +} + +export function registerExperimentsRoutes(ctx: RouteContext): void { + apiRoute(ctx.app, ctx.registry, { + method: "post", + path: "/api/v1/experiments", + tags: ["Experiments"], + summary: "Create an experiment and zero or more new requests", + description: + "Canonical composition endpoint. Request and run IDs are minted before insertion; " + + "the exact revision/execution is then finalized idempotently.", + query: ProjectIdQuerySchema, + body: CreateExperimentInputSchema, + response: CreateExperimentResponseSchema, + errorResponses: { + 409: { description: "Experiment identity conflict" }, + }, + handler: async (req, res) => { + const projectId = getQueryProjectId(req); + const experimentId = uuidv4(); + const experimentExecutionId = uuidv4(); + const experimentRevisionId = uuidv4(); + const now = new Date(); + const identities = req.body.requests.map(() => ({ + requestId: uuidv4(), + runId: uuidv4(), + })); + const requestDocs = await Promise.all( + req.body.requests.map(async (body, index) => { + const identity = identities[index]!; + const taskPrompt = await ctx.taskPromptStore.findOrCreate( + projectId, + body.scenario.task, + ); + return createRequestDocument({ + ...identity, + experimentId, + expectedCount: req.body.requests.length, + index, + projectId, + body, + taskPromptId: taskPrompt._id, + logsUrl: ctx.blobStorage.getLogsBlobUrl( + `${identity.requestId}/runs/${identity.runId}/run.jsonl`, + ), + now, + }); + }), + ); + if (requestDocs.length > 0) { + await ctx.requestCollection.insertMany(requestDocs); + } + const requestStates: ExperimentRequestState[] = requestDocs.map((doc) => ({ + requestId: doc._id, + runId: doc.run!._id, + dependencies: [], + blockers: [], + })); + const metadata: ExperimentMetadata = req.body.metadata; + const experimentStore = store(ctx); + const pending = await experimentStore.createPending({ + experimentId, + projectId, + metadata, + revision: { + experimentRevisionId, + revisionNumber: 1, + memberCount: requestStates.length, + membershipFingerprint: fingerprintExperimentPayload( + requestStates.map(({ requestId, runId }) => ({ requestId, runId })), + ), + definitionFingerprint: fingerprintExperimentPayload({ + metadata, + requests: req.body.requests, + }), + createdAt: now, + }, + experimentExecutionId, + requests: requestStates, + now, + }); + const finalized = await experimentStore.finalizeCreation({ + projectId, + experimentId, + expectedExperimentExecutionId: experimentExecutionId, + expectedStateVersion: pending.stateVersion, + now, + }); + if (!finalized) { + res.status(409).json({ error: "Experiment finalization contention" }); + return; + } + if (requestDocs.length > 0) { + await ctx.requestCollection.updateMany( + { + _id: { $in: requestDocs.map((doc) => doc._id) }, + projectId, + experimentId, + experimentInitialRunId: { + $in: requestDocs.map((doc) => doc.run!._id), + }, + }, + { $set: { experimentFinalizedAt: now } }, + ); + } + res.status(201).json({ + experiment: experimentResponse(finalized), + requestIds: requestDocs.map((doc) => doc._id), + }); + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/experiments", + tags: ["Experiments"], + summary: "List experiments", + query: PageQuerySchema, + response: ExperimentListResponseSchema, + handler: async (req, res) => { + const page = await store(ctx).listPage({ + projectId: getQueryProjectId(req), + limit: req.query.limit, + afterExperimentId: req.query.after, + }); + res.json({ + data: page.experiments.map(experimentResponse), + ...(page.nextCursor ? { nextCursor: page.nextCursor } : {}), + }); + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/experiments/:id", + tags: ["Experiments"], + summary: "Get an experiment", + params: z.object({ id: z.string().min(1) }), + query: ProjectIdQuerySchema, + response: ExperimentResponseSchema, + errorResponses: { 404: { description: "Experiment not found" } }, + handler: async (req, res) => { + const experiment = await store(ctx).get( + getQueryProjectId(req), + req.params.id, + ); + if (!experiment) { + res.status(404).json({ error: "Experiment not found" }); + return; + } + res.json(experimentResponse(experiment)); + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/experiments/:id/executions", + tags: ["Experiments"], + summary: "List current and historical experiment executions", + params: z.object({ id: z.string().min(1) }), + query: z.object({ + limit: z.coerce.number().int().min(1).max(100).default(50), + after: CursorSchema, + }), + response: ExecutionListResponseSchema, + errorResponses: { 404: { description: "Experiment not found" } }, + handler: async (req, res) => { + const experiment = await ctx.experimentCollection.findOne({ + _id: req.params.id, + finalizedAt: { $exists: true }, + }); + if (!experiment) { + res.status(404).json({ error: "Experiment not found" }); + return; + } + const after = decodeExecutionCursor(req.query.after); + const historyFilter: Document = { + projectId: experiment.projectId, + experimentId: req.params.id, + }; + if (after) { + historyFilter.$or = [ + { attemptNumber: { $lt: after.attemptNumber } }, + { + attemptNumber: after.attemptNumber, + experimentExecutionId: { + $gt: after.experimentExecutionId, + }, + }, + ]; + } + const raw = await ctx.experimentExecutionsCollection + .find(historyFilter) + .sort({ attemptNumber: -1, experimentExecutionId: 1 }) + .limit(req.query.limit + 1) + .toArray(); + const executions: ExperimentExecution[] = + after ? raw : [experiment.execution, ...raw]; + const data = executions.slice(0, req.query.limit).map(executionResponse); + const hasMore = executions.length > req.query.limit; + const last = executions[Math.min(data.length, executions.length) - 1]; + res.json({ + data, + ...(hasMore && last + ? { + nextCursor: encodeExecutionCursor({ + attemptNumber: last.attemptNumber, + experimentExecutionId: last.experimentExecutionId, + }), + } + : {}), + }); + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/experiments/:id/executions/:experimentExecutionId", + tags: ["Experiments"], + summary: "Get one exact experiment execution", + params: z.object({ + id: z.string().min(1), + experimentExecutionId: z.string().min(1), + }), + response: ExecutionResponseSchema, + errorResponses: { 404: { description: "Experiment execution not found" } }, + handler: async (req, res) => { + const experiment = await ctx.experimentCollection.findOne({ + _id: req.params.id, + finalizedAt: { $exists: true }, + }); + if (!experiment) { + res.status(404).json({ error: "Experiment execution not found" }); + return; + } + const execution = await store(ctx).getExecution({ + projectId: experiment.projectId, + experimentId: req.params.id, + experimentExecutionId: req.params.experimentExecutionId, + }); + if (!execution) { + res.status(404).json({ error: "Experiment execution not found" }); + return; + } + res.json(executionResponse(execution)); + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "post", + path: "/api/v1/experiments/:id/retry", + tags: ["Experiments"], + summary: "Retry current experiment processing", + params: z.object({ id: z.string().min(1) }), + query: ProjectIdQuerySchema, + body: z.object({ experimentExecutionId: z.string().min(1) }), + response: RetryExperimentResponseSchema, + errorResponses: { + 404: { description: "Experiment not found" }, + 409: { description: "Execution is stale or cannot be retried" }, + }, + handler: async (req, res) => { + const projectId = getQueryProjectId(req); + const experimentStore = store(ctx); + const current = await experimentStore.get(projectId, req.params.id); + if (!current) { + res.status(404).json({ error: "Experiment not found" }); + return; + } + if ( + current.execution.experimentExecutionId !== + req.body.experimentExecutionId + ) { + res.status(409).json({ + error: "Experiment execution is stale", + experimentExecutionId: + current.execution.experimentExecutionId, + blockers: current.execution.blocking, + }); + return; + } + const frozen = await experimentStore.freezeCurrent({ + projectId, + experimentId: req.params.id, + experimentExecutionId: req.body.experimentExecutionId, + expectedExecutionVersion: current.execution.version, + }); + if (!frozen) { + res.status(409).json({ + error: "Experiment retry contention", + experimentExecutionId: + current.execution.experimentExecutionId, + blockers: current.execution.blocking, + }); + return; + } + const retried = await experimentStore.retryCurrent({ + projectId, + experimentId: req.params.id, + expectedExperimentExecutionId: + req.body.experimentExecutionId, + expectedExecutionVersion: frozen.execution.version, + nextExperimentExecutionId: uuidv4(), + }); + if (!retried) { + res.status(409).json({ + error: "Experiment retry could not be published", + experimentExecutionId: + current.execution.experimentExecutionId, + blockers: current.execution.blocking, + }); + return; + } + res.status(201).json({ + experimentId: retried.experimentId, + experimentExecutionId: + retried.execution.experimentExecutionId, + attemptNumber: retried.execution.attemptNumber, + readiness: readiness(retried.execution), + blockers: retried.execution.blocking, + }); + }, + }); +} diff --git a/apps/api/src/routes/profiles.ts b/apps/api/src/routes/profiles.ts index b61c0b808..c8da480c6 100644 --- a/apps/api/src/routes/profiles.ts +++ b/apps/api/src/routes/profiles.ts @@ -107,7 +107,7 @@ apiRoute(ctx.app, ctx.registry, { const profileId = uuidv4(); const versionRef = `${profileId}@1`; - // Resolve extension versions (same pattern as run submission) + // Resolve extension versions (same pattern as request creation) let resolvedExtensions: string[] | undefined; if (extensions && extensions.length > 0) { const extensionClient = new ExtensionClient(""); diff --git a/apps/api/src/routes/requests/archive.test.ts b/apps/api/src/routes/requests/archive.test.ts index e6bc8a176..e9ad843d5 100644 --- a/apps/api/src/routes/requests/archive.test.ts +++ b/apps/api/src/routes/requests/archive.test.ts @@ -304,7 +304,6 @@ function makeFixtureRun(id = "run-fixture-1") { workerType: "coder-acp-copilot", createdAt: new Date("2026-01-01T00:00:00Z"), priority: 0, - submissionId: "submission-abc", run: { _id: id, attemptNumber: 1, @@ -765,7 +764,6 @@ describe("run import/export — round-trip (export → import)", () => { updatedAt: new Date("2026-05-11T23:04:47.446Z"), maxIterations: 1, priority: 0, - submissionId: "sub-rich-1", run: { _id: runId, attemptNumber: 2, // ← not 1; previous code hard-coded 1 diff --git a/apps/api/src/routes/requests/index.ts b/apps/api/src/routes/requests/index.ts index 070a583f3..07987a124 100644 --- a/apps/api/src/routes/requests/index.ts +++ b/apps/api/src/routes/requests/index.ts @@ -37,8 +37,8 @@ import { isCriterionCompatibleWithGate, orderGates, computeTaskPromptId, - ensureNativeSubmission, } from "shared"; +import { trackMetric } from "telemetry"; import type { ProfileDocument, ProfileVersionDocument, @@ -236,6 +236,28 @@ const validatePersistedRequestTarget = (request: RequestDocument) => /** Maximum number of profile variations (including the base profile) allowed in one submit. */ const MAX_PROFILE_VARIATIONS = 25; +const LEGACY_EXPERIMENT_COMPOSITION_ID = + "scope.api.requests.legacy-experiment-composition.v1"; + +function emitLegacyCompositionHeaders( + res: { setHeader(name: string, value: string): void }, + properties: readonly string[], +): void { + if (properties.length === 0) return; + res.setHeader("Deprecation", "true"); + res.setHeader( + "Link", + '; rel="successor-version"; title="POST /api/v1/experiments"', + ); + trackMetric({ + name: "api.deprecated_input_used", + value: 1, + properties: { + identifier: LEGACY_EXPERIMENT_COMPOSITION_ID, + properties: [...properties].sort().join(","), + }, + }); +} /** * Server-side validation of a request's gate configuration (docs/design/gates.md @@ -479,7 +501,16 @@ apiRoute(ctx.app, ctx.registry, { tags: ["Requests"], summary: "Submit request(s)", body: CreateRequestInputSchema.extend({ - count: z.number().min(1).max(10).default(1), + count: z + .number() + .min(1) + .max(10) + .default(1) + .openapi({ + deprecated: true, + description: + "Deprecated for composition. Use POST /api/v1/experiments and its requests array.", + }), skills: z.array(z.string()).optional(), agentVersion: z.string().optional(), codebase: z.string().optional(), @@ -554,6 +585,10 @@ apiRoute(ctx.app, ctx.registry, { ? profileVariations.filter((v: unknown): v is string => typeof v === "string") : []; const isVariationSubmit = typedProfileVariations.length > 0; + emitLegacyCompositionHeaders(res, [ + ...(count > 1 ? ["count"] : []), + ...(isVariationSubmit ? ["profileVariations"] : []), + ]); if (isVariationSubmit) { if (!requestedProfileSpec) { @@ -639,7 +674,7 @@ apiRoute(ctx.app, ctx.registry, { const taskPromptId = taskPrompt._id; // Gates are the shared evaluation harness for the whole comparative - // submission, not a per-variation controlled field: every variation + // request batch, not a per-variation controlled field: every variation // (including the base) runs the same gate configuration. Resolve, validate // and canonicalize them up front so an invalid config fails the whole // submit before any inserts (docs/design/gates.md §4.3), mirroring the @@ -836,10 +871,8 @@ apiRoute(ctx.app, ctx.registry, { }); } - // Pass 2: build documents and insert them under a single shared submissionId. - // All variations belong to the same comparative submission so consumers can - // group them via `submissionId` (matches docs/architecture/app-design.md). - const submissionId = uuidv4(); + // Pass 2: build independent request documents. Comparative composition is + // represented only by POST /experiments. const allNewIds: string[] = []; const newDocs: RequestDocument[] = []; const variationResults: Array<{ profileId: string; label?: string; ids: string[] }> = []; @@ -879,10 +912,6 @@ apiRoute(ctx.app, ctx.registry, { profileId: r.profile._id, profileVersionId: r.profileVersion.ref ?? r.profileVersion._id, ...(persistedVariationGates ? { gates: persistedVariationGates } : {}), - submissionId, - submissionExpectedRequestCount: expectedRequestCount, - submissionRequestIndex: newDocs.length, - submissionInitialRunId: runId, ...agentsMdFields, run: { _id: runId, attemptNumber: 1, status: "pending", logsUrl: ctx.blobStorage.getLogsBlobUrl(`${requestId}/runs/${runId}/run.jsonl`) }, }; @@ -896,25 +925,10 @@ apiRoute(ctx.app, ctx.registry, { } await ctx.requestCollection.insertMany(newDocs); - await ensureNativeSubmission({ - collections: { submissions: ctx.submissionCollection }, - submissionId, - projectId, - requests: newDocs.map((doc) => ({ - requestId: doc._id, - runId: doc.run!._id, - })), - createdAt: newDocs[0]?.createdAt, - }); - await ctx.requestCollection.updateMany( - { _id: { $in: allNewIds }, submissionId }, - { $set: { submissionFinalizedAt: new Date() } }, - ); res.status(201).json({ ids: allNewIds, count: allNewIds.length, - submissionId, variations: variationResults, variationCount: variationEntries.length, status: "pending", @@ -1280,7 +1294,7 @@ apiRoute(ctx.app, ctx.registry, { const taskPrompt = await ctx.taskPromptStore.findOrCreate(projectId, scenario.task); const taskPromptId = taskPrompt._id; - // Resolve AGENTS.md lineage once for this submission (shared across count>1). + // Resolve AGENTS.md lineage once for this request batch (shared across count>1). const agentsMdPromptId = await resolveAgentsMdPromptId(); const agentsMdFields = buildAgentsMdFields(agentsMdPromptId); @@ -1298,9 +1312,6 @@ apiRoute(ctx.app, ctx.registry, { ? [...new Set(resolvedObservations)] : undefined; - // Generate a submission ID to group all runs from this request - const submissionId = uuidv4(); - // Handle multiple runs (count > 1) if (count > 1) { const newIds: string[] = []; @@ -1334,10 +1345,6 @@ apiRoute(ctx.app, ctx.registry, { ...(profileVersionId ? { profileVersionId } : {}), ...(persistedGates ? { gates: persistedGates } : {}), ...(persistedObservations ? { observations: persistedObservations } : {}), - submissionId, - submissionExpectedRequestCount: count, - submissionRequestIndex: i, - submissionInitialRunId: runId, ...agentsMdFields, // Mint a distinct run id for the first attempt. Blob artifacts // are scoped under `{requestId}/runs/{runId}/...` so retries @@ -1349,27 +1356,12 @@ apiRoute(ctx.app, ctx.registry, { // Bulk insert all documents — scheduler will dispatch to queues await ctx.requestCollection.insertMany(newDocs); - await ensureNativeSubmission({ - collections: { submissions: ctx.submissionCollection }, - submissionId, - projectId, - requests: newDocs.map((doc) => ({ - requestId: doc._id, - runId: doc.run!._id, - })), - createdAt: newDocs[0]?.createdAt, - }); - await ctx.requestCollection.updateMany( - { _id: { $in: newIds }, submissionId }, - { $set: { submissionFinalizedAt: new Date() } }, - ); console.log(`Created ${count} ${mode} requests for ${workerType} (priority: ${requestedPriority ?? 0})`); res.status(201).json({ ids: newIds, count, - submissionId, workerType, taskPromptId, ...(model ? { model } : {}), @@ -1414,10 +1406,6 @@ apiRoute(ctx.app, ctx.registry, { ...(profileVersionId ? { profileVersionId } : {}), ...(persistedGates ? { gates: persistedGates } : {}), ...(persistedObservations ? { observations: persistedObservations } : {}), - submissionId, - submissionExpectedRequestCount: 1, - submissionRequestIndex: 0, - submissionInitialRunId: runId, ...agentsMdFields, // Mint a distinct run id for the first attempt. Blob artifacts // are scoped under `{requestId}/runs/{runId}/...` so retries @@ -1427,23 +1415,11 @@ apiRoute(ctx.app, ctx.registry, { // Store in MongoDB — scheduler will dispatch to queue await ctx.requestCollection.insertOne(requestDoc); - await ensureNativeSubmission({ - collections: { submissions: ctx.submissionCollection }, - submissionId, - projectId, - requests: [{ requestId, runId }], - createdAt: requestDoc.createdAt, - }); - await ctx.requestCollection.updateOne( - { _id: requestId, submissionId }, - { $set: { submissionFinalizedAt: new Date() } }, - ); console.log(`Created ${mode} request ${requestId} for ${workerType} (priority: ${requestedPriority ?? 0})`); res.status(201).json({ id: requestId, - submissionId, workerType, ...(model ? { model } : {}), ...(effectiveReasoningEffort ? { reasoningEffort: effectiveReasoningEffort } : {}), @@ -1526,14 +1502,14 @@ apiRoute(ctx.app, ctx.registry, { const projectId = getQueryProjectId(req); const taskPromptIdFilter = req.query.taskPromptId as string; const criteriaFilter = req.query.criteria as string; - const submissionIdFilter = req.query.submissionId as string; + const experimentIdFilter = req.query.experimentId as string; const searchFilter = req.query.search as string | undefined; const turnsFilterRaw = req.query.turns as string | undefined; const turnsOpFilter = (req.query.turnsOp as string | undefined) ?? "eq"; const maxIterationsFilterRaw = req.query.maxIterations as string | undefined; const maxIterationsOpFilter = (req.query.maxIterationsOp as string | undefined) ?? "eq"; const includeDeleted = req.query.includeDeleted === "true"; - const groupByParam = req.query.groupBy as "task" | "submissionId" | "profile" | undefined; + const groupByParam = req.query.groupBy as "task" | "experimentId" | "profile" | undefined; const limit = Math.min(Math.max(Number(req.query.limit) || 10, 1), 100); const afterParam = req.query.after as string | undefined; const beforeParam = req.query.before as string | undefined; @@ -1600,9 +1576,9 @@ apiRoute(ctx.app, ctx.registry, { pushClause(buildMultiClause("priority", priorityValues, { coerceNumber: true })); pushClause(buildMultiClause("agentVersion", agentVersionValues)); - if (submissionIdFilter) { - // Prefix-based matching: allow filtering by partial submission ID. - flat.submissionId = { $regex: `^${submissionIdFilter.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}` }; + if (experimentIdFilter) { + // Prefix-based matching: allow filtering by partial experiment ID. + flat.experimentId = { $regex: `^${experimentIdFilter.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}` }; } if (!includeDeleted) { flat.deletedAt = { $exists: false }; @@ -1699,7 +1675,7 @@ apiRoute(ctx.app, ctx.registry, { // Grouped mode: paginated RunGroup[] via two-phase aggregation if (groupByParam) { - const groupByField = groupByParam === "task" ? "taskPromptId" : groupByParam === "profile" ? "profileId" : "submissionId"; + const groupByField = groupByParam === "task" ? "taskPromptId" : groupByParam === "profile" ? "profileId" : "experimentId"; const groupByAggField = `$${groupByField}`; // Decode group cursor @@ -1932,7 +1908,7 @@ apiRoute(ctx.app, ctx.registry, { taskPromptId: z.string().optional(), criteria: z.string().optional(), features: z.string().optional(), - submissionId: z.string().optional(), + experimentId: z.string().optional(), k: z.string().optional(), }).merge(ProjectIdQuerySchema), response: z.object({}).passthrough().describe("Analysis metrics"), @@ -2116,7 +2092,6 @@ apiRoute(ctx.app, ctx.registry, { const foundIds = new Set(originalRuns.map(r => r._id)); const notFound = ids.filter((id: string) => !foundIds.has(id)); - const submissionId = uuidv4(); const newIds: string[] = []; const newDocs: RequestDocument[] = []; @@ -2255,10 +2230,6 @@ apiRoute(ctx.app, ctx.registry, { // Preserve AGENTS.md lineage across re-submits. ...(original.agentsMdPromptId ? { agentsMdPromptId: original.agentsMdPromptId } : {}), ...(original.agentsMdParentIds && original.agentsMdParentIds.length > 0 ? { agentsMdParentIds: original.agentsMdParentIds } : {}), - submissionId, - submissionExpectedRequestCount: originalRuns.length * count, - submissionRequestIndex: newDocs.length, - submissionInitialRunId: runId, // Bulk re-submit creates a brand-new request — mint a distinct // run id for the first attempt so blob artifacts live under // `{requestId}/runs/{runId}/...` (independent of the original run). @@ -2272,20 +2243,6 @@ apiRoute(ctx.app, ctx.registry, { // Insert all new documents — scheduler will dispatch to queues if (newDocs.length > 0) { await ctx.requestCollection.insertMany(newDocs); - await ensureNativeSubmission({ - collections: { submissions: ctx.submissionCollection }, - submissionId, - projectId: newDocs[0].projectId, - requests: newDocs.map((doc) => ({ - requestId: doc._id, - runId: doc.run!._id, - })), - createdAt: newDocs[0].createdAt, - }); - await ctx.requestCollection.updateMany( - { _id: { $in: newIds }, submissionId }, - { $set: { submissionFinalizedAt: new Date() } }, - ); } console.log(`Bulk re-submitted ${newIds.length} runs from ${originalRuns.length} originals (count=${count})`); @@ -2294,7 +2251,6 @@ apiRoute(ctx.app, ctx.registry, { submitted: newIds.length, failed: notFound, newIds, - submissionId, }); }, }); @@ -2685,7 +2641,7 @@ async function finalizePendingRun( const topChat = run.artifacts.get("run.chat-export.json"); if (topChat) runState.rawChatUrl = topChat; - // Materialize the task-prompt entity for this run. The submission flow + // Materialize the task-prompt entity for this run. The request creation flow // (POST /api/v1/requests) goes through `taskPromptStore.findOrCreate` so // every run has a row in the `task-prompts` collection that backs // features, report triggers, group-by-task, the runs-list task filter and @@ -2717,7 +2673,6 @@ async function finalizePendingRun( createdAt: runDoc.createdAt ?? new Date(), priority: runDoc.priority ?? 0, taskPromptId: resolvedTaskPromptId, - ...(runDoc.submissionId ? {} : { submissionId: uuidv4() }), run: { ...(runState as unknown as RunState), // Per-attempt `run._id` is preserved from the yaml (NOT clobbered @@ -3196,24 +3151,6 @@ apiRoute(ctx.app, ctx.registry, { { _id: { $in: ids }, deletedAt: { $exists: false } }, ).toArray(); const requestMap = new Map(requests.map((r) => [r._id, r])); - const affectedSubmissionIds = [ - ...new Set( - requests.flatMap((request) => - request.submissionId ? [request.submissionId] : [], - ), - ), - ]; - if (affectedSubmissionIds.length > 0) { - await ctx.submissionCollection.updateMany( - { _id: { $in: affectedSubmissionIds } }, - { - $set: { - "run.reconcileAfter": new Date(Date.now() + 60_000), - updatedAt: new Date(), - }, - }, - ); - } const results: Array<{ requestId: string; runId?: string; attemptNumber?: number; error?: string }> = []; let retried = 0; @@ -3295,16 +3232,6 @@ apiRoute(ctx.app, ctx.registry, { retried++; } - if (affectedSubmissionIds.length > 0) { - await ctx.submissionCollection.updateMany( - { _id: { $in: affectedSubmissionIds } }, - { - $unset: { "run.reconcileAfter": "" }, - $set: { updatedAt: new Date() }, - }, - ); - } - res.status(201).json({ retried, skipped, results }); }, }); diff --git a/apps/api/src/routes/submissions.test.ts b/apps/api/src/routes/submissions.test.ts deleted file mode 100644 index 984bdd358..000000000 --- a/apps/api/src/routes/submissions.test.ts +++ /dev/null @@ -1,253 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -import { OpenAPIRegistry } from "@asteasolutions/zod-to-openapi"; -import express from "express"; -import request from "supertest"; -import { describe, expect, it, vi } from "vitest"; -import type { Collection, Db } from "mongodb"; -import type { - RequestDocument, - ServiceHandlerDocument, - SubmissionDocument, - SubmissionRunHistoryDocument, -} from "shared"; -import { computeSubmissionVectorFingerprint } from "shared"; -import type { RouteContext } from "../route-context.js"; -import { registerSubmissionsRoutes } from "./submissions.js"; - -function queryResult(items: T[]) { - return { - sort: () => queryResult(items), - limit: () => queryResult(items), - toArray: async () => items, - }; -} - -function buildApp(input: { - requests: RequestDocument[]; - submission: SubmissionDocument; - history?: SubmissionRunHistoryDocument[]; -}) { - let submission = structuredClone(input.submission); - const services: ServiceHandlerDocument[] = [ - { - _id: "pp-atif", - type: "post-process-handler", - version: 1, - queue: "post-processor", - selector: "atif", - autoBackfill: true, - dependsOn: [], - }, - { - _id: "pp-observations", - type: "post-process-handler", - version: 1, - queue: "post-processor", - selector: "observations", - autoBackfill: true, - dependsOn: ["pp-atif"], - }, - ]; - const submissionCollection = { - findOne: vi.fn(async () => structuredClone(submission)), - find: vi.fn(() => queryResult([structuredClone(submission)])), - findOneAndUpdate: vi.fn(async ( - filter: Record, - update: { $set: Record }, - ) => { - if (filter["run._id"] !== submission.run._id) return null; - submission = { - ...submission, - updatedAt: update.$set.updatedAt as Date, - run: { - ...submission.run, - dependencyProjection: update.$set[ - "run.dependencyProjection" - ] as SubmissionDocument["run"]["dependencyProjection"], - blocking: update.$set[ - "run.blocking" - ] as SubmissionDocument["run"]["blocking"], - updatedAt: update.$set["run.updatedAt"] as Date, - }, - }; - return structuredClone(submission); - }), - } as unknown as Collection; - const db = { - collection: vi.fn(() => ({ - find: vi.fn(() => - queryResult(services.filter((service) => service.type === "post-process-handler")), - ), - findOne: vi.fn(async () => null), - })), - } as unknown as Db; - const ctx = { - app: express(), - registry: new OpenAPIRegistry(), - db, - requestCollection: { - find: vi.fn(() => queryResult(input.requests)), - } as unknown as Collection, - submissionCollection, - submissionRunsCollection: { - find: vi.fn(() => queryResult(input.history ?? [])), - findOne: vi.fn(async (filter: { _id: string }) => - (input.history ?? []).find((run) => run._id === filter._id) ?? null, - ), - } as unknown as Collection, - } as unknown as RouteContext; - ctx.app.use(express.json()); - registerSubmissionsRoutes(ctx); - ctx.app.use( - ( - error: Error, - _req: express.Request, - res: express.Response, - _next: express.NextFunction, - ) => res.status(500).json({ error: error.message }), - ); - return ctx.app; -} - -describe("submission routes", () => { - const requestDocument: RequestDocument = { - _id: "request-1", - projectId: "project-1", - scenario: { task: "Implement feature", criteria: ["criterion"] }, - workerType: "worker", - createdAt: new Date(), - priority: 0, - run: { - _id: "run-1", - attemptNumber: 1, - status: "done", - outcome: "failed", - handlerStatus: { - "pp-observations": { status: "done", version: 1 }, - }, - }, - }; - const submissionDocument: SubmissionDocument = { - _id: "submission-1", - submissionId: "submission-1", - projectId: "project-1", - requestIds: ["request-1"], - createdAt: new Date(), - run: { - _id: "submission-run-1", - attemptNumber: 1, - vectorFingerprint: computeSubmissionVectorFingerprint([ - { requestId: "request-1", runId: "run-1" }, - ]), - requests: [{ requestId: "request-1", runId: "run-1" }], - createdAt: new Date(), - dependencyProjection: [], - blocking: [], - }, - }; - - it("returns readiness and the exact request/run vector", async () => { - const response = await request( - buildApp({ - requests: [requestDocument], - submission: submissionDocument, - }), - ).get("/api/v1/submissions/submission-1"); - - expect(response.status, JSON.stringify(response.body)).toBe(200); - expect(response.body).toMatchObject({ - submissionId: "submission-1", - submissionRunId: "submission-run-1", - readiness: "ready", - requests: [{ requestId: "request-1", runId: "run-1", outcome: "failed" }], - dependencyProjection: [{ - requestId: "request-1", - runId: "run-1", - handlerId: "pp-observations", - state: "ready", - }], - }); - }); - - it("lists current and immutable archived attempts", async () => { - const historical: SubmissionRunHistoryDocument = { - ...submissionDocument.run, - _id: "submission-run-0", - attemptNumber: 1, - reconcileAfter: new Date(), - rollover: { - nextSubmissionRunId: "internal-next", - vectorFingerprint: "internal-fingerprint", - requests: [{ requestId: "request-1", runId: "run-1" }], - reason: "request-vector-changed", - claimedAt: new Date(), - }, - submissionId: "submission-1", - projectId: "project-1", - }; - const response = await request( - buildApp({ - requests: [requestDocument], - submission: submissionDocument, - history: [historical], - }), - ).get("/api/v1/submissions/submission-1/runs"); - - expect(response.status).toBe(200); - expect(response.body.runs.map((run: { _id: string }) => run._id)).toEqual([ - "submission-run-1", - "submission-run-0", - ]); - expect(response.body.runs[1]).not.toHaveProperty("reconcileAfter"); - expect(response.body.runs[1]).not.toHaveProperty("rollover"); - }); - - it("rejects a manual retry targeting a stale submission run", async () => { - const response = await request( - buildApp({ - requests: [requestDocument], - submission: submissionDocument, - }), - ) - .post("/api/v1/submissions/submission-1/retry") - .send({ submissionRunId: "stale-submission-run" }); - - expect(response.status).toBe(409); - expect(response.body).toMatchObject({ - error: "Submission attempt is stale", - submissionRunId: "submission-run-1", - }); - }); - - it("returns exact waiting request/run pairs when retry dependencies are not ready", async () => { - const waitingRequest: RequestDocument = { - ...requestDocument, - run: { - _id: "run-1", - attemptNumber: 1, - status: "done", - handlerStatus: {}, - }, - }; - const response = await request( - buildApp({ - requests: [waitingRequest], - submission: submissionDocument, - }), - ) - .post("/api/v1/submissions/submission-1/retry") - .send({ submissionRunId: "submission-run-1" }); - - expect(response.status).toBe(409); - expect(response.body.blocking).toEqual([ - { - requestId: "request-1", - runId: "run-1", - handlerId: "pp-observations", - reason: "dependency-not-ready", - }, - ]); - }); -}); diff --git a/apps/api/src/routes/submissions.ts b/apps/api/src/routes/submissions.ts deleted file mode 100644 index 201aae5bd..000000000 --- a/apps/api/src/routes/submissions.ts +++ /dev/null @@ -1,245 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -import { z } from "zod"; -import { - SubmissionCoordinator, - SubmissionListResponseSchema, - SubmissionResponseSchema, - SubmissionRetryResponseSchema, - SubmissionRunResponseSchema, - type ServiceHandlerDocument, - deriveSubmissionReadiness, -} from "shared"; -import type { SubmissionDocument } from "../route-context.js"; -import type { RouteContext } from "../route-context.js"; -import { apiRoute } from "../openapi/api-route.js"; -import { ProjectIdQuerySchema, getQueryProjectId } from "../utils/project-scope.js"; - -function coordinator(ctx: RouteContext): SubmissionCoordinator { - return new SubmissionCoordinator({ - requests: ctx.requestCollection, - submissions: ctx.submissionCollection, - submissionRuns: ctx.submissionRunsCollection, - services: ctx.db.collection("services"), - }); -} - -function toRunResponse( - run: T, -): Omit { - const { - reconcileAfter: _reconcileAfter, - rollover: _rollover, - ...response - } = run; - return response; -} - -function retryBlocking(submission: SubmissionDocument) { - if (submission.run.blocking.length > 0) return submission.run.blocking; - return submission.run.dependencyProjection - .filter((dependency) => dependency.state !== "ready") - .map((dependency) => ({ - requestId: dependency.requestId, - runId: dependency.runId, - handlerId: dependency.handlerId, - reason: "dependency-not-ready" as const, - ...(dependency.error ? { error: dependency.error } : {}), - })); -} - -async function toResponse(ctx: RouteContext, submission: SubmissionDocument) { - const requestDocs = await ctx.requestCollection - .find({ - _id: { $in: submission.requestIds }, - projectId: submission.projectId, - deletedAt: { $exists: false }, - }) - .toArray(); - const byId = new Map(requestDocs.map((request) => [request._id, request])); - const requests = submission.run.requests.flatMap((ref) => { - const request = byId.get(ref.requestId); - if (!request) return []; - return [{ - requestId: request._id, - runId: ref.runId, - task: request.scenario.task, - status: request.run?.status ?? "unknown", - ...(request.run?.outcome ? { outcome: request.run.outcome } : {}), - workerType: request.workerType, - ...(request.model ? { model: request.model } : {}), - ...(request.profileId ? { profileId: request.profileId } : {}), - }]; - }); - return { - submissionId: submission._id, - projectId: submission.projectId, - requestIds: submission.requestIds, - createdAt: submission.createdAt, - ...(submission.updatedAt ? { updatedAt: submission.updatedAt } : {}), - submissionRunId: submission.run._id, - attemptNumber: submission.run.attemptNumber, - readiness: deriveSubmissionReadiness(submission.run), - requests, - dependencyProjection: submission.run.dependencyProjection, - blocking: submission.run.blocking, - ...(submission.run.handlerStatus - ? { handlerProgress: submission.run.handlerStatus } - : {}), - ...(submission.run.report ? { report: submission.run.report } : {}), - ...(submission.run.supersededAt - ? { supersededAt: submission.run.supersededAt } - : {}), - ...(submission.run.supersededBySubmissionRunId - ? { supersededBySubmissionRunId: submission.run.supersededBySubmissionRunId } - : {}), - ...(submission.run.supersededReason - ? { supersededReason: submission.run.supersededReason } - : {}), - }; -} - -export function registerSubmissionsRoutes(ctx: RouteContext): void { - apiRoute(ctx.app, ctx.registry, { - method: "get", - path: "/api/v1/submissions", - tags: ["Submissions"], - summary: "List submissions", - query: ProjectIdQuerySchema.extend({ - limit: z.coerce.number().int().min(1).max(100).default(50), - }), - response: SubmissionListResponseSchema, - handler: async (req, res) => { - const projectId = getQueryProjectId(req); - const docs = await ctx.submissionCollection - .find({ projectId }) - .sort({ createdAt: -1, _id: -1 }) - .limit(req.query.limit) - .toArray(); - res.json({ - submissions: await Promise.all(docs.map((doc) => toResponse(ctx, doc))), - }); - }, - }); - - apiRoute(ctx.app, ctx.registry, { - method: "get", - path: "/api/v1/submissions/:id", - tags: ["Submissions"], - summary: "Get a submission", - params: z.object({ id: z.string() }), - response: SubmissionResponseSchema, - errorResponses: { 404: { description: "Submission not found" } }, - handler: async (req, res) => { - const reconciled = await coordinator(ctx).reconcile(req.params.id); - if (!reconciled) { - res.status(404).json({ error: "Submission not found" }); - return; - } - res.json(await toResponse(ctx, reconciled.submission)); - }, - }); - - apiRoute(ctx.app, ctx.registry, { - method: "get", - path: "/api/v1/submissions/:id/runs", - tags: ["Submissions"], - summary: "List current and historical submission attempts", - params: z.object({ id: z.string() }), - response: z.object({ runs: z.array(SubmissionRunResponseSchema) }), - errorResponses: { 404: { description: "Submission not found" } }, - handler: async (req, res) => { - const submission = await ctx.submissionCollection.findOne({ _id: req.params.id }); - if (!submission) { - res.status(404).json({ error: "Submission not found" }); - return; - } - const historical = await ctx.submissionRunsCollection - .find({ submissionId: req.params.id }) - .sort({ attemptNumber: -1 }) - .toArray(); - res.json({ - runs: [toRunResponse(submission.run), ...historical.map(toRunResponse)], - }); - }, - }); - - apiRoute(ctx.app, ctx.registry, { - method: "get", - path: "/api/v1/submissions/:id/runs/:submissionRunId", - tags: ["Submissions"], - summary: "Get one submission attempt", - params: z.object({ id: z.string(), submissionRunId: z.string() }), - response: SubmissionRunResponseSchema, - errorResponses: { 404: { description: "Submission attempt not found" } }, - handler: async (req, res) => { - const submission = await ctx.submissionCollection.findOne({ _id: req.params.id }); - if (submission?.run._id === req.params.submissionRunId) { - res.json(toRunResponse(submission.run)); - return; - } - const historical = await ctx.submissionRunsCollection.findOne({ - _id: req.params.submissionRunId, - submissionId: req.params.id, - }); - if (!historical) { - res.status(404).json({ error: "Submission attempt not found" }); - return; - } - res.json(toRunResponse(historical)); - }, - }); - - apiRoute(ctx.app, ctx.registry, { - method: "post", - path: "/api/v1/submissions/:id/retry", - tags: ["Submissions"], - summary: "Retry current submission processing", - params: z.object({ id: z.string() }), - body: z.object({ submissionRunId: z.string() }), - response: SubmissionRetryResponseSchema, - errorResponses: { - 404: { description: "Submission not found" }, - 409: { description: "Submission attempt is stale or dependencies are blocked" }, - }, - handler: async (req, res) => { - const current = await ctx.submissionCollection.findOne({ _id: req.params.id }); - if (!current) { - res.status(404).json({ error: "Submission not found" }); - return; - } - if (current.run._id !== req.body.submissionRunId) { - res.status(409).json({ - error: "Submission attempt is stale", - submissionRunId: current.run._id, - blocking: retryBlocking(current), - }); - return; - } - const result = await coordinator(ctx).retryCurrent( - req.params.id, - req.body.submissionRunId, - ); - if (!result) { - res.status(404).json({ error: "Submission not found" }); - return; - } - if (result.readiness === "blocked" || result.readiness === "waiting") { - res.status(409).json({ - error: "Submission dependencies are not ready", - submissionRunId: result.submission.run._id, - blocking: retryBlocking(result.submission), - }); - return; - } - res.status(201).json({ - submissionId: result.submission._id, - submissionRunId: result.submission.run._id, - attemptNumber: result.submission.run.attemptNumber, - readiness: result.readiness, - blocking: result.submission.run.blocking, - }); - }, - }); -} diff --git a/apps/api/src/test-helpers.ts b/apps/api/src/test-helpers.ts index 74447d6e7..64229a113 100644 --- a/apps/api/src/test-helpers.ts +++ b/apps/api/src/test-helpers.ts @@ -222,8 +222,9 @@ export function createAllMockDependencies() { const db = createMockDb(); const collection = createMockCollection(); const runsCollection = createMockCollection(); - const submissionCollection = createMockCollection(); - const submissionRunsCollection = createMockCollection(); + const experimentCollection = createMockCollection(); + const experimentExecutionsCollection = createMockCollection(); + const experimentChunksCollection = createMockCollection(); const criteriaCollection = createMockCollection(); const promptFeatureCollection = createMockCollection(); const reportCollection = createMockCollection(); @@ -252,8 +253,9 @@ export function createAllMockDependencies() { db, collection, runsCollection, - submissionCollection, - submissionRunsCollection, + experimentCollection, + experimentExecutionsCollection, + experimentChunksCollection, criteriaCollection, promptFeatureCollection, reportCollection, @@ -284,8 +286,9 @@ export function createAllMockDependencies() { db: ReturnType; collection: Collection; runsCollection: Collection; - submissionCollection: Collection; - submissionRunsCollection: Collection; + experimentCollection: Collection; + experimentExecutionsCollection: Collection; + experimentChunksCollection: Collection; criteriaCollection: Collection; promptFeatureCollection: Collection; reportCollection: Collection; diff --git a/apps/api/src/utils/agent-helpers.ts b/apps/api/src/utils/agent-helpers.ts index 768a635cf..95f130278 100644 --- a/apps/api/src/utils/agent-helpers.ts +++ b/apps/api/src/utils/agent-helpers.ts @@ -41,7 +41,7 @@ export interface ValidateAgentTargetOptions { } /** - * Resolve a submission target entirely from the agent registry. + * Resolve a request target entirely from the agent registry. * * Existence, deletion, explicit availability, active version, and advertised * queue are always enforced. Capability flags are enforced only while the diff --git a/apps/api/src/utils/codebase-helpers.ts b/apps/api/src/utils/codebase-helpers.ts index 1cb5d504d..fd9d69f0f 100644 --- a/apps/api/src/utils/codebase-helpers.ts +++ b/apps/api/src/utils/codebase-helpers.ts @@ -84,7 +84,7 @@ export interface CodebaseSpecResolveContext { } /** - * Resolve a codebase spec given at run-submission time into a concrete revision id. + * Resolve a codebase spec given at request creation time into a concrete revision id. * * Accepted forms: * - a raw revision UUID (`_id`) → validated as-is. diff --git a/apps/scheduler/src/experiment-coordinator.test.ts b/apps/scheduler/src/experiment-coordinator.test.ts new file mode 100644 index 000000000..570839e67 --- /dev/null +++ b/apps/scheduler/src/experiment-coordinator.test.ts @@ -0,0 +1,365 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { Collection, Document } from "mongodb"; +import { + EXPERIMENT_STORAGE_PROFILE_VERSION, + ExperimentStore, + FixedPayloadStore, + fingerprintExperimentPayload, + type ExperimentDocument, + type ExperimentExecutionHistoryDocument, + type ExperimentRequestState, + type RequestDocument, + type ServiceHandlerDocument, +} from "shared"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + ExperimentCoordinator, + type ExperimentCoordinatorCollections, +} from "./experiment-coordinator.js"; + +const now = new Date("2026-01-01T00:00:00.000Z"); + +function experiment( + requests: ExperimentRequestState[], + overrides: Partial = {}, +): ExperimentDocument { + const fingerprint = fingerprintExperimentPayload(requests); + return { + _id: "experiment-1", + experimentId: "experiment-1", + projectId: "project-1", + metadata: { name: "Experiment" }, + storageProfileVersion: EXPERIMENT_STORAGE_PROFILE_VERSION, + stateVersion: 1, + revision: { + experimentRevisionId: "revision-1", + revisionNumber: 1, + memberCount: requests.length, + membershipFingerprint: fingerprint, + definitionFingerprint: "definition-1", + createdAt: now, + }, + execution: { + experimentExecutionId: "execution-1", + attemptNumber: 1, + version: 1, + requests: { + storage: "inline", + entryCount: requests.length, + payloadFingerprint: fingerprint, + entries: requests, + }, + vectorFingerprint: fingerprint, + handlerSummary: {}, + blocking: [], + createdAt: now, + updatedAt: now, + ...overrides, + }, + createdAt: now, + updatedAt: now, + finalizedAt: now, + }; +} + +function cursor(documents: T[]) { + return { + sort: vi.fn().mockReturnThis(), + limit: vi.fn().mockReturnThis(), + toArray: vi.fn().mockResolvedValue(documents), + }; +} + +function collections(input: { + requests: RequestDocument[]; + services: ServiceHandlerDocument[]; + updatedExperiments?: ExperimentDocument[]; + currentExperiment?: ExperimentDocument; +}): ExperimentCoordinatorCollections { + const updates = [...(input.updatedExperiments ?? [])]; + return { + requests: { + find: vi.fn().mockReturnValue(cursor(input.requests)), + updateMany: vi.fn(), + } as unknown as Collection, + experiments: { + findOne: vi.fn().mockResolvedValue(input.currentExperiment ?? null), + findOneAndUpdate: vi.fn().mockImplementation(async () => updates.shift() ?? null), + } as unknown as Collection, + executions: {} as Collection, + chunks: {} as Collection, + services: { + find: vi.fn().mockImplementation((filter: Document) => { + const type = filter.type; + return cursor(input.services.filter((service) => service.type === type)); + }), + } as unknown as Collection, + }; +} + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("ExperimentCoordinator", () => { + it("projects authoritative run status and dispatches an exact execution message", async () => { + const initialRequest: ExperimentRequestState = { + requestId: "request-1", + runId: "run-1", + dependencies: [], + blockers: [], + }; + const projectedRequest: ExperimentRequestState = { + ...initialRequest, + dependencies: [ + { + handlerId: "pp-observations", + state: "ready", + version: 3, + updatedAt: now, + }, + ], + }; + const initial = experiment([initialRequest]); + const projected = experiment([projectedRequest], { version: 2 }); + const queued = experiment([projectedRequest], { + version: 3, + handlerSummary: { + "experiment-processing": { + status: "queued", + version: 2, + claimToken: "claim-1", + updatedAt: now, + }, + }, + }); + vi.spyOn(ExperimentStore.prototype, "get").mockResolvedValue(initial); + vi.spyOn(FixedPayloadStore.prototype, "readRequests").mockResolvedValue([ + initialRequest, + ]); + const enqueue = vi.fn().mockResolvedValue(undefined); + const coordinator = new ExperimentCoordinator( + collections({ + requests: [ + { + _id: "request-1", + projectId: "project-1", + scenario: { task: "task", criteria: [] }, + workerType: "worker", + createdAt: now, + taskPromptId: "prompt-1", + priority: 0, + run: { + _id: "run-1", + attemptNumber: 1, + status: "done", + logsUrl: "https://logs", + handlerStatus: { + "pp-observations": { + status: "done", + version: 3, + updatedAt: now, + }, + }, + }, + }, + ], + services: [ + { + _id: "pp-observations", + type: "post-process-handler", + version: 3, + queue: "queue-pp-observations", + selector: "observations", + autoBackfill: false, + dependsOn: [], + }, + { + _id: "experiment-processing", + type: "experiment-handler", + version: 2, + queue: "queue-experiment-processing", + selector: "experiment-processing", + autoBackfill: false, + dependsOn: ["pp-observations"], + dependencyMode: "all-runs", + }, + ], + updatedExperiments: [projected, queued], + }), + enqueue, + ); + + const result = await coordinator.reconcile("project-1", "experiment-1"); + + expect(result?.dispatched).toBe(true); + expect(enqueue).toHaveBeenCalledWith( + expect.objectContaining({ _id: "experiment-processing" }), + { + type: "experiment-processing", + projectId: "project-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-1", + handlerId: "experiment-processing", + handlerVersion: 2, + claimToken: expect.any(String), + executionVersion: 3, + }, + ); + }); + + it("re-enqueues a dropped notice without changing its exact identity", async () => { + const requestState: ExperimentRequestState = { + requestId: "request-1", + runId: "run-1", + dependencies: [ + { + handlerId: "pp-observations", + state: "ready", + updatedAt: now, + }, + ], + blockers: [], + }; + const queuedAt = new Date(Date.now() - 60_000); + const current = experiment([requestState], { + version: 7, + handlerSummary: { + "experiment-processing": { + status: "queued", + version: 2, + claimToken: "claim-original", + updatedAt: queuedAt, + }, + }, + }); + vi.spyOn(ExperimentStore.prototype, "get").mockResolvedValue(current); + vi.spyOn(FixedPayloadStore.prototype, "readRequests").mockResolvedValue([ + requestState, + ]); + const enqueue = vi.fn().mockResolvedValue(undefined); + const coordinator = new ExperimentCoordinator( + collections({ + requests: [ + { + _id: "request-1", + projectId: "project-1", + scenario: { task: "task", criteria: [] }, + workerType: "worker", + createdAt: now, + taskPromptId: "prompt-1", + priority: 0, + run: { + _id: "run-1", + attemptNumber: 1, + status: "done", + logsUrl: "https://logs", + handlerStatus: { + "pp-observations": { + status: "done", + updatedAt: now, + }, + }, + }, + }, + ], + services: [ + { + _id: "pp-observations", + type: "post-process-handler", + version: 1, + queue: "queue-pp-observations", + selector: "observations", + autoBackfill: false, + dependsOn: [], + }, + { + _id: "experiment-processing", + type: "experiment-handler", + version: 2, + queue: "queue-experiment-processing", + selector: "experiment-processing", + autoBackfill: false, + dependsOn: ["pp-observations"], + dependencyMode: "all-runs", + }, + ], + }), + enqueue, + ); + + const result = await coordinator.reconcile("project-1", "experiment-1"); + + expect(result?.dispatched).toBe(true); + expect(enqueue).toHaveBeenCalledWith( + expect.any(Object), + expect.objectContaining({ + experimentExecutionId: "execution-1", + executionVersion: 7, + claimToken: "claim-original", + }), + ); + }); + + it("finalizes an interrupted pending creation before marking its requests", async () => { + const requestState: ExperimentRequestState = { + requestId: "request-1", + runId: "run-1", + dependencies: [], + blockers: [], + }; + const pending = experiment([requestState]); + delete pending.finalizedAt; + const finalized = { ...pending, finalizedAt: now }; + const experimentCollections = collections({ + requests: [ + { + _id: "request-1", + projectId: "project-1", + scenario: { task: "task", criteria: [] }, + workerType: "worker", + createdAt: now, + taskPromptId: "prompt-1", + priority: 0, + experimentId: "experiment-1", + experimentExpectedRequestCount: 1, + experimentRequestIndex: 0, + experimentInitialRunId: "run-1", + run: { + _id: "run-1", + attemptNumber: 1, + status: "pending", + logsUrl: "https://logs", + }, + }, + ], + services: [], + currentExperiment: pending, + }); + const finalize = vi + .spyOn(ExperimentStore.prototype, "finalizeCreation") + .mockResolvedValue(finalized); + const coordinator = new ExperimentCoordinator(experimentCollections); + + const recovered = await coordinator.recoverInterruptedCreations(); + + expect(recovered).toBe(1); + expect(finalize).toHaveBeenCalledWith( + expect.objectContaining({ + projectId: "project-1", + experimentId: "experiment-1", + expectedExperimentExecutionId: "execution-1", + expectedStateVersion: 1, + }), + ); + expect(experimentCollections.requests.updateMany).toHaveBeenCalledWith( + expect.objectContaining({ + projectId: "project-1", + experimentId: "experiment-1", + }), + { $set: { experimentFinalizedAt: expect.any(Date) } }, + ); + }); +}); diff --git a/apps/scheduler/src/experiment-coordinator.ts b/apps/scheduler/src/experiment-coordinator.ts new file mode 100644 index 000000000..f88cf469f --- /dev/null +++ b/apps/scheduler/src/experiment-coordinator.ts @@ -0,0 +1,527 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { randomUUID } from "node:crypto"; +import type { Collection, Document } from "mongodb"; +import { + DependencyGraph, + ExperimentStore, + FixedPayloadStore, + fingerprintExperimentPayload, + type ExperimentBlocker, + type ExperimentDocument, + type ExperimentExecutionHistoryDocument, + type ExperimentHandlerServiceDocument, + type ExperimentQueueMessagePayload, + type ExperimentRequestState, + type HandlerRunStatus, + type HandlerServiceDocument, + type RequestDocument, + type ServiceHandlerDocument, +} from "shared"; + +export interface ExperimentCoordinatorCollections { + requests: Collection; + experiments: Collection; + executions: Collection; + chunks: Collection; + services: Collection; +} + +export interface ExperimentDispatchMessage + extends ExperimentQueueMessagePayload { + projectId: string; + handlerId: string; + handlerVersion: number; + claimToken: string; + executionVersion: number; +} + +export type EnqueueExperiment = ( + service: ExperimentHandlerServiceDocument, + message: ExperimentDispatchMessage, +) => Promise; + +export type ExperimentReadiness = + | "empty" + | "waiting" + | "blocked" + | "ready" + | "processing" + | "done" + | "failed"; + +export interface ReconcileExperimentResult { + experiment: ExperimentDocument; + readiness: ExperimentReadiness; + rolled: boolean; + dispatched: boolean; +} + +function namedHandlerState( + handlerId: string, + handlers: readonly HandlerServiceDocument[], + status: Record | undefined, +): "waiting" | "ready" | "failed" | "blocked" { + const direct = status?.[handlerId]?.status; + if (direct === "done") return "ready"; + if (direct === "failed") return "failed"; + if (!handlers.some((handler) => handler._id === handlerId)) return "waiting"; + const graph = new DependencyGraph( + handlers.map((handler) => ({ + id: handler._id, + dependsOn: handler.dependsOn, + })), + ); + for (const ancestorId of graph.getAncestors(handlerId)) { + if (status?.[ancestorId]?.status === "failed") return "blocked"; + } + return "waiting"; +} + +function currentReadiness( + experiment: ExperimentDocument, +): ExperimentReadiness { + if (experiment.execution.requests.entryCount === 0) return "empty"; + if (experiment.execution.discardedAt) return "failed"; + if ( + experiment.execution.blocking.some( + (blocker) => blocker.reason === "dependency-failed", + ) + ) { + return "failed"; + } + if ( + experiment.execution.blocking.length > 0 && + experiment.execution.blocking.every( + (blocker) => blocker.reason === "dependency-not-ready", + ) + ) { + return "waiting"; + } + if (experiment.execution.blocking.length > 0) return "blocked"; + const statuses = Object.values(experiment.execution.handlerSummary); + if (statuses.some((status) => status.status === "failed")) return "failed"; + if ( + statuses.some( + (status) => + status.status === "queued" || status.status === "processing", + ) + ) { + return "processing"; + } + if ( + statuses.length > 0 && + statuses.every((status) => status.status === "done") + ) { + return "done"; + } + return statuses.length === 0 ? "ready" : "waiting"; +} + +function sameProjection( + left: readonly ExperimentRequestState[], + right: readonly ExperimentRequestState[], +): boolean { + return fingerprintExperimentPayload(left) === fingerprintExperimentPayload(right); +} + +export class ExperimentCoordinator { + private readonly store: ExperimentStore; + private readonly payloads: FixedPayloadStore; + + constructor( + private readonly collections: ExperimentCoordinatorCollections, + private readonly enqueue?: EnqueueExperiment, + ) { + this.store = new ExperimentStore({ + experiments: collections.experiments, + executions: collections.executions, + chunks: collections.chunks, + }); + this.payloads = new FixedPayloadStore(collections.chunks); + } + + async reconcile( + projectId: string, + experimentId: string, + ): Promise { + let rolled = false; + for (let attempt = 0; attempt < 4; attempt += 1) { + let experiment = await this.store.get(projectId, experimentId); + if (!experiment) return null; + if (experiment.execution.rollover) { + const repaired = await this.store.repairRollover(projectId, experimentId); + if (!repaired) return null; + experiment = repaired; + rolled = true; + } + + const requests = await this.readRequests(experiment); + const requestDocs = await this.collections.requests + .find({ + _id: { $in: requests.map((request) => request.requestId) }, + projectId, + deletedAt: { $exists: false }, + } as Document) + .toArray(); + const byId = new Map(requestDocs.map((request) => [request._id, request])); + const runVectorChanged = requests.some((request) => { + const current = byId.get(request.requestId); + return current?.run?._id !== request.runId; + }); + const handlers = await this.loadRunHandlers(); + + if (runVectorChanged) { + const nextRequests = requests.map((request) => { + const current = byId.get(request.requestId); + if (!current?.run || current.run._id === request.runId) return request; + return this.projectRequest( + request.requestId, + current.run._id, + current, + handlers, + ); + }); + const frozen = await this.store.freezeCurrent({ + projectId, + experimentId, + experimentExecutionId: + experiment.execution.experimentExecutionId, + expectedExecutionVersion: experiment.execution.version, + }); + if (!frozen) { + const latest = await this.store.get(projectId, experimentId); + if ( + !latest || + latest.execution.experimentExecutionId !== + experiment.execution.experimentExecutionId + ) { + return latest + ? { + experiment: latest, + readiness: currentReadiness(latest), + rolled: true, + dispatched: false, + } + : null; + } + continue; + } + const next = await this.store.rollover({ + projectId, + experimentId, + expectedExperimentExecutionId: + experiment.execution.experimentExecutionId, + expectedExecutionVersion: frozen.execution.version, + nextExperimentExecutionId: randomUUID(), + requests: nextRequests, + reason: "rollover", + }); + if (!next) continue; + rolled = true; + continue; + } + + const projected = requests.map((request) => + this.projectRequest( + request.requestId, + request.runId, + byId.get(request.requestId), + handlers, + ), + ); + const blockers = projected.flatMap((request) => request.blockers); + if (!sameProjection(requests, projected)) { + if (experiment.execution.requests.storage !== "inline") { + throw new Error( + `Chunked experiment ${experimentId} requires a rollover before dependency projection changes`, + ); + } + const now = new Date(); + const payloadFingerprint = fingerprintExperimentPayload(projected); + const updated = await this.collections.experiments.findOneAndUpdate( + { + _id: experimentId, + projectId, + "execution.experimentExecutionId": + experiment.execution.experimentExecutionId, + "execution.version": experiment.execution.version, + "execution.frozenAt": { $exists: false }, + "execution.rollover": { $exists: false }, + } as Document, + { + $set: { + "execution.requests.entries": projected, + "execution.requests.payloadFingerprint": payloadFingerprint, + "execution.vectorFingerprint": payloadFingerprint, + "execution.blocking": blockers, + "execution.updatedAt": now, + updatedAt: now, + }, + $inc: { "execution.version": 1, stateVersion: 1 }, + } as Document, + { returnDocument: "after" }, + ); + if (!updated) continue; + experiment = updated; + } else if ( + fingerprintExperimentPayload(experiment.execution.blocking) !== + fingerprintExperimentPayload(blockers) + ) { + const updated = await this.collections.experiments.findOneAndUpdate( + { + _id: experimentId, + projectId, + "execution.experimentExecutionId": + experiment.execution.experimentExecutionId, + "execution.version": experiment.execution.version, + } as Document, + { + $set: { + "execution.blocking": blockers, + "execution.updatedAt": new Date(), + updatedAt: new Date(), + }, + $inc: { "execution.version": 1, stateVersion: 1 }, + } as Document, + { returnDocument: "after" }, + ); + if (!updated) continue; + experiment = updated; + } + + const dispatch = await this.dispatchReadyHandlers(experiment, projected); + const latest = + (await this.store.get(projectId, experimentId)) ?? experiment; + return { + experiment: latest, + readiness: currentReadiness(latest), + rolled, + dispatched: dispatch, + }; + } + throw new Error( + `Experiment ${experimentId} invariant failure after bounded same-identity contention`, + ); + } + + async recoverInterruptedCreations(limit = 30): Promise { + const candidates = await this.collections.requests + .find({ + experimentId: { $exists: true }, + experimentExpectedRequestCount: { $exists: true }, + experimentFinalizedAt: { $exists: false }, + deletedAt: { $exists: false }, + } as Document) + .sort({ _id: 1 }) + .limit(limit) + .toArray(); + let recovered = 0; + for (const experimentId of new Set( + candidates.flatMap((request) => + request.experimentId ? [request.experimentId] : [], + ), + )) { + const member = candidates.find( + (request) => request.experimentId === experimentId, + ); + if (!member) continue; + const raw = await this.collections.experiments.findOne({ + _id: experimentId, + projectId: member.projectId, + } as Document); + if (!raw) continue; + const existing = + raw.finalizedAt + ? raw + : await this.store.finalizeCreation({ + projectId: member.projectId, + experimentId, + expectedExperimentExecutionId: + raw.execution.experimentExecutionId, + expectedStateVersion: raw.stateVersion, + }); + if (!existing) continue; + await this.collections.requests.updateMany( + { + projectId: member.projectId, + experimentId, + experimentFinalizedAt: { $exists: false }, + } as Document, + { $set: { experimentFinalizedAt: new Date() } } as Document, + ); + recovered += 1; + } + return recovered; + } + + private async readRequests( + experiment: ExperimentDocument, + ): Promise { + return this.payloads.readRequests( + { + projectId: experiment.projectId, + experimentId: experiment.experimentId, + ownerId: experiment.execution.experimentExecutionId, + generation: experiment.execution.attemptNumber, + }, + experiment.execution.requests, + ); + } + + private async loadRunHandlers(): Promise { + const docs = await this.collections.services + .find({ type: "post-process-handler" } as Document) + .toArray(); + return docs.filter( + (doc): doc is HandlerServiceDocument => + doc.type === "post-process-handler", + ); + } + + private async loadExperimentHandlers(): Promise< + ExperimentHandlerServiceDocument[] + > { + const docs = await this.collections.services + .find({ type: "experiment-handler" } as Document) + .toArray(); + return docs.filter( + (doc): doc is ExperimentHandlerServiceDocument => + doc.type === "experiment-handler", + ); + } + + private projectRequest( + requestId: string, + runId: string, + request: RequestDocument | undefined, + handlers: readonly HandlerServiceDocument[], + ): ExperimentRequestState { + if (!request?.run) { + return { + requestId, + runId, + dependencies: [], + blockers: [ + { + requestId, + runId, + handlerId: "request", + reason: "request-missing", + }, + ], + }; + } + const experimentHandlers = request.run.handlerStatus ?? {}; + const dependencyIds = handlers + .filter((handler) => handler.selector === "observations") + .map((handler) => handler._id); + const effectiveDependencyIds = + dependencyIds.length > 0 ? dependencyIds : ["pp-observations"]; + const dependencies = effectiveDependencyIds.map((handlerId) => { + const source = experimentHandlers[handlerId]; + const state = namedHandlerState( + handlerId, + handlers, + request.run?.handlerStatus, + ); + return { + handlerId, + state, + ...(source?.version !== undefined ? { version: source.version } : {}), + ...(source?.updatedAt ? { updatedAt: source.updatedAt } : {}), + ...(source?.error ? { error: source.error } : {}), + }; + }); + const blockers: ExperimentBlocker[] = dependencies.flatMap((dependency) => { + if (dependency.state === "ready") return []; + const reason = + dependency.state === "failed" + ? ("dependency-failed" as const) + : dependency.state === "blocked" + ? ("dependency-blocked" as const) + : ("dependency-not-ready" as const); + return [ + { + requestId, + runId, + handlerId: dependency.handlerId, + reason, + ...(dependency.error ? { error: dependency.error } : {}), + }, + ]; + }); + return { requestId, runId, dependencies, blockers }; + } + + private async dispatchReadyHandlers( + experiment: ExperimentDocument, + requests: readonly ExperimentRequestState[], + ): Promise { + if (!this.enqueue || requests.length === 0) return false; + const services = await this.loadExperimentHandlers(); + let dispatched = false; + for (const service of services) { + const existing = experiment.execution.handlerSummary[service._id]; + const allReady = requests.every((request) => + service.dependsOn.every( + (handlerId) => + request.dependencies.find( + (dependency) => dependency.handlerId === handlerId, + )?.state === "ready", + ), + ); + const retryDroppedNotice = + existing?.status === "queued" && + Date.now() - existing.updatedAt.getTime() >= 30_000; + if ((!existing && !allReady) || (existing && !retryDroppedNotice)) continue; + + const claimToken = existing?.claimToken ?? randomUUID(); + const now = new Date(); + let current = experiment; + if (!existing) { + const path = `execution.handlerSummary.${service._id}`; + const queued = await this.collections.experiments.findOneAndUpdate( + { + _id: experiment.experimentId, + projectId: experiment.projectId, + "execution.experimentExecutionId": + experiment.execution.experimentExecutionId, + "execution.version": experiment.execution.version, + [`${path}.status`]: { $exists: false }, + } as Document, + { + $set: { + [path]: { + status: "queued", + version: service.version, + claimToken, + updatedAt: now, + }, + "execution.updatedAt": now, + updatedAt: now, + }, + $inc: { "execution.version": 1, stateVersion: 1 }, + } as Document, + { returnDocument: "after" }, + ); + if (!queued) continue; + current = queued; + } + await this.enqueue(service, { + type: service.selector, + projectId: current.projectId, + experimentId: current.experimentId, + experimentExecutionId: + current.execution.experimentExecutionId, + handlerId: service._id, + handlerVersion: service.version, + claimToken, + executionVersion: current.execution.version, + }); + dispatched = true; + } + return dispatched; + } +} diff --git a/apps/scheduler/src/experiment-dispatcher.ts b/apps/scheduler/src/experiment-dispatcher.ts new file mode 100644 index 000000000..4e0df8091 --- /dev/null +++ b/apps/scheduler/src/experiment-dispatcher.ts @@ -0,0 +1,169 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { QueueClient } from "@azure/storage-queue"; +import type { Collection, Db } from "mongodb"; +import type { + ExperimentDocument, + ExperimentExecutionHistoryDocument, + ExperimentHandlerServiceDocument, + RequestDocument, + ServiceHandlerDocument, +} from "shared"; +import { ExperimentStore } from "shared"; +import { + ExperimentCoordinator, + type ExperimentDispatchMessage, +} from "./experiment-coordinator.js"; + +export class ExperimentDispatcher { + private interval: ReturnType | null = null; + private dispatching = false; + private scanAfterExperimentId: string | null = null; + private readonly queueClients = new Map(); + private readonly coordinator: ExperimentCoordinator; + + constructor( + private readonly requests: Collection, + private readonly experiments: Collection, + private readonly executions: Collection, + private readonly chunks: Collection, + db: Db, + private readonly createQueueClient: (queueName: string) => QueueClient, + private readonly pollIntervalMs = 30_000, + private readonly batchSize = 30, + ) { + this.coordinator = new ExperimentCoordinator( + { + requests, + experiments, + executions, + chunks, + services: db.collection("services"), + }, + (service, message) => this.enqueue(service, message), + ); + } + + start(): void { + if (this.interval) return; + this.interval = setInterval(() => void this.poll(), this.pollIntervalMs); + void this.poll(); + } + + async stop(): Promise { + if (this.interval) { + clearInterval(this.interval); + this.interval = null; + } + const deadline = Date.now() + 5_000; + while (this.dispatching && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 100)); + } + } + + async onRequestChanged(requestId: string): Promise { + const request = await this.requests.findOne( + { _id: requestId }, + { projection: { projectId: 1, experimentId: 1 } }, + ); + if (request?.experimentId) { + await this.coordinator.reconcile( + request.projectId, + request.experimentId, + ); + } + } + + async onHandlerComplete(input: { + projectId: string; + experimentId: string; + experimentExecutionId: string; + executionVersion: number; + handlerId: string; + handlerVersion: number; + claimToken: string; + status: "done" | "failed"; + error?: string; + }): Promise { + const updated = await new ExperimentStore({ + experiments: this.experiments, + executions: this.executions, + chunks: this.chunks, + }).completeHandler({ + ...input, + expectedExecutionVersion: input.executionVersion, + }); + return updated !== null; + } + + async onHandlerClaim(input: { + projectId: string; + experimentId: string; + experimentExecutionId: string; + executionVersion: number; + handlerId: string; + handlerVersion: number; + claimToken: string; + }): Promise { + const updated = await new ExperimentStore({ + experiments: this.experiments, + executions: this.executions, + chunks: this.chunks, + }).claimHandler({ + ...input, + expectedExecutionVersion: input.executionVersion, + }); + return updated?.execution.version ?? null; + } + + private async poll(): Promise { + if (this.dispatching) return; + this.dispatching = true; + try { + await this.coordinator.recoverInterruptedCreations(this.batchSize); + const candidates = await this.experiments + .find( + this.scanAfterExperimentId + ? { + _id: { $gt: this.scanAfterExperimentId }, + finalizedAt: { $exists: true }, + } + : { finalizedAt: { $exists: true } }, + ) + .sort({ _id: 1 }) + .limit(this.batchSize) + .project({ _id: 1, projectId: 1 }) + .toArray(); + this.scanAfterExperimentId = + candidates.length === this.batchSize + ? candidates.at(-1)?._id ?? null + : null; + for (const candidate of candidates) { + await this.coordinator.reconcile( + candidate.projectId, + candidate._id, + ); + } + } catch (error) { + console.error("[ExperimentDispatcher] Poll failed:", error); + } finally { + this.dispatching = false; + } + } + + private async enqueue( + service: ExperimentHandlerServiceDocument, + message: ExperimentDispatchMessage, + ): Promise { + let client = this.queueClients.get(service.queue); + if (!client) { + client = this.createQueueClient(service.queue); + await client.createIfNotExists(); + this.queueClients.set(service.queue, client); + } + await client.sendMessage( + Buffer.from(JSON.stringify(message)).toString("base64"), + ); + } +} diff --git a/apps/scheduler/src/handler-dispatcher.ts b/apps/scheduler/src/handler-dispatcher.ts index d744092c3..c6415dada 100644 --- a/apps/scheduler/src/handler-dispatcher.ts +++ b/apps/scheduler/src/handler-dispatcher.ts @@ -9,10 +9,30 @@ import type { HandlerRunStatus, ServiceHandlerDocument, } from "shared"; -import { DependencyGraph, isHandlerDagDrained, withRetry } from "shared"; +import { DependencyGraph, withRetry } from "shared"; import type { RetryOptions } from "shared"; import type { NotifyHandler } from "./notify-routes.js"; +function isHandlerDagDrained( + handlers: readonly HandlerServiceDocument[], + handlerStatus: Record | undefined, +): boolean { + const graph = new DependencyGraph( + handlers.map((handler) => ({ + id: handler._id, + dependsOn: handler.dependsOn, + })), + ); + return handlers.every((handler) => { + const status = handlerStatus?.[handler._id]?.status; + if (status === "done" || status === "failed") return true; + if (status !== undefined) return false; + return [...graph.getAncestors(handler._id)].some( + (ancestorId) => handlerStatus?.[ancestorId]?.status === "failed", + ); + }); +} + /** * HandlerDispatcher — DAG-aware post-processing orchestrator. * @@ -157,12 +177,12 @@ export class HandlerDispatcher implements NotifyHandler { } async registerHandler(doc: ServiceHandlerDocument): Promise { - if (doc.type === "submission-handler") { + if (doc.type === "experiment-handler") { const postProcessHandlers = await this.loadHandlers(true); for (const dependency of doc.dependsOn) { if (!postProcessHandlers.some((handler) => handler._id === dependency)) { throw new Error( - `Submission handler '${doc._id}' depends on unknown run handler '${dependency}'`, + `Experiment handler '${doc._id}' depends on unknown run handler '${dependency}'`, ); } } @@ -173,7 +193,7 @@ export class HandlerDispatcher implements NotifyHandler { { upsert: true }, ); console.log( - `[HandlerDispatcher] Registered submission handler ${_id} (version=${doc.version}, queue=${doc.queue}, dependsOn=[${doc.dependsOn.join(", ")}])`, + `[HandlerDispatcher] Registered experiment handler ${_id} (version=${doc.version}, queue=${doc.queue}, dependsOn=[${doc.dependsOn.join(", ")}])`, ); return; } diff --git a/apps/scheduler/src/index.ts b/apps/scheduler/src/index.ts index 9fe0fbf79..ff1068a86 100644 --- a/apps/scheduler/src/index.ts +++ b/apps/scheduler/src/index.ts @@ -8,15 +8,15 @@ import { DefaultAzureCredential } from "@azure/identity"; import { RequestScheduler } from "./request-scheduler.js"; import { PostProcessorDispatcher } from "./post-processor-dispatcher.js"; import { HandlerDispatcher } from "./handler-dispatcher.js"; -import { SubmissionDispatcher } from "./submission-dispatcher.js"; +import { ExperimentDispatcher } from "./experiment-dispatcher.js"; import { createHttpServer, type NotifyHandler } from "./notify-routes.js"; import { StuckRunReaper } from "./stuck-run-reaper.js"; import { RedisHeartbeatStore, type HeartbeatStore } from "shared"; import type { CodingAgentDocument, RequestDocument, - SubmissionDocument, - SubmissionRunHistoryDocument, + ExperimentDocument, + ExperimentExecutionHistoryDocument, } from "shared"; import { initTelemetry, trackMetric, trackEvent, shutdownTelemetry } from "telemetry"; @@ -134,9 +134,13 @@ async function main(): Promise { const db = mongoClient.db(MONGO_DATABASE); const collection = db.collection(MONGO_COLLECTION); const agentCollection = db.collection("agents"); - const submissionCollection = db.collection("submissions"); - const submissionRunsCollection = - db.collection("submission-runs"); + const experimentCollection = + db.collection("experiments"); + const experimentExecutionsCollection = + db.collection( + "experiment-executions", + ); + const experimentChunksCollection = db.collection("experiment-chunks"); // Start the scheduler const scheduler = new RequestScheduler( @@ -200,16 +204,17 @@ async function main(): Promise { `[Scheduler] Handler dispatcher (DAG) started (poll=${ppPollIntervalMs}ms, topologyCacheTtl=${handlerCacheTtlMs}ms)`, ); - const submissionDispatcher = new SubmissionDispatcher( + const experimentDispatcher = new ExperimentDispatcher( collection, - submissionCollection, - submissionRunsCollection, + experimentCollection, + experimentExecutionsCollection, + experimentChunksCollection, db, createQueueClient, ppPollIntervalMs, ); - submissionDispatcher.start(); - console.log(`[Scheduler] Submission dispatcher started (poll=${ppPollIntervalMs}ms)`); + experimentDispatcher.start(); + console.log(`[Scheduler] Experiment dispatcher started (poll=${ppPollIntervalMs}ms)`); // Start the stuck-run reaper (authoritative backstop). Redis is non-fatal: // on misconfig or connection failure the reaper self-disables (its sweeps @@ -248,11 +253,11 @@ async function main(): Promise { } // Run notifications update the run DAG and immediately reconcile the parent - // submission. The polling loop remains the durable safety net. + // experiment. The polling loop remains the durable safety net. const notifyHandler: NotifyHandler = { onRunTerminal: async (requestId, runId) => { await handlerDispatcher.onRunTerminal(requestId, runId); - await submissionDispatcher.onRequestChanged(requestId); + await experimentDispatcher.onRequestChanged(requestId); }, onHandlerComplete: async (requestId, runId, handlerId, status) => { await handlerDispatcher.onHandlerComplete( @@ -261,8 +266,12 @@ async function main(): Promise { handlerId, status, ); - await submissionDispatcher.onRequestChanged(requestId); + await experimentDispatcher.onRequestChanged(requestId); }, + onExperimentHandlerClaim: (input) => + experimentDispatcher.onHandlerClaim(input), + onExperimentHandlerComplete: (input) => + experimentDispatcher.onHandlerComplete(input), registerHandler: (doc) => handlerDispatcher.registerHandler(doc), }; const httpServer = createHttpServer(notifyHandler); @@ -276,7 +285,7 @@ async function main(): Promise { await scheduler.stop(); await postProcessorDispatcher.stop(); await handlerDispatcher.stop(); - await submissionDispatcher.stop(); + await experimentDispatcher.stop(); if (stuckRunReaper) await stuckRunReaper.stop(); if (heartbeatStore) await heartbeatStore.close(); httpServer.close(); diff --git a/apps/scheduler/src/notify-routes.test.ts b/apps/scheduler/src/notify-routes.test.ts index 2d7ef741b..284bf081a 100644 --- a/apps/scheduler/src/notify-routes.test.ts +++ b/apps/scheduler/src/notify-routes.test.ts @@ -121,6 +121,161 @@ describe("createHttpServer", () => { expect(handler.onRunTerminal).not.toHaveBeenCalled(); }); + it("accepts exact experiment handler completions", async () => { + const onExperimentHandlerComplete = vi.fn().mockResolvedValue(true); + const handler: NotifyHandler = { + onRunTerminal: vi.fn().mockResolvedValue(undefined), + onHandlerComplete: vi.fn().mockResolvedValue(undefined), + onExperimentHandlerComplete, + registerHandler: vi.fn().mockResolvedValue(undefined), + }; + const { server, baseUrl } = await startServer(handler); + servers.push(server); + const body = { + projectId: "project-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-1", + executionVersion: 4, + handlerId: "experiment-report", + handlerVersion: 2, + claimToken: "claim-1", + status: "done", + }; + + const response = await fetch( + `${baseUrl}/notify/experiment-handler-complete`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }, + ); + + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ ok: true }); + expect(onExperimentHandlerComplete).toHaveBeenCalledWith(body); + }); + + it("claims an exact experiment handler execution and returns its new version", async () => { + const onExperimentHandlerClaim = vi.fn().mockResolvedValue(5); + const handler: NotifyHandler = { + onRunTerminal: vi.fn().mockResolvedValue(undefined), + onHandlerComplete: vi.fn().mockResolvedValue(undefined), + onExperimentHandlerClaim, + registerHandler: vi.fn().mockResolvedValue(undefined), + }; + const { server, baseUrl } = await startServer(handler); + servers.push(server); + const body = { + projectId: "project-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-1", + executionVersion: 4, + handlerId: "experiment-report", + handlerVersion: 2, + claimToken: "claim-1", + }; + + const response = await fetch( + `${baseUrl}/notify/experiment-handler-claim`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }, + ); + + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ + ok: true, + executionVersion: 5, + }); + expect(onExperimentHandlerClaim).toHaveBeenCalledWith(body); + }); + + it("rejects stale experiment handler claims", async () => { + const handler: NotifyHandler = { + onRunTerminal: vi.fn().mockResolvedValue(undefined), + onHandlerComplete: vi.fn().mockResolvedValue(undefined), + onExperimentHandlerClaim: vi.fn().mockResolvedValue(null), + registerHandler: vi.fn().mockResolvedValue(undefined), + }; + const { server, baseUrl } = await startServer(handler); + servers.push(server); + + const response = await fetch( + `${baseUrl}/notify/experiment-handler-claim`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + projectId: "project-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-1", + executionVersion: 4, + handlerId: "experiment-report", + handlerVersion: 2, + claimToken: "claim-1", + }), + }, + ); + + expect(response.status).toBe(409); + expect(await response.json()).toEqual({ + error: "Stale experiment handler claim discarded", + }); + }); + + it("rejects stale and malformed experiment handler completions", async () => { + const onExperimentHandlerComplete = vi.fn().mockResolvedValue(false); + const handler: NotifyHandler = { + onRunTerminal: vi.fn().mockResolvedValue(undefined), + onHandlerComplete: vi.fn().mockResolvedValue(undefined), + onExperimentHandlerComplete, + registerHandler: vi.fn().mockResolvedValue(undefined), + }; + const { server, baseUrl } = await startServer(handler); + servers.push(server); + + const stale = await fetch( + `${baseUrl}/notify/experiment-handler-complete`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + projectId: "project-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-1", + executionVersion: 4, + handlerId: "experiment-report", + handlerVersion: 2, + claimToken: "claim-1", + status: "failed", + error: "worker failed", + }), + }, + ); + const malformed = await fetch( + `${baseUrl}/notify/experiment-handler-complete`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + projectId: "project-1", + experimentId: "experiment-1", + status: "done", + }), + }, + ); + + expect(stale.status).toBe(409); + expect(await stale.json()).toEqual({ + error: "Stale experiment handler completion discarded", + }); + expect(malformed.status).toBe(400); + expect(onExperimentHandlerComplete).toHaveBeenCalledTimes(1); + }); + it("returns 400 when required fields are missing", async () => { const handler: NotifyHandler = { onRunTerminal: vi.fn().mockResolvedValue(undefined), @@ -193,6 +348,39 @@ describe("createHttpServer", () => { expect(handler.registerHandler).toHaveBeenCalledWith(doc); }); + it("registers experiment-processing handlers", async () => { + const handler: NotifyHandler = { + onRunTerminal: vi.fn().mockResolvedValue(undefined), + onHandlerComplete: vi.fn().mockResolvedValue(undefined), + registerHandler: vi.fn().mockResolvedValue(undefined), + }; + const { server, baseUrl } = await startServer(handler); + servers.push(server); + const doc = { + _id: "experiment-processing", + type: "experiment-handler", + version: 1, + queue: "experiment-processing-queue", + selector: "experiment-processing", + autoBackfill: false, + dependsOn: ["pp-observations"], + dependencyMode: "all-runs", + }; + + const response = await fetch(`${baseUrl}/handlers/register`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(doc), + }); + + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ + ok: true, + handlerId: "experiment-processing", + }); + expect(handler.registerHandler).toHaveBeenCalledWith(doc); + }); + it("returns 400 for invalid handler registrations", async () => { const handler: NotifyHandler = { onRunTerminal: vi.fn().mockResolvedValue(undefined), diff --git a/apps/scheduler/src/notify-routes.ts b/apps/scheduler/src/notify-routes.ts index 6810a90c0..35c01eb1d 100644 --- a/apps/scheduler/src/notify-routes.ts +++ b/apps/scheduler/src/notify-routes.ts @@ -29,6 +29,10 @@ export interface NotifyHandler { status: "done" | "failed", ): Promise; registerHandler(doc: ServiceHandlerDocument): Promise; + onExperimentHandlerClaim?( + input: ExperimentHandlerClaimRequest, + ): Promise; + onExperimentHandlerComplete?(input: ExperimentHandlerCompleteRequest): Promise; } interface RunTerminalRequest { @@ -43,6 +47,22 @@ interface HandlerCompleteRequest { status: "done" | "failed"; } +export interface ExperimentHandlerClaimRequest { + projectId: string; + experimentId: string; + experimentExecutionId: string; + executionVersion: number; + handlerId: string; + handlerVersion: number; + claimToken: string; +} + +export interface ExperimentHandlerCompleteRequest + extends ExperimentHandlerClaimRequest { + status: "done" | "failed"; + error?: string; +} + function sendJson( res: http.ServerResponse, statusCode: number, @@ -98,6 +118,50 @@ function parseHandlerCompleteRequest(body: unknown): HandlerCompleteRequest | nu }; } +function parseExperimentHandlerCompleteRequest( + body: unknown, +): ExperimentHandlerCompleteRequest | null { + const claim = parseExperimentHandlerClaimRequest(body); + if (!claim || !isRecord(body)) return null; + if ( + (body.status !== "done" && body.status !== "failed") || + (body.error !== undefined && typeof body.error !== "string") + ) { + return null; + } + return { + ...claim, + status: body.status, + ...(typeof body.error === "string" ? { error: body.error } : {}), + }; +} + +function parseExperimentHandlerClaimRequest( + body: unknown, +): ExperimentHandlerClaimRequest | null { + if (!isRecord(body)) return null; + if ( + !isNonEmptyString(body.projectId) || + !isNonEmptyString(body.experimentId) || + !isNonEmptyString(body.experimentExecutionId) || + !Number.isInteger(body.executionVersion) || + !isNonEmptyString(body.handlerId) || + !Number.isInteger(body.handlerVersion) || + !isNonEmptyString(body.claimToken) + ) { + return null; + } + return { + projectId: body.projectId, + experimentId: body.experimentId, + experimentExecutionId: body.experimentExecutionId, + executionVersion: body.executionVersion as number, + handlerId: body.handlerId, + handlerVersion: body.handlerVersion as number, + claimToken: body.claimToken, + }; +} + function parseHandlerRegisterRequest(body: unknown): ServiceHandlerDocument | null { if (!isRecord(body)) { return null; @@ -115,7 +179,7 @@ function parseHandlerRegisterRequest(body: unknown): ServiceHandlerDocument | nu if ( !isNonEmptyString(_id) || - (type !== "post-process-handler" && type !== "submission-handler") || + (type !== "post-process-handler" && type !== "experiment-handler") || typeof version !== "number" || !Number.isFinite(version) || !isNonEmptyString(queue) || @@ -135,7 +199,7 @@ function parseHandlerRegisterRequest(body: unknown): ServiceHandlerDocument | nu autoBackfill, dependsOn: dependsOn as string[], }; - if (type === "submission-handler") { + if (type === "experiment-handler") { if (body.dependencyMode !== "all-runs") return null; return { ...common, @@ -265,12 +329,64 @@ export function createHttpServer( return; } + if ( + method === "POST" && + url.pathname === "/notify/experiment-handler-claim" + ) { + const parsedBody = parseExperimentHandlerClaimRequest( + await readJsonBody(req, maxBodyBytes), + ); + if (!parsedBody || !handler.onExperimentHandlerClaim) { + sendJson(res, 400, { + error: + "Exact experiment execution, handler version, and claim token are required", + }); + return; + } + const executionVersion = + await handler.onExperimentHandlerClaim(parsedBody); + if (executionVersion === null) { + sendJson(res, 409, { + error: "Stale experiment handler claim discarded", + }); + return; + } + sendJson(res, 200, { ok: true, executionVersion }); + return; + } + + if ( + method === "POST" && + url.pathname === "/notify/experiment-handler-complete" + ) { + const parsedBody = parseExperimentHandlerCompleteRequest( + await readJsonBody(req, maxBodyBytes), + ); + if (!parsedBody || !handler.onExperimentHandlerComplete) { + sendJson(res, 400, { + error: + "Exact experiment execution, handler version, claim token, and terminal status are required", + }); + return; + } + const completed = + await handler.onExperimentHandlerComplete(parsedBody); + if (!completed) { + sendJson(res, 409, { + error: "Stale experiment handler completion discarded", + }); + return; + } + sendJson(res, 200, { ok: true }); + return; + } + if (method === "POST" && url.pathname === "/handlers/register") { const parsedBody = parseHandlerRegisterRequest(await readJsonBody(req, maxBodyBytes)); if (!parsedBody) { sendJson(res, 400, { error: - "Invalid handler registration: _id, supported type, version, queue, selector, autoBackfill, dependsOn, and submission dependencyMode when applicable are required", + "Invalid handler registration: _id, supported type, version, queue, selector, autoBackfill, dependsOn, and experiment dependencyMode when applicable are required", }); return; } diff --git a/apps/scheduler/src/submission-dispatcher.ts b/apps/scheduler/src/submission-dispatcher.ts deleted file mode 100644 index cd2914853..000000000 --- a/apps/scheduler/src/submission-dispatcher.ts +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -import type { QueueClient } from "@azure/storage-queue"; -import type { Collection, Db } from "mongodb"; -import { - SubmissionCoordinator, - type RequestDocument, - type SubmissionDocument, - type SubmissionHandlerServiceDocument, - type SubmissionQueueMessagePayload, - type SubmissionRunHistoryDocument, - type ServiceHandlerDocument, -} from "shared"; - -export class SubmissionDispatcher { - private interval: ReturnType | null = null; - private dispatching = false; - private scanAfterSubmissionId: string | null = null; - private readonly queueClients = new Map(); - private readonly coordinator: SubmissionCoordinator; - - constructor( - private readonly requests: Collection, - private readonly submissions: Collection, - submissionRuns: Collection, - db: Db, - private readonly createQueueClient: (queueName: string) => QueueClient, - private readonly pollIntervalMs = 30_000, - private readonly batchSize = 30, - ) { - this.coordinator = new SubmissionCoordinator( - { - requests, - submissions, - submissionRuns, - services: db.collection("services"), - }, - (service, message) => this.enqueue(service, message), - ); - } - - start(): void { - if (this.interval) return; - this.interval = setInterval(() => this.poll(), this.pollIntervalMs); - void this.poll(); - } - - async stop(): Promise { - if (this.interval) { - clearInterval(this.interval); - this.interval = null; - } - const deadline = Date.now() + 5_000; - while (this.dispatching && Date.now() < deadline) { - await new Promise((resolve) => setTimeout(resolve, 100)); - } - } - - async onRequestChanged(requestId: string): Promise { - const request = await this.requests.findOne( - { _id: requestId }, - { projection: { submissionId: 1 } }, - ); - if (request?.submissionId) { - await this.coordinator.reconcile(request.submissionId); - } - } - - private async poll(): Promise { - if (this.dispatching) return; - this.dispatching = true; - try { - await this.coordinator.recoverMissingNativeSubmissions(this.batchSize); - const candidates = await this.submissions - .find( - this.scanAfterSubmissionId - ? { _id: { $gt: this.scanAfterSubmissionId } } - : {}, - ) - .sort({ _id: 1 }) - .limit(this.batchSize) - .project({ _id: 1 }) - .toArray(); - this.scanAfterSubmissionId = - candidates.length === this.batchSize - ? candidates.at(-1)?._id ?? null - : null; - for (const candidate of candidates) { - await this.coordinator.reconcile(candidate._id); - } - } catch (error) { - console.error("[SubmissionDispatcher] Poll failed:", error); - } finally { - this.dispatching = false; - } - } - - private async enqueue( - service: SubmissionHandlerServiceDocument, - message: SubmissionQueueMessagePayload, - ): Promise { - let client = this.queueClients.get(service.queue); - if (!client) { - client = this.createQueueClient(service.queue); - await client.createIfNotExists(); - this.queueClients.set(service.queue, client); - } - const encoded = Buffer.from(JSON.stringify(message)).toString("base64"); - await client.sendMessage(encoded); - } -} diff --git a/packages/shared/src/cursor.test.ts b/packages/shared/src/cursor.test.ts index b06f0bfaf..b097915e9 100644 --- a/packages/shared/src/cursor.test.ts +++ b/packages/shared/src/cursor.test.ts @@ -18,8 +18,10 @@ describe("encodeCursor / decodeCursor", () => { }); it("preserves colons in values", () => { - const encoded = encodeCursor({ submissionId: "sub:with:colons" }); - expect(decodeCursor(encoded)).toEqual({ submissionId: "sub:with:colons" }); + const encoded = encodeCursor({ experimentId: "experiment:with:colons" }); + expect(decodeCursor(encoded)).toEqual({ + experimentId: "experiment:with:colons", + }); }); it("throws on missing tilde", () => { diff --git a/packages/shared/src/experiments/experiment-store.test.ts b/packages/shared/src/experiments/experiment-store.test.ts index ef925baa0..f9e0e0e1c 100644 --- a/packages/shared/src/experiments/experiment-store.test.ts +++ b/packages/shared/src/experiments/experiment-store.test.ts @@ -115,7 +115,12 @@ function experiment(): ExperimentDocument { }, vectorFingerprint: "vector", handlerSummary: { - report: { status: "queued", version: 1, updatedAt: now }, + report: { + status: "queued", + version: 1, + claimToken: "claim-1", + updatedAt: now, + }, }, blocking: [], createdAt: now, @@ -146,6 +151,7 @@ describe("ExperimentStore CAS contracts", () => { expectedExecutionVersion: 1, handlerId: "report", handlerVersion: 1, + claimToken: "claim-1", }), store.freezeCurrent({ projectId: "project-1", @@ -177,6 +183,7 @@ describe("ExperimentStore CAS contracts", () => { expectedExecutionVersion: 1, handlerId: "report", handlerVersion: 1, + claimToken: "claim-1", }), ).resolves.toBeNull(); await expect( @@ -187,7 +194,58 @@ describe("ExperimentStore CAS contracts", () => { expectedExecutionVersion: 1, handlerId: "report", handlerVersion: 1, + claimToken: "claim-1", }), ).resolves.toBeNull(); }); + + it("rejects stale completion tokens and accepts the exact claimed version", async () => { + const experiments = atomicCollection(experiment()); + const store = new ExperimentStore({ + experiments: + experiments as unknown as Collection, + executions: + {} as Collection, + chunks: {} as Collection, + }); + const claimed = await store.claimHandler({ + projectId: "project-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-1", + expectedExecutionVersion: 1, + handlerId: "report", + handlerVersion: 1, + claimToken: "claim-1", + }); + expect(claimed?.execution.version).toBe(2); + + await expect( + store.completeHandler({ + projectId: "project-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-1", + expectedExecutionVersion: 2, + handlerId: "report", + handlerVersion: 1, + claimToken: "stale-token", + status: "done", + }), + ).resolves.toBeNull(); + + const completed = await store.completeHandler({ + projectId: "project-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-1", + expectedExecutionVersion: 2, + handlerId: "report", + handlerVersion: 1, + claimToken: "claim-1", + status: "done", + }); + expect(completed?.execution.handlerSummary.report).toMatchObject({ + status: "done", + version: 1, + claimToken: "claim-1", + }); + }); }); diff --git a/packages/shared/src/experiments/experiment-store.ts b/packages/shared/src/experiments/experiment-store.ts index 43b13ca7f..ce9ab4f1e 100644 --- a/packages/shared/src/experiments/experiment-store.ts +++ b/packages/shared/src/experiments/experiment-store.ts @@ -305,6 +305,7 @@ export class ExperimentStore { expectedExecutionVersion: number; handlerId: string; handlerVersion: number; + claimToken: string; now?: Date; }): Promise { assertHandlerId(input.handlerId); @@ -322,12 +323,14 @@ export class ExperimentStore { "execution.rollover": { $exists: false }, [`${path}.status`]: "queued", [`${path}.version`]: input.handlerVersion, + [`${path}.claimToken`]: input.claimToken, } as Document, { $set: { [path]: { status: "processing", version: input.handlerVersion, + claimToken: input.claimToken, updatedAt: now, }, "execution.startedAt": now, @@ -348,6 +351,7 @@ export class ExperimentStore { expectedExecutionVersion: number; handlerId: string; handlerVersion: number; + claimToken: string; status: Extract; error?: string; now?: Date; @@ -366,12 +370,14 @@ export class ExperimentStore { "execution.frozenAt": { $exists: false }, [`${path}.status`]: "processing", [`${path}.version`]: input.handlerVersion, + [`${path}.claimToken`]: input.claimToken, } as Document, { $set: { [path]: { status: input.status, version: input.handlerVersion, + claimToken: input.claimToken, updatedAt: now, ...(input.error ? { error: input.error } : {}), }, diff --git a/packages/shared/src/experiments/schemas.ts b/packages/shared/src/experiments/schemas.ts index 8491cf493..0022421f3 100644 --- a/packages/shared/src/experiments/schemas.ts +++ b/packages/shared/src/experiments/schemas.ts @@ -22,6 +22,8 @@ export const ExperimentDependencyProjectionSchema = z export const ExperimentBlockerSchema = z .object({ + requestId: IdSchema.optional(), + runId: IdSchema.optional(), handlerId: IdSchema, reason: z.enum([ "request-missing", @@ -67,6 +69,7 @@ export const ExperimentHandlerSummarySchema = z .object({ status: z.enum(["queued", "processing", "done", "failed"]), version: z.number().int().nonnegative(), + claimToken: IdSchema.optional(), updatedAt: z.coerce.date(), error: ErrorSchema.optional(), }) diff --git a/packages/shared/src/experiments/types.ts b/packages/shared/src/experiments/types.ts index e31b2f328..8d8595259 100644 --- a/packages/shared/src/experiments/types.ts +++ b/packages/shared/src/experiments/types.ts @@ -50,6 +50,8 @@ export type ExperimentBlockerReason = | "dependency-blocked"; export interface ExperimentBlocker { + requestId?: string; + runId?: string; handlerId: string; reason: ExperimentBlockerReason; error?: string; @@ -88,6 +90,7 @@ export type ExperimentHandlerStatus = export interface ExperimentHandlerSummary { status: ExperimentHandlerStatus; version: number; + claimToken?: string; updatedAt: Date; error?: string; } diff --git a/packages/shared/src/schemas/request.ts b/packages/shared/src/schemas/request.ts index ac7766f2d..029c03e09 100644 --- a/packages/shared/src/schemas/request.ts +++ b/packages/shared/src/schemas/request.ts @@ -137,11 +137,18 @@ export const CreateRequestInputSchema = z /** Resources to provision for this run, in setup order. Each entry is a * bare spec (slug, `slug@rN`, or revision id) or an object carrying * parameter values. Resolved at submit time and shared by every variation - * in a grouped submission, so each profile gets an identical environment. */ + * in an experiment, so each profile gets an identical environment. */ resources: z.array(ResourceBindingSpecSchema).optional(), extensions: z.array(z.string()).optional(), profileId: z.string().optional(), - profileVariations: z.array(z.string()).optional(), + profileVariations: z + .array(z.string()) + .optional() + .openapi({ + deprecated: true, + description: + "Deprecated for composition. Use POST /api/v1/experiments and its requests array.", + }), priority: z.number().int().optional(), // Raw AGENTS.md body to deliver to the worker workspace. When present, the // API findOrCreates an `agents.md`-typed prompt and stores its id on the @@ -181,9 +188,7 @@ export const RequestResponseSchema = z profileId: z.string().optional(), profileVersionId: z.string().optional(), experimentId: z.string().optional(), - experimentRevisionId: z.string().optional(), - experimentExecutionId: z.string().optional(), - experimentExpectedRequestCount: z.number().int().min(1).optional(), + experimentExpectedRequestCount: z.number().int().min(0).optional(), experimentRequestIndex: z.number().int().min(0).optional(), experimentInitialRunId: z.string().optional(), experimentFinalizedAt: z.coerce.date().optional(), @@ -289,7 +294,7 @@ export const ListRequestsQuerySchema = z worker: MultiValueParam, taskPromptId: z.string().optional(), criteria: z.string().optional(), - submissionId: z.string().optional(), + experimentId: z.string().optional(), profileId: MultiValueParam, status: MultiValueParam, outcome: MultiValueParam, @@ -299,7 +304,7 @@ export const ListRequestsQuerySchema = z agentVersion: MultiValueParam, // Free-text, case-insensitive search across run id, scenario task, model, worker. search: z.string().optional(), - groupBy: z.enum(["task", "submissionId", "profile"]).optional(), + groupBy: z.enum(["task", "experimentId", "profile"]).optional(), limit: z.coerce.number().int().min(1).max(100).optional(), after: z.string().optional(), before: z.string().optional(), @@ -372,7 +377,7 @@ export const GroupUniformValuesSchema = z codebaseRevisionId: z.string().optional(), extensions: z.array(z.string()).optional(), status: RequestStatusSchema.optional(), - submissionId: z.string().optional(), + experimentId: z.string().optional(), task: z.string().optional(), }) .openapi("GroupUniformValues"); diff --git a/packages/shared/src/types/types.ts b/packages/shared/src/types/types.ts index 9108cb690..93d76d525 100644 --- a/packages/shared/src/types/types.ts +++ b/packages/shared/src/types/types.ts @@ -384,16 +384,18 @@ export interface RequestDocument { agentVersion?: string; // Agent software version prefix (e.g. "copilot-0.0.415") — FK → AgentVersion.agentVersion profileId?: string; // FK → ProfileDocument._id (the profile lineage) profileVersionId?: string; // FK → ProfileVersionDocument._id (exact version used) - experimentId?: string; - experimentRevisionId?: string; - experimentExecutionId?: string; - /** Expected request count for interrupted Experiment finalization. */ + experimentId?: string; // FK → ExperimentDocument._id + /** Expected native request count for interrupted ExperimentDocument finalization. + * Historical/imported requests omit it and remain view-only if their parent + * experiment document is missing. */ experimentExpectedRequestCount?: number; - /** Stable position within the Experiment's immutable request vector. */ + /** Stable position within the experiment's immutable request vector. */ experimentRequestIndex?: number; - /** Initial run id retained so recovery can rebuild the exact first vector. */ + /** Run id minted for the experiment's first execution. It remains unchanged + * across request retries so recovery can rebuild the exact initial vector. */ experimentInitialRunId?: string; - /** Set only after the parent Experiment has been published. */ + /** Set after the parent ExperimentDocument has been idempotently finalized. + * Missing on interrupted native writes so the scheduler can recover them. */ experimentFinalizedAt?: Date; /** Scheduling priority. Higher = processed first. Default: 0. */ priority: number; @@ -592,6 +594,12 @@ export type ServiceHandlerDocument = | HandlerServiceDocument | ExperimentHandlerServiceDocument; +export interface ExperimentQueueMessagePayload { + type: string; + experimentId: string; + experimentExecutionId: string; +} + /** * Field names whose values move from `RequestDocument` (top-level, legacy * shape) into `RequestDocument.run: RunState` (new shape introduced for From aedbc18ce22e227522900f14d6593a77a1922bd1 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Thu, 15 Oct 2026 23:11:58 -0700 Subject: [PATCH 08/25] fix(requests): preserve legacy batch correlation Restore submissionId as a request-batch correlation contract while keeping Experiment as the only aggregate. Require project scope for nested Experiment execution reads and publish the shared composition deprecation identifier. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b05e74ff-44da-4e78-9f05-0e256009e7ee --- .../openapi-snapshot.test.ts.snap | 47 ++++++++++++++++++ apps/api/src/endpoints.test.ts | 49 +++++++++++++++++++ apps/api/src/experiment-openapi.test.ts | 7 +++ apps/api/src/grouping.test.ts | 9 ++++ apps/api/src/grouping.ts | 18 +++++-- apps/api/src/routes/experiments.test.ts | 30 ++++++++++++ apps/api/src/routes/experiments.ts | 7 ++- apps/api/src/routes/requests/archive.test.ts | 5 +- apps/api/src/routes/requests/index.ts | 44 +++++++++++++---- packages/shared/src/schemas/request.ts | 7 ++- packages/shared/src/types/types.ts | 12 ++++- 11 files changed, 219 insertions(+), 16 deletions(-) diff --git a/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap b/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap index c1d80e316..2b987b988 100644 --- a/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap +++ b/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap @@ -2207,6 +2207,9 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` ], "type": "string", }, + "submissionId": { + "type": "string", + }, "task": { "type": "string", }, @@ -3407,6 +3410,9 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, "type": "array", }, + "submissionId": { + "type": "string", + }, "taskPromptId": { "type": "string", }, @@ -5335,6 +5341,14 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "type": "string", }, }, + { + "in": "query", + "name": "submissionId", + "required": false, + "schema": { + "type": "string", + }, + }, { "in": "query", "name": "experimentId", @@ -6366,6 +6380,18 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "type": "string", }, }, + { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "in": "query", + "name": "projectId", + "required": true, + "schema": { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, + "type": "string", + }, + }, { "in": "query", "name": "limit", @@ -6429,6 +6455,18 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "type": "string", }, }, + { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "in": "query", + "name": "projectId", + "required": true, + "schema": { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, + "type": "string", + }, + }, ], "responses": { "200": { @@ -9782,6 +9820,14 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "type": "string", }, }, + { + "in": "query", + "name": "submissionId", + "required": false, + "schema": { + "type": "string", + }, + }, { "in": "query", "name": "experimentId", @@ -9931,6 +9977,7 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "schema": { "enum": [ "task", + "submissionId", "experimentId", "profile", ], diff --git a/apps/api/src/endpoints.test.ts b/apps/api/src/endpoints.test.ts index 893288464..3e1f50e5a 100644 --- a/apps/api/src/endpoints.test.ts +++ b/apps/api/src/endpoints.test.ts @@ -781,6 +781,36 @@ describe("API Endpoints", () => { expect(res.body.data[0]).toHaveProperty("key", "experiment-1"); }); + it("preserves grouped results when groupBy=submissionId", async () => { + const groupedDocs = [ + { + key: "submission-1", + label: "submission-1", + aggregates: { + count: 2, + turns: null, + duration: null, + promptTokens: null, + completionTokens: null, + }, + uniform: {}, + }, + ]; + (mocks.collection.aggregate as any) + .mockReturnValueOnce({ + toArray: vi.fn().mockResolvedValue([{ _id: "submission-1" }]), + }) + .mockReturnValueOnce({ toArray: vi.fn().mockResolvedValue(groupedDocs) }) + .mockReturnValueOnce({ toArray: vi.fn().mockResolvedValue([]) }) + .mockReturnValueOnce({ toArray: vi.fn().mockResolvedValue([]) }); + + const res = await request(testServer()).get( + `/api/v1/requests?groupBy=submissionId&projectId=${TEST_PROJECT_ID}`, + ); + expect(res.status).toBe(200); + expect(res.body.data[0]).toHaveProperty("key", "submission-1"); + }); + it("calls aggregate pipeline when groupBy is provided", async () => { (mocks.collection.aggregate as any) .mockReturnValueOnce({ toArray: vi.fn().mockResolvedValue([]) }); // keys (empty) @@ -1956,6 +1986,8 @@ describe("API Endpoints", () => { const doc = (mocks.collection.insertOne as any).mock.calls[0][0]; expect(doc).toHaveProperty("reasoningEffort", "high"); expect(doc).not.toHaveProperty("experimentId"); + expect(doc).toHaveProperty("submissionId", res.body.submissionId); + expect(res.body.submissionId).toEqual(expect.any(String)); expect(mocks.experimentCollection.updateOne).not.toHaveBeenCalled(); expect(res.headers).not.toHaveProperty("deprecation"); expect(res.headers).not.toHaveProperty("link"); @@ -1974,6 +2006,13 @@ describe("API Endpoints", () => { const docs = (mocks.collection.insertMany as any).mock.calls[0][0]; expect(docs).toHaveLength(3); expect(docs.every((doc: object) => !("experimentId" in doc))).toBe(true); + expect(res.body.submissionId).toEqual(expect.any(String)); + expect( + docs.every( + (doc: { submissionId?: string }) => + doc.submissionId === res.body.submissionId, + ), + ).toBe(true); expect(mocks.experimentCollection.updateOne).not.toHaveBeenCalled(); expect(res.headers.deprecation).toBe("true"); expect(res.headers.link).toBe( @@ -2285,6 +2324,7 @@ describe("API Endpoints", () => { ); for (const doc of docs) { expect(doc.experimentId).toBeUndefined(); + expect(doc.submissionId).toBe(res.body.submissionId); expect(Array.isArray(doc.gates)).toBe(true); const selectGate = doc.gates.find((g: { gate: string }) => g.gate === "select"); const buildGate = doc.gates.find((g: { gate: string }) => g.gate === "build"); @@ -2296,6 +2336,10 @@ describe("API Endpoints", () => { expect(buildGate.promptText).toBeUndefined(); } expect(mocks.experimentCollection.updateOne).not.toHaveBeenCalled(); + expect(res.body.submissionId).toEqual(expect.any(String)); + expect(res.headers.deprecation).toBe("true"); + expect(res.headers.link).toContain("POST /api/v1/experiments"); + expect(res.headers).not.toHaveProperty("sunset"); // Response surfaces the configured gate count. expect(res.body.gates).toBe(2); }); @@ -2454,6 +2498,11 @@ describe("API Endpoints", () => { expect(insertCall[0]).toHaveProperty("model", "gpt-4o"); expect(insertCall[0]).toHaveProperty("maxIterations", 5); expect(insertCall[0]).not.toHaveProperty("experimentId"); + expect(insertCall[0]).toHaveProperty( + "submissionId", + res.body.submissionId, + ); + expect(res.body.submissionId).toEqual(expect.any(String)); expect(mocks.experimentCollection.updateOne).not.toHaveBeenCalled(); }); diff --git a/apps/api/src/experiment-openapi.test.ts b/apps/api/src/experiment-openapi.test.ts index fd5288a78..7a6a1df3d 100644 --- a/apps/api/src/experiment-openapi.test.ts +++ b/apps/api/src/experiment-openapi.test.ts @@ -3,6 +3,7 @@ import { beforeAll, describe, expect, it, vi } from "vitest"; import type { OpenAPIObject } from "openapi3-ts/oas31"; +import { LEGACY_REQUEST_COMPOSITION_DEPRECATION_ID } from "shared"; vi.mock("db-migrations/check-migrations", () => ({ checkMigrations: vi.fn().mockResolvedValue({ @@ -34,6 +35,12 @@ beforeAll(async () => { }); describe("Experiment OpenAPI contract", () => { + it("exports the shared legacy composition deprecation identifier", () => { + expect(LEGACY_REQUEST_COMPOSITION_DEPRECATION_ID).toBe( + "legacy-request-composition", + ); + }); + it("publishes canonical experiment operations", () => { expect(document.paths).toHaveProperty("/api/v1/experiments"); expect(document.paths).toHaveProperty("/api/v1/experiments/{id}"); diff --git a/apps/api/src/grouping.test.ts b/apps/api/src/grouping.test.ts index 746498c58..9ecc86bf6 100644 --- a/apps/api/src/grouping.test.ts +++ b/apps/api/src/grouping.test.ts @@ -19,6 +19,14 @@ describe("buildGroupingPipeline", () => { expect(pipeline).toHaveLength(4); }); + it("preserves submissionId grouping", () => { + const pipeline = buildGroupingPipeline("submissionId"); + expect(pipeline).toHaveLength(4); + expect( + (pipeline[0].$addFields as Record)._groupKey, + ).toBeDefined(); + }); + it("$sort stage orders by key ascending", () => { const pipeline = buildGroupingPipeline("task"); expect(pipeline[3]).toEqual({ $sort: { key: 1 } }); @@ -74,6 +82,7 @@ describe("buildGroupingPipeline", () => { expect(group._agentVersions).toBeDefined(); expect(group._platforms).toBeDefined(); expect(group._statuses).toBeDefined(); + expect(group._submissionIds).toBeDefined(); expect(group._experimentIds).toBeDefined(); expect(group._tasks).toBeDefined(); expect(group._mcpKeys).toBeDefined(); diff --git a/apps/api/src/grouping.ts b/apps/api/src/grouping.ts index d3ae0f60b..307e2bedc 100644 --- a/apps/api/src/grouping.ts +++ b/apps/api/src/grouping.ts @@ -12,12 +12,22 @@ import type { GroupByKey, RunGroup, AggregateStats, GroupUniformValues } from "s * The pipeline is appended after a $match stage with the caller's filter. */ export function buildGroupingPipeline( - groupBy: "task" | "experimentId" | "profile", + groupBy: "task" | "submissionId" | "experimentId" | "profile", ): Document[] { const groupField = - groupBy === "task" ? "$taskPromptId" : groupBy === "profile" ? "$profileId" : "$experimentId"; + groupBy === "task" + ? "$taskPromptId" + : groupBy === "profile" + ? "$profileId" + : groupBy === "submissionId" + ? "$submissionId" + : "$experimentId"; const fallbackGroupField = - groupBy === "profile" ? { $literal: "no-profile" } : { $literal: "no-experiment" }; + groupBy === "profile" + ? { $literal: "no-profile" } + : groupBy === "submissionId" + ? { $literal: "no-submission" } + : { $literal: "no-experiment" }; // `task` groups purely by the always-present `taskPromptId` — it is // materialized on every run-creation path (findOrCreate → taskPrompt._id) and // backfilled onto legacy docs by migration 001, so the group key is always a @@ -131,6 +141,7 @@ export function buildGroupingPipeline( _agentVersions: { $addToSet: { $ifNull: ["$agentVersion", ""] } }, _platforms: { $addToSet: { $ifNull: ["$run.os.platform", ""] } }, _statuses: { $addToSet: "$run.status" }, + _submissionIds: { $addToSet: { $ifNull: ["$submissionId", ""] } }, _experimentIds: { $addToSet: { $ifNull: ["$experimentId", ""] } }, _tasks: { $addToSet: { $ifNull: ["$scenario.task", ""] } }, _mcpKeys: { $addToSet: "$_mcpKey" }, @@ -183,6 +194,7 @@ export function buildGroupingPipeline( agentVersion: { $cond: { if: { $and: [{ $eq: [{ $size: "$_agentVersions" }, 1] }, { $ne: [{ $arrayElemAt: ["$_agentVersions", 0] }, ""] }] }, then: { $arrayElemAt: ["$_agentVersions", 0] }, else: "$$REMOVE" } }, platform: { $cond: { if: { $and: [{ $eq: [{ $size: "$_platforms" }, 1] }, { $ne: [{ $arrayElemAt: ["$_platforms", 0] }, ""] }] }, then: { $arrayElemAt: ["$_platforms", 0] }, else: "$$REMOVE" } }, status: { $cond: { if: { $eq: [{ $size: "$_statuses" }, 1] }, then: { $arrayElemAt: ["$_statuses", 0] }, else: "$$REMOVE" } }, + submissionId: { $cond: { if: { $and: [{ $eq: [{ $size: "$_submissionIds" }, 1] }, { $ne: [{ $arrayElemAt: ["$_submissionIds", 0] }, ""] }] }, then: { $arrayElemAt: ["$_submissionIds", 0] }, else: "$$REMOVE" } }, experimentId: { $cond: { if: { $and: [{ $eq: [{ $size: "$_experimentIds" }, 1] }, { $ne: [{ $arrayElemAt: ["$_experimentIds", 0] }, ""] }] }, then: { $arrayElemAt: ["$_experimentIds", 0] }, else: "$$REMOVE" } }, task: { $cond: { if: { $and: [{ $eq: [{ $size: "$_tasks" }, 1] }, { $ne: [{ $arrayElemAt: ["$_tasks", 0] }, ""] }] }, then: { $arrayElemAt: ["$_tasks", 0] }, else: "$$REMOVE" } }, mcpServers: { $cond: { if: { $and: [{ $eq: [{ $size: "$_mcpKeys" }, 1] }, { $ne: [{ $arrayElemAt: ["$_mcpKeys", 0] }, ""] }] }, then: "$_firstMcpServers", else: "$$REMOVE" } }, diff --git a/apps/api/src/routes/experiments.test.ts b/apps/api/src/routes/experiments.test.ts index 677e1633f..d778f48c2 100644 --- a/apps/api/src/routes/experiments.test.ts +++ b/apps/api/src/routes/experiments.test.ts @@ -94,6 +94,36 @@ describe("Experiment API contract", () => { expect(retry.status).toBe(400); }); + it("requires and verifies project scope on execution routes", async () => { + const findOne = vi.fn().mockResolvedValue(null); + const { app } = harness({ + experimentCollection: { findOne }, + } as unknown as Partial); + + const historyWithoutProject = await request(app).get( + "/api/v1/experiments/experiment-1/executions", + ); + const exactWithoutProject = await request(app).get( + "/api/v1/experiments/experiment-1/executions/execution-1", + ); + const wrongProject = await request(app).get( + "/api/v1/experiments/experiment-1/executions?projectId=wrong-project", + ); + const wrongProjectExact = await request(app).get( + "/api/v1/experiments/experiment-1/executions/execution-1?projectId=wrong-project", + ); + + expect(historyWithoutProject.status).toBe(400); + expect(exactWithoutProject.status).toBe(400); + expect(wrongProject.status).toBe(404); + expect(wrongProjectExact.status).toBe(404); + expect(findOne).toHaveBeenCalledWith({ + _id: "experiment-1", + projectId: "wrong-project", + finalizedAt: { $exists: true }, + }); + }); + it("enforces explicit empty and compose modes", async () => { const { app } = harness(); diff --git a/apps/api/src/routes/experiments.ts b/apps/api/src/routes/experiments.ts index 824b1442b..4739afe58 100644 --- a/apps/api/src/routes/experiments.ts +++ b/apps/api/src/routes/experiments.ts @@ -525,15 +525,17 @@ export function registerExperimentsRoutes(ctx: RouteContext): void { tags: ["Experiments"], summary: "List current and historical experiment executions", params: z.object({ id: z.string().min(1) }), - query: z.object({ + query: ProjectIdQuerySchema.extend({ limit: z.coerce.number().int().min(1).max(100).default(50), after: CursorSchema, }), response: ExecutionListResponseSchema, errorResponses: { 404: { description: "Experiment not found" } }, handler: async (req, res) => { + const projectId = getQueryProjectId(req); const experiment = await ctx.experimentCollection.findOne({ _id: req.params.id, + projectId, finalizedAt: { $exists: true }, }); if (!experiment) { @@ -589,11 +591,14 @@ export function registerExperimentsRoutes(ctx: RouteContext): void { id: z.string().min(1), experimentExecutionId: z.string().min(1), }), + query: ProjectIdQuerySchema, response: ExecutionResponseSchema, errorResponses: { 404: { description: "Experiment execution not found" } }, handler: async (req, res) => { + const projectId = getQueryProjectId(req); const experiment = await ctx.experimentCollection.findOne({ _id: req.params.id, + projectId, finalizedAt: { $exists: true }, }); if (!experiment) { diff --git a/apps/api/src/routes/requests/archive.test.ts b/apps/api/src/routes/requests/archive.test.ts index e9ad843d5..bdb1fce40 100644 --- a/apps/api/src/routes/requests/archive.test.ts +++ b/apps/api/src/routes/requests/archive.test.ts @@ -304,6 +304,7 @@ function makeFixtureRun(id = "run-fixture-1") { workerType: "coder-acp-copilot", createdAt: new Date("2026-01-01T00:00:00Z"), priority: 0, + submissionId: "submission-abc", run: { _id: id, attemptNumber: 1, @@ -764,6 +765,7 @@ describe("run import/export — round-trip (export → import)", () => { updatedAt: new Date("2026-05-11T23:04:47.446Z"), maxIterations: 1, priority: 0, + submissionId: "sub-rich-1", run: { _id: runId, attemptNumber: 2, // ← not 1; previous code hard-coded 1 @@ -954,7 +956,7 @@ describe("run import/export — round-trip (export → import)", () => { describe("run import — task-prompt entity creation (#832)", () => { // Task-prompt entities back features, report triggers, group-by-task, // the runs-list task filter and the criteria/MDP analysis. The submission - // flow (POST /api/v1/requests) materializes them via + // request flow (POST /api/v1/requests) materializes them via // taskPromptStore.findOrCreate(scenario.task). Imported runs used to skip // this step entirely \u2014 `taskPromptId` was preserved from `run.yaml` but // no row was ever created, leaving a dangling reference. @@ -1001,6 +1003,7 @@ describe("run import — task-prompt entity creation (#832)", () => { // The inserted request points at it. const inserted = reqCollection.docs.get(id); expect(inserted.taskPromptId).toBe(expectedId); + expect(inserted.submissionId).toEqual(expect.any(String)); }); it("reuses an existing task prompt and does not create a duplicate", async () => { diff --git a/apps/api/src/routes/requests/index.ts b/apps/api/src/routes/requests/index.ts index 07987a124..0b5d2df81 100644 --- a/apps/api/src/routes/requests/index.ts +++ b/apps/api/src/routes/requests/index.ts @@ -37,6 +37,7 @@ import { isCriterionCompatibleWithGate, orderGates, computeTaskPromptId, + LEGACY_REQUEST_COMPOSITION_DEPRECATION_ID, } from "shared"; import { trackMetric } from "telemetry"; import type { @@ -236,9 +237,6 @@ const validatePersistedRequestTarget = (request: RequestDocument) => /** Maximum number of profile variations (including the base profile) allowed in one submit. */ const MAX_PROFILE_VARIATIONS = 25; -const LEGACY_EXPERIMENT_COMPOSITION_ID = - "scope.api.requests.legacy-experiment-composition.v1"; - function emitLegacyCompositionHeaders( res: { setHeader(name: string, value: string): void }, properties: readonly string[], @@ -253,7 +251,7 @@ function emitLegacyCompositionHeaders( name: "api.deprecated_input_used", value: 1, properties: { - identifier: LEGACY_EXPERIMENT_COMPOSITION_ID, + identifier: LEGACY_REQUEST_COMPOSITION_DEPRECATION_ID, properties: [...properties].sort().join(","), }, }); @@ -871,14 +869,14 @@ apiRoute(ctx.app, ctx.registry, { }); } - // Pass 2: build independent request documents. Comparative composition is - // represented only by POST /experiments. + // Preserve the legacy request-batch correlation value. Experiments remain + // explicit resources created only by POST /experiments. + const submissionId = uuidv4(); const allNewIds: string[] = []; const newDocs: RequestDocument[] = []; const variationResults: Array<{ profileId: string; label?: string; ids: string[] }> = []; const agentsMdPromptId = await resolveAgentsMdPromptId(); const agentsMdFields = buildAgentsMdFields(agentsMdPromptId); - const expectedRequestCount = resolved.length * count; for (const r of resolved) { const newIds: string[] = []; @@ -912,6 +910,7 @@ apiRoute(ctx.app, ctx.registry, { profileId: r.profile._id, profileVersionId: r.profileVersion.ref ?? r.profileVersion._id, ...(persistedVariationGates ? { gates: persistedVariationGates } : {}), + submissionId, ...agentsMdFields, run: { _id: runId, attemptNumber: 1, status: "pending", logsUrl: ctx.blobStorage.getLogsBlobUrl(`${requestId}/runs/${runId}/run.jsonl`) }, }; @@ -929,6 +928,7 @@ apiRoute(ctx.app, ctx.registry, { res.status(201).json({ ids: allNewIds, count: allNewIds.length, + submissionId, variations: variationResults, variationCount: variationEntries.length, status: "pending", @@ -1311,6 +1311,7 @@ apiRoute(ctx.app, ctx.registry, { const persistedObservations: string[] | undefined = observationsProvided ? [...new Set(resolvedObservations)] : undefined; + const submissionId = uuidv4(); // Handle multiple runs (count > 1) if (count > 1) { @@ -1345,6 +1346,7 @@ apiRoute(ctx.app, ctx.registry, { ...(profileVersionId ? { profileVersionId } : {}), ...(persistedGates ? { gates: persistedGates } : {}), ...(persistedObservations ? { observations: persistedObservations } : {}), + submissionId, ...agentsMdFields, // Mint a distinct run id for the first attempt. Blob artifacts // are scoped under `{requestId}/runs/{runId}/...` so retries @@ -1362,6 +1364,7 @@ apiRoute(ctx.app, ctx.registry, { res.status(201).json({ ids: newIds, count, + submissionId, workerType, taskPromptId, ...(model ? { model } : {}), @@ -1406,6 +1409,7 @@ apiRoute(ctx.app, ctx.registry, { ...(profileVersionId ? { profileVersionId } : {}), ...(persistedGates ? { gates: persistedGates } : {}), ...(persistedObservations ? { observations: persistedObservations } : {}), + submissionId, ...agentsMdFields, // Mint a distinct run id for the first attempt. Blob artifacts // are scoped under `{requestId}/runs/{runId}/...` so retries @@ -1420,6 +1424,7 @@ apiRoute(ctx.app, ctx.registry, { res.status(201).json({ id: requestId, + submissionId, workerType, ...(model ? { model } : {}), ...(effectiveReasoningEffort ? { reasoningEffort: effectiveReasoningEffort } : {}), @@ -1502,6 +1507,7 @@ apiRoute(ctx.app, ctx.registry, { const projectId = getQueryProjectId(req); const taskPromptIdFilter = req.query.taskPromptId as string; const criteriaFilter = req.query.criteria as string; + const submissionIdFilter = req.query.submissionId as string; const experimentIdFilter = req.query.experimentId as string; const searchFilter = req.query.search as string | undefined; const turnsFilterRaw = req.query.turns as string | undefined; @@ -1509,7 +1515,12 @@ apiRoute(ctx.app, ctx.registry, { const maxIterationsFilterRaw = req.query.maxIterations as string | undefined; const maxIterationsOpFilter = (req.query.maxIterationsOp as string | undefined) ?? "eq"; const includeDeleted = req.query.includeDeleted === "true"; - const groupByParam = req.query.groupBy as "task" | "experimentId" | "profile" | undefined; + const groupByParam = req.query.groupBy as + | "task" + | "submissionId" + | "experimentId" + | "profile" + | undefined; const limit = Math.min(Math.max(Number(req.query.limit) || 10, 1), 100); const afterParam = req.query.after as string | undefined; const beforeParam = req.query.before as string | undefined; @@ -1576,6 +1587,11 @@ apiRoute(ctx.app, ctx.registry, { pushClause(buildMultiClause("priority", priorityValues, { coerceNumber: true })); pushClause(buildMultiClause("agentVersion", agentVersionValues)); + if (submissionIdFilter) { + flat.submissionId = { + $regex: `^${submissionIdFilter.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}`, + }; + } if (experimentIdFilter) { // Prefix-based matching: allow filtering by partial experiment ID. flat.experimentId = { $regex: `^${experimentIdFilter.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}` }; @@ -1675,7 +1691,12 @@ apiRoute(ctx.app, ctx.registry, { // Grouped mode: paginated RunGroup[] via two-phase aggregation if (groupByParam) { - const groupByField = groupByParam === "task" ? "taskPromptId" : groupByParam === "profile" ? "profileId" : "experimentId"; + const groupByField = + groupByParam === "task" + ? "taskPromptId" + : groupByParam === "profile" + ? "profileId" + : groupByParam; const groupByAggField = `$${groupByField}`; // Decode group cursor @@ -1908,6 +1929,7 @@ apiRoute(ctx.app, ctx.registry, { taskPromptId: z.string().optional(), criteria: z.string().optional(), features: z.string().optional(), + submissionId: z.string().optional(), experimentId: z.string().optional(), k: z.string().optional(), }).merge(ProjectIdQuerySchema), @@ -2092,6 +2114,7 @@ apiRoute(ctx.app, ctx.registry, { const foundIds = new Set(originalRuns.map(r => r._id)); const notFound = ids.filter((id: string) => !foundIds.has(id)); + const submissionId = uuidv4(); const newIds: string[] = []; const newDocs: RequestDocument[] = []; @@ -2230,6 +2253,7 @@ apiRoute(ctx.app, ctx.registry, { // Preserve AGENTS.md lineage across re-submits. ...(original.agentsMdPromptId ? { agentsMdPromptId: original.agentsMdPromptId } : {}), ...(original.agentsMdParentIds && original.agentsMdParentIds.length > 0 ? { agentsMdParentIds: original.agentsMdParentIds } : {}), + submissionId, // Bulk re-submit creates a brand-new request — mint a distinct // run id for the first attempt so blob artifacts live under // `{requestId}/runs/{runId}/...` (independent of the original run). @@ -2251,6 +2275,7 @@ apiRoute(ctx.app, ctx.registry, { submitted: newIds.length, failed: notFound, newIds, + submissionId, }); }, }); @@ -2673,6 +2698,7 @@ async function finalizePendingRun( createdAt: runDoc.createdAt ?? new Date(), priority: runDoc.priority ?? 0, taskPromptId: resolvedTaskPromptId, + ...(runDoc.submissionId ? {} : { submissionId: uuidv4() }), run: { ...(runState as unknown as RunState), // Per-attempt `run._id` is preserved from the yaml (NOT clobbered diff --git a/packages/shared/src/schemas/request.ts b/packages/shared/src/schemas/request.ts index 029c03e09..bf4c75743 100644 --- a/packages/shared/src/schemas/request.ts +++ b/packages/shared/src/schemas/request.ts @@ -187,6 +187,7 @@ export const RequestResponseSchema = z agentVersion: z.string().optional(), profileId: z.string().optional(), profileVersionId: z.string().optional(), + submissionId: z.string().optional(), experimentId: z.string().optional(), experimentExpectedRequestCount: z.number().int().min(0).optional(), experimentRequestIndex: z.number().int().min(0).optional(), @@ -294,6 +295,7 @@ export const ListRequestsQuerySchema = z worker: MultiValueParam, taskPromptId: z.string().optional(), criteria: z.string().optional(), + submissionId: z.string().optional(), experimentId: z.string().optional(), profileId: MultiValueParam, status: MultiValueParam, @@ -304,7 +306,9 @@ export const ListRequestsQuerySchema = z agentVersion: MultiValueParam, // Free-text, case-insensitive search across run id, scenario task, model, worker. search: z.string().optional(), - groupBy: z.enum(["task", "experimentId", "profile"]).optional(), + groupBy: z + .enum(["task", "submissionId", "experimentId", "profile"]) + .optional(), limit: z.coerce.number().int().min(1).max(100).optional(), after: z.string().optional(), before: z.string().optional(), @@ -377,6 +381,7 @@ export const GroupUniformValuesSchema = z codebaseRevisionId: z.string().optional(), extensions: z.array(z.string()).optional(), status: RequestStatusSchema.optional(), + submissionId: z.string().optional(), experimentId: z.string().optional(), task: z.string().optional(), }) diff --git a/packages/shared/src/types/types.ts b/packages/shared/src/types/types.ts index 93d76d525..016e980f8 100644 --- a/packages/shared/src/types/types.ts +++ b/packages/shared/src/types/types.ts @@ -384,6 +384,7 @@ export interface RequestDocument { agentVersion?: string; // Agent software version prefix (e.g. "copilot-0.0.415") — FK → AgentVersion.agentVersion profileId?: string; // FK → ProfileDocument._id (the profile lineage) profileVersionId?: string; // FK → ProfileVersionDocument._id (exact version used) + submissionId?: string; // Legacy request-batch correlation identifier (not a resource FK) experimentId?: string; // FK → ExperimentDocument._id /** Expected native request count for interrupted ExperimentDocument finalization. * Historical/imported requests omit it and remain view-only if their parent @@ -600,6 +601,9 @@ export interface ExperimentQueueMessagePayload { experimentExecutionId: string; } +export const LEGACY_REQUEST_COMPOSITION_DEPRECATION_ID = + "legacy-request-composition" as const; + /** * Field names whose values move from `RequestDocument` (top-level, legacy * shape) into `RequestDocument.run: RunState` (new shape introduced for @@ -733,7 +737,12 @@ export interface QueueProcessorConfig extends BaseQueueProcessorConfig { // --- Runs grouping types (shared between API and portal) --- -export type GroupByKey = "none" | "task" | "experimentId" | "profile"; +export type GroupByKey = + | "none" + | "task" + | "submissionId" + | "experimentId" + | "profile"; export interface AggregateStats { min: number; @@ -751,6 +760,7 @@ export interface GroupUniformValues { skillRevisions?: string[]; extensions?: string[]; status?: "pending" | "processing" | "done"; + submissionId?: string; experimentId?: string; task?: string; } From 48f244b80774d0f1fc4b9bc3b42a209b0cf62fa9 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Thu, 15 Oct 2026 23:14:34 -0700 Subject: [PATCH 09/25] feat(shared): add experiment payload CAS updates Add representation-preserving request projection updates with root publication claims, versioned bucket CAS, resumable partial-failure outcomes, and inline/chunked concurrency coverage. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 8fe76b9e-1205-4910-b51b-e8fa51538125 --- .../src/experiments/experiment-store.test.ts | 309 +++++++++++++++++- .../src/experiments/experiment-store.ts | 270 ++++++++++++++- .../experiments/fixed-payload-store.test.ts | 237 +++++++++++++- .../src/experiments/fixed-payload-store.ts | 228 ++++++++++++- packages/shared/src/experiments/layout.ts | 28 +- packages/shared/src/experiments/routing.ts | 9 + .../shared/src/experiments/schemas.test.ts | 1 + packages/shared/src/experiments/schemas.ts | 11 + packages/shared/src/experiments/types.ts | 36 ++ 9 files changed, 1100 insertions(+), 29 deletions(-) diff --git a/packages/shared/src/experiments/experiment-store.test.ts b/packages/shared/src/experiments/experiment-store.test.ts index f9e0e0e1c..fe823a92a 100644 --- a/packages/shared/src/experiments/experiment-store.test.ts +++ b/packages/shared/src/experiments/experiment-store.test.ts @@ -4,10 +4,16 @@ import type { Collection, Document } from "mongodb"; import { describe, expect, it } from "vitest"; import { ExperimentStore } from "./experiment-store.js"; +import { FixedPayloadStore } from "./fixed-payload-store.js"; +import { + fingerprintExperimentPayload, + fingerprintExperimentRequestVector, +} from "./routing.js"; import { EXPERIMENT_STORAGE_PROFILE_VERSION, type ExperimentDocument, type ExperimentExecutionHistoryDocument, + type ExperimentRequestState, } from "./types.js"; function getPath(source: object, path: string): unknown { @@ -27,11 +33,23 @@ function setPath(target: object, path: string, value: unknown): void { if (typeof next !== "object" || next === null) { current[segment] = {}; } + current = current[segment] as Record; } current[segments.at(-1)!] = value; } +function unsetPath(target: object, path: string): void { + const segments = path.split("."); + let current = target as Record; + for (const segment of segments.slice(0, -1)) { + const next = current[segment]; + if (typeof next !== "object" || next === null) return; + current = next as Record; + } + delete current[segments.at(-1)!]; +} + function matches(document: object, filter: Document): boolean { return Object.entries(filter).every(([path, expected]) => { const actual = getPath(document, path); @@ -51,8 +69,13 @@ function matches(document: object, filter: Document): boolean { function atomicCollection(initial: ExperimentDocument) { let document = structuredClone(initial); + let findOneAndUpdateCalls = 0; + let failAtCall: number | undefined; return { current: () => structuredClone(document), + failOnFindOneAndUpdate(call: number) { + failAtCall = call; + }, async findOne(filter: Document) { return matches(document, filter) ? structuredClone(document) : null; }, @@ -61,8 +84,13 @@ function atomicCollection(initial: ExperimentDocument) { update: { $set?: Record; $inc?: Record; + $unset?: Record; }, ) { + findOneAndUpdateCalls += 1; + if (findOneAndUpdateCalls === failAtCall) { + throw new Error("injected root publication failure"); + } if (!matches(document, filter)) return null; const next = structuredClone(document); for (const [path, value] of Object.entries(update.$set ?? {})) { @@ -73,14 +101,39 @@ function atomicCollection(initial: ExperimentDocument) { if (typeof value !== "number") throw new Error(`Non-number ${path}`); setPath(next, path, value + increment); } + for (const path of Object.keys(update.$unset ?? {})) { + unsetPath(next, path); + } document = next; return structuredClone(document); }, + async updateOne( + filter: Document, + update: { + $set?: Record; + $inc?: Record; + $unset?: Record; + }, + ) { + const updated = await this.findOneAndUpdate(filter, update); + return { + matchedCount: updated ? 1 : 0, + modifiedCount: updated ? 1 : 0, + }; + }, }; } function experiment(): ExperimentDocument { const now = new Date("2026-01-01T00:00:00.000Z"); + const entries = [ + { + requestId: "request-1", + runId: "run-1", + dependencies: [], + blockers: [], + }, + ]; return { _id: "experiment-1", experimentId: "experiment-1", @@ -103,17 +156,11 @@ function experiment(): ExperimentDocument { requests: { storage: "inline", entryCount: 1, - payloadFingerprint: "payload", - entries: [ - { - requestId: "request-1", - runId: "run-1", - dependencies: [], - blockers: [], - }, - ], + payloadFingerprint: fingerprintExperimentPayload(entries), + payloadVersion: 1, + entries, }, - vectorFingerprint: "vector", + vectorFingerprint: fingerprintExperimentRequestVector(entries), handlerSummary: { report: { status: "queued", @@ -132,6 +179,44 @@ function experiment(): ExperimentDocument { }; } +function chunkCollection() { + const documents = new Map(); + return { + async updateOne( + filter: Document, + update: { $setOnInsert?: Document; $set?: Document }, + ) { + const id = String(filter._id); + const existing = documents.get(id); + if (!existing && update.$setOnInsert) { + documents.set(id, structuredClone(update.$setOnInsert)); + return { matchedCount: 1, modifiedCount: 1 }; + } + if (!existing) return { matchedCount: 0, modifiedCount: 0 }; + const matches = Object.entries(filter).every(([key, expected]) => { + const actual = existing[key]; + if ( + typeof expected === "object" && + expected !== null && + "$exists" in expected + ) { + return ( + (actual !== undefined) === + Boolean((expected as { $exists: unknown }).$exists) + ); + } + return actual === expected; + }); + if (!matches) return { matchedCount: 0, modifiedCount: 0 }; + if (update.$set) Object.assign(existing, structuredClone(update.$set)); + return { matchedCount: 1, modifiedCount: 1 }; + }, + async findOne(filter: Document) { + return documents.get(String(filter._id)) ?? null; + }, + }; +} + describe("ExperimentStore CAS contracts", () => { it("allows exactly one winner in a writer-freezer race", async () => { const experiments = atomicCollection(experiment()); @@ -248,4 +333,208 @@ describe("ExperimentStore CAS contracts", () => { claimToken: "claim-1", }); }); + + it("serializes request-state writers against the freezer", async () => { + const experiments = atomicCollection(experiment()); + const store = new ExperimentStore({ + experiments: + experiments as unknown as Collection, + executions: + {} as Collection, + chunks: {} as Collection, + }); + const projected = [ + { + requestId: "request-1", + runId: "run-1", + dependencies: [ + { handlerId: "observations", state: "ready" as const }, + ], + blockers: [], + }, + ]; + + const [writer, freezer] = await Promise.all([ + store.casUpdateRequestStates({ + projectId: "project-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-1", + expectedExecutionVersion: 1, + updateId: "projection-1", + requests: projected, + }), + store.freezeCurrent({ + projectId: "project-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-1", + expectedExecutionVersion: 1, + }), + ]); + + const writerWon = writer.status === "applied"; + expect(writerWon || freezer !== null).toBe(true); + expect(writerWon && freezer !== null).toBe(false); + if (writerWon) { + expect(writer.requests).toEqual(projected); + expect(writer.experiment.execution.requests.storage).toBe("inline"); + expect(writer.experiment.execution.version).toBe(3); + } else { + expect(writer).toMatchObject({ status: "stale" }); + expect(freezer?.execution.frozenAt).toBeInstanceOf(Date); + } + }); + + it("rejects request/run vector changes as an invariant violation", async () => { + const experiments = atomicCollection(experiment()); + const store = new ExperimentStore({ + experiments: + experiments as unknown as Collection, + executions: + {} as Collection, + chunks: {} as Collection, + }); + + await expect( + store.casUpdateRequestStates({ + projectId: "project-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-1", + expectedExecutionVersion: 1, + updateId: "projection-invalid", + requests: [ + { + requestId: "request-1", + runId: "different-run", + dependencies: [], + blockers: [], + }, + ], + }), + ).resolves.toMatchObject({ + status: "invariant", + message: + "Request-state update cannot change the immutable request/run vector", + }); + }); + + it("surfaces a root publication fault and resumes the same update id", async () => { + const experiments = atomicCollection(experiment()); + experiments.failOnFindOneAndUpdate(2); + const store = new ExperimentStore({ + experiments: + experiments as unknown as Collection, + executions: + {} as Collection, + chunks: {} as Collection, + }); + const projected = [ + { + requestId: "request-1", + runId: "run-1", + dependencies: [ + { handlerId: "observations", state: "ready" as const }, + ], + blockers: [], + }, + ]; + const input = { + projectId: "project-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-1", + expectedExecutionVersion: 1, + updateId: "projection-retry", + requests: projected, + }; + + await expect(store.casUpdateRequestStates(input)).resolves.toMatchObject({ + status: "partial-failure", + updateId: "projection-retry", + retryable: true, + message: "injected root publication failure", + }); + expect( + experiments.current().execution.requestStateUpdate?.updateId, + ).toBe("projection-retry"); + + const repaired = await store.casUpdateRequestStates(input); + expect(repaired.status).toBe("applied"); + if (repaired.status !== "applied") throw new Error("expected applied"); + expect(repaired.requests).toEqual(projected); + expect( + repaired.experiment.execution.requestStateUpdate, + ).toBeUndefined(); + }); + + it("publishes the same projection semantics through a chunked execution", async () => { + const chunks = chunkCollection(); + const payloadStore = new FixedPayloadStore( + chunks as unknown as Collection, + ); + const entries: ExperimentRequestState[] = Array.from( + { length: 70 }, + (_, index) => ({ + requestId: `request-${index}`, + runId: `run-${index}`, + dependencies: [], + blockers: [], + }), + ); + const identity = { + projectId: "project-1", + experimentId: "experiment-1", + ownerId: "execution-1", + generation: 1, + }; + const payload = await payloadStore.prepareRequests({ + identity, + entries, + inlineDocument: (value) => ({ + value, + padding: "x".repeat(256 * 1024), + }), + }); + if (payload.storage !== "chunked") throw new Error("expected chunked"); + const root = experiment(); + root.revision.memberCount = entries.length; + root.execution.requests = payload; + root.execution.vectorFingerprint = + fingerprintExperimentRequestVector(entries); + const experiments = atomicCollection(root); + const store = new ExperimentStore({ + experiments: + experiments as unknown as Collection, + executions: + {} as Collection, + chunks: chunks as unknown as Collection, + }); + const projected = entries.map((entry) => ({ + ...entry, + dependencies: [ + { handlerId: "observations", state: "ready" as const }, + ], + })); + + const result = await store.casUpdateRequestStates({ + projectId: "project-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-1", + expectedExecutionVersion: 1, + updateId: "chunked-projection", + requests: projected, + }); + + expect(result.status).toBe("applied"); + if (result.status !== "applied") throw new Error("expected applied"); + const updatedPayload = result.experiment.execution.requests; + expect(updatedPayload.storage).toBe("chunked"); + if (updatedPayload.storage !== "chunked") { + throw new Error("expected chunked"); + } + expect(updatedPayload.bucketCount).toBe(payload.bucketCount); + expect(updatedPayload.assignmentVersion).toBe(payload.assignmentVersion); + expect(updatedPayload.chunkIds).toEqual(payload.chunkIds); + expect( + await payloadStore.readRequests(identity, updatedPayload), + ).toEqual(projected); + }); }); diff --git a/packages/shared/src/experiments/experiment-store.ts b/packages/shared/src/experiments/experiment-store.ts index ce9ab4f1e..da21e733d 100644 --- a/packages/shared/src/experiments/experiment-store.ts +++ b/packages/shared/src/experiments/experiment-store.ts @@ -11,7 +11,10 @@ import { ExperimentRequestStateSchema, ExperimentRevisionSchema, } from "./schemas.js"; -import { fingerprintExperimentPayload } from "./routing.js"; +import { + fingerprintExperimentPayload, + fingerprintExperimentRequestVector, +} from "./routing.js"; import { EXPERIMENT_STORAGE_PROFILE_VERSION, type ExperimentDocument, @@ -21,6 +24,7 @@ import { type ExperimentMetadata, type ExperimentPage, type ExperimentRequestState, + type ExperimentRequestStateUpdateResult, type ExperimentRevision, type FixedPayload, } from "./types.js"; @@ -128,7 +132,7 @@ export class ExperimentStore { attemptNumber: 1, version: 1, requests: payload, - vectorFingerprint: fingerprintExperimentPayload(requests), + vectorFingerprint: fingerprintExperimentRequestVector(requests), handlerSummary: input.handlerSummary ?? {}, blocking: input.blocking ?? [], createdAt: now, @@ -409,6 +413,7 @@ export class ExperimentStore { "execution.version": input.expectedExecutionVersion, "execution.frozenAt": { $exists: false }, "execution.rollover": { $exists: false }, + "execution.requestStateUpdate": { $exists: false }, } as Document, { $set: { @@ -423,6 +428,244 @@ export class ExperimentStore { return updated ? parseExperiment(updated) : null; } + async casUpdateRequestStates(input: { + projectId: string; + experimentId: string; + experimentExecutionId: string; + expectedExecutionVersion: number; + updateId: string; + requests: readonly ExperimentRequestState[]; + now?: Date; + }): Promise { + if (input.updateId.trim().length === 0 || input.updateId.length > 200) { + return { + status: "invariant", + message: "Request-state updateId must contain 1-200 characters", + }; + } + let requests: ExperimentRequestState[]; + try { + requests = input.requests.map((request) => + ExperimentRequestStateSchema.parse(request), + ) as ExperimentRequestState[]; + } catch (error) { + return { + status: "invariant", + message: error instanceof Error ? error.message : String(error), + }; + } + if ( + new Set(requests.map((request) => request.requestId)).size !== + requests.length + ) { + return { + status: "invariant", + message: "Request-state update contains duplicate request ids", + }; + } + const nextPayloadFingerprint = fingerprintExperimentPayload(requests); + const nextVectorFingerprint = + fingerprintExperimentRequestVector(requests); + let experiment = await this.getRaw(input.projectId, input.experimentId); + if (!experiment) return { status: "stale", reason: "experiment" }; + if ( + experiment.execution.experimentExecutionId !== + input.experimentExecutionId + ) { + return { status: "stale", reason: "execution" }; + } + if (experiment.execution.frozenAt) { + return { status: "stale", reason: "frozen" }; + } + if ( + experiment.execution.vectorFingerprint !== nextVectorFingerprint || + experiment.execution.requests.entryCount !== requests.length + ) { + return { + status: "invariant", + message: + "Request-state update cannot change the immutable request/run vector", + }; + } + + const activeClaim = experiment.execution.requestStateUpdate; + if (activeClaim) { + if (activeClaim.updateId !== input.updateId) { + return { + status: "contention", + activeUpdateId: activeClaim.updateId, + }; + } + if ( + activeClaim.nextPayloadFingerprint !== nextPayloadFingerprint || + activeClaim.expectedPayloadFingerprint !== + experiment.execution.requests.payloadFingerprint + ) { + return { + status: "invariant", + message: "Retried request-state update does not match its claim", + }; + } + if ( + experiment.execution.version !== + input.expectedExecutionVersion + 1 + ) { + return { status: "stale", reason: "version" }; + } + } else { + if ( + experiment.execution.version !== input.expectedExecutionVersion + ) { + return { status: "stale", reason: "version" }; + } + const claimedAt = input.now ?? new Date(); + let claimed: ExperimentDocument | null; + try { + claimed = await this.collections.experiments.findOneAndUpdate( + { + _id: input.experimentId, + projectId: input.projectId, + "execution.experimentExecutionId": + input.experimentExecutionId, + "execution.version": input.expectedExecutionVersion, + "execution.frozenAt": { $exists: false }, + "execution.rollover": { $exists: false }, + "execution.requestStateUpdate": { $exists: false }, + } as Document, + { + $set: { + "execution.requestStateUpdate": { + updateId: input.updateId, + expectedPayloadFingerprint: + experiment.execution.requests.payloadFingerprint, + nextPayloadFingerprint, + claimedAt, + }, + "execution.updatedAt": claimedAt, + updatedAt: claimedAt, + }, + $inc: { "execution.version": 1, stateVersion: 1 }, + } as Document, + { returnDocument: "after" }, + ); + } catch (error) { + return { + status: "partial-failure", + updateId: input.updateId, + updatedChunkIds: [], + message: error instanceof Error ? error.message : String(error), + retryable: true, + }; + } + if (!claimed) { + const raced = await this.getRaw( + input.projectId, + input.experimentId, + ); + if (raced?.execution.frozenAt) { + return { status: "stale", reason: "frozen" }; + } + if (raced?.execution.requestStateUpdate) { + return { + status: "contention", + activeUpdateId: + raced.execution.requestStateUpdate.updateId, + }; + } + return { status: "stale", reason: "version" }; + } + experiment = parseExperiment(claimed); + } + + const payloadResult = await this.payloads.casUpdateRequests({ + identity: this.executionIdentity(experiment), + payload: experiment.execution.requests, + entries: requests, + now: input.now, + }); + if (payloadResult.status !== "applied") { + if ( + payloadResult.status !== "partial-failure" && + payloadResult.updatedChunkIds.length === 0 + ) { + try { + await this.abortRequestStateUpdate(experiment, input.updateId); + } catch (error) { + return { + status: "partial-failure", + updateId: input.updateId, + updatedChunkIds: [], + message: error instanceof Error ? error.message : String(error), + retryable: true, + }; + } + } + if (payloadResult.status === "invariant") { + return { + status: "invariant", + message: payloadResult.message, + }; + } + if (payloadResult.status === "stale") { + return { status: "stale", reason: "version" }; + } + return { + status: "partial-failure", + updateId: input.updateId, + updatedChunkIds: payloadResult.updatedChunkIds, + message: payloadResult.message, + retryable: true, + }; + } + + const now = input.now ?? new Date(); + try { + const finalized = await this.collections.experiments.findOneAndUpdate( + { + _id: input.experimentId, + projectId: input.projectId, + "execution.experimentExecutionId": + input.experimentExecutionId, + "execution.version": input.expectedExecutionVersion + 1, + "execution.frozenAt": { $exists: false }, + "execution.requestStateUpdate.updateId": input.updateId, + } as Document, + { + $set: { + "execution.requests": payloadResult.payload, + "execution.updatedAt": now, + updatedAt: now, + }, + $unset: { "execution.requestStateUpdate": "" }, + $inc: { "execution.version": 1, stateVersion: 1 }, + } as Document, + { returnDocument: "after" }, + ); + if (!finalized) { + return { + status: "partial-failure", + updateId: input.updateId, + updatedChunkIds: payloadResult.updatedChunkIds, + message: "Request-state payload updated but root publication CAS failed", + retryable: true, + }; + } + return { + status: "applied", + experiment: parseExperiment(finalized), + requests, + }; + } catch (error) { + return { + status: "partial-failure", + updateId: input.updateId, + updatedChunkIds: payloadResult.updatedChunkIds, + message: error instanceof Error ? error.message : String(error), + retryable: true, + }; + } + } + async archiveCurrent(input: { projectId: string; experimentId: string; @@ -631,7 +874,7 @@ export class ExperimentStore { attemptNumber: current.attemptNumber + 1, version: 1, requests, - vectorFingerprint: fingerprintExperimentPayload(entries), + vectorFingerprint: fingerprintExperimentRequestVector(entries), handlerSummary: {}, blocking: [], createdAt: now, @@ -686,4 +929,25 @@ export class ExperimentStore { } return persisted; } + + private async abortRequestStateUpdate( + experiment: ExperimentDocument, + updateId: string, + ): Promise { + await this.collections.experiments.updateOne( + { + _id: experiment.experimentId, + projectId: experiment.projectId, + "execution.experimentExecutionId": + experiment.execution.experimentExecutionId, + "execution.version": experiment.execution.version, + "execution.requestStateUpdate.updateId": updateId, + } as Document, + { + $unset: { "execution.requestStateUpdate": "" }, + $set: { updatedAt: new Date() }, + $inc: { stateVersion: 1 }, + } as Document, + ); + } } diff --git a/packages/shared/src/experiments/fixed-payload-store.test.ts b/packages/shared/src/experiments/fixed-payload-store.test.ts index 9339dbecb..142415a30 100644 --- a/packages/shared/src/experiments/fixed-payload-store.test.ts +++ b/packages/shared/src/experiments/fixed-payload-store.test.ts @@ -18,16 +18,47 @@ function request(index: number): ExperimentRequestState { function chunkCollection(failAt?: number) { const documents = new Map(); let writes = 0; + let nextFailure = failAt; + function matches(document: Document, filter: Document): boolean { + return Object.entries(filter).every(([key, expected]) => { + const actual = document[key]; + if ( + typeof expected === "object" && + expected !== null && + "$exists" in expected + ) { + return ( + (actual !== undefined) === + Boolean((expected as { $exists: unknown }).$exists) + ); + } + return actual === expected; + }); + } return { documents, + failOnWrite(write: number) { + nextFailure = write; + }, + writeCount() { + return writes; + }, async updateOne( filter: Document, - update: { $setOnInsert: Document }, + update: { $setOnInsert?: Document; $set?: Document }, ) { writes += 1; - if (writes === failAt) throw new Error("injected write failure"); + if (writes === nextFailure) throw new Error("injected write failure"); const id = String(filter._id); - if (!documents.has(id)) documents.set(id, structuredClone(update.$setOnInsert)); + const existing = documents.get(id); + if (!existing && update.$setOnInsert) { + documents.set(id, structuredClone(update.$setOnInsert)); + return { matchedCount: 1, modifiedCount: 1 }; + } + if (!existing || !matches(existing, filter)) { + return { matchedCount: 0, modifiedCount: 0 }; + } + if (update.$set) Object.assign(existing, structuredClone(update.$set)); return { matchedCount: 1, modifiedCount: 1 }; }, async findOne(filter: Document) { @@ -127,4 +158,204 @@ describe("FixedPayloadStore", () => { store.readRequests({ ...identity, projectId: "other-project" }, payload), ).rejects.toThrow(/identity mismatch/); }); + + it.each(["inline", "chunked"] as const)( + "applies the same logical projection update for %s payloads", + async (storage) => { + const collection = chunkCollection(); + const store = new FixedPayloadStore( + collection as unknown as Collection, + ); + const ownerId = `execution-${storage}`; + const updateIdentity = { ...identity, ownerId }; + const entries = Array.from({ length: 70 }, (_, index) => request(index)); + const payload = await store.prepareRequests({ + identity: updateIdentity, + entries, + inlineDocument: (value) => ({ + value, + ...(storage === "chunked" + ? { padding: "x".repeat(256 * 1024) } + : {}), + }), + }); + expect(payload.storage).toBe(storage); + const projected = entries.map((entry, index) => ({ + ...entry, + dependencies: [ + { + handlerId: "observations", + state: index % 2 === 0 ? ("ready" as const) : ("waiting" as const), + }, + ], + blockers: + index % 2 === 0 + ? [] + : [ + { + handlerId: "observations", + reason: "dependency-not-ready" as const, + }, + ], + })); + + const result = await store.casUpdateRequests({ + identity: updateIdentity, + payload, + entries: projected, + }); + + expect(result.status).toBe("applied"); + if (result.status !== "applied") throw new Error("expected applied"); + expect(result.payload.storage).toBe(storage); + expect(result.payload.payloadVersion).toBe(payload.payloadVersion + 1); + if (payload.storage === "chunked" && result.payload.storage === "chunked") { + expect(result.payload.bucketCount).toBe(payload.bucketCount); + expect(result.payload.assignmentVersion).toBe(payload.assignmentVersion); + expect(result.payload.chunkIds).toEqual(payload.chunkIds); + } + expect( + await store.readRequests(updateIdentity, result.payload), + ).toEqual(projected); + }, + ); + + it("reports and resumes a fault-injected partial chunk update", async () => { + const collection = chunkCollection(); + const store = new FixedPayloadStore( + collection as unknown as Collection, + ); + const entries = Array.from({ length: 100 }, (_, index) => ({ + ...request(index), + dependencies: [ + { + handlerId: "observations", + state: "waiting" as const, + error: "x".repeat(1_500), + }, + ], + })); + const payload = await store.prepareRequests({ + identity, + entries, + inlineDocument: (value) => ({ + value, + padding: "x".repeat(256 * 1024), + }), + }); + expect(payload.storage).toBe("chunked"); + const projected = entries.map((entry) => ({ + ...entry, + dependencies: [{ handlerId: "observations", state: "ready" as const }], + })); + collection.failOnWrite(collection.writeCount() + 2); + + const failed = await store.casUpdateRequests({ + identity, + payload, + entries: projected, + }); + expect(failed).toMatchObject({ + status: "partial-failure", + message: "injected write failure", + }); + if (failed.status !== "partial-failure") { + throw new Error("expected partial failure"); + } + expect(failed.updatedChunkIds).toHaveLength(1); + + collection.failOnWrite(-1); + const repaired = await store.casUpdateRequests({ + identity, + payload, + entries: projected, + }); + expect(repaired.status).toBe("applied"); + if (repaired.status !== "applied") throw new Error("expected applied"); + expect(await store.readRequests(identity, repaired.payload)).toEqual( + projected, + ); + }); + + it("rejects frozen or ownership-mismatched buckets without success", async () => { + const collection = chunkCollection(); + const store = new FixedPayloadStore( + collection as unknown as Collection, + ); + const entries = [request(1)]; + const payload = await store.prepareRequests({ + identity, + entries, + inlineDocument: (value) => ({ + value, + padding: "x".repeat(256 * 1024), + }), + }); + if (payload.storage !== "chunked") throw new Error("expected chunked"); + const chunk = collection.documents.get(payload.chunkIds[0]!); + if (!chunk) throw new Error("missing chunk"); + chunk.frozenAt = new Date(); + + await expect( + store.casUpdateRequests({ + identity, + payload, + entries: [ + { + ...entries[0]!, + dependencies: [ + { handlerId: "observations", state: "ready" }, + ], + }, + ], + }), + ).resolves.toMatchObject({ status: "stale", updatedChunkIds: [] }); + + delete chunk.frozenAt; + chunk.ownerId = "other-execution"; + await expect( + store.casUpdateRequests({ + identity, + payload, + entries, + }), + ).resolves.toMatchObject({ + status: "invariant", + updatedChunkIds: [], + }); + }); + + it("rejects an update that exceeds the fixed entry budget", async () => { + const collection = chunkCollection(); + const store = new FixedPayloadStore( + collection as unknown as Collection, + ); + const payload = await store.prepareRequests({ + identity, + entries: [request(1)], + inlineDocument: (value) => ({ value }), + }); + + await expect( + store.casUpdateRequests({ + identity, + payload, + entries: [ + { + ...request(1), + dependencies: [ + { + handlerId: "observations", + state: "failed", + error: "x".repeat(4_096), + }, + ], + }, + ], + }), + ).resolves.toMatchObject({ + status: "invariant", + updatedChunkIds: [], + }); + }); }); diff --git a/packages/shared/src/experiments/fixed-payload-store.ts b/packages/shared/src/experiments/fixed-payload-store.ts index 9d7b91340..60097dd3f 100644 --- a/packages/shared/src/experiments/fixed-payload-store.ts +++ b/packages/shared/src/experiments/fixed-payload-store.ts @@ -3,11 +3,17 @@ import type { Collection, Document } from "mongodb"; import { z } from "zod"; -import { selectFixedPayloadLayout } from "./layout.js"; +import { + assertExperimentRequestEntryBudget, + bsonSize, + experimentPayloadBudgetBytes, + selectFixedPayloadLayout, +} from "./layout.js"; import { ExperimentRequestStateSchema, } from "./schemas.js"; import { + assignExperimentRequestBucket, computeExperimentChunkId, fingerprintExperimentPayload, } from "./routing.js"; @@ -30,8 +36,11 @@ interface ExperimentChunkDocument { storageProfileVersion: typeof EXPERIMENT_STORAGE_PROFILE_VERSION; assignmentVersion: typeof EXPERIMENT_REQUEST_BUCKET_ASSIGNMENT_VERSION; payloadFingerprint: string; + version: number; entries: Array<{ position: number; state: ExperimentRequestState }>; createdAt: Date; + updatedAt: Date; + frozenAt?: Date; } const ExperimentChunkDocumentSchema = z @@ -49,6 +58,7 @@ const ExperimentChunkDocumentSchema = z EXPERIMENT_REQUEST_BUCKET_ASSIGNMENT_VERSION, ), payloadFingerprint: z.string().min(1), + version: z.number().int().positive(), entries: z.array( z .object({ @@ -58,6 +68,8 @@ const ExperimentChunkDocumentSchema = z .strict(), ), createdAt: z.coerce.date(), + updatedAt: z.coerce.date(), + frozenAt: z.coerce.date().optional(), }) .strict(); @@ -84,6 +96,7 @@ function sameChunk( left.storageProfileVersion === right.storageProfileVersion && left.assignmentVersion === right.assignmentVersion && left.payloadFingerprint === right.payloadFingerprint && + left.version === right.version && fingerprintExperimentPayload(left.entries) === fingerprintExperimentPayload(right.entries) ); @@ -117,6 +130,7 @@ export class FixedPayloadStore { storage: "inline", entryCount: inlineEntries.length, payloadFingerprint, + payloadVersion: 1, entries: inlineEntries, }), chunkDocument: (chunkIndex, bucketCount, bucketEntries) => @@ -129,6 +143,7 @@ export class FixedPayloadStore { state, })), payloadFingerprint, + version: 1, createdAt: input.now ?? new Date(0), }), }); @@ -137,6 +152,7 @@ export class FixedPayloadStore { storage: "inline", entryCount: entries.length, payloadFingerprint, + payloadVersion: 1, entries, }; } @@ -152,6 +168,7 @@ export class FixedPayloadStore { state, })), payloadFingerprint, + version: 1, createdAt, }), ); @@ -167,6 +184,7 @@ export class FixedPayloadStore { storage: "chunked", entryCount: entries.length, payloadFingerprint, + payloadVersion: 1, assignmentVersion: EXPERIMENT_REQUEST_BUCKET_ASSIGNMENT_VERSION, bucketCount: layout.bucketCount, chunkIds: documents.map((document) => document._id), @@ -220,6 +238,208 @@ export class FixedPayloadStore { await this.readRequests(identity, payload); } + async casUpdateRequests(input: { + identity: FixedPayloadIdentity; + payload: FixedPayload; + entries: readonly ExperimentRequestState[]; + now?: Date; + }): Promise< + | { + status: "applied"; + payload: FixedPayload; + updatedChunkIds: string[]; + } + | { status: "stale"; updatedChunkIds: string[] } + | { status: "invariant"; message: string; updatedChunkIds: string[] } + | { + status: "partial-failure"; + message: string; + updatedChunkIds: string[]; + } + > { + let entries: ExperimentRequestState[]; + try { + entries = input.entries.map((entry) => { + const parsed = ExperimentRequestStateSchema.parse(entry); + assertExperimentRequestEntryBudget(parsed); + return parsed; + }); + if (entries.length !== input.payload.entryCount) { + return { + status: "invariant", + message: "Request-state update cannot change entry count", + updatedChunkIds: [], + }; + } + } catch (error) { + return { + status: "invariant", + message: error instanceof Error ? error.message : String(error), + updatedChunkIds: [], + }; + } + + const nextFingerprint = fingerprintExperimentPayload(entries); + const nextVersion = input.payload.payloadVersion + 1; + if (input.payload.storage === "inline") { + return { + status: "applied", + payload: { + ...input.payload, + payloadFingerprint: nextFingerprint, + payloadVersion: nextVersion, + entries, + }, + updatedChunkIds: [], + }; + } + const chunkedPayload = input.payload; + + const positionedBuckets = Array.from( + { length: chunkedPayload.bucketCount }, + () => + [] as Array<{ position: number; state: ExperimentRequestState }>, + ); + entries.forEach((state, position) => { + positionedBuckets[ + assignExperimentRequestBucket(state.requestId, chunkedPayload.bucketCount) + ]!.push({ position, state }); + }); + const expectedChunkIds = positionedBuckets.map((_entries, chunkIndex) => + computeExperimentChunkId({ + ...input.identity, + kind: "requests", + chunkIndex, + }), + ); + if ( + expectedChunkIds.length !== chunkedPayload.chunkIds.length || + expectedChunkIds.some( + (chunkId, index) => chunkId !== chunkedPayload.chunkIds[index], + ) + ) { + return { + status: "invariant", + message: "Chunk topology does not match the immutable ownership tuple", + updatedChunkIds: [], + }; + } + + const now = input.now ?? new Date(); + const desired = positionedBuckets.map((bucketEntries, chunkIndex) => + this.chunkDocument({ + identity: input.identity, + chunkIndex, + bucketCount: chunkedPayload.bucketCount, + entries: bucketEntries, + payloadFingerprint: nextFingerprint, + version: nextVersion, + createdAt: now, + }), + ); + const budget = experimentPayloadBudgetBytes(); + if ( + desired.some( + (document) => + bsonSize(document) > budget, + ) + ) { + return { + status: "invariant", + message: + "Projection update exceeds the existing fixed bucket BSON budget", + updatedChunkIds: [], + }; + } + + const updatedChunkIds: string[] = []; + for (const document of desired) { + try { + const result = await this.chunks.updateOne( + { + _id: document._id, + projectId: input.identity.projectId, + experimentId: input.identity.experimentId, + kind: "requests", + ownerId: input.identity.ownerId, + generation: input.identity.generation, + chunkIndex: document.chunkIndex, + bucketCount: chunkedPayload.bucketCount, + assignmentVersion: chunkedPayload.assignmentVersion, + payloadFingerprint: chunkedPayload.payloadFingerprint, + version: chunkedPayload.payloadVersion, + frozenAt: { $exists: false }, + } as Document, + { + $set: { + entries: document.entries, + payloadFingerprint: nextFingerprint, + version: nextVersion, + updatedAt: now, + }, + } as Document, + ); + if (result.modifiedCount === 1) { + updatedChunkIds.push(document._id); + continue; + } + const raw: unknown = await this.chunks.findOne({ + _id: document._id, + } as Document); + if (!raw) { + return { + status: updatedChunkIds.length > 0 ? "partial-failure" : "invariant", + message: `Missing experiment chunk ${document._id}`, + updatedChunkIds, + }; + } + const actual = ExperimentChunkDocumentSchema.parse( + raw, + ) as ExperimentChunkDocument; + if ( + actual.version === nextVersion && + actual.payloadFingerprint === nextFingerprint && + sameChunk(actual, document) + ) { + updatedChunkIds.push(document._id); + continue; + } + try { + this.assertIdentity(actual, input.identity, chunkedPayload); + } catch (error) { + return { + status: updatedChunkIds.length > 0 ? "partial-failure" : "invariant", + message: error instanceof Error ? error.message : String(error), + updatedChunkIds, + }; + } + if (updatedChunkIds.length > 0) { + return { + status: "partial-failure", + message: `Chunk ${document._id} lost its CAS`, + updatedChunkIds, + }; + } + return { status: "stale", updatedChunkIds }; + } catch (error) { + return { + status: "partial-failure", + message: error instanceof Error ? error.message : String(error), + updatedChunkIds, + }; + } + } + return { + status: "applied", + payload: { + ...chunkedPayload, + payloadFingerprint: nextFingerprint, + payloadVersion: nextVersion, + }, + updatedChunkIds, + }; + } + async discardUnpublished( payload: FixedPayload, ): Promise { @@ -236,6 +456,7 @@ export class FixedPayloadStore { bucketCount: number; entries: Array<{ position: number; state: ExperimentRequestState }>; payloadFingerprint: string; + version: number; createdAt: Date; }): ExperimentChunkDocument { return { @@ -251,8 +472,10 @@ export class FixedPayloadStore { storageProfileVersion: EXPERIMENT_STORAGE_PROFILE_VERSION, assignmentVersion: EXPERIMENT_REQUEST_BUCKET_ASSIGNMENT_VERSION, payloadFingerprint: input.payloadFingerprint, + version: input.version, entries: input.entries, createdAt: input.createdAt, + updatedAt: input.createdAt, }; } @@ -297,7 +520,8 @@ export class FixedPayloadStore { document.generation !== identity.generation || document.bucketCount !== payload.bucketCount || document.assignmentVersion !== payload.assignmentVersion || - document.payloadFingerprint !== payload.payloadFingerprint + document.payloadFingerprint !== payload.payloadFingerprint || + document.version !== payload.payloadVersion ) { throw new Error(`Experiment chunk ${document._id} identity mismatch`); } diff --git a/packages/shared/src/experiments/layout.ts b/packages/shared/src/experiments/layout.ts index c2ce92b07..562694a3f 100644 --- a/packages/shared/src/experiments/layout.ts +++ b/packages/shared/src/experiments/layout.ts @@ -25,7 +25,7 @@ export function bsonSize(value: object): number { return BSON.calculateObjectSize(value); } -function payloadBudgetBytes(): number { +export function experimentPayloadBudgetBytes(): number { return Math.floor( (EXPERIMENT_STORAGE_PROFILE.targetDocumentBytes - EXPERIMENT_STORAGE_PROFILE.metadataReserveBytes) * @@ -33,6 +33,20 @@ function payloadBudgetBytes(): number { ); } +export function assertExperimentRequestEntryBudget( + entry: ExperimentRequestState, +): void { + const bytes = bsonSize({ + position: Number.MAX_SAFE_INTEGER, + state: entry, + }); + if (bytes > EXPERIMENT_STORAGE_PROFILE.maxRequestEntryBytes) { + throw new Error( + `Experiment request ${entry.requestId} is ${bytes} bytes; maximum is ${EXPERIMENT_STORAGE_PROFILE.maxRequestEntryBytes}`, + ); + } +} + export function selectFixedPayloadLayout(input: { entries: readonly ExperimentRequestState[]; inlineDocument: (entries: ExperimentRequestState[]) => object; @@ -49,15 +63,7 @@ export function selectFixedPayloadLayout(input: { throw new Error(`Duplicate experiment requestId '${entry.requestId}'`); } ids.add(entry.requestId); - const bytes = bsonSize({ - position: Number.MAX_SAFE_INTEGER, - state: entry, - }); - if (bytes > EXPERIMENT_STORAGE_PROFILE.maxRequestEntryBytes) { - throw new Error( - `Experiment request ${entry.requestId} is ${bytes} bytes; maximum is ${EXPERIMENT_STORAGE_PROFILE.maxRequestEntryBytes}`, - ); - } + assertExperimentRequestEntryBudget(entry); } const inlineBytes = bsonSize(input.inlineDocument(entries)); @@ -76,7 +82,7 @@ export function selectFixedPayloadLayout(input: { throw new Error("An empty fixed payload must be stored inline"); } - const budget = payloadBudgetBytes(); + const budget = experimentPayloadBudgetBytes(); const maxEntriesPerBucket = Math.max( 1, Math.floor( diff --git a/packages/shared/src/experiments/routing.ts b/packages/shared/src/experiments/routing.ts index bd9afd8fe..dc5cacbf9 100644 --- a/packages/shared/src/experiments/routing.ts +++ b/packages/shared/src/experiments/routing.ts @@ -2,6 +2,7 @@ // Licensed under the MIT License. import { createHash } from "node:crypto"; +import type { ExperimentRequestState } from "./types.js"; const REQUEST_BUCKET_HASH_PREFIX = "experiment-request-bucket:v1\0"; @@ -49,3 +50,11 @@ export function fingerprintExperimentPayload(value: unknown): string { .update(JSON.stringify(value), "utf8") .digest("base64url"); } + +export function fingerprintExperimentRequestVector( + requests: readonly Pick[], +): string { + return fingerprintExperimentPayload( + requests.map(({ requestId, runId }) => ({ requestId, runId })), + ); +} diff --git a/packages/shared/src/experiments/schemas.test.ts b/packages/shared/src/experiments/schemas.test.ts index 9a4b6fb2d..64b80727c 100644 --- a/packages/shared/src/experiments/schemas.test.ts +++ b/packages/shared/src/experiments/schemas.test.ts @@ -64,6 +64,7 @@ describe("experiment schemas", () => { storage: "inline", entryCount: 0, payloadFingerprint: "f", + payloadVersion: 1, entries: [], }, vectorFingerprint: "v", diff --git a/packages/shared/src/experiments/schemas.ts b/packages/shared/src/experiments/schemas.ts index 0022421f3..d3a96e088 100644 --- a/packages/shared/src/experiments/schemas.ts +++ b/packages/shared/src/experiments/schemas.ts @@ -83,6 +83,7 @@ export const ExperimentFixedRequestPayloadSchema = z.discriminatedUnion( storage: z.literal("inline"), entryCount: z.number().int().nonnegative(), payloadFingerprint: IdSchema, + payloadVersion: z.number().int().positive(), entries: z.array(ExperimentRequestStateSchema), }) .strict() @@ -94,6 +95,7 @@ export const ExperimentFixedRequestPayloadSchema = z.discriminatedUnion( storage: z.literal("chunked"), entryCount: z.number().int().positive(), payloadFingerprint: IdSchema, + payloadVersion: z.number().int().positive(), assignmentVersion: z.literal( EXPERIMENT_REQUEST_BUCKET_ASSIGNMENT_VERSION, ), @@ -130,6 +132,15 @@ const ExperimentExecutionBaseSchema = z supersededAt: z.coerce.date().optional(), supersededByExperimentExecutionId: IdSchema.optional(), supersededReason: z.enum(["rollover", "retry"]).optional(), + requestStateUpdate: z + .object({ + updateId: IdSchema, + expectedPayloadFingerprint: IdSchema, + nextPayloadFingerprint: IdSchema, + claimedAt: z.coerce.date(), + }) + .strict() + .optional(), }) .strict(); diff --git a/packages/shared/src/experiments/types.ts b/packages/shared/src/experiments/types.ts index 8d8595259..a3b64d2fa 100644 --- a/packages/shared/src/experiments/types.ts +++ b/packages/shared/src/experiments/types.ts @@ -12,6 +12,7 @@ export interface InlineFixedPayload { storage: "inline"; entryCount: number; payloadFingerprint: string; + payloadVersion: number; entries: T[]; } @@ -19,6 +20,7 @@ export interface ChunkedFixedPayload { storage: "chunked"; entryCount: number; payloadFingerprint: string; + payloadVersion: number; assignmentVersion: typeof EXPERIMENT_REQUEST_BUCKET_ASSIGNMENT_VERSION; bucketCount: number; chunkIds: string[]; @@ -101,6 +103,13 @@ export interface ExperimentRolloverClaim { claimedAt: Date; } +export interface ExperimentRequestStateUpdateClaim { + updateId: string; + expectedPayloadFingerprint: string; + nextPayloadFingerprint: string; + claimedAt: Date; +} + export interface ExperimentExecution { experimentExecutionId: string; attemptNumber: number; @@ -119,6 +128,7 @@ export interface ExperimentExecution { supersededAt?: Date; supersededByExperimentExecutionId?: string; supersededReason?: "rollover" | "retry"; + requestStateUpdate?: ExperimentRequestStateUpdateClaim; rollover?: ExperimentRolloverClaim; } @@ -150,3 +160,29 @@ export interface ExperimentPage { experiments: ExperimentDocument[]; nextCursor?: string; } + +export type ExperimentRequestStateUpdateResult = + | { + status: "applied"; + experiment: ExperimentDocument; + requests: ExperimentRequestState[]; + } + | { + status: "stale"; + reason: "experiment" | "execution" | "version" | "frozen"; + } + | { + status: "contention"; + activeUpdateId?: string; + } + | { + status: "invariant"; + message: string; + } + | { + status: "partial-failure"; + updateId: string; + updatedChunkIds: string[]; + message: string; + retryable: true; + }; From a55c388120a803aa0ceb191f1a9acca0ed6d79ff Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Fri, 16 Oct 2026 00:10:23 -0700 Subject: [PATCH 10/25] feat: add Experiment clients and documentation Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4e02b3c3-5ec3-44f8-b88d-642673088e41 --- .../__snapshots__/cli-commands.test.ts.snap | 74 +- apps/cli/src/bundle.integration.test.ts | 2 +- apps/cli/src/commands/experiment.test.ts | 198 ++++ apps/cli/src/commands/experiment.ts | 882 ++++++++++++++++++ apps/cli/src/commands/run.test.ts | 65 +- apps/cli/src/commands/run.ts | 50 +- apps/cli/src/commands/submission.ts | 204 ---- apps/cli/src/index.ts | 4 +- apps/portal/src/App.tsx | 7 +- .../ExperimentStatusCard.stories.tsx | 59 ++ .../components/ExperimentStatusCard.test.tsx | 52 ++ .../src/components/ExperimentStatusCard.tsx | 56 ++ apps/portal/src/components/Layout.tsx | 1 + .../SubmissionStatusCard.stories.tsx | 79 -- .../components/SubmissionStatusCard.test.tsx | 68 -- .../src/components/SubmissionStatusCard.tsx | 54 -- apps/portal/src/lib/api.ts | 195 +++- apps/portal/src/lib/experiments.ts | 321 +++++++ apps/portal/src/pages/ExperimentDetail.tsx | 316 +++++++ apps/portal/src/pages/ExperimentsList.tsx | 120 +++ apps/portal/src/pages/RunDetail.tsx | 8 +- apps/portal/src/pages/RunsList.tsx | 70 +- apps/portal/src/pages/SubmissionDetail.tsx | 162 ---- apps/portal/src/pages/SubmitRun.tsx | 194 +++- apps/portal/src/types.ts | 71 +- website/astro.config.mjs | 2 + .../docs/guides/creating-experiments.mdx | 161 ++++ .../docs/guides/migrating-to-experiments.mdx | 147 +++ .../docs/guides/submitting-requests-api.mdx | 15 +- .../docs/guides/submitting-requests-cli.md | 17 +- .../docs/guides/submitting-requests-portal.md | 14 + .../src/content/docs/reference/rest-api.md | 17 +- 32 files changed, 2907 insertions(+), 778 deletions(-) create mode 100644 apps/cli/src/commands/experiment.test.ts create mode 100644 apps/cli/src/commands/experiment.ts delete mode 100644 apps/cli/src/commands/submission.ts create mode 100644 apps/portal/src/components/ExperimentStatusCard.stories.tsx create mode 100644 apps/portal/src/components/ExperimentStatusCard.test.tsx create mode 100644 apps/portal/src/components/ExperimentStatusCard.tsx delete mode 100644 apps/portal/src/components/SubmissionStatusCard.stories.tsx delete mode 100644 apps/portal/src/components/SubmissionStatusCard.test.tsx delete mode 100644 apps/portal/src/components/SubmissionStatusCard.tsx create mode 100644 apps/portal/src/lib/experiments.ts create mode 100644 apps/portal/src/pages/ExperimentDetail.tsx create mode 100644 apps/portal/src/pages/ExperimentsList.tsx delete mode 100644 apps/portal/src/pages/SubmissionDetail.tsx create mode 100644 website/src/content/docs/guides/creating-experiments.mdx create mode 100644 website/src/content/docs/guides/migrating-to-experiments.mdx diff --git a/apps/cli/src/__snapshots__/cli-commands.test.ts.snap b/apps/cli/src/__snapshots__/cli-commands.test.ts.snap index f972e2107..907676b78 100644 --- a/apps/cli/src/__snapshots__/cli-commands.test.ts.snap +++ b/apps/cli/src/__snapshots__/cli-commands.test.ts.snap @@ -200,6 +200,76 @@ exports[`CLI command registration > full command tree matches snapshot 1`] = ` }, }, }, + "experiment": { + "subcommands": { + "create": { + "options": [ + "--agent-version", + "--agents-md", + "--codebase", + "--count", + "--criteria", + "--description", + "--empty", + "--extensions", + "--gates", + "--max-iterations", + "--mcp-servers", + "--message", + "--model", + "--name", + "--observations", + "--output", + "--persona", + "--preview", + "--profile", + "--profile-variations-file", + "--project", + "--reasoning-effort", + "--resource-param", + "--resources", + "--scenario", + "--skills", + "--tags", + "--traits", + "--url", + "--worker", + ], + }, + "executions": { + "options": [ + "--cursor", + "--limit", + "--output", + "--project", + "--url", + ], + }, + "get": { + "options": [ + "--output", + "--project", + "--url", + ], + }, + "list": { + "options": [ + "--cursor", + "--limit", + "--output", + "--project", + "--url", + ], + }, + "retry": { + "options": [ + "--execution", + "--project", + "--url", + ], + }, + }, + }, "extension": { "subcommands": { "delete": { @@ -764,11 +834,11 @@ exports[`CLI command registration > full command tree matches snapshot 1`] = ` "download-batch": { "options": [ "--dir", + "--experiment-id", "--extract", "--ids", "--output", "--project", - "--submission-id", "--url", ], }, @@ -785,6 +855,7 @@ exports[`CLI command registration > full command tree matches snapshot 1`] = ` "--created-after", "--created-before", "--criteria", + "--experiment-id", "--include-deleted", "--max-iterations", "--model", @@ -798,7 +869,6 @@ exports[`CLI command registration > full command tree matches snapshot 1`] = ` "--sort-by", "--sort-dir", "--status", - "--submission-id", "--task", "--turns", "--url", diff --git a/apps/cli/src/bundle.integration.test.ts b/apps/cli/src/bundle.integration.test.ts index 11d8e0714..c1ca6b791 100644 --- a/apps/cli/src/bundle.integration.test.ts +++ b/apps/cli/src/bundle.integration.test.ts @@ -26,7 +26,7 @@ const MOCK_RESPONSES: Record = { id: "req-test-001", workerType: "coder-acp-copilot", run: { status: "done", outcome: "succeeded" }, - submissionId: "sub-abc123", + experimentId: "experiment-abc123", }, ], pageInfo: { hasNextPage: false, hasPreviousPage: false }, diff --git a/apps/cli/src/commands/experiment.test.ts b/apps/cli/src/commands/experiment.test.ts new file mode 100644 index 000000000..e21981f91 --- /dev/null +++ b/apps/cli/src/commands/experiment.test.ts @@ -0,0 +1,198 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { Command } from "commander"; +import { registerExperimentCommands } from "./experiment.js"; + +interface CapturedRequest { + url: string; + method: string; + body: string; +} + +const experiment = { + experimentId: "experiment-1", + projectId: "project-1", + metadata: { name: "Experiment 1" }, + revision: { + experimentRevisionId: "revision-1", + revisionNumber: 1, + memberCount: 0, + }, + execution: { + experimentExecutionId: "execution-1", + attemptNumber: 1, + readiness: "ready", + requests: [], + handlerProgress: {}, + blocking: [], + createdAt: "2026-01-01T00:00:00.000Z", + }, + readiness: "ready", + blocking: [], + createdRequestIds: [], + createdAt: "2026-01-01T00:00:00.000Z", +}; + +function makeProgram(): Command { + const program = new Command(); + registerExperimentCommands(program); + return program; +} + +function mockFetchWith( + body: unknown, +): { requests: CapturedRequest[] } { + const requests: CapturedRequest[] = []; + vi.stubGlobal( + "fetch", + vi.fn(async (request: Request) => { + requests.push({ + url: request.url, + method: request.method, + body: await request.text(), + }); + return new Response(JSON.stringify(body), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + }), + ); + return { requests }; +} + +describe("experiment commands", () => { + beforeEach(() => { + vi.restoreAllMocks(); + process.exitCode = undefined; + }); + + afterEach(() => { + vi.restoreAllMocks(); + process.exitCode = undefined; + }); + + it("previews composition without calling the API or warning", async () => { + const fetchSpy = vi.fn(); + vi.stubGlobal("fetch", fetchSpy); + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + + await makeProgram().parseAsync( + [ + "experiment", + "create", + "--name", + "Nightly", + "--message", + "Run tests", + "--count", + "2", + "--preview", + "--output", + "json", + "--project", + "project-1", + "--url", + "http://localhost:3100", + ], + { from: "user" }, + ); + + expect(fetchSpy).not.toHaveBeenCalled(); + expect(warn).not.toHaveBeenCalled(); + expect(log.mock.calls.flat().join("\n")).toContain('"expandedRunCount": 2'); + expect(log.mock.calls.flat().join("\n")).toContain('"count": 2'); + }); + + it("creates an explicitly empty project-scoped experiment", async () => { + const { requests } = mockFetchWith(experiment); + vi.spyOn(console, "log").mockImplementation(() => {}); + + await makeProgram().parseAsync( + [ + "experiment", + "create", + "--name", + "Empty", + "--empty", + "--project", + "project-1", + "--url", + "http://localhost:3100", + ], + { from: "user" }, + ); + + expect(requests).toHaveLength(1); + expect(requests[0]).toMatchObject({ + method: "POST", + body: JSON.stringify({ + metadata: { name: "Empty" }, + empty: true, + }), + }); + expect(requests[0]?.url).toBe( + "http://localhost:3100/api/v1/experiments?projectId=project-1", + ); + }); + + it("gets an exact project-scoped execution", async () => { + const { requests } = mockFetchWith(experiment.execution); + vi.spyOn(console, "log").mockImplementation(() => {}); + + await makeProgram().parseAsync( + [ + "experiment", + "executions", + "experiment-1", + "execution-1", + "--project", + "project-1", + "--url", + "http://localhost:3100", + ], + { from: "user" }, + ); + + expect(requests[0]?.url).toBe( + "http://localhost:3100/api/v1/experiments/experiment-1/executions/execution-1?projectId=project-1", + ); + }); + + it("retries with the current execution identifier", async () => { + const { requests } = mockFetchWith({ + ...experiment, + execution: { + ...experiment.execution, + experimentExecutionId: "execution-2", + attemptNumber: 2, + }, + }); + vi.spyOn(console, "log").mockImplementation(() => {}); + + await makeProgram().parseAsync( + [ + "experiment", + "retry", + "experiment-1", + "--execution", + "execution-1", + "--project", + "project-1", + "--url", + "http://localhost:3100", + ], + { from: "user" }, + ); + + expect(requests[0]).toMatchObject({ + method: "POST", + body: JSON.stringify({ experimentExecutionId: "execution-1" }), + }); + expect(requests[0]?.url).toBe( + "http://localhost:3100/api/v1/experiments/experiment-1/retry?projectId=project-1", + ); + }); +}); diff --git a/apps/cli/src/commands/experiment.ts b/apps/cli/src/commands/experiment.ts new file mode 100644 index 000000000..9fd62faaf --- /dev/null +++ b/apps/cli/src/commands/experiment.ts @@ -0,0 +1,882 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { Command } from "commander"; +import { stringify as stringifyYaml } from "yaml"; +import { resolveScenarioAndPersona } from "../config-loader.js"; +import { apiFetch } from "../utils/api-client.js"; +import { requireProjectId } from "../utils/config.js"; +import { formatData, isMachineReadable } from "../utils/formatters.js"; +import { parseGatesOption } from "../utils/gates.js"; +import { configureHelp } from "../utils/helpFormatter.js"; +import { buildResourceBindingSpecs, collectRepeatable } from "../utils/resources.js"; +import { + getDefaultApiUrl, + withOutputOption, + withProjectOption, +} from "../utils/shared.js"; +import { errorText, label, successText, value } from "../utils/style.js"; +import type { DisplayField, OutputFormat } from "../utils/types.js"; + +interface ExperimentMetadata { + name: string; + description?: string; + tags?: string[]; +} + +interface ExperimentExecution { + experimentExecutionId: string; + attemptNumber: number; + version?: number; + readiness?: string; + requestCount: number; + requests: ExperimentRequestSummary[]; + handlerProgress: JsonRecord; + blocking: ExperimentBlocker[]; + currentReport?: { + reportId?: string; + url?: string; + updatedAt?: string; + }; + createdAt: string; + updatedAt?: string; + startedAt?: string; + finishedAt?: string; + discardedAt?: string; + discardReason?: string; + supersededAt?: string; + supersededByExperimentExecutionId?: string; + supersededReason?: "rollover" | "retry"; + archivedAt?: string; +} + +interface ExperimentBlocker { + handlerId: string; + reason: string; + requestId?: string; + runId?: string; + error?: string; +} + +interface ExperimentRequestSummary { + requestId: string; + runId: string; + task?: string; + status?: string; + outcome?: string; + workerType?: string; + model?: string; + profileId?: string; +} + +interface ExperimentView { + experimentId: string; + projectId: string; + metadata: ExperimentMetadata; + revision: { + experimentRevisionId: string; + revisionNumber: number; + memberCount: number; + membershipFingerprint?: string; + definitionFingerprint?: string; + createdAt?: string; + }; + execution: ExperimentExecution; + readiness?: string; + blocking: ExperimentBlocker[]; + createdRequestIds: string[]; + createdAt: string; + updatedAt?: string; + finalizedAt?: string; +} + +interface ExperimentPage { + items: ExperimentView[]; + nextCursor?: string; +} + +interface ExperimentExecutionPage { + items: ExperimentExecution[]; + nextCursor?: string; +} + +type JsonRecord = Record; + +function asRecord(value: unknown, name: string): JsonRecord { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`Invalid ${name} response from Scope`); + } + return value as JsonRecord; +} + +function stringField( + value: JsonRecord, + key: string, + name: string, + optional = false, +): string | undefined { + const field = value[key]; + if (field === undefined && optional) return undefined; + if (typeof field !== "string" || field.length === 0) { + throw new Error(`Invalid ${name}.${key} response from Scope`); + } + return field; +} + +function numberField(value: JsonRecord, key: string, name: string): number { + const field = value[key]; + if (typeof field !== "number" || !Number.isFinite(field)) { + throw new Error(`Invalid ${name}.${key} response from Scope`); + } + return field; +} + +function parseBlocker(value: unknown): ExperimentBlocker { + const blocker = asRecord(value, "experiment blocker"); + return { + handlerId: stringField(blocker, "handlerId", "experiment blocker")!, + reason: stringField(blocker, "reason", "experiment blocker")!, + requestId: stringField(blocker, "requestId", "experiment blocker", true), + runId: stringField(blocker, "runId", "experiment blocker", true), + error: stringField(blocker, "error", "experiment blocker", true), + }; +} + +function parseRequestSummary(value: unknown): ExperimentRequestSummary { + const request = asRecord(value, "experiment request"); + return { + requestId: stringField(request, "requestId", "experiment request")!, + runId: stringField(request, "runId", "experiment request")!, + task: stringField(request, "task", "experiment request", true), + status: stringField(request, "status", "experiment request", true), + outcome: stringField(request, "outcome", "experiment request", true), + workerType: stringField(request, "workerType", "experiment request", true), + model: stringField(request, "model", "experiment request", true), + profileId: stringField(request, "profileId", "experiment request", true), + }; +} + +function parseExecution(value: unknown): ExperimentExecution { + const execution = asRecord(value, "experiment execution"); + const requests = execution.requests; + const requestVector = Array.isArray(requests) + ? { requestCount: requests.length, requests: requests.map(parseRequestSummary) } + : (() => { + const payload = requests === undefined + ? {} + : asRecord(requests, "experiment requests"); + const entries = Array.isArray(payload.entries) + ? payload.entries.map(parseRequestSummary) + : []; + return { + requestCount: + typeof payload.entryCount === "number" + ? payload.entryCount + : entries.length, + requests: entries, + }; + })(); + const blocking = Array.isArray(execution.blocking) + ? execution.blocking.map(parseBlocker) + : []; + const progress = asRecord( + execution.handlerProgress ?? execution.handlerSummary ?? {}, + "experiment handler progress", + ); + const report = execution.currentReport === undefined + ? undefined + : asRecord(execution.currentReport, "experiment report"); + return { + experimentExecutionId: stringField( + execution, + "experimentExecutionId", + "experiment execution", + )!, + attemptNumber: numberField( + execution, + "attemptNumber", + "experiment execution", + ), + readiness: stringField( + execution, + "readiness", + "experiment execution", + true, + ), + version: + typeof execution.version === "number" ? execution.version : undefined, + requestCount: requestVector.requestCount, + requests: requestVector.requests, + handlerProgress: progress, + blocking, + currentReport: report + ? { + reportId: stringField(report, "reportId", "experiment report", true), + url: stringField(report, "url", "experiment report", true), + updatedAt: stringField(report, "updatedAt", "experiment report", true), + } + : undefined, + createdAt: stringField( + execution, + "createdAt", + "experiment execution", + )!, + updatedAt: stringField(execution, "updatedAt", "experiment execution", true), + startedAt: stringField(execution, "startedAt", "experiment execution", true), + finishedAt: stringField(execution, "finishedAt", "experiment execution", true), + discardedAt: stringField(execution, "discardedAt", "experiment execution", true), + discardReason: stringField(execution, "discardReason", "experiment execution", true), + supersededAt: stringField(execution, "supersededAt", "experiment execution", true), + supersededByExperimentExecutionId: stringField( + execution, + "supersededByExperimentExecutionId", + "experiment execution", + true, + ), + supersededReason: + execution.supersededReason === "rollover" || + execution.supersededReason === "retry" + ? execution.supersededReason + : undefined, + archivedAt: stringField(execution, "archivedAt", "experiment execution", true), + }; +} + +function parseExperiment(value: unknown): ExperimentView { + const experiment = asRecord(value, "experiment"); + const metadata = asRecord(experiment.metadata, "experiment metadata"); + const revision = asRecord(experiment.revision, "experiment revision"); + const execution = parseExecution( + experiment.execution ?? experiment.currentExecution, + ); + const blockingValue = experiment.blocking ?? execution.blocking; + const blocking = Array.isArray(blockingValue) + ? blockingValue.map(parseBlocker) + : []; + const createdRequestIds = Array.isArray(experiment.createdRequestIds) + ? experiment.createdRequestIds.map((id) => { + if (typeof id !== "string") { + throw new Error("Invalid experiment.createdRequestIds response from Scope"); + } + return id; + }) + : []; + const tags = Array.isArray(metadata.tags) + ? metadata.tags.map((tag) => { + if (typeof tag !== "string") { + throw new Error("Invalid experiment metadata tag response from Scope"); + } + return tag; + }) + : undefined; + + return { + experimentId: stringField(experiment, "experimentId", "experiment")!, + projectId: stringField(experiment, "projectId", "experiment")!, + metadata: { + name: stringField(metadata, "name", "experiment metadata")!, + description: stringField( + metadata, + "description", + "experiment metadata", + true, + ), + tags, + }, + revision: { + experimentRevisionId: stringField( + revision, + "experimentRevisionId", + "experiment revision", + )!, + revisionNumber: numberField( + revision, + "revisionNumber", + "experiment revision", + ), + memberCount: + typeof revision.memberCount === "number" + ? revision.memberCount + : execution.requestCount, + membershipFingerprint: stringField( + revision, + "membershipFingerprint", + "experiment revision", + true, + ), + definitionFingerprint: stringField( + revision, + "definitionFingerprint", + "experiment revision", + true, + ), + createdAt: stringField(revision, "createdAt", "experiment revision", true), + }, + execution, + readiness: + stringField(experiment, "readiness", "experiment", true) ?? + execution.readiness, + blocking, + createdRequestIds, + createdAt: stringField(experiment, "createdAt", "experiment")!, + updatedAt: stringField(experiment, "updatedAt", "experiment", true), + finalizedAt: stringField(experiment, "finalizedAt", "experiment", true), + }; +} + +function parsePage(value: unknown): ExperimentPage { + const page = asRecord(value, "experiment page"); + if (!Array.isArray(page.items)) { + throw new Error("Invalid experiment page items response from Scope"); + } + return { + items: page.items.map(parseExperiment), + nextCursor: stringField(page, "nextCursor", "experiment page", true), + }; +} + +function parseExecutionPage(value: unknown): ExperimentExecutionPage { + const page = asRecord(value, "experiment execution page"); + if (!Array.isArray(page.items)) { + throw new Error("Invalid experiment execution page items response from Scope"); + } + return { + items: page.items.map(parseExecution), + nextCursor: stringField( + page, + "nextCursor", + "experiment execution page", + true, + ), + }; +} + +async function json(response: Response): Promise { + return response.json() as Promise; +} + +async function throwApiError(response: Response): Promise { + const value = await json(response); + const body = asRecord(value, "error"); + throw new Error( + typeof body.error === "string" ? body.error : `HTTP ${response.status}`, + ); +} + +function resolveTextOrFile(input: string): string { + return input.startsWith("@") + ? readFileSync(resolve(input.slice(1)), "utf8") + : input; +} + +const fields: DisplayField[] = [ + { key: "experimentId", label: "Experiment" }, + { + key: "metadata", + label: "Name", + formatter: (row: JsonRecord) => + String((row.metadata as ExperimentMetadata | undefined)?.name ?? ""), + }, + { + key: "execution", + label: "Attempt", + formatter: (row: JsonRecord) => + String((row.execution as ExperimentExecution | undefined)?.attemptNumber ?? ""), + }, + { key: "readiness", label: "Readiness" }, + { + key: "revision", + label: "Runs", + formatter: (row: JsonRecord) => + String( + (row.revision as ExperimentView["revision"] | undefined)?.memberCount ?? 0, + ), + }, + { key: "createdAt", label: "Created" }, +]; + +function addRunConfigurationOptions(command: Command): Command { + return command + .option("-s, --scenario ", "Scenario YAML file") + .option("-p, --persona ", "Persona YAML file") + .option("-t, --traits ", "Traits YAML file") + .option("-m, --message ", "Task prompt") + .option("-w, --worker ", "Registered worker ID") + .option("-c, --criteria ", "Evaluation criteria") + .option("--max-iterations ", "Max judge iterations", Number.parseInt) + .option("--model ", "Coding-agent model") + .option("--reasoning-effort ", "Reasoning effort") + .option("--mcp-servers ", "MCP server slugs") + .option("--skills ", "Skill slugs") + .option("--codebase ", "Codebase revision reference") + .option("--resources ", "Resource binding specs") + .option( + "--resource-param :=", + "Resource parameter override (repeatable)", + collectRepeatable, + [], + ) + .option("--extensions ", "VS Code extension IDs") + .option("--agent-version ", "Agent version") + .option("--profile ", "Base profile ID or version ref") + .option( + "--profile-variations-file ", + "JSON file containing profile variation refs", + ) + .option( + "--agents-md ", + "AGENTS.md content; prefix a path with @", + ) + .option("--gates ", "Gate configuration JSON or file") + .option("--observations ", "Observation criterion IDs") + .option( + "--count ", + "Occurrences per configuration (1-10)", + (value: string) => Number.parseInt(value, 10), + ); +} + +function buildRequest(options: Record): { + worker?: string; + request: JsonRecord; + profileCount: number; + occurrences: number; +} { + let message = + typeof options.message === "string" ? options.message : undefined; + let criteria = Array.isArray(options.criteria) + ? options.criteria.filter((item): item is string => typeof item === "string") + : undefined; + let personaInstructions: string | undefined; + let persona: Record | undefined; + if (typeof options.scenario === "string") { + const resolved = resolveScenarioAndPersona( + options.scenario, + typeof options.persona === "string" ? options.persona : undefined, + typeof options.traits === "string" ? options.traits : undefined, + ); + message ??= resolved.task; + if (!criteria?.length) criteria = resolved.criteria; + personaInstructions = resolved.personaInstructions; + persona = resolved.persona; + } + if (!message?.trim()) { + throw new Error("--message or --scenario is required unless --empty is used"); + } + + const occurrences = + typeof options.count === "number" ? options.count : 1; + if (!Number.isInteger(occurrences) || occurrences < 1 || occurrences > 10) { + throw new Error("--count must be an integer between 1 and 10"); + } + + const request: JsonRecord = { + scenario: { task: message.trim(), criteria: criteria ?? [] }, + }; + const copyString = (source: string, target = source): void => { + if (typeof options[source] === "string" && options[source]) { + request[target] = options[source]; + } + }; + copyString("model"); + copyString("reasoningEffort"); + copyString("agentVersion"); + copyString("profile", "profileId"); + copyString("codebase"); + if (typeof options.maxIterations === "number") { + request.maxIterations = options.maxIterations; + } + if (personaInstructions) request.personaInstructions = personaInstructions; + if (persona) request.persona = persona; + if (occurrences > 1) request.count = occurrences; + for (const [source, target] of [ + ["mcpServers", "mcpServers"], + ["skills", "skills"], + ["extensions", "extensions"], + ["observations", "observations"], + ] as const) { + if (Array.isArray(options[source]) && options[source].length > 0) { + request[target] = options[source]; + } + } + const resources = buildResourceBindingSpecs( + Array.isArray(options.resources) + ? options.resources.filter( + (item): item is string => typeof item === "string", + ) + : undefined, + Array.isArray(options.resourceParam) + ? options.resourceParam.filter( + (item): item is string => typeof item === "string", + ) + : [], + ); + if (resources) request.resources = resources; + if (typeof options.agentsMd === "string") { + const content = resolveTextOrFile(options.agentsMd); + if (content.trim()) request.agentsMd = content; + } + if (typeof options.gates === "string") { + request.gates = parseGatesOption( + options.gates, + typeof options.maxIterations === "number" + ? options.maxIterations + : undefined, + ); + } + + let profileCount = typeof options.profile === "string" ? 1 : 0; + if (typeof options.profileVariationsFile === "string") { + if (typeof options.profile !== "string") { + throw new Error( + "--profile is required with --profile-variations-file", + ); + } + const parsed: unknown = JSON.parse( + readFileSync(resolve(options.profileVariationsFile), "utf8"), + ); + if (!Array.isArray(parsed) || !parsed.every((item) => typeof item === "string")) { + throw new Error("profile variations file must be a JSON string array"); + } + request.profileVariations = parsed; + profileCount += parsed.length; + } + if (profileCount === 0) profileCount = 1; + + return { + worker: typeof options.worker === "string" ? options.worker : undefined, + request, + profileCount, + occurrences, + }; +} + +export function registerExperimentCommands(program: Command): void { + const experiment = program + .command("experiment") + .description("Create and inspect benchmark experiments") + .action(() => experiment.help()); + configureHelp(experiment); + + const create = addRunConfigurationOptions( + withProjectOption( + withOutputOption( + experiment + .command("create") + .description("Create an experiment from a run configuration") + .requiredOption("--name ", "Experiment name") + .option("--description ", "Experiment description") + .option("--tags ", "Experiment tags") + .option("--empty", "Create an explicit empty experiment") + .option( + "--preview", + "Print the composition and request body without creating", + ) + .option("-u, --url ", "API base URL", getDefaultApiUrl()), + ), + ), + ); + + create.action(async (options) => { + try { + const projectId = requireProjectId(options.project); + const metadata: ExperimentMetadata = { + name: options.name, + ...(options.description ? { description: options.description } : {}), + ...(options.tags?.length ? { tags: options.tags } : {}), + }; + if (options.empty) { + const conflicting = [ + "scenario", + "message", + "worker", + "criteria", + "profile", + "profileVariationsFile", + "count", + ].filter((key) => options[key] !== undefined); + if (conflicting.length > 0) { + throw new Error( + `--empty cannot be combined with run configuration options: ${conflicting.join(", ")}`, + ); + } + } + const built = options.empty ? undefined : buildRequest(options); + const body = options.empty + ? { metadata, empty: true } + : { metadata, request: built!.request }; + const expandedRunCount = built + ? built.profileCount * built.occurrences + : 0; + if (options.preview) { + const preview = { + expandedRunCount, + body, + }; + const format = (options.output ?? "json") as OutputFormat; + if (format === "json") { + console.log(JSON.stringify(preview, null, 2)); + return; + } + if (format === "yaml") { + console.log(stringifyYaml(preview).trimEnd()); + return; + } + console.log( + formatData( + [preview], + [ + { key: "expandedRunCount", label: "Runs" }, + { + key: "body", + label: "Create body", + formatter: () => JSON.stringify(body), + }, + ], + format, + ), + ); + return; + } + const workerQuery = built?.worker + ? `?worker=${encodeURIComponent(built.worker)}` + : ""; + const response = await apiFetch( + options.url, + `/experiments${workerQuery}`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + projectId, + }, + ); + if (!response.ok) await throwApiError(response); + const created = parseExperiment(await json(response)); + const format = (options.output ?? "table") as OutputFormat; + if (isMachineReadable(format)) { + console.log(formatData([created], fields, format)); + return; + } + console.log( + `${successText("Experiment created:")} ${value(created.experimentId)}`, + ); + console.log(`${label("Name:")} ${value(created.metadata.name)}`); + console.log( + `${label("Runs:")} ${value(String(created.revision.memberCount))}`, + ); + console.log( + `${label("Execution:")} ${value(created.execution.experimentExecutionId)}`, + ); + } catch (error) { + console.error( + errorText("Error:"), + error instanceof Error ? error.message : error, + ); + process.exitCode = 1; + } + }); + + withProjectOption( + withOutputOption( + experiment + .command("list") + .description("List experiments") + .option("--cursor ", "Continue from a list cursor") + .option("--limit ", "Maximum items", Number.parseInt) + .option("-u, --url ", "API base URL", getDefaultApiUrl()), + ), + ).action(async (options) => { + try { + const projectId = requireProjectId(options.project); + const params = new URLSearchParams(); + if (options.cursor) params.set("cursor", options.cursor); + if (options.limit) params.set("limit", String(options.limit)); + const response = await apiFetch( + options.url, + `/experiments${params.size ? `?${params}` : ""}`, + { projectId }, + ); + if (!response.ok) await throwApiError(response); + const page = parsePage(await json(response)); + console.log( + formatData( + page.items, + fields, + (options.output ?? "table") as OutputFormat, + ), + ); + if (page.nextCursor && !isMachineReadable(options.output ?? "table")) { + console.log(`${label("Next cursor:")} ${value(page.nextCursor)}`); + } + } catch (error) { + console.error( + errorText("Error:"), + error instanceof Error ? error.message : error, + ); + process.exitCode = 1; + } + }); + + withProjectOption( + withOutputOption( + experiment + .command("get") + .description("Get an experiment and its current execution") + .argument("", "Experiment ID") + .option("-u, --url ", "API base URL", getDefaultApiUrl()), + ), + ).action(async (id: string, options) => { + try { + const projectId = requireProjectId(options.project); + const response = await apiFetch( + options.url, + `/experiments/${encodeURIComponent(id)}`, + { projectId }, + ); + if (!response.ok) await throwApiError(response); + const item = parseExperiment(await json(response)); + const format = (options.output ?? "table") as OutputFormat; + if (isMachineReadable(format)) { + console.log(formatData([item], fields, format)); + return; + } + console.log(`${label("Experiment:")} ${value(item.experimentId)}`); + console.log(`${label("Name:")} ${value(item.metadata.name)}`); + console.log( + `${label("Execution:")} ${value(item.execution.experimentExecutionId)}`, + ); + console.log( + `${label("Attempt:")} ${value(String(item.execution.attemptNumber))}`, + ); + console.log( + `${label("Readiness:")} ${value(item.readiness ?? "unknown")}`, + ); + console.log( + `${label("Runs:")} ${value(String(item.revision.memberCount))}`, + ); + } catch (error) { + console.error( + errorText("Error:"), + error instanceof Error ? error.message : error, + ); + process.exitCode = 1; + } + }); + + withProjectOption( + withOutputOption( + experiment + .command("executions") + .description("List current and historical experiment executions") + .argument("", "Experiment ID") + .argument("[executionId]", "Get one exact execution") + .option("--cursor ", "Continue from a list cursor") + .option("--limit ", "Maximum items", Number.parseInt) + .option("-u, --url ", "API base URL", getDefaultApiUrl()), + ), + ).action(async (id: string, executionId: string | undefined, options) => { + try { + const projectId = requireProjectId(options.project); + if (executionId) { + const response = await apiFetch( + options.url, + `/experiments/${encodeURIComponent(id)}/executions/${encodeURIComponent(executionId)}`, + { projectId }, + ); + if (!response.ok) await throwApiError(response); + const item = parseExecution(await json(response)); + console.log( + formatData( + [item], + [ + { key: "experimentExecutionId", label: "Execution" }, + { key: "attemptNumber", label: "Attempt" }, + { key: "readiness", label: "Readiness" }, + { key: "requestCount", label: "Runs" }, + { key: "supersededReason", label: "Superseded" }, + { key: "createdAt", label: "Created" }, + ], + (options.output ?? "table") as OutputFormat, + ), + ); + return; + } + const params = new URLSearchParams(); + if (options.cursor) params.set("cursor", options.cursor); + if (options.limit) params.set("limit", String(options.limit)); + const response = await apiFetch( + options.url, + `/experiments/${encodeURIComponent(id)}/executions${ + params.size ? `?${params}` : "" + }`, + { projectId }, + ); + if (!response.ok) await throwApiError(response); + const page = parseExecutionPage(await json(response)); + console.log( + formatData( + page.items, + [ + { key: "experimentExecutionId", label: "Execution" }, + { key: "attemptNumber", label: "Attempt" }, + { key: "readiness", label: "Readiness" }, + { key: "requestCount", label: "Runs" }, + { key: "supersededReason", label: "Superseded" }, + { key: "createdAt", label: "Created" }, + ], + (options.output ?? "table") as OutputFormat, + ), + ); + } catch (error) { + console.error( + errorText("Error:"), + error instanceof Error ? error.message : error, + ); + process.exitCode = 1; + } + }); + + withProjectOption( + experiment + .command("retry") + .description("Retry the current experiment execution") + .argument("", "Experiment ID") + .requiredOption( + "--execution ", + "Current experiment execution ID", + ) + .option("-u, --url ", "API base URL", getDefaultApiUrl()), + ).action(async (id: string, options) => { + try { + const projectId = requireProjectId(options.project); + const response = await apiFetch( + options.url, + `/experiments/${encodeURIComponent(id)}/retry`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + experimentExecutionId: options.execution, + }), + projectId, + }, + ); + if (!response.ok) await throwApiError(response); + const item = parseExperiment(await json(response)); + console.log( + successText( + `Experiment retry started: attempt ${item.execution.attemptNumber} (${item.execution.experimentExecutionId})`, + ), + ); + } catch (error) { + console.error( + errorText("Error:"), + error instanceof Error ? error.message : error, + ); + process.exitCode = 1; + } + }); +} diff --git a/apps/cli/src/commands/run.test.ts b/apps/cli/src/commands/run.test.ts index b26334bab..bc40eb363 100644 --- a/apps/cli/src/commands/run.test.ts +++ b/apps/cli/src/commands/run.test.ts @@ -75,7 +75,7 @@ describe("run list", () => { id: "req-paginated-1", workerType: "coder-acp-copilot", run: { status: "done", outcome: "succeeded" }, - submissionId: "submission-12345678", + experimentId: "experiment-12345678", }, ], pageInfo: { @@ -99,7 +99,7 @@ describe("run list", () => { id: "req-array-1", workerType: "coder-acp-claude-code", run: { status: "running", outcome: "finished" }, - submissionId: "submission-abcdef12", + experimentId: "experiment-abcdef12", }, ]); @@ -117,7 +117,7 @@ describe("run list", () => { id: "req-json-1", workerType: "coder-acp-copilot", run: { status: "done", outcome: "succeeded" }, - submissionId: "submission-feedbeef", + experimentId: "experiment-feedbeef", }, ], pageInfo: { @@ -139,7 +139,7 @@ describe("run list", () => { id: "req-tsv-1", workerType: "coder-acp-copilot", run: { status: "done", outcome: "succeeded" }, - submissionId: "submission-11223344", + experimentId: "experiment-11223344", }, ], pageInfo: { @@ -149,7 +149,7 @@ describe("run list", () => { }); const output = await runListAndCaptureOutput(["-o", "tsv"]); - expect(output).toContain("req-tsv-1\tcoder-acp-copilot\tdone\tsubmissi"); + expect(output).toContain("req-tsv-1\tcoder-acp-copilot\tdone\texperime"); expect(output).not.toContain("Found 1 request(s):"); }); @@ -160,7 +160,7 @@ describe("run list", () => { id: "req-yaml-1", workerType: "coder-acp-copilot", run: { status: "done", outcome: "succeeded" }, - submissionId: "submission-a1b2c3d4", + experimentId: "experiment-a1b2c3d4", }, ], pageInfo: { @@ -236,6 +236,59 @@ describe("run submit", () => { const { body } = captureSubmit(); expect(body).not.toHaveProperty("agentsMd"); }); + + it("warns on deprecated multi-request composition", async () => { + mockFetchWith({ + ids: ["req-submit-3", "req-submit-4"], + count: 2, + status: "queued", + }); + vi.spyOn(console, "log").mockImplementation(() => {}); + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + + await makeProgram().parseAsync( + [ + "run", "submit", + "-m", "repeat task", + "--worker", "dynamic-worker", + "--count", "2", + "--no-stream", + "-u", "http://localhost:3100", + "--project", "proj-test", + ], + { from: "user" }, + ); + + expect(warnSpy).toHaveBeenCalledOnce(); + expect(warnSpy.mock.calls.flat().join(" ")).toContain( + "legacy-request-composition", + ); + }); + + it("does not warn for an ordinary single request", async () => { + mockFetchWith({ + id: "req-submit-5", + workerType: "coder-acp-copilot", + status: "queued", + }); + vi.spyOn(console, "log").mockImplementation(() => {}); + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + + await makeProgram().parseAsync( + [ + "run", "submit", + "-m", "single task", + "--worker", "dynamic-worker", + "--count", "1", + "--no-stream", + "-u", "http://localhost:3100", + "--project", "proj-test", + ], + { from: "user" }, + ); + + expect(warnSpy).not.toHaveBeenCalled(); + }); }); describe("run retry", () => { diff --git a/apps/cli/src/commands/run.ts b/apps/cli/src/commands/run.ts index ab2f7c15d..0c559a223 100644 --- a/apps/cli/src/commands/run.ts +++ b/apps/cli/src/commands/run.ts @@ -21,6 +21,9 @@ import { apiFetch, getApiBasePath } from "../utils/api-client.js"; import { parseGatesOption } from "../utils/gates.js"; import { buildResourceBindingSpecs, collectRepeatable } from "../utils/resources.js"; +const LEGACY_REQUEST_COMPOSITION_DEPRECATION = + "legacy-request-composition" as const; + /** * Resolve a CLI option that may be either a literal string or a `@path` * reference to a file whose contents should be read. Used for flags like @@ -213,6 +216,17 @@ run body.profileVariations = parsed; } + if ( + (typeof body.count === "number" && body.count > 1) || + (Array.isArray(body.profileVariations) && body.profileVariations.length > 0) + ) { + console.warn( + `${label("Deprecation:")} [${LEGACY_REQUEST_COMPOSITION_DEPRECATION}] ` + + "Use `scope experiment create` for composed benchmarks. " + + "`scope run submit` remains permanent for a single request.", + ); + } + // In variation mode the API derives the worker per-variation from each // profile's workerType, and explicitly rejects `?worker=`. Skip the // query param so the request isn't 400'd, and warn if --worker was @@ -250,13 +264,13 @@ run const result = await response.json(); // A variation submit creates one request per profile and returns - // { ids, variations, submissionId, ... } with no `id` or `workerType`. + // { ids, variations, experimentId, ... } with no `id` or `workerType`. // Printing it through the single-request path rendered `undefined` and - // then crashed in chalk, so the whole submission looked like it failed + // then crashed in chalk, so the whole request expansion looked like it failed // when in fact every request had been created. if (Array.isArray(result.ids) && result.ids.length > 0 && !result.id) { console.log(`${successText('Submitted:')} ${value(String(result.count ?? result.ids.length))} request(s) across ${value(String(result.variationCount ?? result.ids.length))} profile(s)`); - if (result.submissionId) console.log(`${label('Submission:')} ${value(result.submissionId)}`); + if (result.experimentId) console.log(`${label('Experiment:')} ${value(result.experimentId)}`); for (const variation of (result.variations ?? [])) { const ids: string[] = variation.ids ?? (variation.id ? [variation.id] : []); const name = variation.label ?? variation.profileName ?? variation.profileId ?? 'variation'; @@ -266,13 +280,13 @@ run for (const warning of (Array.isArray(result.warnings) ? result.warnings : [])) { console.log(`${errorText('⚠ Warning:')} ${warning}`); } - // Streaming follows a single request; a submission has several. + // Streaming follows a single request; an expanded composition has several. printFollowUpCommands(result.ids[0]); return; } console.log(`${successText('Request submitted:')} ${value(result.id)}`); - if (result.submissionId) console.log(`${label('Submission:')} ${value(result.submissionId)}`); + if (result.experimentId) console.log(`${label('Experiment:')} ${value(result.experimentId)}`); console.log(`${label('Worker:')} ${value(result.workerType)}`); if (result.model) console.log(`${label('Model:')} ${value(result.model)}`); if (result.reasoningEffort) console.log(`${label('Reasoning Effort:')} ${value(result.reasoningEffort)}`); @@ -509,7 +523,7 @@ run .option("--search ", "Free-text search over id, task, model, and worker") .option("--created-after ", "Only runs created at/after this ISO-8601 datetime") .option("--created-before ", "Only runs created at/before this ISO-8601 datetime") - .option("--submission-id ", "Filter by submission ID") + .option("--experiment-id ", "Filter by experiment ID") .option("--turns ", "Filter by actual turns (e.g. '>=5', '<=10', '=3')") .option("--max-iterations ", "Filter by configured maxIterations (e.g. '>=5', '<=10', '=3')") .option("--sort-by ", "Sort field: created, updated, priority, worker, status, id, duration") @@ -550,8 +564,8 @@ run if (options.createdBefore) { params.set("createdBefore", options.createdBefore); } - if (options.submissionId) { - params.set("submissionId", options.submissionId); + if (options.experimentId) { + params.set("experimentId", options.experimentId); } const parseIterExpr = (raw: string, name: string): { op: "eq" | "gte" | "lte"; value: number } => { const m = /^(>=|<=|=)?\s*(\d+)$/.exec(String(raw).trim()); @@ -623,8 +637,8 @@ run return o === 'succeeded' ? successText(s) : o === 'failed' || o === 'finished' ? errorText(s) : value(s); }, }, - { key: 'submissionId', label: 'Submission', - formatter: (req: any) => req.submissionId ? req.submissionId.substring(0, 8) : '–', + { key: 'experimentId', label: 'Experiment', + formatter: (req: Record) => typeof req.experimentId === "string" ? req.experimentId.substring(0, 8) : '–', }, { key: 'createdAt', label: 'Created', formatter: (req: any) => formatDate(req.createdAt), @@ -792,24 +806,24 @@ run .command("download-batch") .description("Download multiple runs at once into a single batch archive") .option("-i, --ids ", "One or more run IDs") - .option("--submission-id ", "Download all runs for a submission") + .option("--experiment-id ", "Download all runs for an experiment") .option("-o, --output ", "Output file path (default: batch-.tar.gz)") .option("-e, --extract", "Extract the archive after downloading") .option("-d, --dir ", "Extraction directory (implies --extract)") .option("-u, --url ", "API base URL", getDefaultApiUrl()) - .option("--project ", "Project ID for scoped operations (required with --submission-id)") + .option("--project ", "Project ID for scoped operations (required with --experiment-id)") .action(async (options) => { const { url } = options; const shouldExtract = options.extract || !!options.dir; try { - // Resolve IDs: either from --ids or by querying --submission-id + // Resolve IDs: either from --ids or by querying --experiment-id let ids: string[] = options.ids || []; - if (options.submissionId) { - console.log(`${label('Fetching runs for submission')} ${value(options.submissionId)}...`); + if (options.experimentId) { + console.log(`${label('Fetching runs for experiment')} ${value(options.experimentId)}...`); const projectId = requireProjectId(options.project); - const listResp = await apiFetch(url, `/requests?submissionId=${encodeURIComponent(options.submissionId)}&limit=1000`, { projectId }); + const listResp = await apiFetch(url, `/requests?experimentId=${encodeURIComponent(options.experimentId)}&limit=1000`, { projectId }); if (!listResp.ok) { const error = await listResp.json(); console.error(errorText("Error fetching runs:"), error); @@ -818,7 +832,7 @@ run const listData = await listResp.json(); const runs = listData.data || listData; if (!Array.isArray(runs) || runs.length === 0) { - console.error(errorText("No runs found for this submission")); + console.error(errorText("No runs found for this experiment")); process.exit(1); } ids = runs.map((r: { _id: string }) => r._id); @@ -826,7 +840,7 @@ run } if (ids.length === 0) { - console.error(errorText("Error: provide --ids or --submission-id")); + console.error(errorText("Error: provide --ids or --experiment-id")); process.exit(1); } diff --git a/apps/cli/src/commands/submission.ts b/apps/cli/src/commands/submission.ts deleted file mode 100644 index 835e7f3ed..000000000 --- a/apps/cli/src/commands/submission.ts +++ /dev/null @@ -1,204 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -import { Command } from "commander"; -import { configureHelp } from "../utils/helpFormatter.js"; -import { formatData, isMachineReadable } from "../utils/formatters.js"; -import type { DisplayField, OutputFormat } from "../utils/types.js"; -import { - getDefaultApiUrl, - withOutputOption, - withProjectOption, -} from "../utils/shared.js"; -import { requireProjectId } from "../utils/config.js"; -import { apiFetch } from "../utils/api-client.js"; -import { errorText, label, successText, value } from "../utils/style.js"; - -interface SubmissionSummary { - submissionId: string; - submissionRunId: string; - attemptNumber: number; - readiness: string; - createdAt: string; - requestIds: string[]; - requests: Array<{ - requestId: string; - runId: string; - task: string; - status: string; - outcome?: string; - }>; - blocking: Array<{ - requestId: string; - runId: string; - reason: string; - }>; -} -async function responseJson(response: Response): Promise { - return response.json() as Promise; -} - -async function throwApiError(response: Response): Promise { - const body = await responseJson<{ error?: string }>(response); - throw new Error(body.error ?? `HTTP ${response.status}`); -} - -const fields: DisplayField[] = [ - { key: "submissionId", label: "Submission" }, - { key: "attemptNumber", label: "Attempt" }, - { key: "readiness", label: "Readiness" }, - { - key: "requestIds", - label: "Runs", - formatter: (row: Record) => - String((row.requestIds as string[] | undefined)?.length ?? 0), - }, - { key: "createdAt", label: "Created" }, -]; - -export function registerSubmissionCommands(program: Command): void { - const submission = program - .command("submission") - .description("Inspect and retry grouped run submissions") - .action(() => submission.help()); - configureHelp(submission); - - withProjectOption( - withOutputOption( - submission - .command("list") - .description("List submissions") - .option("-u, --url ", "API base URL", getDefaultApiUrl()), - ), - ).action(async (options) => { - try { - const projectId = requireProjectId(options.project); - const response = await apiFetch(options.url, "/submissions", { projectId }); - if (!response.ok) await throwApiError(response); - const body = await responseJson<{ submissions: SubmissionSummary[] }>(response); - console.log( - formatData(body.submissions, fields, (options.output ?? "table") as OutputFormat), - ); - } catch (error) { - console.error(errorText("Error:"), error instanceof Error ? error.message : error); - process.exitCode = 1; - } - }); - - withOutputOption( - submission - .command("get") - .description("Get a submission and its current request/run vector") - .argument("", "Submission ID") - .option("-u, --url ", "API base URL", getDefaultApiUrl()), - ).action(async (id: string, options) => { - try { - const response = await apiFetch( - options.url, - `/submissions/${encodeURIComponent(id)}`, - ); - if (!response.ok) await throwApiError(response); - const doc = await responseJson(response); - const format = (options.output ?? "table") as OutputFormat; - if (isMachineReadable(format)) { - console.log(formatData([doc], fields, format)); - return; - } - console.log(`${label("Submission:")} ${value(doc.submissionId)}`); - console.log(`${label("Attempt:")} ${value(String(doc.attemptNumber))}`); - console.log(`${label("Readiness:")} ${value(doc.readiness)}`); - console.log(`${label("Submission run:")} ${value(doc.submissionRunId)}`); - for (const request of doc.requests) { - console.log( - ` ${value(request.requestId)} ${request.status}` + - `${request.outcome ? `/${request.outcome}` : ""} run=${request.runId}`, - ); - } - for (const blocking of doc.blocking) { - console.log( - ` blocked ${blocking.requestId}/${blocking.runId}: ${blocking.reason}`, - ); - } - } catch (error) { - console.error(errorText("Error:"), error instanceof Error ? error.message : error); - process.exitCode = 1; - } - }); - - withOutputOption( - submission - .command("runs") - .description("List current and archived submission attempts") - .argument("", "Submission ID") - .option("-u, --url ", "API base URL", getDefaultApiUrl()), - ).action(async (id: string, options) => { - try { - const response = await apiFetch( - options.url, - `/submissions/${encodeURIComponent(id)}/runs`, - ); - if (!response.ok) await throwApiError(response); - const body = await responseJson<{ - runs: Array<{ - _id: string; - attemptNumber: number; - createdAt: string; - requests: unknown[]; - supersededReason?: string; - }>; - }>(response); - const runFields: DisplayField[] = [ - { key: "_id", label: "Submission Run" }, - { key: "attemptNumber", label: "Attempt" }, - { - key: "requests", - label: "Runs", - formatter: (row: Record) => - String((row.requests as unknown[] | undefined)?.length ?? 0), - }, - { key: "supersededReason", label: "Superseded" }, - { key: "createdAt", label: "Created" }, - ]; - console.log( - formatData(body.runs, runFields, (options.output ?? "table") as OutputFormat), - ); - } catch (error) { - console.error(errorText("Error:"), error instanceof Error ? error.message : error); - process.exitCode = 1; - } - }); - - submission - .command("retry") - .description("Retry the current submission processing attempt") - .argument("", "Submission ID") - .requiredOption("--submission-run ", "Current submission run ID") - .option("-u, --url ", "API base URL", getDefaultApiUrl()) - .action(async (id: string, options) => { - try { - const response = await apiFetch( - options.url, - `/submissions/${encodeURIComponent(id)}/retry`, - { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ submissionRunId: options.submissionRun }), - }, - ); - if (!response.ok) await throwApiError(response); - const result = await responseJson<{ - submissionRunId: string; - attemptNumber: number; - readiness: string; - }>(response); - console.log( - successText( - `Submission retry started: attempt ${result.attemptNumber} (${result.submissionRunId}, ${result.readiness})`, - ), - ); - } catch (error) { - console.error(errorText("Error:"), error instanceof Error ? error.message : error); - process.exitCode = 1; - } - }); -} diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts index 6d59d10a6..a9278391a 100644 --- a/apps/cli/src/index.ts +++ b/apps/cli/src/index.ts @@ -25,7 +25,7 @@ import { registerTaskPromptCommands } from "./commands/task-prompt.js"; import { registerProfileCommands } from "./commands/profile.js"; import { registerProjectCommands } from "./commands/project.js"; import { registerSecretCommands } from "./commands/secret.js"; -import { registerSubmissionCommands } from "./commands/submission.js"; +import { registerExperimentCommands } from "./commands/experiment.js"; import { registerUpdateCommand } from "./commands/update.js"; import { checkForUpdates } from "./utils/update-check.js"; import { errorText } from "./utils/style.js"; @@ -71,7 +71,7 @@ configureHelp(program); registerSecretCommands(program); registerProjectCommands(program); registerRunCommands(program); -registerSubmissionCommands(program); +registerExperimentCommands(program); registerCriteriaCommands(program); registerPromptFeatureCommands(program); registerReportCommands(program); diff --git a/apps/portal/src/App.tsx b/apps/portal/src/App.tsx index 56e6a2d79..195a75400 100644 --- a/apps/portal/src/App.tsx +++ b/apps/portal/src/App.tsx @@ -6,7 +6,8 @@ import { Layout } from "@/components/Layout"; import { RunsList } from "@/pages/RunsList"; import { RunDetail } from "@/pages/RunDetail"; import { SubmitRun } from "@/pages/SubmitRun"; -import { SubmissionDetail } from "@/pages/SubmissionDetail"; +import { ExperimentDetail } from "@/pages/ExperimentDetail"; +import { ExperimentsList } from "@/pages/ExperimentsList"; import { CriteriaList } from "@/pages/CriteriaList"; import { CriterionDetail } from "@/pages/CriterionDetail"; import { CriterionPreviewPanel } from "@/pages/CriterionPreviewPanel"; @@ -86,7 +87,9 @@ export function App() { } /> } /> - } /> + } /> + } /> + } /> }> } /> diff --git a/apps/portal/src/components/ExperimentStatusCard.stories.tsx b/apps/portal/src/components/ExperimentStatusCard.stories.tsx new file mode 100644 index 000000000..af93dd5c9 --- /dev/null +++ b/apps/portal/src/components/ExperimentStatusCard.stories.tsx @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { Meta, StoryObj } from "@storybook/react-vite"; +import type { ExperimentView } from "@/lib/experiments"; +import { ExperimentStatusCard } from "./ExperimentStatusCard"; + +const experiment: ExperimentView = { + experimentId: "experiment-123", + projectId: "project-123", + metadata: { name: "Profile comparison", tags: ["nightly"] }, + revision: { + experimentRevisionId: "revision-1", + revisionNumber: 1, + memberCount: 2, + }, + createdAt: "2026-10-15T00:00:00.000Z", + createdRequestIds: ["request-1", "request-2"], + readiness: "processing", + execution: { + experimentExecutionId: "execution-1", + attemptNumber: 2, + requestCount: 2, + createdAt: "2026-10-15T00:00:00.000Z", + requests: [], + handlerProgress: { + observations: { status: "done" }, + report: { status: "processing" }, + }, + blocking: [], + }, + blocking: [], +}; + +const meta = { + title: "Experiments/ExperimentStatusCard", + component: ExperimentStatusCard, + args: { experiment }, +} satisfies Meta; + +export default meta; +type Story = StoryObj; + +export const Processing: Story = {}; + +export const Blocked: Story = { + args: { + experiment: { + ...experiment, + readiness: "blocked", + blocking: [ + { + handlerId: "observations", + reason: "dependency-failed", + }, + ], + }, + }, +}; diff --git a/apps/portal/src/components/ExperimentStatusCard.test.tsx b/apps/portal/src/components/ExperimentStatusCard.test.tsx new file mode 100644 index 000000000..e01e90b3c --- /dev/null +++ b/apps/portal/src/components/ExperimentStatusCard.test.tsx @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// @vitest-environment happy-dom +import { cleanup, render, screen } from "@testing-library/react"; +import { afterEach, describe, expect, it } from "vitest"; +import type { ExperimentView } from "@/lib/experiments"; +import { ExperimentStatusCard } from "./ExperimentStatusCard"; + +afterEach(cleanup); + +describe("ExperimentStatusCard", () => { + it("renders readiness and exact handler progress", () => { + const experiment: ExperimentView = { + experimentId: "experiment-1", + projectId: "project-1", + metadata: { name: "Compiler comparison" }, + revision: { + experimentRevisionId: "revision-1", + revisionNumber: 1, + memberCount: 2, + }, + createdAt: "2026-10-15T00:00:00.000Z", + createdRequestIds: ["request-1", "request-2"], + readiness: "blocked", + execution: { + experimentExecutionId: "execution-1", + attemptNumber: 3, + requestCount: 2, + createdAt: "2026-10-15T00:00:00.000Z", + requests: [], + handlerProgress: { + observations: { status: "done" }, + report: { status: "processing" }, + }, + blocking: [], + }, + blocking: [ + { + handlerId: "observations", + reason: "dependency-failed", + }, + ], + }; + + render(); + + expect(screen.getByText("blocked")).toBeDefined(); + expect(screen.getByText("3")).toBeDefined(); + expect(screen.getByText("1/2")).toBeDefined(); + }); +}); diff --git a/apps/portal/src/components/ExperimentStatusCard.tsx b/apps/portal/src/components/ExperimentStatusCard.tsx new file mode 100644 index 000000000..5128838cc --- /dev/null +++ b/apps/portal/src/components/ExperimentStatusCard.tsx @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { Badge } from "@/components/ui/badge"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import type { ExperimentView } from "@/lib/experiments"; + +export function ExperimentStatusCard({ + experiment, +}: { + experiment: ExperimentView; +}) { + const readiness = experiment.readiness ?? "unknown"; + const variant = + readiness === "failed" || readiness === "blocked" + ? "destructive" + : readiness === "processing" || readiness === "done" + ? "default" + : readiness === "ready" + ? "outline" + : "secondary"; + const handlers = Object.values(experiment.execution.handlerProgress); + const completedHandlers = handlers.filter( + (handler) => + handler && + typeof handler === "object" && + "status" in handler && + handler.status === "done", + ).length; + return ( + + + Processing + {readiness} + + +
+
Attempt
+
+ {experiment.execution.attemptNumber} +
+
+
+
Handlers complete
+
+ {completedHandlers}/{handlers.length} +
+
+
+
Blocking runs
+
{experiment.blocking.length}
+
+ + + ); +} diff --git a/apps/portal/src/components/Layout.tsx b/apps/portal/src/components/Layout.tsx index 2a08d36cd..c3eb9e9be 100644 --- a/apps/portal/src/components/Layout.tsx +++ b/apps/portal/src/components/Layout.tsx @@ -95,6 +95,7 @@ const navGroups: NavGroup[] = [ scoped: true, items: [ { to: "/runs", label: "Runs", icon: List }, + { to: "/experiments", label: "Experiments", icon: FlaskConical }, { to: "/statistics", label: "Statistics", icon: BarChart3 }, { to: "/reports", label: "Reports", icon: FileText }, { to: "/insights", label: "Insights", icon: Lightbulb }, diff --git a/apps/portal/src/components/SubmissionStatusCard.stories.tsx b/apps/portal/src/components/SubmissionStatusCard.stories.tsx deleted file mode 100644 index 0c465e7e9..000000000 --- a/apps/portal/src/components/SubmissionStatusCard.stories.tsx +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -import type { Meta, StoryObj } from "@storybook/react-vite"; -import { SubmissionStatusCard } from "./SubmissionStatusCard"; -import type { Submission } from "@/types"; - -const submission: Submission = { - submissionId: "submission-123", - projectId: "project-123", - requestIds: ["request-1", "request-2"], - createdAt: new Date().toISOString(), - submissionRunId: "submission-run-1", - attemptNumber: 2, - readiness: "processing", - requests: [ - { - requestId: "request-1", - runId: "run-1", - task: "Implement the feature", - status: "done", - outcome: "succeeded", - workerType: "coder-acp-copilot", - }, - { - requestId: "request-2", - runId: "run-2", - task: "Implement the feature", - status: "done", - outcome: "failed", - workerType: "coder-acp-copilot", - }, - ], - dependencyProjection: [ - { - requestId: "request-1", - runId: "run-1", - handlerId: "pp-observations", - state: "ready", - }, - { - requestId: "request-2", - runId: "run-2", - handlerId: "pp-observations", - state: "ready", - }, - ], - blocking: [], - handlerProgress: { - "pp-readout": { status: "processing", version: 1 }, - }, -}; -const meta = { - title: "Submissions/SubmissionStatusCard", - component: SubmissionStatusCard, - args: { submission }, -} satisfies Meta; - -export default meta; -type Story = StoryObj; - -export const Processing: Story = {}; - -export const Blocked: Story = { - args: { - submission: { - ...submission, - readiness: "blocked", - blocking: [ - { - requestId: "request-2", - runId: "run-2", - handlerId: "pp-observations", - reason: "dependency-failed", - }, - ], - }, - }, -}; diff --git a/apps/portal/src/components/SubmissionStatusCard.test.tsx b/apps/portal/src/components/SubmissionStatusCard.test.tsx deleted file mode 100644 index b735e762e..000000000 --- a/apps/portal/src/components/SubmissionStatusCard.test.tsx +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -// @vitest-environment happy-dom -import { cleanup, render, screen } from "@testing-library/react"; -import { afterEach, describe, expect, it } from "vitest"; -import type { Submission } from "@/types"; -import { SubmissionStatusCard } from "./SubmissionStatusCard"; - -afterEach(cleanup); - -describe("SubmissionStatusCard", () => { - it("renders readiness and exact observation progress", () => { - const submission: Submission = { - submissionId: "submission-1", - projectId: "project-1", - requestIds: ["request-1", "request-2"], - createdAt: new Date().toISOString(), - submissionRunId: "submission-run-1", - attemptNumber: 3, - readiness: "blocked", - requests: [ - { - requestId: "request-1", - runId: "run-1", - task: "Task", - status: "done", - workerType: "worker", - }, - { - requestId: "request-2", - runId: "run-2", - task: "Task", - status: "done", - workerType: "worker", - }, - ], - dependencyProjection: [ - { - requestId: "request-1", - runId: "run-1", - handlerId: "pp-observations", - state: "ready", - }, - { - requestId: "request-2", - runId: "run-2", - handlerId: "pp-observations", - state: "failed", - }, - ], - blocking: [ - { - requestId: "request-2", - runId: "run-2", - handlerId: "pp-observations", - reason: "dependency-failed", - }, - ], - }; - - render(); - - expect(screen.getByText("blocked")).toBeDefined(); - expect(screen.getByText("3")).toBeDefined(); - expect(screen.getByText("1/2")).toBeDefined(); - }); -}); diff --git a/apps/portal/src/components/SubmissionStatusCard.tsx b/apps/portal/src/components/SubmissionStatusCard.tsx deleted file mode 100644 index f711bdd1c..000000000 --- a/apps/portal/src/components/SubmissionStatusCard.tsx +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -import { Badge } from "@/components/ui/badge"; -import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; -import type { Submission } from "@/types"; - -const readinessVariant: Record< - Submission["readiness"], - "default" | "secondary" | "destructive" | "outline" -> = { - waiting: "secondary", - ready: "outline", - blocked: "destructive", - processing: "default", - done: "default", - failed: "destructive", -}; - -export function SubmissionStatusCard({ - submission, -}: { - submission: Submission; -}) { - const ready = submission.dependencyProjection.filter( - (dependency) => dependency.state === "ready", - ).length; - return ( - - - Processing - - {submission.readiness} - - - -
-
Attempt
-
{submission.attemptNumber}
-
-
-
Observations ready
-
- {ready}/{submission.requests.length} -
-
-
-
Blocking runs
-
{submission.blocking.length}
-
- - - ); -} diff --git a/apps/portal/src/lib/api.ts b/apps/portal/src/lib/api.ts index 1258ac38e..8dc195932 100644 --- a/apps/portal/src/lib/api.ts +++ b/apps/portal/src/lib/api.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import type { Run, RunState, CriteriaDocument, CriteriaGraphData, GeneratePromptResponse, AnalysisResponse, PromptFeatureDocument, Report, BulkReportStatus, BulkReportSummary, ReportTemplate, ReportTrigger, ReportTemplateSystemPrompt, KeyDocument, KeyValidationResult, CreateKeyRequest, UpdateKeyRequest, CodingAgent, AgentVersion, McpServerDocument, CreateMcpServerRequest, UpdateMcpServerRequest, BulkResubmitOverrides, Insight, InsightWithReference, TaskPrompt, TaskPromptFeatureExtractionResult, Model, FeatureFlag, SkillDocument, SkillSearchResult, SkillDiscoveryResult, SkillRevisionDocument, CodebaseDocument, CodebaseRevisionDocument, CodebaseSourceType, ResourceDocument, ResourceRevisionDocument, ResourceBindingSpec, CreateResourceBody, CreateResourceRevisionBody, ExtensionDocument, ExtensionSearchResult, ExtensionVersionInfo, MdpResponse, AccountDocument, CreateAccountRequest, UpdateAccountRequest, ProfileWithVersion, ProfileVersionDocument, ProfileDocument, RunGroup, RunFacetsResponse, CursorPaginatedResponse, IterationOp, GateConfig, GateId, PromptType, RunSortField, RunSortDir, CriterionKind, TaxonomyElementId, CriterionSubject, Submission } from "@/types"; +import type { Run, RunState, CriteriaDocument, CriteriaGraphData, GeneratePromptResponse, AnalysisResponse, PromptFeatureDocument, Report, BulkReportStatus, BulkReportSummary, ReportTemplate, ReportTrigger, ReportTemplateSystemPrompt, KeyDocument, KeyValidationResult, CreateKeyRequest, UpdateKeyRequest, CodingAgent, AgentVersion, McpServerDocument, CreateMcpServerRequest, UpdateMcpServerRequest, BulkResubmitOverrides, Insight, InsightWithReference, TaskPrompt, TaskPromptFeatureExtractionResult, Model, FeatureFlag, SkillDocument, SkillSearchResult, SkillDiscoveryResult, SkillRevisionDocument, CodebaseDocument, CodebaseRevisionDocument, CodebaseSourceType, ResourceDocument, ResourceRevisionDocument, ResourceBindingSpec, CreateResourceBody, CreateResourceRevisionBody, ExtensionDocument, ExtensionSearchResult, ExtensionVersionInfo, MdpResponse, AccountDocument, CreateAccountRequest, UpdateAccountRequest, ProfileWithVersion, ProfileVersionDocument, ProfileDocument, RunGroup, RunFacetsResponse, CursorPaginatedResponse, IterationOp, GateConfig, GateId, PromptType, RunSortField, RunSortDir, CriterionKind, TaxonomyElementId, CriterionSubject } from "@/types"; import type { Project, CreateProjectRequest, UpdateProjectRequest } from "@/types"; import { qs } from "./url"; @@ -9,6 +9,17 @@ import { recordServerDate } from "./serverClock"; import { apiClient } from "./api-client"; import { getSelectedProjectId, ProjectRequiredError } from "./project-scope"; import { MAX_ARCHIVE_UPLOAD_LABEL } from "./codebaseUpload"; +import { + parseExperiment, + parseExperimentExecution, + parseExperimentExecutionPage, + parseExperimentPage, + type ExperimentExecutionPageView, + type ExperimentExecutionView, + type ExperimentMetadataView, + type ExperimentPageView, + type ExperimentView, +} from "./experiments"; const BASE = "/api/v1"; @@ -85,7 +96,7 @@ export interface RunFilterParams { profileId?: string | string[]; taskPromptId?: string; criteria?: string; - submissionId?: string; + experimentId?: string; search?: string; createdAfter?: string; createdBefore?: string; @@ -108,7 +119,7 @@ function runFilterQs(f: RunFilterParams): Record => @@ -178,8 +219,8 @@ export const api = { })}`, undefined, { scoped: true }); }, - /** List runs grouped by task/profile/submissionId, with cursor pagination and server-side filtering */ - listRunGroups: (opts: RunFilterParams & { groupBy: "task" | "submissionId" | "profile"; sortBy?: RunSortField; sortDir?: RunSortDir; limit?: number; after?: string; before?: string; last?: boolean }): Promise> => { + /** List runs grouped by task/profile/experimentId, with cursor pagination and server-side filtering */ + listRunGroups: (opts: RunFilterParams & { groupBy: "task" | "experimentId" | "profile"; sortBy?: RunSortField; sortDir?: RunSortDir; limit?: number; after?: string; before?: string; last?: boolean }): Promise> => { return request(`/requests${qs({ groupBy: opts.groupBy, ...runFilterQs(opts), @@ -209,29 +250,19 @@ export const api = { }, /** Submit a new run (or multiple runs if count > 1) */ - submitRun: (body: { - scenario: { task: string; criteria: string[]; version?: "v1" | "v2" }; - observations?: string[]; - worker?: string; - model?: string; - reasoningEffort?: string; - maxIterations?: number; - personaInstructions?: string; - persona?: { personality: string; experience: string; verbosity: string; type: string }; - count?: number; - mcpServers?: string[]; - skills?: string[]; - extensions?: string[]; - resources?: ResourceBindingSpec[]; - agentVersion?: string; - profileId?: string; - profileVariations?: string[]; - agentsMd?: string; - agentsMdParentIds?: string[]; - gates?: GateConfig[]; - codebase?: string; - codebaseRevisionId?: string; - }): Promise<(Run & { message: string; submissionId: string }) | { ids: string[]; count: number; message: string; submissionId: string }> => { + submitRun: (body: RunCreateInput): Promise< + (Run & { + message: string; + experimentId?: string; + submissionId?: string; + }) | { + ids: string[]; + count: number; + message: string; + experimentId?: string; + submissionId?: string; + } + > => { const { worker, ...payload } = body; const url = worker ? `/requests?worker=${encodeURIComponent(worker)}` : `/requests`; return request(url, { @@ -240,26 +271,98 @@ export const api = { }, { scoped: true }); }, - listSubmissions: (): Promise<{ submissions: Submission[] }> => - request("/submissions", undefined, { scoped: true }), - - getSubmission: (id: string): Promise => - request(`/submissions/${id}`), + createExperiment: async (body: { + metadata: ExperimentMetadataView; + request?: RunCreateInput; + empty?: boolean; + }): Promise => { + const worker = body.request?.worker; + const requestBody = body.request + ? { ...body.request, worker: undefined } + : undefined; + const path = worker + ? `/experiments?worker=${encodeURIComponent(worker)}` + : "/experiments"; + const value = await request( + path, + { + method: "POST", + body: JSON.stringify({ + metadata: body.metadata, + ...(requestBody ? { request: requestBody } : {}), + ...(body.empty ? { empty: true } : {}), + }), + }, + { scoped: true }, + ); + return parseExperiment(value); + }, + + listExperiments: async (opts?: { + cursor?: string; + limit?: number; + }): Promise => { + const value = await request( + `/experiments${qs({ + cursor: opts?.cursor, + limit: opts?.limit ? String(opts.limit) : undefined, + })}`, + undefined, + { scoped: true }, + ); + return parseExperimentPage(value); + }, + + getExperiment: async (id: string): Promise => { + const value = await request( + `/experiments/${encodeURIComponent(id)}`, + undefined, + { scoped: true }, + ); + return parseExperiment(value); + }, + + listExperimentExecutions: async ( + id: string, + opts?: { cursor?: string; limit?: number }, + ): Promise => { + const value = await request( + `/experiments/${encodeURIComponent(id)}/executions${qs({ + cursor: opts?.cursor, + limit: opts?.limit ? String(opts.limit) : undefined, + })}`, + undefined, + { scoped: true }, + ); + return parseExperimentExecutionPage(value); + }, + + getExperimentExecution: async ( + id: string, + experimentExecutionId: string, + ): Promise => { + const value = await request( + `/experiments/${encodeURIComponent(id)}/executions/${encodeURIComponent(experimentExecutionId)}`, + undefined, + { scoped: true }, + ); + return parseExperimentExecution(value); + }, - retrySubmission: ( + retryExperiment: async ( id: string, - submissionRunId: string, - ): Promise<{ - submissionId: string; - submissionRunId: string; - attemptNumber: number; - readiness: Submission["readiness"]; - blocking: Submission["blocking"]; - }> => - request(`/submissions/${id}/retry`, { - method: "POST", - body: JSON.stringify({ submissionRunId }), - }), + experimentExecutionId: string, + ): Promise => { + const value = await request( + `/experiments/${encodeURIComponent(id)}/retry`, + { + method: "POST", + body: JSON.stringify({ experimentExecutionId }), + }, + { scoped: true }, + ); + return parseExperiment(value); + }, /** Soft-delete a run */ deleteRun: (id: string): Promise<{ id: string; deleted: boolean }> => { diff --git a/apps/portal/src/lib/experiments.ts b/apps/portal/src/lib/experiments.ts new file mode 100644 index 000000000..9132075db --- /dev/null +++ b/apps/portal/src/lib/experiments.ts @@ -0,0 +1,321 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +export const LEGACY_REQUEST_COMPOSITION_DEPRECATION = + "legacy-request-composition" as const; + +export interface ExperimentMetadataView { + name: string; + description?: string; + tags?: string[]; + createdBy?: string; +} + +export interface ExperimentRevisionView { + experimentRevisionId: string; + revisionNumber: number; + memberCount: number; + membershipFingerprint?: string; + definitionFingerprint?: string; + createdAt?: string; +} + +export interface ExperimentBlockerView { + handlerId: string; + reason: string; + requestId?: string; + runId?: string; + error?: string; +} + +export interface ExperimentRequestSummaryView { + requestId: string; + runId: string; + task?: string; + status?: string; + outcome?: string; + workerType?: string; + model?: string; + profileId?: string; +} + +export interface ExperimentExecutionView { + experimentExecutionId: string; + attemptNumber: number; + version?: number; + readiness?: string; + requestCount: number; + requests: ExperimentRequestSummaryView[]; + handlerProgress: Record; + blocking: ExperimentBlockerView[]; + currentReport?: { reportId?: string; url?: string; updatedAt?: string }; + createdAt: string; + updatedAt?: string; + startedAt?: string; + finishedAt?: string; + discardedAt?: string; + discardReason?: string; + supersededAt?: string; + supersededByExperimentExecutionId?: string; + supersededReason?: "rollover" | "retry"; + archivedAt?: string; +} + +export interface ExperimentView { + experimentId: string; + projectId: string; + metadata: ExperimentMetadataView; + revision: ExperimentRevisionView; + execution: ExperimentExecutionView; + readiness?: string; + blocking: ExperimentBlockerView[]; + createdRequestIds: string[]; + createdAt: string; + updatedAt?: string; + finalizedAt?: string; +} + +export interface ExperimentPageView { + items: ExperimentView[]; + nextCursor?: string; +} + +export interface ExperimentExecutionPageView { + items: ExperimentExecutionView[]; + nextCursor?: string; +} + +function record(value: unknown, name: string): Record { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`Invalid ${name} response from Scope`); + } + return value as Record; +} + +function requiredString( + value: Record, + key: string, + name: string, +): string { + const candidate = value[key]; + if (typeof candidate !== "string" || candidate.length === 0) { + throw new Error(`Invalid ${name}.${key} response from Scope`); + } + return candidate; +} + +function optionalString( + value: Record, + key: string, +): string | undefined { + const candidate = value[key]; + if (candidate === undefined || candidate === null) return undefined; + if (typeof candidate !== "string") { + throw new Error(`Invalid ${key} response from Scope`); + } + return candidate; +} + +function requiredNumber( + value: Record, + key: string, + name: string, +): number { + const candidate = value[key]; + if (typeof candidate !== "number" || !Number.isFinite(candidate)) { + throw new Error(`Invalid ${name}.${key} response from Scope`); + } + return candidate; +} + +function parseBlocker(value: unknown): ExperimentBlockerView { + const item = record(value, "experiment blocker"); + return { + handlerId: requiredString(item, "handlerId", "experiment blocker"), + reason: requiredString(item, "reason", "experiment blocker"), + requestId: optionalString(item, "requestId"), + runId: optionalString(item, "runId"), + error: optionalString(item, "error"), + }; +} + +function parseRequestSummary(value: unknown): ExperimentRequestSummaryView { + const item = record(value, "experiment request"); + return { + requestId: requiredString(item, "requestId", "experiment request"), + runId: requiredString(item, "runId", "experiment request"), + task: optionalString(item, "task"), + status: optionalString(item, "status"), + outcome: optionalString(item, "outcome"), + workerType: optionalString(item, "workerType"), + model: optionalString(item, "model"), + profileId: optionalString(item, "profileId"), + }; +} + +function parseRequestVector( + value: unknown, +): { requestCount: number; requests: ExperimentRequestSummaryView[] } { + if (Array.isArray(value)) { + const requests = value.map(parseRequestSummary); + return { requestCount: requests.length, requests }; + } + const payload = record(value, "experiment request vector"); + const entries = Array.isArray(payload.entries) + ? payload.entries.map(parseRequestSummary) + : []; + const entryCount = + typeof payload.entryCount === "number" ? payload.entryCount : entries.length; + return { requestCount: entryCount, requests: entries }; +} + +export function parseExperimentExecution( + value: unknown, +): ExperimentExecutionView { + const execution = record(value, "experiment execution"); + const vector = parseRequestVector(execution.requests ?? []); + const blocking = Array.isArray(execution.blocking) + ? execution.blocking.map(parseBlocker) + : []; + const progressValue = execution.handlerProgress ?? execution.handlerSummary ?? {}; + const handlerProgress = record(progressValue, "experiment handler progress"); + const reportValue = execution.currentReport; + const currentReport = + reportValue === undefined + ? undefined + : (() => { + const report = record(reportValue, "experiment report"); + return { + reportId: optionalString(report, "reportId"), + url: optionalString(report, "url"), + updatedAt: optionalString(report, "updatedAt"), + }; + })(); + + return { + experimentExecutionId: requiredString( + execution, + "experimentExecutionId", + "experiment execution", + ), + attemptNumber: requiredNumber( + execution, + "attemptNumber", + "experiment execution", + ), + version: + typeof execution.version === "number" ? execution.version : undefined, + readiness: optionalString(execution, "readiness"), + requestCount: vector.requestCount, + requests: vector.requests, + handlerProgress, + blocking, + currentReport, + createdAt: requiredString(execution, "createdAt", "experiment execution"), + updatedAt: optionalString(execution, "updatedAt"), + startedAt: optionalString(execution, "startedAt"), + finishedAt: optionalString(execution, "finishedAt"), + discardedAt: optionalString(execution, "discardedAt"), + discardReason: optionalString(execution, "discardReason"), + supersededAt: optionalString(execution, "supersededAt"), + supersededByExperimentExecutionId: optionalString( + execution, + "supersededByExperimentExecutionId", + ), + supersededReason: + execution.supersededReason === "rollover" || + execution.supersededReason === "retry" + ? execution.supersededReason + : undefined, + archivedAt: optionalString(execution, "archivedAt"), + }; +} + +export function parseExperiment(value: unknown): ExperimentView { + const experiment = record(value, "experiment"); + const metadata = record(experiment.metadata, "experiment metadata"); + const revision = record(experiment.revision, "experiment revision"); + const executionValue = experiment.execution ?? experiment.currentExecution; + const execution = parseExperimentExecution(executionValue); + const blockingValue = experiment.blocking ?? execution.blocking; + const blocking = Array.isArray(blockingValue) + ? blockingValue.map(parseBlocker) + : []; + const createdRequestIds = Array.isArray(experiment.createdRequestIds) + ? experiment.createdRequestIds.map((id) => { + if (typeof id !== "string") { + throw new Error("Invalid experiment.createdRequestIds response from Scope"); + } + return id; + }) + : []; + + return { + experimentId: requiredString(experiment, "experimentId", "experiment"), + projectId: requiredString(experiment, "projectId", "experiment"), + metadata: { + name: requiredString(metadata, "name", "experiment metadata"), + description: optionalString(metadata, "description"), + tags: Array.isArray(metadata.tags) + ? metadata.tags.map((tag) => { + if (typeof tag !== "string") { + throw new Error("Invalid experiment metadata tag response from Scope"); + } + return tag; + }) + : undefined, + createdBy: optionalString(metadata, "createdBy"), + }, + revision: { + experimentRevisionId: requiredString( + revision, + "experimentRevisionId", + "experiment revision", + ), + revisionNumber: requiredNumber( + revision, + "revisionNumber", + "experiment revision", + ), + memberCount: + typeof revision.memberCount === "number" + ? revision.memberCount + : execution.requestCount, + membershipFingerprint: optionalString(revision, "membershipFingerprint"), + definitionFingerprint: optionalString(revision, "definitionFingerprint"), + createdAt: optionalString(revision, "createdAt"), + }, + execution, + readiness: + optionalString(experiment, "readiness") ?? execution.readiness, + blocking, + createdRequestIds, + createdAt: requiredString(experiment, "createdAt", "experiment"), + updatedAt: optionalString(experiment, "updatedAt"), + finalizedAt: optionalString(experiment, "finalizedAt"), + }; +} + +export function parseExperimentPage(value: unknown): ExperimentPageView { + const page = record(value, "experiment page"); + if (!Array.isArray(page.items)) { + throw new Error("Invalid experiment page items response from Scope"); + } + return { + items: page.items.map(parseExperiment), + nextCursor: optionalString(page, "nextCursor"), + }; +} + +export function parseExperimentExecutionPage( + value: unknown, +): ExperimentExecutionPageView { + const page = record(value, "experiment execution page"); + if (!Array.isArray(page.items)) { + throw new Error("Invalid experiment execution page items response from Scope"); + } + return { + items: page.items.map(parseExperimentExecution), + nextCursor: optionalString(page, "nextCursor"), + }; +} diff --git a/apps/portal/src/pages/ExperimentDetail.tsx b/apps/portal/src/pages/ExperimentDetail.tsx new file mode 100644 index 000000000..857633ba3 --- /dev/null +++ b/apps/portal/src/pages/ExperimentDetail.tsx @@ -0,0 +1,316 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { useState } from "react"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { ArrowLeft, FileText, RotateCcw } from "lucide-react"; +import { Link, useParams } from "react-router-dom"; +import { toast } from "sonner"; +import { ExperimentStatusCard } from "@/components/ExperimentStatusCard"; +import { ShortId } from "@/components/ShortId"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { ApiError, api } from "@/lib/api"; +import type { ExperimentExecutionView } from "@/lib/experiments"; +import { formatDate } from "@/lib/utils"; + +function ExecutionSummary({ + execution, +}: { + execution: ExperimentExecutionView; +}) { + return ( + + + + Execution attempt {execution.attemptNumber} + + + +
+
Execution ID
+ +
+
+
Runs
+
{execution.requestCount}
+
+
+
Created
+
{formatDate(execution.createdAt)}
+
+ + + ); +} + +export function ExperimentDetail() { + const { id } = useParams<{ id: string }>(); + const queryClient = useQueryClient(); + const [selectedExecutionId, setSelectedExecutionId] = useState(); + const experimentQuery = useQuery({ + queryKey: ["experiment", id], + queryFn: () => api.getExperiment(id!), + enabled: Boolean(id), + refetchInterval: (query) => { + const readiness = query.state.data?.readiness; + return readiness === "done" || + readiness === "failed" || + readiness === "blocked" + ? false + : 5_000; + }, + }); + const executionsQuery = useQuery({ + queryKey: ["experiment-executions", id], + queryFn: () => api.listExperimentExecutions(id!, { limit: 50 }), + enabled: Boolean(id) && experimentQuery.isSuccess, + }); + const selectedExecutionQuery = useQuery({ + queryKey: ["experiment-execution", id, selectedExecutionId], + queryFn: () => api.getExperimentExecution(id!, selectedExecutionId!), + enabled: Boolean(id && selectedExecutionId), + }); + const retry = useMutation({ + mutationFn: () => + api.retryExperiment( + experimentQuery.data!.experimentId, + experimentQuery.data!.execution.experimentExecutionId, + ), + onSuccess: () => { + toast.success("Experiment processing restarted"); + void queryClient.invalidateQueries({ queryKey: ["experiment", id] }); + void queryClient.invalidateQueries({ + queryKey: ["experiment-executions", id], + }); + }, + onError: (error) => { + toast.error(error instanceof Error ? error.message : "Retry failed"); + }, + }); + + if (experimentQuery.isLoading) { + return ( +
+ Loading experiment… +
+ ); + } + if (experimentQuery.isError || !experimentQuery.data) { + const correlationOnly = + experimentQuery.error instanceof ApiError && + experimentQuery.error.status === 404; + return ( +
+ + + + + {correlationOnly + ? "Historical experiment correlation" + : "Experiment could not be loaded"} + + + + {correlationOnly ? ( + <> +

+ {id} is + attached to historical runs but has no canonical Experiment + document. +

+

+ This correlation is view-only. It cannot be retried or used to + trigger processing. +

+ + ) : ( +

+ {experimentQuery.error instanceof Error + ? experimentQuery.error.message + : "The experiment request failed."} +

+ )} + + +
+ ); + } + + const experiment = experimentQuery.data; + const canRetry = !["waiting", "processing"].includes( + experiment.readiness ?? "", + ); + + return ( +
+
+
+ +
+

+ {experiment.metadata.name} +

+ +
+

+ Created {formatDate(experiment.createdAt)} +

+
+ +
+ + {experiment.metadata.description && ( +

+ {experiment.metadata.description} +

+ )} + {experiment.metadata.tags && experiment.metadata.tags.length > 0 && ( +
+ {experiment.metadata.tags.map((tag) => ( + + {tag} + + ))} +
+ )} + + + + {experiment.blocking.length > 0 && ( + + + Blocking dependencies + + + {experiment.blocking.map((blocking, index) => ( +
+ {blocking.requestId && ( + + {blocking.requestId.slice(0, 8)} + + )} + {blocking.reason} + + {blocking.handlerId} + + {blocking.error && {blocking.error}} +
+ ))} + + + )} + + + + Current runs + + + {experiment.execution.requests.length === 0 ? ( +

+ {experiment.execution.requestCount === 0 + ? "This experiment is empty." + : `${experiment.execution.requestCount} linked runs are stored outside the inline response.`} +

+ ) : ( + experiment.execution.requests.map((request) => ( + +
+
+ + + {request.task ?? request.requestId} + +
+
+ {request.requestId} · {request.runId} +
+
+
+ {request.status && ( + {request.status} + )} + {request.outcome && ( + {request.outcome} + )} +
+ + )) + )} + + + + + + Execution history + + + {executionsQuery.isLoading && ( +

+ Loading execution history… +

+ )} + {executionsQuery.data?.items.map((execution) => ( + + ))} + + + + {selectedExecutionQuery.isLoading && ( +

Loading execution…

+ )} + {selectedExecutionQuery.data && ( + + )} +
+ ); +} diff --git a/apps/portal/src/pages/ExperimentsList.tsx b/apps/portal/src/pages/ExperimentsList.tsx new file mode 100644 index 000000000..5df215565 --- /dev/null +++ b/apps/portal/src/pages/ExperimentsList.tsx @@ -0,0 +1,120 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { useState } from "react"; +import { useQuery } from "@tanstack/react-query"; +import { FlaskConical, Plus } from "lucide-react"; +import { Link } from "react-router-dom"; +import { ShortId } from "@/components/ShortId"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { + Card, + CardContent, + CardHeader, + CardTitle, +} from "@/components/ui/card"; +import { api } from "@/lib/api"; +import { formatDate } from "@/lib/utils"; + +export function ExperimentsList() { + const [cursor, setCursor] = useState(); + const experimentsQuery = useQuery({ + queryKey: ["experiments", cursor], + queryFn: () => api.listExperiments({ cursor, limit: 50 }), + }); + + return ( +
+
+
+

Experiments

+

+ Reproducible benchmark compositions and their processing history. +

+
+ +
+ + {experimentsQuery.isLoading && ( +

Loading experiments…

+ )} + {experimentsQuery.isError && ( +

+ {experimentsQuery.error instanceof Error + ? experimentsQuery.error.message + : "Experiments could not be loaded."} +

+ )} + {experimentsQuery.data?.items.length === 0 && ( + + + +
+

No experiments yet

+

+ Create one from a run configuration or start explicitly empty. +

+
+ + + )} +
+ {experimentsQuery.data?.items.map((experiment) => ( + + + +
+ + {experiment.metadata.name} + +
+ + Created {formatDate(experiment.createdAt)} +
+
+ + {experiment.readiness ?? "unknown"} + + + + {experiment.revision.memberCount} runs + Revision {experiment.revision.revisionNumber} + Attempt {experiment.execution.attemptNumber} + + + + ))} +
+ + {experimentsQuery.data?.nextCursor && ( +
+ +
+ )} +
+ ); +} diff --git a/apps/portal/src/pages/RunDetail.tsx b/apps/portal/src/pages/RunDetail.tsx index 9eaed350f..81d3429bb 100644 --- a/apps/portal/src/pages/RunDetail.tsx +++ b/apps/portal/src/pages/RunDetail.tsx @@ -905,15 +905,15 @@ export function RunDetail() { /> )} - {run.submissionId && ( + {run.experimentId && ( - {run.submissionId.slice(0, 8)} + {run.experimentId.slice(0, 8)} )} /> diff --git a/apps/portal/src/pages/RunsList.tsx b/apps/portal/src/pages/RunsList.tsx index c7257fc97..31007a293 100644 --- a/apps/portal/src/pages/RunsList.tsx +++ b/apps/portal/src/pages/RunsList.tsx @@ -56,7 +56,7 @@ import { isAgentAvailable, isAgentVersionAvailable, STATUS_LIST, OUTCOME_LIST } import type { Run, RunStatus, RunOutcome, IterationOp, BulkResubmitOverrides, RunSortField, RunFacetBucket } from "@/types"; import { buildRunWorkerFilterOptions } from "./run-worker-filter-options"; -const FILTER_KEYS = ["worker", "status", "outcome", "taskPromptId", "submissionId", "criteria", "model", "profile", "os", "priority", "version", "dateFrom", "dateTo", "groupBy", "turns", "turnsOp", "maxIter", "maxIterOp"] as const; +const FILTER_KEYS = ["worker", "status", "outcome", "taskPromptId", "experimentId", "criteria", "model", "profile", "os", "priority", "version", "dateFrom", "dateTo", "groupBy", "turns", "turnsOp", "maxIter", "maxIterOp"] as const; /** * Describes how to fetch a given (1-based) page from the cursor-paginated @@ -132,18 +132,18 @@ const GROUP_MEMBER_PAGE = 100; /** * Map a server group key (issue #1138 server-side grouping) to the extra * server-side filter that selects that group's member runs. `task` groups by - * `taskPromptId`, `profile` by `profileId`, `submissionId` by `submissionId` - * (the literal `no-submission`/`no-profile` keys mean "missing", expressed as + * `taskPromptId`, `profile` by `profileId`, `experimentId` by `experimentId` + * (the literal `no-experiment`/`no-profile` keys mean "missing", expressed as * the `__empty__` sentinel so the API matches rows lacking the field). */ function groupKeyExtraFilter( - groupBy: "task" | "submissionId" | "profile", + groupBy: "task" | "experimentId" | "profile", key: string, -): { taskPromptId?: string; submissionId?: string; profileId?: string[] } { +): { taskPromptId?: string; experimentId?: string; profileId?: string[] } { if (groupBy === "task") return { taskPromptId: key }; if (groupBy === "profile") return { profileId: [key === "no-profile" ? EMPTY_FILTER_VALUE : key] }; - return { submissionId: key === "no-submission" ? EMPTY_FILTER_VALUE : key }; + return { experimentId: key === "no-experiment" ? EMPTY_FILTER_VALUE : key }; } /** Compact inline badges with overflow (+N) dropdown for dense table cells. */ @@ -313,7 +313,7 @@ const COLUMN_OPTIONS: CustomizeColumnsOption[] = [ // growth-ecosystems/scope-project#127). { id: "status", label: "Status" }, { id: "outcome", label: "Outcome" }, - { id: "submission", label: "Submission" }, + { id: "experiment", label: "Experiment" }, { id: "task", label: "Task" }, { id: "criteria", label: "Criteria" }, { id: "worker", label: "Worker" }, @@ -339,18 +339,18 @@ const COLUMN_IDS = COLUMN_OPTIONS.map((o) => o.id); // Segmented toggle used in the list header for the Group By control. Kept // inline because it is only used here. -const GROUP_BY_OPTIONS: ReadonlyArray<{ value: "profile" | "task" | "submissionId"; label: string }> = [ +const GROUP_BY_OPTIONS: ReadonlyArray<{ value: "profile" | "task" | "experimentId"; label: string }> = [ { value: "profile", label: "Profile" }, { value: "task", label: "Task" }, - { value: "submissionId", label: "Submission" }, + { value: "experimentId", label: "Experiment" }, ]; function GroupByToggle({ value, onChange, }: { - value: "none" | "profile" | "task" | "submissionId"; - onChange: (next: "none" | "profile" | "task" | "submissionId") => void; + value: "none" | "profile" | "task" | "experimentId"; + onChange: (next: "none" | "profile" | "task" | "experimentId") => void; }) { return (