diff --git a/AGENTS.md b/AGENTS.md index 409016367..119d758fc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -253,6 +253,7 @@ not open the PR against the fork unless the user explicitly asks you to. |----------|-------------| | [docs/architecture/overview.md](docs/architecture/overview.md) | System architecture, component interactions, data flow | | [docs/architecture/app-design.md](docs/architecture/app-design.md) | Data models, API design, package dependency graph | +| [docs/architecture/experiments.md](docs/architecture/experiments.md) | Canonical Experiment revisions, execution attempts, fixed-vector storage, publication, and client contract | | [docs/architecture/prompt-evaluations.md](docs/architecture/prompt-evaluations.md) | Static prompt quality, datasets, commands, and maintenance rules | | [docs/architecture/data-organization-projects.md](docs/architecture/data-organization-projects.md) | Projects (a single container) to isolate/group data within a cluster; composes with data-tags and auth-rbac | | [docs/architecture/auth-rbac.md](docs/architecture/auth-rbac.md) | Explicit-login IdP auth, Redis user-access cache, Portal handshake; deferred RBAC/internal-token roadmap | diff --git a/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap b/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap index 317154c30..d602570fc 100644 --- a/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap +++ b/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap @@ -721,6 +721,150 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` ], "type": "object", }, + "CreateExperimentInput": { + "properties": { + "metadata": { + "additionalProperties": false, + "properties": { + "description": { + "maxLength": 4096, + "type": "string", + }, + "name": { + "maxLength": 256, + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "name", + ], + "type": "object", + }, + "mode": { + "enum": [ + "compose", + "empty", + ], + "type": "string", + }, + "requests": { + "default": [], + "items": { + "properties": { + "agentsMd": { + "type": "string", + }, + "agentsMdParentIds": { + "items": { + "type": "string", + }, + "type": "array", + }, + "codebaseRevisionId": { + "type": "string", + }, + "extensions": { + "items": { + "type": "string", + }, + "type": "array", + }, + "gates": { + "items": { + "$ref": "#/components/schemas/GateConfig", + }, + "type": "array", + }, + "maxIterations": { + "maximum": 50, + "minimum": 1, + "type": "integer", + }, + "mcpServers": { + "items": { + "type": "string", + }, + "type": "array", + }, + "model": { + "type": "string", + }, + "observations": { + "items": { + "type": "string", + }, + "type": "array", + }, + "persona": { + "$ref": "#/components/schemas/Persona", + }, + "personaInstructions": { + "type": "string", + }, + "priority": { + "type": "integer", + }, + "profileId": { + "type": "string", + }, + "reasoningEffort": { + "type": "string", + }, + "resources": { + "items": { + "$ref": "#/components/schemas/ResourceBindingSpec", + }, + "type": "array", + }, + "scenario": { + "$ref": "#/components/schemas/Scenario", + }, + "skillRevisions": { + "items": { + "type": "string", + }, + "type": "array", + }, + "workerType": { + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "scenario", + "workerType", + ], + "type": "object", + }, + "maxItems": 250000, + "type": "array", + }, + }, + "required": [ + "mode", + "metadata", + ], + "type": "object", + }, + "CreateExperimentResponse": { + "properties": { + "experiment": { + "$ref": "#/components/schemas/ExperimentResponse", + }, + "requestIds": { + "items": { + "type": "string", + }, + "type": "array", + }, + }, + "required": [ + "experiment", + "requestIds", + ], + "type": "object", + }, "CreateExtensionInput": { "properties": { "_id": { @@ -986,6 +1130,8 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "type": "string", }, "profileVariations": { + "deprecated": true, + "description": "Deprecated for composition. Use POST /api/v1/experiments and its requests array.", "items": { "type": "string", }, @@ -1349,11 +1495,55 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` ], "type": "object", }, - "ExtensionResponse": { + "ExperimentExecutionListResponse": { "properties": { - "_id": { + "data": { + "items": { + "$ref": "#/components/schemas/ExperimentExecutionResponse", + }, + "type": "array", + }, + "nextCursor": { "type": "string", }, + }, + "required": [ + "data", + ], + "type": "object", + }, + "ExperimentExecutionResponse": { + "properties": { + "attemptNumber": { + "type": "integer", + }, + "blockers": { + "items": { + "properties": { + "error": { + "type": "string", + }, + "handlerId": { + "type": "string", + }, + "reason": { + "type": "string", + }, + "requestId": { + "type": "string", + }, + "runId": { + "type": "string", + }, + }, + "required": [ + "handlerId", + "reason", + ], + "type": "object", + }, + "type": "array", + }, "createdAt": { "format": "date-time", "type": [ @@ -1361,33 +1551,91 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "null", ], }, - "deletedAt": { + "discardReason": { + "type": "string", + }, + "discardedAt": { "format": "date-time", "type": [ "string", "null", ], }, - "description": { + "experimentExecutionId": { "type": "string", }, - "name": { - "type": "string", + "finishedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], }, - "origin": { + "progress": { + "additionalProperties": { + "properties": { + "error": { + "type": "string", + }, + "status": { + "enum": [ + "queued", + "processing", + "done", + "failed", + ], + "type": "string", + }, + "updatedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + }, + "required": [ + "status", + "updatedAt", + ], + "type": "object", + }, + "type": "object", + }, + "readiness": { "enum": [ - "marketplace", - "manual", + "empty", + "waiting", + "blocked", + "ready", + "processing", + "done", + "failed", ], "type": "string", }, - "projectId": { - "type": "string", + "startedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], }, - "publisher": { + "supersededAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "supersededByExperimentExecutionId": { "type": "string", }, - "slug": { + "supersededReason": { + "enum": [ + "rollover", + "retry", + ], "type": "string", }, "updatedAt": { @@ -1399,88 +1647,393 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, }, "required": [ - "_id", - "slug", - "publisher", - "name", - "origin", + "experimentExecutionId", + "attemptNumber", + "readiness", + "blockers", + "progress", "createdAt", - "projectId", - ], - "type": "object", - }, - "ExtensionSearchResult": { - "properties": { - "description": { - "type": "string", - }, - "id": { - "type": "string", - }, - "internal": { - "type": "boolean", - }, - "name": { - "type": "string", - }, - "publisher": { - "type": "string", - }, - "version": { - "type": "string", - }, - }, - "required": [ - "id", - "name", - "publisher", - "internal", + "updatedAt", ], "type": "object", }, - "ExtensionVersionInfo": { + "ExperimentListResponse": { "properties": { - "lastUpdated": { - "type": "string", - }, - "preRelease": { - "type": "boolean", + "data": { + "items": { + "$ref": "#/components/schemas/ExperimentResponse", + }, + "type": "array", }, - "version": { + "nextCursor": { "type": "string", }, }, "required": [ - "version", - "preRelease", - "lastUpdated", + "data", ], "type": "object", }, - "FeatureFlagResponse": { + "ExperimentResponse": { "properties": { - "enabled": { - "type": "boolean", - }, - "key": { - "type": "string", - }, - "label": { - "type": "string", - }, - "updatedAt": { + "createdAt": { "format": "date-time", "type": [ "string", "null", ], }, - }, - "required": [ - "key", - "label", - "enabled", - "updatedAt", + "currentExecution": { + "properties": { + "attemptNumber": { + "type": "integer", + }, + "blockers": { + "items": { + "properties": { + "error": { + "type": "string", + }, + "handlerId": { + "type": "string", + }, + "reason": { + "type": "string", + }, + "requestId": { + "type": "string", + }, + "runId": { + "type": "string", + }, + }, + "required": [ + "handlerId", + "reason", + ], + "type": "object", + }, + "type": "array", + }, + "createdAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "discardReason": { + "type": "string", + }, + "discardedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "experimentExecutionId": { + "type": "string", + }, + "finishedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "progress": { + "additionalProperties": { + "properties": { + "error": { + "type": "string", + }, + "status": { + "enum": [ + "queued", + "processing", + "done", + "failed", + ], + "type": "string", + }, + "updatedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + }, + "required": [ + "status", + "updatedAt", + ], + "type": "object", + }, + "type": "object", + }, + "readiness": { + "enum": [ + "empty", + "waiting", + "blocked", + "ready", + "processing", + "done", + "failed", + ], + "type": "string", + }, + "report": { + "properties": { + "reportId": { + "type": "string", + }, + "updatedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "url": { + "type": "string", + }, + }, + "type": "object", + }, + "startedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "updatedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + }, + "required": [ + "experimentExecutionId", + "attemptNumber", + "readiness", + "blockers", + "progress", + "createdAt", + "updatedAt", + ], + "type": "object", + }, + "experimentId": { + "type": "string", + }, + "metadata": { + "additionalProperties": false, + "properties": { + "createdBy": { + "type": "string", + }, + "description": { + "maxLength": 4096, + "type": "string", + }, + "name": { + "maxLength": 256, + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "name", + ], + "type": "object", + }, + "projectId": { + "type": "string", + }, + "revision": { + "properties": { + "createdAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "experimentRevisionId": { + "type": "string", + }, + "memberCount": { + "type": "integer", + }, + "revisionNumber": { + "type": "integer", + }, + }, + "required": [ + "experimentRevisionId", + "revisionNumber", + "memberCount", + "createdAt", + ], + "type": "object", + }, + "updatedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + }, + "required": [ + "experimentId", + "projectId", + "metadata", + "revision", + "currentExecution", + "createdAt", + "updatedAt", + ], + "type": "object", + }, + "ExtensionResponse": { + "properties": { + "_id": { + "type": "string", + }, + "createdAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "deletedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "description": { + "type": "string", + }, + "name": { + "type": "string", + }, + "origin": { + "enum": [ + "marketplace", + "manual", + ], + "type": "string", + }, + "projectId": { + "type": "string", + }, + "publisher": { + "type": "string", + }, + "slug": { + "type": "string", + }, + "updatedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + }, + "required": [ + "_id", + "slug", + "publisher", + "name", + "origin", + "createdAt", + "projectId", + ], + "type": "object", + }, + "ExtensionSearchResult": { + "properties": { + "description": { + "type": "string", + }, + "id": { + "type": "string", + }, + "internal": { + "type": "boolean", + }, + "name": { + "type": "string", + }, + "publisher": { + "type": "string", + }, + "version": { + "type": "string", + }, + }, + "required": [ + "id", + "name", + "publisher", + "internal", + ], + "type": "object", + }, + "ExtensionVersionInfo": { + "properties": { + "lastUpdated": { + "type": "string", + }, + "preRelease": { + "type": "boolean", + }, + "version": { + "type": "string", + }, + }, + "required": [ + "version", + "preRelease", + "lastUpdated", + ], + "type": "object", + }, + "FeatureFlagResponse": { + "properties": { + "enabled": { + "type": "boolean", + }, + "key": { + "type": "string", + }, + "label": { + "type": "string", + }, + "updatedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + }, + "required": [ + "key", + "label", + "enabled", + "updatedAt", ], "type": "object", }, @@ -1601,6 +2154,9 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "codebaseRevisionId": { "type": "string", }, + "experimentId": { + "type": "string", + }, "extensions": { "items": { "type": "string", @@ -2741,6 +3297,27 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "null", ], }, + "experimentExpectedRequestCount": { + "minimum": 0, + "type": "integer", + }, + "experimentFinalizedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "experimentId": { + "type": "string", + }, + "experimentInitialRunId": { + "type": "string", + }, + "experimentRequestIndex": { + "minimum": 0, + "type": "integer", + }, "extensions": { "items": { "type": "string", @@ -3109,6 +3686,66 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, "type": "object", }, + "RetryExperimentResponse": { + "properties": { + "attemptNumber": { + "type": "integer", + }, + "blockers": { + "items": { + "properties": { + "error": { + "type": "string", + }, + "handlerId": { + "type": "string", + }, + "reason": { + "type": "string", + }, + "requestId": { + "type": "string", + }, + "runId": { + "type": "string", + }, + }, + "required": [ + "handlerId", + "reason", + ], + "type": "object", + }, + "type": "array", + }, + "experimentExecutionId": { + "type": "string", + }, + "experimentId": { + "type": "string", + }, + "readiness": { + "enum": [ + "empty", + "waiting", + "blocked", + "ready", + "processing", + "done", + "failed", + ], + "type": "string", + }, + }, + "required": [ + "experimentId", + "experimentExecutionId", + "attemptNumber", + "readiness", + "blockers", + ], + "type": "object", + }, "RunFacetBucket": { "properties": { "count": { @@ -4696,6 +5333,14 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "type": "string", }, }, + { + "in": "query", + "name": "experimentId", + "required": false, + "schema": { + "type": "string", + }, + }, { "in": "query", "name": "k", @@ -4705,14 +5350,221 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, }, { - "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "in": "query", + "name": "projectId", + "required": true, + "schema": { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, + "type": "string", + }, + }, + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": {}, + "description": "Analysis metrics", + "properties": {}, + "type": "object", + }, + }, + }, + "description": "Success", + }, + }, + "summary": "Compute pass@k / success@T metrics", + "tags": [ + "Requests", + ], + }, + }, + "/api/v1/codebase-revisions/{id}": { + "get": { + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CodebaseRevisionResponse", + }, + }, + }, + "description": "Success", + }, + "404": { + "description": "Revision not found", + }, + }, + "summary": "Get codebase revision by id", + "tags": [ + "Codebase Revisions", + ], + }, + }, + "/api/v1/codebase-revisions/{id}/archive": { + "get": { + "responses": { + "200": { + "description": "Binary tar.gz archive", + }, + "404": { + "description": "Revision or archive not found", + }, + "500": { + "description": "Blob storage error", + }, + }, + "summary": "Download codebase revision archive", + "tags": [ + "Codebase Revisions", + ], + }, + }, + "/api/v1/codebases": { + "get": { + "parameters": [ + { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "in": "query", + "name": "projectId", + "required": true, + "schema": { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, + "type": "string", + }, + }, + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/CodebaseResponse", + }, + "type": "array", + }, + }, + }, + "description": "Success", + }, + }, + "summary": "List all codebases", + "tags": [ + "Codebases", + ], + }, + "post": { + "parameters": [ + { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "in": "query", + "name": "projectId", + "required": true, + "schema": { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, + "type": "string", + }, + }, + ], + "responses": { + "201": { + "description": "Success", + }, + "400": { + "description": "Invalid input or missing archive", + }, + }, + "summary": "Create a codebase (archive codebases require the archive file)", + "tags": [ + "Codebases", + ], + }, + }, + "/api/v1/codebases/{id}": { + "delete": { + "responses": { + "204": { + "description": "Success", + }, + "404": { + "description": "Codebase not found", + }, + }, + "summary": "Delete a codebase", + "tags": [ + "Codebases", + ], + }, + "get": { + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CodebaseResponse", + }, + }, + }, + "description": "Success", + }, + "404": { + "description": "Codebase not found", + }, + }, + "summary": "Get a codebase", + "tags": [ + "Codebases", + ], + }, + "patch": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateCodebaseInput", + }, + }, + }, + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CodebaseResponse", + }, + }, + }, + "description": "Success", + }, + "404": { + "description": "Codebase not found", + }, + }, + "summary": "Update a codebase", + "tags": [ + "Codebases", + ], + }, + }, + "/api/v1/codebases/{id}/revisions": { + "get": { + "parameters": [ + { "in": "query", - "name": "projectId", - "required": true, + "name": "limit", + "required": false, "schema": { - "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", - "example": "00000000-0000-0000-0000-000000000000", - "minLength": 1, "type": "string", }, }, @@ -4722,26 +5574,36 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "content": { "application/json": { "schema": { - "additionalProperties": {}, - "description": "Analysis metrics", - "properties": {}, - "type": "object", + "items": { + "$ref": "#/components/schemas/CodebaseRevisionResponse", + }, + "type": "array", }, }, }, "description": "Success", }, + "404": { + "description": "Codebase not found", + }, }, - "summary": "Compute pass@k / success@T metrics", + "summary": "List codebase revisions", "tags": [ - "Requests", + "Codebases", ], }, - }, - "/api/v1/codebase-revisions/{id}": { - "get": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ResolveCodebaseRevisionInput", + }, + }, + }, + }, "responses": { - "200": { + "201": { "content": { "application/json": { "schema": { @@ -4751,38 +5613,90 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, "description": "Success", }, + "400": { + "description": "Not a git codebase", + }, "404": { - "description": "Revision not found", + "description": "Codebase not found", + }, + "502": { + "description": "GitHub resolution failed", }, }, - "summary": "Get codebase revision by id", + "summary": "Resolve a new git codebase revision", "tags": [ - "Codebase Revisions", + "Codebases", ], }, }, - "/api/v1/codebase-revisions/{id}/archive": { - "get": { + "/api/v1/codebases/{id}/upload": { + "post": { "responses": { - "200": { - "description": "Binary tar.gz archive", + "201": { + "description": "Success", }, - "404": { - "description": "Revision or archive not found", + "400": { + "description": "Missing archive or not an archive codebase", }, - "500": { - "description": "Blob storage error", + "404": { + "description": "Codebase not found", }, }, - "summary": "Download codebase revision archive", + "summary": "Upload a codebase archive as a new revision", "tags": [ - "Codebase Revisions", + "Codebases", ], }, }, - "/api/v1/codebases": { + "/api/v1/criteria": { "get": { "parameters": [ + { + "in": "query", + "name": "q", + "required": false, + "schema": { + "type": "string", + }, + }, + { + "description": "Comma-separated criterion IDs to include", + "in": "query", + "name": "ids", + "required": false, + "schema": { + "description": "Comma-separated criterion IDs to include", + "type": "string", + }, + }, + { + "description": "When true and ids is set, also include dependency ancestors", + "in": "query", + "name": "ancestors", + "required": false, + "schema": { + "description": "When true and ids is set, also include dependency ancestors", + "enum": [ + "true", + "false", + ], + "type": "string", + }, + }, + { + "description": "Filter by criterion kind", + "in": "query", + "name": "kind", + "required": false, + "schema": { + "description": "Filter by criterion kind", + "enum": [ + "gate", + "observation", + ], + "type": "string", + }, + }, { "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", "in": "query", @@ -4802,7 +5716,7 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "application/json": { "schema": { "items": { - "$ref": "#/components/schemas/CodebaseResponse", + "$ref": "#/components/schemas/CriteriaResponse", }, "type": "array", }, @@ -4811,9 +5725,9 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "description": "Success", }, }, - "summary": "List all codebases", + "summary": "List criteria", "tags": [ - "Codebases", + "Criteria", ], }, "post": { @@ -4831,224 +5745,202 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, }, ], - "responses": { - "201": { - "description": "Success", - }, - "400": { - "description": "Invalid input or missing archive", - }, - }, - "summary": "Create a codebase (archive codebases require the archive file)", - "tags": [ - "Codebases", - ], - }, - }, - "/api/v1/codebases/{id}": { - "delete": { - "responses": { - "204": { - "description": "Success", - }, - "404": { - "description": "Codebase not found", - }, - }, - "summary": "Delete a codebase", - "tags": [ - "Codebases", - ], - }, - "get": { - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CodebaseResponse", - }, - }, - }, - "description": "Success", - }, - "404": { - "description": "Codebase not found", - }, - }, - "summary": "Get a codebase", - "tags": [ - "Codebases", - ], - }, - "patch": { "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/UpdateCodebaseInput", + "$ref": "#/components/schemas/CreateCriteriaInput", }, }, }, }, "responses": { - "200": { + "201": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/CodebaseResponse", + "$ref": "#/components/schemas/CriteriaResponse", }, }, }, "description": "Success", }, - "404": { - "description": "Codebase not found", + "409": { + "description": "Criterion already exists", }, }, - "summary": "Update a codebase", + "summary": "Create criterion", "tags": [ - "Codebases", + "Criteria", ], }, }, - "/api/v1/codebases/{id}/revisions": { - "get": { + "/api/v1/criteria/generate-prompt": { + "post": { "parameters": [ { + "description": "Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.", "in": "query", - "name": "limit", + "name": "projectId", "required": false, "schema": { + "description": "Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, "type": "string", }, }, ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "behavior": { + "type": "string", + }, + "currentId": { + "type": "string", + }, + "gates": { + "items": { + "type": "string", + }, + "type": "array", + }, + }, + "required": [ + "behavior", + ], + "type": "object", + }, + }, + }, + }, "responses": { - "200": { + "201": { "content": { "application/json": { "schema": { - "items": { - "$ref": "#/components/schemas/CodebaseRevisionResponse", + "properties": { + "prompt": { + "type": "string", + }, + "suggestedChildren": { + "items": { + "type": "string", + }, + "type": "array", + }, + "suggestedId": { + "type": "string", + }, + "suggestedParents": { + "items": { + "type": "string", + }, + "type": "array", + }, }, - "type": "array", + "required": [ + "prompt", + "suggestedId", + "suggestedParents", + "suggestedChildren", + ], + "type": "object", }, }, }, "description": "Success", }, - "404": { - "description": "Codebase not found", + "400": { + "description": "Empty behavior string", + }, + "503": { + "description": "LLM not configured", }, }, - "summary": "List codebase revisions", + "summary": "Generate criterion prompt from behavior", "tags": [ - "Codebases", + "Criteria", ], }, - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ResolveCodebaseRevisionInput", - }, + }, + "/api/v1/criteria/graph": { + "get": { + "parameters": [ + { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "in": "query", + "name": "projectId", + "required": true, + "schema": { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, + "type": "string", }, }, - }, + ], "responses": { - "201": { + "200": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/CodebaseRevisionResponse", + "$ref": "#/components/schemas/CriteriaGraph", }, }, }, "description": "Success", }, - "400": { - "description": "Not a git codebase", - }, - "404": { - "description": "Codebase not found", - }, - "502": { - "description": "GitHub resolution failed", - }, - }, - "summary": "Resolve a new git codebase revision", - "tags": [ - "Codebases", - ], - }, - }, - "/api/v1/codebases/{id}/upload": { - "post": { - "responses": { - "201": { - "description": "Success", - }, - "400": { - "description": "Missing archive or not an archive codebase", - }, - "404": { - "description": "Codebase not found", - }, }, - "summary": "Upload a codebase archive as a new revision", + "summary": "Get criteria DAG", "tags": [ - "Codebases", + "Criteria", ], }, }, - "/api/v1/criteria": { + "/api/v1/criteria/mdp": { "get": { "parameters": [ { "in": "query", - "name": "q", + "name": "criteria", "required": false, "schema": { "type": "string", }, }, { - "description": "Comma-separated criterion IDs to include", "in": "query", - "name": "ids", + "name": "features", "required": false, "schema": { - "description": "Comma-separated criterion IDs to include", "type": "string", }, }, { - "description": "When true and ids is set, also include dependency ancestors", "in": "query", - "name": "ancestors", + "name": "since", "required": false, "schema": { - "description": "When true and ids is set, also include dependency ancestors", - "enum": [ - "true", - "false", - ], "type": "string", }, }, { - "description": "Filter by criterion kind", "in": "query", - "name": "kind", + "name": "worker", + "required": false, + "schema": { + "type": "string", + }, + }, + { + "in": "query", + "name": "taskPromptId", "required": false, "schema": { - "description": "Filter by criterion kind", - "enum": [ - "gate", - "observation", - ], "type": "string", }, }, @@ -5070,21 +5962,22 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "content": { "application/json": { "schema": { - "items": { - "$ref": "#/components/schemas/CriteriaResponse", - }, - "type": "array", + "additionalProperties": {}, + "properties": {}, + "type": "object", }, }, }, "description": "Success", }, }, - "summary": "List criteria", + "summary": "Compute MDP transitions", "tags": [ "Criteria", ], }, + }, + "/api/v1/criteria/seed": { "post": { "parameters": [ { @@ -5104,7 +5997,18 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/CreateCriteriaInput", + "properties": { + "criteria": { + "items": { + "$ref": "#/components/schemas/CreateCriteriaInput", + }, + "type": "array", + }, + }, + "required": [ + "criteria", + ], + "type": "object", }, }, }, @@ -5114,94 +6018,77 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/CriteriaResponse", + "properties": { + "errors": { + "items": { + "type": "string", + }, + "type": "array", + }, + "seeded": { + "type": "number", + }, + }, + "required": [ + "seeded", + "errors", + ], + "type": "object", }, }, }, "description": "Success", }, - "409": { - "description": "Criterion already exists", + "400": { + "description": "Seed batch would introduce a dependency cycle", }, }, - "summary": "Create criterion", + "summary": "Seed criteria in bulk", "tags": [ "Criteria", ], }, }, - "/api/v1/criteria/generate-prompt": { - "post": { + "/api/v1/criteria/{id}": { + "delete": { "parameters": [ { - "description": "Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.", + "in": "path", + "name": "id", + "required": true, + "schema": { + "type": "string", + }, + }, + { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", "in": "query", "name": "projectId", - "required": false, + "required": true, "schema": { - "description": "Optional project scope. When provided, a slug/id is resolved within that project (slugs may repeat across projects); when omitted, resolution falls back to a legacy global lookup for backward compatibility.", + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", "example": "00000000-0000-0000-0000-000000000000", "minLength": 1, "type": "string", }, }, ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "behavior": { - "type": "string", - }, - "currentId": { - "type": "string", - }, - "gates": { - "items": { - "type": "string", - }, - "type": "array", - }, - }, - "required": [ - "behavior", - ], - "type": "object", - }, - }, - }, - }, "responses": { - "201": { + "200": { "content": { "application/json": { "schema": { "properties": { - "prompt": { - "type": "string", - }, - "suggestedChildren": { - "items": { - "type": "string", - }, - "type": "array", + "deleted": { + "type": "boolean", }, - "suggestedId": { + "id": { "type": "string", }, - "suggestedParents": { - "items": { - "type": "string", - }, - "type": "array", - }, }, "required": [ - "prompt", - "suggestedId", - "suggestedParents", - "suggestedChildren", + "id", + "deleted", ], "type": "object", }, @@ -5209,22 +6096,71 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, "description": "Success", }, - "400": { - "description": "Empty behavior string", - }, - "503": { - "description": "LLM not configured", + "404": { + "description": "Criterion not found", + }, + "409": { + "description": "Criterion has dependents", + }, + }, + "summary": "Soft-delete criterion", + "tags": [ + "Criteria", + ], + }, + "get": { + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "type": "string", + }, + }, + { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "in": "query", + "name": "projectId", + "required": true, + "schema": { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, + "type": "string", + }, + }, + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CriteriaResponse", + }, + }, + }, + "description": "Success", + }, + "404": { + "description": "Criterion not found", }, }, - "summary": "Generate criterion prompt from behavior", + "summary": "Get criterion", "tags": [ "Criteria", ], }, - }, - "/api/v1/criteria/graph": { - "get": { + "put": { "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "type": "string", + }, + }, { "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", "in": "query", @@ -5238,67 +6174,95 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, }, ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateCriteriaInput", + }, + }, + }, + }, "responses": { "200": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/CriteriaGraph", + "$ref": "#/components/schemas/CriteriaResponse", }, }, }, "description": "Success", }, + "400": { + "description": "Invalid dependency reference or self-reference", + }, + "404": { + "description": "Criterion not found", + }, }, - "summary": "Get criteria DAG", + "summary": "Update criterion", "tags": [ "Criteria", ], }, }, - "/api/v1/criteria/mdp": { + "/api/v1/experiments": { "get": { "parameters": [ { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", "in": "query", - "name": "criteria", - "required": false, - "schema": { - "type": "string", - }, - }, - { - "in": "query", - "name": "features", - "required": false, + "name": "projectId", + "required": true, "schema": { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, "type": "string", }, }, { "in": "query", - "name": "since", + "name": "limit", "required": false, "schema": { - "type": "string", + "default": 50, + "maximum": 100, + "minimum": 1, + "type": "integer", }, }, { "in": "query", - "name": "worker", + "name": "after", "required": false, "schema": { + "minLength": 1, "type": "string", }, }, - { - "in": "query", - "name": "taskPromptId", - "required": false, - "schema": { - "type": "string", + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ExperimentListResponse", + }, + }, }, + "description": "Success", }, + }, + "summary": "List experiments", + "tags": [ + "Experiments", + ], + }, + "post": { + "description": "Canonical composition endpoint. Request and run IDs are minted before insertion; the exact revision/execution is then finalized idempotently.", + "parameters": [ { "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", "in": "query", @@ -5312,29 +6276,48 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, }, ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateExperimentInput", + }, + }, + }, + }, "responses": { - "200": { + "201": { "content": { "application/json": { "schema": { - "additionalProperties": {}, - "properties": {}, - "type": "object", + "$ref": "#/components/schemas/CreateExperimentResponse", }, }, }, "description": "Success", }, + "409": { + "description": "Experiment identity conflict", + }, }, - "summary": "Compute MDP transitions", + "summary": "Create an experiment and zero or more new requests", "tags": [ - "Criteria", + "Experiments", ], }, }, - "/api/v1/criteria/seed": { - "post": { + "/api/v1/experiments/{id}": { + "get": { "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "minLength": 1, + "type": "string", + }, + }, { "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", "in": "query", @@ -5348,70 +6331,36 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, }, ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "criteria": { - "items": { - "$ref": "#/components/schemas/CreateCriteriaInput", - }, - "type": "array", - }, - }, - "required": [ - "criteria", - ], - "type": "object", - }, - }, - }, - }, "responses": { - "201": { + "200": { "content": { "application/json": { "schema": { - "properties": { - "errors": { - "items": { - "type": "string", - }, - "type": "array", - }, - "seeded": { - "type": "number", - }, - }, - "required": [ - "seeded", - "errors", - ], - "type": "object", + "$ref": "#/components/schemas/ExperimentResponse", }, }, }, "description": "Success", }, - "400": { - "description": "Seed batch would introduce a dependency cycle", + "404": { + "description": "Experiment not found", }, }, - "summary": "Seed criteria in bulk", + "summary": "Get an experiment", "tags": [ - "Criteria", + "Experiments", ], }, }, - "/api/v1/criteria/{id}": { - "delete": { + "/api/v1/experiments/{id}/executions": { + "get": { "parameters": [ { "in": "path", "name": "id", "required": true, "schema": { + "minLength": 1, "type": "string", }, }, @@ -5427,42 +6376,49 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "type": "string", }, }, + { + "in": "query", + "name": "limit", + "required": false, + "schema": { + "default": 50, + "maximum": 100, + "minimum": 1, + "type": "integer", + }, + }, + { + "in": "query", + "name": "after", + "required": false, + "schema": { + "minLength": 1, + "type": "string", + }, + }, ], "responses": { "200": { "content": { "application/json": { "schema": { - "properties": { - "deleted": { - "type": "boolean", - }, - "id": { - "type": "string", - }, - }, - "required": [ - "id", - "deleted", - ], - "type": "object", + "$ref": "#/components/schemas/ExperimentExecutionListResponse", }, }, }, "description": "Success", }, "404": { - "description": "Criterion not found", - }, - "409": { - "description": "Criterion has dependents", + "description": "Experiment not found", }, }, - "summary": "Soft-delete criterion", + "summary": "List current and historical experiment executions", "tags": [ - "Criteria", + "Experiments", ], }, + }, + "/api/v1/experiments/{id}/executions/{experimentExecutionId}": { "get": { "parameters": [ { @@ -5470,6 +6426,16 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "name": "id", "required": true, "schema": { + "minLength": 1, + "type": "string", + }, + }, + { + "in": "path", + "name": "experimentExecutionId", + "required": true, + "schema": { + "minLength": 1, "type": "string", }, }, @@ -5491,28 +6457,31 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/CriteriaResponse", + "$ref": "#/components/schemas/ExperimentExecutionResponse", }, }, }, "description": "Success", }, "404": { - "description": "Criterion not found", + "description": "Experiment execution not found", }, }, - "summary": "Get criterion", + "summary": "Get one exact experiment execution", "tags": [ - "Criteria", + "Experiments", ], }, - "put": { + }, + "/api/v1/experiments/{id}/retry": { + "post": { "parameters": [ { "in": "path", "name": "id", "required": true, "schema": { + "minLength": 1, "type": "string", }, }, @@ -5533,32 +6502,41 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/UpdateCriteriaInput", + "properties": { + "experimentExecutionId": { + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "experimentExecutionId", + ], + "type": "object", }, }, }, }, "responses": { - "200": { + "201": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/CriteriaResponse", + "$ref": "#/components/schemas/RetryExperimentResponse", }, }, }, "description": "Success", }, - "400": { - "description": "Invalid dependency reference or self-reference", - }, "404": { - "description": "Criterion not found", + "description": "Experiment not found", + }, + "409": { + "description": "Execution is stale or cannot be retried", }, }, - "summary": "Update criterion", + "summary": "Retry current experiment processing", "tags": [ - "Criteria", + "Experiments", ], }, }, @@ -8834,6 +9812,14 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "type": "string", }, }, + { + "in": "query", + "name": "experimentId", + "required": false, + "schema": { + "type": "string", + }, + }, { "in": "query", "name": "profileId", @@ -8976,6 +9962,7 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "enum": [ "task", "submissionId", + "experimentId", "profile", ], "type": "string", @@ -9194,6 +10181,8 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, "count": { "default": 1, + "deprecated": true, + "description": "Deprecated for composition. Use POST /api/v1/experiments and its requests array.", "maximum": 10, "minimum": 1, "type": "number", diff --git a/apps/api/src/endpoints.test.ts b/apps/api/src/endpoints.test.ts index 6af7b8cb5..3e1f50e5a 100644 --- a/apps/api/src/endpoints.test.ts +++ b/apps/api/src/endpoints.test.ts @@ -766,19 +766,49 @@ describe("API Endpoints", () => { expect(res.body.data[0]).toHaveProperty("uniform"); }); - it("returns grouped results when groupBy=submissionId", async () => { + it("returns grouped results when groupBy=experimentId", async () => { const groupedDocs = [ - { key: "sub-1", label: "sub-1", aggregates: { count: 2, turns: null, duration: null, promptTokens: null, completionTokens: null }, uniform: {} }, + { key: "experiment-1", label: "experiment-1", aggregates: { count: 2, turns: null, duration: null, promptTokens: null, completionTokens: null }, uniform: {} }, ]; (mocks.collection.aggregate as any) - .mockReturnValueOnce({ toArray: vi.fn().mockResolvedValue([{ _id: "sub-1" }]) }) + .mockReturnValueOnce({ toArray: vi.fn().mockResolvedValue([{ _id: "experiment-1" }]) }) .mockReturnValueOnce({ toArray: vi.fn().mockResolvedValue(groupedDocs) }) .mockReturnValueOnce({ toArray: vi.fn().mockResolvedValue([]) }) .mockReturnValueOnce({ toArray: vi.fn().mockResolvedValue([]) }); - const res = await request(testServer()).get(`/api/v1/requests?groupBy=submissionId&projectId=${TEST_PROJECT_ID}`); + const res = await request(testServer()).get(`/api/v1/requests?groupBy=experimentId&projectId=${TEST_PROJECT_ID}`); expect(res.status).toBe(200); - expect(res.body.data[0]).toHaveProperty("key", "sub-1"); + expect(res.body.data[0]).toHaveProperty("key", "experiment-1"); + }); + + it("preserves grouped results when groupBy=submissionId", async () => { + const groupedDocs = [ + { + key: "submission-1", + label: "submission-1", + aggregates: { + count: 2, + turns: null, + duration: null, + promptTokens: null, + completionTokens: null, + }, + uniform: {}, + }, + ]; + (mocks.collection.aggregate as any) + .mockReturnValueOnce({ + toArray: vi.fn().mockResolvedValue([{ _id: "submission-1" }]), + }) + .mockReturnValueOnce({ toArray: vi.fn().mockResolvedValue(groupedDocs) }) + .mockReturnValueOnce({ toArray: vi.fn().mockResolvedValue([]) }) + .mockReturnValueOnce({ toArray: vi.fn().mockResolvedValue([]) }); + + const res = await request(testServer()).get( + `/api/v1/requests?groupBy=submissionId&projectId=${TEST_PROJECT_ID}`, + ); + expect(res.status).toBe(200); + expect(res.body.data[0]).toHaveProperty("key", "submission-1"); }); it("calls aggregate pipeline when groupBy is provided", async () => { @@ -1955,6 +1985,40 @@ describe("API Endpoints", () => { expect(res.status).toBe(201); const doc = (mocks.collection.insertOne as any).mock.calls[0][0]; expect(doc).toHaveProperty("reasoningEffort", "high"); + expect(doc).not.toHaveProperty("experimentId"); + expect(doc).toHaveProperty("submissionId", res.body.submissionId); + expect(res.body.submissionId).toEqual(expect.any(String)); + expect(mocks.experimentCollection.updateOne).not.toHaveBeenCalled(); + expect(res.headers).not.toHaveProperty("deprecation"); + expect(res.headers).not.toHaveProperty("link"); + expect(res.headers).not.toHaveProperty("sunset"); + }); + + it("keeps count fan-out independent from experiments", async () => { + const res = await request(testServer()) + .post(`/api/v1/requests?worker=coder-acp-copilot&projectId=${TEST_PROJECT_ID}`) + .send({ + scenario: { task: "Build something", criteria: ["works"] }, + count: 3, + }); + + expect(res.status).toBe(201); + const docs = (mocks.collection.insertMany as any).mock.calls[0][0]; + expect(docs).toHaveLength(3); + expect(docs.every((doc: object) => !("experimentId" in doc))).toBe(true); + expect(res.body.submissionId).toEqual(expect.any(String)); + expect( + docs.every( + (doc: { submissionId?: string }) => + doc.submissionId === res.body.submissionId, + ), + ).toBe(true); + expect(mocks.experimentCollection.updateOne).not.toHaveBeenCalled(); + expect(res.headers.deprecation).toBe("true"); + expect(res.headers.link).toBe( + '; rel="successor-version"; title="POST /api/v1/experiments"', + ); + expect(res.headers).not.toHaveProperty("sunset"); }); it("profile reasoningEffort takes precedence over client-provided effort", async () => { @@ -2259,6 +2323,8 @@ describe("API Endpoints", () => { new Set(["base-profile", "var-profile"]), ); for (const doc of docs) { + expect(doc.experimentId).toBeUndefined(); + expect(doc.submissionId).toBe(res.body.submissionId); expect(Array.isArray(doc.gates)).toBe(true); const selectGate = doc.gates.find((g: { gate: string }) => g.gate === "select"); const buildGate = doc.gates.find((g: { gate: string }) => g.gate === "build"); @@ -2269,6 +2335,11 @@ describe("API Endpoints", () => { expect(buildGate.promptId).toBe("tp-build"); expect(buildGate.promptText).toBeUndefined(); } + expect(mocks.experimentCollection.updateOne).not.toHaveBeenCalled(); + expect(res.body.submissionId).toEqual(expect.any(String)); + expect(res.headers.deprecation).toBe("true"); + expect(res.headers.link).toContain("POST /api/v1/experiments"); + expect(res.headers).not.toHaveProperty("sunset"); // Response surfaces the configured gate count. expect(res.body.gates).toBe(2); }); @@ -2426,6 +2497,13 @@ describe("API Endpoints", () => { expect(insertCall[0]).toHaveProperty("taskPromptId", "tp-123"); expect(insertCall[0]).toHaveProperty("model", "gpt-4o"); expect(insertCall[0]).toHaveProperty("maxIterations", 5); + expect(insertCall[0]).not.toHaveProperty("experimentId"); + expect(insertCall[0]).toHaveProperty( + "submissionId", + res.body.submissionId, + ); + expect(res.body.submissionId).toEqual(expect.any(String)); + expect(mocks.experimentCollection.updateOne).not.toHaveBeenCalled(); }); it("applies profile override: uses profile fields for worker, model, mcpServers, skillRevisions, extensions", async () => { @@ -2802,11 +2880,13 @@ describe("API Endpoints", () => { it("retries eligible requests and skips non-terminal ones", async () => { const doc1 = { _id: "req-1", + experimentId: "experiment-1", workerType: "coder-acp-copilot", run: { _id: "run-1", attemptNumber: 1, status: "done", outcome: "failed" }, }; const doc2 = { _id: "req-2", + experimentId: "experiment-1", workerType: "coder-acp-copilot", run: { _id: "run-2", attemptNumber: 1, status: "processing" }, }; @@ -2832,6 +2912,7 @@ describe("API Endpoints", () => { $set: expect.objectContaining({ agentVersion: "1.0.0" }), }), ); + expect(mocks.experimentCollection.updateMany).not.toHaveBeenCalled(); }); it("returns 400 when ids array is empty", async () => { diff --git a/apps/api/src/experiment-coordinator.test.ts b/apps/api/src/experiment-coordinator.test.ts new file mode 100644 index 000000000..30a0afd09 --- /dev/null +++ b/apps/api/src/experiment-coordinator.test.ts @@ -0,0 +1,475 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { Collection, Document } from "mongodb"; +import { + EXPERIMENT_STORAGE_PROFILE_VERSION, + ExperimentStore, + FixedPayloadStore, + fingerprintExperimentPayload, + type ExperimentDocument, + type ExperimentExecutionHistoryDocument, + type ExperimentRequestState, + type RequestDocument, + type ServiceHandlerDocument, +} from "shared"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + ExperimentCoordinator, + type ExperimentCoordinatorCollections, +} from "./experiment-coordinator.js"; + +const now = new Date("2026-01-01T00:00:00.000Z"); + +function experiment( + requests: ExperimentRequestState[], + overrides: Partial = {}, +): ExperimentDocument { + const fingerprint = fingerprintExperimentPayload(requests); + return { + _id: "experiment-1", + experimentId: "experiment-1", + projectId: "project-1", + metadata: { name: "Experiment" }, + storageProfileVersion: EXPERIMENT_STORAGE_PROFILE_VERSION, + stateVersion: 1, + revision: { + experimentRevisionId: "revision-1", + revisionNumber: 1, + memberCount: requests.length, + membershipFingerprint: fingerprint, + definitionFingerprint: "definition-1", + createdAt: now, + }, + execution: { + experimentExecutionId: "execution-1", + attemptNumber: 1, + version: 1, + requests: { + storage: "inline", + entryCount: requests.length, + payloadFingerprint: fingerprint, + payloadVersion: 1, + entries: requests, + }, + vectorFingerprint: fingerprint, + handlerSummary: {}, + blocking: [], + createdAt: now, + updatedAt: now, + ...overrides, + }, + createdAt: now, + updatedAt: now, + finalizedAt: now, + }; +} + +function cursor(documents: T[]) { + return { + sort: vi.fn().mockReturnThis(), + limit: vi.fn().mockReturnThis(), + toArray: vi.fn().mockResolvedValue(documents), + }; +} + +function collections(input: { + requests: RequestDocument[]; + services: ServiceHandlerDocument[]; + updatedExperiments?: ExperimentDocument[]; + currentExperiment?: ExperimentDocument; +}): ExperimentCoordinatorCollections { + const updates = [...(input.updatedExperiments ?? [])]; + return { + requests: { + find: vi.fn().mockReturnValue(cursor(input.requests)), + updateMany: vi.fn(), + } as unknown as Collection, + experiments: { + findOne: vi.fn().mockResolvedValue(input.currentExperiment ?? null), + findOneAndUpdate: vi.fn().mockImplementation(async () => updates.shift() ?? null), + } as unknown as Collection, + executions: {} as Collection, + chunks: {} as Collection, + services: { + find: vi.fn().mockImplementation((filter: Document) => { + const type = filter.type; + return cursor(input.services.filter((service) => service.type === type)); + }), + } as unknown as Collection, + }; +} + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("ExperimentCoordinator", () => { + it("projects authoritative run status and dispatches an exact execution message", async () => { + const initialRequest: ExperimentRequestState = { + requestId: "request-1", + runId: "run-1", + dependencies: [], + blockers: [], + }; + const projectedRequest: ExperimentRequestState = { + ...initialRequest, + dependencies: [ + { + handlerId: "pp-observations", + state: "ready", + version: 3, + updatedAt: now, + }, + ], + }; + const initial = experiment([initialRequest]); + const projected = experiment([projectedRequest], { version: 2 }); + const queued = experiment([projectedRequest], { + version: 3, + handlerSummary: { + "experiment-processing": { + status: "queued", + version: 2, + claimToken: "claim-1", + updatedAt: now, + }, + }, + }); + + vi.spyOn(ExperimentStore.prototype, "get").mockResolvedValue(initial); + vi.spyOn(FixedPayloadStore.prototype, "readRequests").mockResolvedValue([ + initialRequest, + ]); + vi.spyOn( + ExperimentStore.prototype, + "casUpdateRequestStates", + ).mockResolvedValue({ + status: "applied", + experiment: projected, + requests: [projectedRequest], + }); + const enqueue = vi.fn().mockResolvedValue(undefined); + const coordinator = new ExperimentCoordinator( + collections({ + requests: [ + { + _id: "request-1", + projectId: "project-1", + scenario: { task: "task", criteria: [] }, + workerType: "worker", + createdAt: now, + taskPromptId: "prompt-1", + priority: 0, + run: { + _id: "run-1", + attemptNumber: 1, + status: "done", + logsUrl: "https://logs", + handlerStatus: { + "pp-observations": { + status: "done", + version: 3, + updatedAt: now, + }, + }, + }, + }, + ], + services: [ + { + _id: "pp-observations", + type: "post-process-handler", + version: 3, + queue: "queue-pp-observations", + selector: "observations", + autoBackfill: false, + dependsOn: [], + }, + { + _id: "experiment-processing", + type: "experiment-handler", + version: 2, + queue: "queue-experiment-processing", + selector: "experiment-processing", + autoBackfill: false, + dependsOn: ["pp-observations"], + dependencyMode: "all-runs", + }, + ], + updatedExperiments: [queued], + }), + enqueue, + ); + + const result = await coordinator.reconcile("project-1", "experiment-1"); + + expect(result?.dispatched).toBe(true); + expect(enqueue).toHaveBeenCalledWith( + expect.objectContaining({ _id: "experiment-processing" }), + { + type: "experiment-processing", + projectId: "project-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-1", + handlerId: "experiment-processing", + handlerVersion: 2, + claimToken: expect.any(String), + executionVersion: 3, + }, + ); + }); + + it("bounds aggregate execution blockers while retaining exact request projections", async () => { + const requests: ExperimentRequestState[] = Array.from( + { length: 40 }, + (_, index) => ({ + requestId: `request-${index}`, + runId: `run-${index}`, + dependencies: [], + blockers: [], + }), + ); + const projectedRequests: ExperimentRequestState[] = requests.map( + (request) => ({ + ...request, + dependencies: [ + { + handlerId: "pp-observations", + state: "waiting", + }, + ], + blockers: [ + { + requestId: request.requestId, + runId: request.runId, + handlerId: "pp-observations", + reason: "dependency-not-ready", + }, + ], + }), + ); + const initial = experiment(requests); + const projected = experiment(projectedRequests, { version: 2 }); + const bounded = experiment(projectedRequests, { + version: 3, + blocking: projectedRequests + .flatMap((request) => request.blockers) + .slice(0, 32), + }); + vi.spyOn(ExperimentStore.prototype, "get") + .mockResolvedValueOnce(initial) + .mockResolvedValueOnce(bounded); + vi.spyOn(FixedPayloadStore.prototype, "readRequests").mockResolvedValue( + requests, + ); + vi.spyOn( + ExperimentStore.prototype, + "casUpdateRequestStates", + ).mockResolvedValue({ + status: "applied", + experiment: projected, + requests: projectedRequests, + }); + const coordinatorCollections = collections({ + requests: requests.map((request) => ({ + _id: request.requestId, + projectId: "project-1", + scenario: { task: "task", criteria: [] }, + workerType: "worker", + createdAt: now, + taskPromptId: "prompt-1", + priority: 0, + run: { + _id: request.runId, + attemptNumber: 1, + status: "done", + logsUrl: "https://logs", + handlerStatus: {}, + }, + })), + services: [ + { + _id: "pp-observations", + type: "post-process-handler", + version: 1, + queue: "queue-pp-observations", + selector: "observations", + autoBackfill: false, + dependsOn: [], + }, + ], + updatedExperiments: [bounded], + }); + const coordinator = new ExperimentCoordinator(coordinatorCollections); + + const result = await coordinator.reconcile("project-1", "experiment-1"); + + expect(result?.experiment.execution.blocking).toHaveLength(32); + expect( + vi.mocked(coordinatorCollections.experiments.findOneAndUpdate).mock + .calls[0]?.[1], + ).toMatchObject({ + $set: { + "execution.blocking": bounded.execution.blocking, + }, + }); + expect( + vi.mocked(ExperimentStore.prototype.casUpdateRequestStates).mock + .calls[0]?.[0].requests, + ).toHaveLength(40); + }); + + it("re-enqueues a dropped notice without changing its exact identity", async () => { + const requestState: ExperimentRequestState = { + requestId: "request-1", + runId: "run-1", + dependencies: [ + { + handlerId: "pp-observations", + state: "ready", + updatedAt: now, + }, + ], + blockers: [], + }; + const queuedAt = new Date(Date.now() - 60_000); + const current = experiment([requestState], { + version: 7, + handlerSummary: { + "experiment-processing": { + status: "queued", + version: 2, + claimToken: "claim-original", + updatedAt: queuedAt, + }, + }, + }); + vi.spyOn(ExperimentStore.prototype, "get").mockResolvedValue(current); + vi.spyOn(FixedPayloadStore.prototype, "readRequests").mockResolvedValue([ + requestState, + ]); + const enqueue = vi.fn().mockResolvedValue(undefined); + const coordinator = new ExperimentCoordinator( + collections({ + requests: [ + { + _id: "request-1", + projectId: "project-1", + scenario: { task: "task", criteria: [] }, + workerType: "worker", + createdAt: now, + taskPromptId: "prompt-1", + priority: 0, + run: { + _id: "run-1", + attemptNumber: 1, + status: "done", + logsUrl: "https://logs", + handlerStatus: { + "pp-observations": { + status: "done", + updatedAt: now, + }, + }, + }, + }, + ], + services: [ + { + _id: "pp-observations", + type: "post-process-handler", + version: 1, + queue: "queue-pp-observations", + selector: "observations", + autoBackfill: false, + dependsOn: [], + }, + { + _id: "experiment-processing", + type: "experiment-handler", + version: 2, + queue: "queue-experiment-processing", + selector: "experiment-processing", + autoBackfill: false, + dependsOn: ["pp-observations"], + dependencyMode: "all-runs", + }, + ], + }), + enqueue, + ); + + const result = await coordinator.reconcile("project-1", "experiment-1"); + + expect(result?.dispatched).toBe(true); + expect(enqueue).toHaveBeenCalledWith( + expect.any(Object), + expect.objectContaining({ + experimentExecutionId: "execution-1", + executionVersion: 7, + claimToken: "claim-original", + }), + ); + }); + + it("finalizes an interrupted pending creation before marking its requests", async () => { + const requestState: ExperimentRequestState = { + requestId: "request-1", + runId: "run-1", + dependencies: [], + blockers: [], + }; + const pending = experiment([requestState]); + delete pending.finalizedAt; + const finalized = { ...pending, finalizedAt: now }; + const experimentCollections = collections({ + requests: [ + { + _id: "request-1", + projectId: "project-1", + scenario: { task: "task", criteria: [] }, + workerType: "worker", + createdAt: now, + taskPromptId: "prompt-1", + priority: 0, + experimentId: "experiment-1", + experimentExpectedRequestCount: 1, + experimentRequestIndex: 0, + experimentInitialRunId: "run-1", + run: { + _id: "run-1", + attemptNumber: 1, + status: "pending", + logsUrl: "https://logs", + }, + }, + ], + services: [], + currentExperiment: pending, + }); + const finalize = vi + .spyOn(ExperimentStore.prototype, "finalizeCreation") + .mockResolvedValue(finalized); + const coordinator = new ExperimentCoordinator(experimentCollections); + + const recovered = await coordinator.recoverInterruptedCreations(); + + expect(recovered).toBe(1); + expect(finalize).toHaveBeenCalledWith( + expect.objectContaining({ + projectId: "project-1", + experimentId: "experiment-1", + expectedExperimentExecutionId: "execution-1", + expectedStateVersion: 1, + }), + ); + expect(experimentCollections.requests.updateMany).toHaveBeenCalledWith( + expect.objectContaining({ + projectId: "project-1", + experimentId: "experiment-1", + }), + { $set: { experimentFinalizedAt: expect.any(Date) } }, + ); + }); +}); diff --git a/apps/api/src/experiment-coordinator.ts b/apps/api/src/experiment-coordinator.ts new file mode 100644 index 000000000..50dd06857 --- /dev/null +++ b/apps/api/src/experiment-coordinator.ts @@ -0,0 +1,522 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { randomUUID } from "node:crypto"; +import type { Collection, Document } from "mongodb"; +import { + DependencyGraph, + ExperimentStore, + FixedPayloadStore, + fingerprintExperimentPayload, + type ExperimentBlocker, + type ExperimentDocument, + type ExperimentExecutionHistoryDocument, + type ExperimentHandlerServiceDocument, + type ExperimentQueueMessagePayload, + type ExperimentRequestState, + type HandlerRunStatus, + type HandlerServiceDocument, + type RequestDocument, + type ServiceHandlerDocument, +} from "shared"; + +export interface ExperimentCoordinatorCollections { + requests: Collection; + experiments: Collection; + executions: Collection; + chunks: Collection; + services: Collection; +} + +export interface ExperimentDispatchMessage + extends ExperimentQueueMessagePayload { + projectId: string; + handlerId: string; + handlerVersion: number; + claimToken: string; + executionVersion: number; +} + +export type EnqueueExperiment = ( + service: ExperimentHandlerServiceDocument, + message: ExperimentDispatchMessage, +) => Promise; + +export type ExperimentReadiness = + | "empty" + | "waiting" + | "blocked" + | "ready" + | "processing" + | "done" + | "failed"; + +export interface ReconcileExperimentResult { + experiment: ExperimentDocument; + readiness: ExperimentReadiness; + rolled: boolean; + dispatched: boolean; +} + +function namedHandlerState( + handlerId: string, + handlers: readonly HandlerServiceDocument[], + status: Record | undefined, +): "waiting" | "ready" | "failed" | "blocked" { + const direct = status?.[handlerId]?.status; + if (direct === "done") return "ready"; + if (direct === "failed") return "failed"; + if (!handlers.some((handler) => handler._id === handlerId)) return "waiting"; + const graph = new DependencyGraph( + handlers.map((handler) => ({ + id: handler._id, + dependsOn: handler.dependsOn, + })), + ); + for (const ancestorId of graph.getAncestors(handlerId)) { + if (status?.[ancestorId]?.status === "failed") return "blocked"; + } + return "waiting"; +} + +function currentReadiness( + experiment: ExperimentDocument, +): ExperimentReadiness { + if (experiment.execution.requests.entryCount === 0) return "empty"; + if (experiment.execution.discardedAt) return "failed"; + if ( + experiment.execution.blocking.some( + (blocker) => blocker.reason === "dependency-failed", + ) + ) { + return "failed"; + } + if ( + experiment.execution.blocking.length > 0 && + experiment.execution.blocking.every( + (blocker) => blocker.reason === "dependency-not-ready", + ) + ) { + return "waiting"; + } + if (experiment.execution.blocking.length > 0) return "blocked"; + const statuses = Object.values(experiment.execution.handlerSummary); + if (statuses.some((status) => status.status === "failed")) return "failed"; + if ( + statuses.some( + (status) => + status.status === "queued" || status.status === "processing", + ) + ) { + return "processing"; + } + if ( + statuses.length > 0 && + statuses.every((status) => status.status === "done") + ) { + return "done"; + } + return statuses.length === 0 ? "ready" : "waiting"; +} + +function sameProjection( + left: readonly ExperimentRequestState[], + right: readonly ExperimentRequestState[], +): boolean { + return fingerprintExperimentPayload(left) === fingerprintExperimentPayload(right); +} + +export class ExperimentCoordinator { + private readonly store: ExperimentStore; + private readonly payloads: FixedPayloadStore; + + constructor( + private readonly collections: ExperimentCoordinatorCollections, + private readonly enqueue?: EnqueueExperiment, + ) { + this.store = new ExperimentStore({ + experiments: collections.experiments, + executions: collections.executions, + chunks: collections.chunks, + }); + this.payloads = new FixedPayloadStore(collections.chunks); + } + + async reconcile( + projectId: string, + experimentId: string, + ): Promise { + let rolled = false; + for (let attempt = 0; attempt < 4; attempt += 1) { + let experiment = await this.store.get(projectId, experimentId); + if (!experiment) return null; + if (experiment.execution.rollover) { + const repaired = await this.store.repairRollover(projectId, experimentId); + if (!repaired) return null; + experiment = repaired; + rolled = true; + } + + const requests = await this.readRequests(experiment); + const requestDocs = await this.collections.requests + .find({ + _id: { $in: requests.map((request) => request.requestId) }, + projectId, + deletedAt: { $exists: false }, + } as Document) + .toArray(); + const byId = new Map(requestDocs.map((request) => [request._id, request])); + const runVectorChanged = requests.some((request) => { + const current = byId.get(request.requestId); + return current?.run?._id !== request.runId; + }); + const handlers = await this.loadRunHandlers(); + + if (runVectorChanged) { + const nextRequests = requests.map((request) => { + const current = byId.get(request.requestId); + if (!current?.run || current.run._id === request.runId) return request; + return this.projectRequest( + request.requestId, + current.run._id, + current, + handlers, + ); + }); + const frozen = await this.store.freezeCurrent({ + projectId, + experimentId, + experimentExecutionId: + experiment.execution.experimentExecutionId, + expectedExecutionVersion: experiment.execution.version, + }); + if (!frozen) { + const latest = await this.store.get(projectId, experimentId); + if ( + !latest || + latest.execution.experimentExecutionId !== + experiment.execution.experimentExecutionId + ) { + return latest + ? { + experiment: latest, + readiness: currentReadiness(latest), + rolled: true, + dispatched: false, + } + : null; + } + continue; + } + const next = await this.store.rollover({ + projectId, + experimentId, + expectedExperimentExecutionId: + experiment.execution.experimentExecutionId, + expectedExecutionVersion: frozen.execution.version, + nextExperimentExecutionId: randomUUID(), + requests: nextRequests, + reason: "rollover", + }); + if (!next) continue; + rolled = true; + continue; + } + + const projected = requests.map((request) => + this.projectRequest( + request.requestId, + request.runId, + byId.get(request.requestId), + handlers, + ), + ); + const blockers = projected + .flatMap((request) => request.blockers) + .slice(0, 32); + if (!sameProjection(requests, projected)) { + const projectionFingerprint = fingerprintExperimentPayload(projected); + const updated = await this.store.casUpdateRequestStates({ + projectId, + experimentId, + experimentExecutionId: + experiment.execution.experimentExecutionId, + expectedExecutionVersion: experiment.execution.version, + updateId: `projection:${projectionFingerprint}`, + requests: projected, + }); + if (updated.status === "stale" || updated.status === "contention") { + continue; + } + if (updated.status === "invariant") { + throw new Error( + `Experiment ${experimentId} projection invariant failed: ${updated.message}`, + ); + } + if (updated.status === "partial-failure") { + throw new Error( + `Experiment ${experimentId} projection update ${updated.updateId} is incomplete: ${updated.message}`, + ); + } + experiment = updated.experiment; + } + if ( + fingerprintExperimentPayload(experiment.execution.blocking) !== + fingerprintExperimentPayload(blockers) + ) { + const updated = await this.collections.experiments.findOneAndUpdate( + { + _id: experimentId, + projectId, + "execution.experimentExecutionId": + experiment.execution.experimentExecutionId, + "execution.version": experiment.execution.version, + } as Document, + { + $set: { + "execution.blocking": blockers, + "execution.updatedAt": new Date(), + updatedAt: new Date(), + }, + $inc: { "execution.version": 1, stateVersion: 1 }, + } as Document, + { returnDocument: "after" }, + ); + if (!updated) continue; + experiment = updated; + } + + const dispatch = await this.dispatchReadyHandlers(experiment, projected); + const latest = + (await this.store.get(projectId, experimentId)) ?? experiment; + return { + experiment: latest, + readiness: currentReadiness(latest), + rolled, + dispatched: dispatch, + }; + } + throw new Error( + `Experiment ${experimentId} invariant failure after bounded same-identity contention`, + ); + } + + async recoverInterruptedCreations(limit = 30): Promise { + const candidates = await this.collections.requests + .find({ + experimentId: { $exists: true }, + experimentExpectedRequestCount: { $exists: true }, + experimentFinalizedAt: { $exists: false }, + deletedAt: { $exists: false }, + } as Document) + .sort({ _id: 1 }) + .limit(limit) + .toArray(); + let recovered = 0; + for (const experimentId of new Set( + candidates.flatMap((request) => + request.experimentId ? [request.experimentId] : [], + ), + )) { + const member = candidates.find( + (request) => request.experimentId === experimentId, + ); + if (!member) continue; + const raw = await this.collections.experiments.findOne({ + _id: experimentId, + projectId: member.projectId, + } as Document); + if (!raw) continue; + const existing = + raw.finalizedAt + ? raw + : await this.store.finalizeCreation({ + projectId: member.projectId, + experimentId, + expectedExperimentExecutionId: + raw.execution.experimentExecutionId, + expectedStateVersion: raw.stateVersion, + }); + if (!existing) continue; + await this.collections.requests.updateMany( + { + projectId: member.projectId, + experimentId, + experimentFinalizedAt: { $exists: false }, + } as Document, + { $set: { experimentFinalizedAt: new Date() } } as Document, + ); + recovered += 1; + } + return recovered; + } + + private async readRequests( + experiment: ExperimentDocument, + ): Promise { + return this.payloads.readRequests( + { + projectId: experiment.projectId, + experimentId: experiment.experimentId, + ownerId: experiment.execution.experimentExecutionId, + generation: experiment.execution.attemptNumber, + }, + experiment.execution.requests, + ); + } + + private async loadRunHandlers(): Promise { + const docs = await this.collections.services + .find({ type: "post-process-handler" } as Document) + .toArray(); + return docs.filter( + (doc): doc is HandlerServiceDocument => + doc.type === "post-process-handler", + ); + } + + private async loadExperimentHandlers(): Promise< + ExperimentHandlerServiceDocument[] + > { + const docs = await this.collections.services + .find({ type: "experiment-handler" } as Document) + .toArray(); + return docs.filter( + (doc): doc is ExperimentHandlerServiceDocument => + doc.type === "experiment-handler", + ); + } + + private projectRequest( + requestId: string, + runId: string, + request: RequestDocument | undefined, + handlers: readonly HandlerServiceDocument[], + ): ExperimentRequestState { + if (!request?.run) { + return { + requestId, + runId, + dependencies: [], + blockers: [ + { + requestId, + runId, + handlerId: "request", + reason: "request-missing", + }, + ], + }; + } + const experimentHandlers = request.run.handlerStatus ?? {}; + const dependencyIds = handlers + .filter((handler) => handler.selector === "observations") + .map((handler) => handler._id); + const effectiveDependencyIds = + dependencyIds.length > 0 ? dependencyIds : ["pp-observations"]; + const dependencies = effectiveDependencyIds.map((handlerId) => { + const source = experimentHandlers[handlerId]; + const state = namedHandlerState( + handlerId, + handlers, + request.run?.handlerStatus, + ); + return { + handlerId, + state, + ...(source?.version !== undefined ? { version: source.version } : {}), + ...(source?.updatedAt ? { updatedAt: source.updatedAt } : {}), + ...(source?.error ? { error: source.error } : {}), + }; + }); + const blockers: ExperimentBlocker[] = dependencies.flatMap((dependency) => { + if (dependency.state === "ready") return []; + const reason = + dependency.state === "failed" + ? ("dependency-failed" as const) + : dependency.state === "blocked" + ? ("dependency-blocked" as const) + : ("dependency-not-ready" as const); + return [ + { + requestId, + runId, + handlerId: dependency.handlerId, + reason, + ...(dependency.error ? { error: dependency.error } : {}), + }, + ]; + }); + return { requestId, runId, dependencies, blockers }; + } + + private async dispatchReadyHandlers( + experiment: ExperimentDocument, + requests: readonly ExperimentRequestState[], + ): Promise { + if (!this.enqueue || requests.length === 0) return false; + const services = await this.loadExperimentHandlers(); + let dispatched = false; + for (const service of services) { + const existing = experiment.execution.handlerSummary[service._id]; + const allReady = requests.every((request) => + service.dependsOn.every( + (handlerId) => + request.dependencies.find( + (dependency) => dependency.handlerId === handlerId, + )?.state === "ready", + ), + ); + const retryDroppedNotice = + existing?.status === "queued" && + Date.now() - existing.updatedAt.getTime() >= 30_000; + if ((!existing && !allReady) || (existing && !retryDroppedNotice)) continue; + + const claimToken = existing?.claimToken ?? randomUUID(); + const now = new Date(); + let current = experiment; + if (!existing) { + const path = `execution.handlerSummary.${service._id}`; + const queued = await this.collections.experiments.findOneAndUpdate( + { + _id: experiment.experimentId, + projectId: experiment.projectId, + "execution.experimentExecutionId": + experiment.execution.experimentExecutionId, + "execution.version": experiment.execution.version, + [`${path}.status`]: { $exists: false }, + } as Document, + { + $set: { + [path]: { + status: "queued", + version: service.version, + claimToken, + updatedAt: now, + }, + "execution.updatedAt": now, + updatedAt: now, + }, + $inc: { "execution.version": 1, stateVersion: 1 }, + } as Document, + { returnDocument: "after" }, + ); + if (!queued) continue; + current = queued; + } + await this.enqueue(service, { + type: service.selector, + projectId: current.projectId, + experimentId: current.experimentId, + experimentExecutionId: + current.execution.experimentExecutionId, + handlerId: service._id, + handlerVersion: service.version, + claimToken, + executionVersion: current.execution.version, + }); + dispatched = true; + } + return dispatched; + } +} diff --git a/apps/api/src/experiment-dispatcher.ts b/apps/api/src/experiment-dispatcher.ts new file mode 100644 index 000000000..4e0df8091 --- /dev/null +++ b/apps/api/src/experiment-dispatcher.ts @@ -0,0 +1,169 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { QueueClient } from "@azure/storage-queue"; +import type { Collection, Db } from "mongodb"; +import type { + ExperimentDocument, + ExperimentExecutionHistoryDocument, + ExperimentHandlerServiceDocument, + RequestDocument, + ServiceHandlerDocument, +} from "shared"; +import { ExperimentStore } from "shared"; +import { + ExperimentCoordinator, + type ExperimentDispatchMessage, +} from "./experiment-coordinator.js"; + +export class ExperimentDispatcher { + private interval: ReturnType | null = null; + private dispatching = false; + private scanAfterExperimentId: string | null = null; + private readonly queueClients = new Map(); + private readonly coordinator: ExperimentCoordinator; + + constructor( + private readonly requests: Collection, + private readonly experiments: Collection, + private readonly executions: Collection, + private readonly chunks: Collection, + db: Db, + private readonly createQueueClient: (queueName: string) => QueueClient, + private readonly pollIntervalMs = 30_000, + private readonly batchSize = 30, + ) { + this.coordinator = new ExperimentCoordinator( + { + requests, + experiments, + executions, + chunks, + services: db.collection("services"), + }, + (service, message) => this.enqueue(service, message), + ); + } + + start(): void { + if (this.interval) return; + this.interval = setInterval(() => void this.poll(), this.pollIntervalMs); + void this.poll(); + } + + async stop(): Promise { + if (this.interval) { + clearInterval(this.interval); + this.interval = null; + } + const deadline = Date.now() + 5_000; + while (this.dispatching && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 100)); + } + } + + async onRequestChanged(requestId: string): Promise { + const request = await this.requests.findOne( + { _id: requestId }, + { projection: { projectId: 1, experimentId: 1 } }, + ); + if (request?.experimentId) { + await this.coordinator.reconcile( + request.projectId, + request.experimentId, + ); + } + } + + async onHandlerComplete(input: { + projectId: string; + experimentId: string; + experimentExecutionId: string; + executionVersion: number; + handlerId: string; + handlerVersion: number; + claimToken: string; + status: "done" | "failed"; + error?: string; + }): Promise { + const updated = await new ExperimentStore({ + experiments: this.experiments, + executions: this.executions, + chunks: this.chunks, + }).completeHandler({ + ...input, + expectedExecutionVersion: input.executionVersion, + }); + return updated !== null; + } + + async onHandlerClaim(input: { + projectId: string; + experimentId: string; + experimentExecutionId: string; + executionVersion: number; + handlerId: string; + handlerVersion: number; + claimToken: string; + }): Promise { + const updated = await new ExperimentStore({ + experiments: this.experiments, + executions: this.executions, + chunks: this.chunks, + }).claimHandler({ + ...input, + expectedExecutionVersion: input.executionVersion, + }); + return updated?.execution.version ?? null; + } + + private async poll(): Promise { + if (this.dispatching) return; + this.dispatching = true; + try { + await this.coordinator.recoverInterruptedCreations(this.batchSize); + const candidates = await this.experiments + .find( + this.scanAfterExperimentId + ? { + _id: { $gt: this.scanAfterExperimentId }, + finalizedAt: { $exists: true }, + } + : { finalizedAt: { $exists: true } }, + ) + .sort({ _id: 1 }) + .limit(this.batchSize) + .project({ _id: 1, projectId: 1 }) + .toArray(); + this.scanAfterExperimentId = + candidates.length === this.batchSize + ? candidates.at(-1)?._id ?? null + : null; + for (const candidate of candidates) { + await this.coordinator.reconcile( + candidate.projectId, + candidate._id, + ); + } + } catch (error) { + console.error("[ExperimentDispatcher] Poll failed:", error); + } finally { + this.dispatching = false; + } + } + + private async enqueue( + service: ExperimentHandlerServiceDocument, + message: ExperimentDispatchMessage, + ): Promise { + let client = this.queueClients.get(service.queue); + if (!client) { + client = this.createQueueClient(service.queue); + await client.createIfNotExists(); + this.queueClients.set(service.queue, client); + } + await client.sendMessage( + Buffer.from(JSON.stringify(message)).toString("base64"), + ); + } +} diff --git a/apps/api/src/experiment-openapi.test.ts b/apps/api/src/experiment-openapi.test.ts new file mode 100644 index 000000000..aeaddb2bd --- /dev/null +++ b/apps/api/src/experiment-openapi.test.ts @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { beforeAll, describe, expect, it, vi } from "vitest"; +import type { OpenAPIObject } from "openapi3-ts/oas31"; +import { LEGACY_REQUEST_COMPOSITION_DEPRECATION_ID } from "shared"; + +vi.mock("db-migrations/check-migrations", () => ({ + checkMigrations: vi.fn().mockResolvedValue({ + ready: true, + applied: [], + pending: [], + }), +})); +vi.mock("./llm.js", () => ({ + isLlmAvailable: vi.fn().mockReturnValue(false), + generateCriteriaPrompt: vi.fn(), +})); +vi.mock("./prompt-feature-llm.js", () => ({ + isLlmAvailable: vi.fn().mockReturnValue(false), + generatePromptFeaturePrompt: vi.fn(), + extractPromptFeatures: vi.fn(), +})); +vi.mock("./task-prompt-llm.js", () => ({ + isTaskPromptLlmAvailable: vi.fn().mockReturnValue(false), + generateTaskPrompt: vi.fn(), +})); + +let document: OpenAPIObject; + +beforeAll(async () => { + await import("./index.js"); + const { generateOpenAPIDocument } = await import("./openapi/index.js"); + document = generateOpenAPIDocument(); +}); + +describe("Experiment OpenAPI contract", () => { + it("exports the shared legacy composition deprecation identifier", () => { + expect(LEGACY_REQUEST_COMPOSITION_DEPRECATION_ID).toBe( + "legacy-request-composition", + ); + }); + + it("publishes canonical experiment operations", () => { + expect(document.paths).toHaveProperty("/api/v1/experiments"); + expect(document.paths).toHaveProperty("/api/v1/experiments/{id}"); + expect(document.paths).toHaveProperty( + "/api/v1/experiments/{id}/executions", + ); + expect(document.paths).toHaveProperty( + "/api/v1/experiments/{id}/executions/{experimentExecutionId}", + ); + expect(document.paths).toHaveProperty("/api/v1/experiments/{id}/retry"); + expect( + Object.keys(document.paths ?? {}).filter((path) => + path.includes("submission"), + ), + ).toEqual([]); + expect( + JSON.stringify(document.components?.schemas?.CreateExperimentInput), + ).not.toContain('"tags"'); + expect( + JSON.stringify(document.components?.schemas?.ExperimentResponse), + ).not.toContain('"tags"'); + }); + + it("keeps POST /requests active while deprecating only composition properties", () => { + const operation = document.paths?.["/api/v1/requests"]?.post; + expect(operation?.deprecated).not.toBe(true); + const requestSchema = document.components?.schemas?.CreateRequestInput; + const operationJson = JSON.stringify(operation); + const requestSchemaJson = JSON.stringify(requestSchema); + + expect(operationJson).toContain('"count"'); + expect(operationJson).toContain('"deprecated":true'); + expect(requestSchemaJson).toContain('"profileVariations"'); + expect(requestSchemaJson).toContain('"deprecated":true'); + expect(operationJson).toContain("POST /api/v1/experiments"); + expect(requestSchemaJson).toContain("POST /api/v1/experiments"); + }); +}); diff --git a/apps/api/src/grouping.test.ts b/apps/api/src/grouping.test.ts index 56a834ce3..9ecc86bf6 100644 --- a/apps/api/src/grouping.test.ts +++ b/apps/api/src/grouping.test.ts @@ -14,9 +14,17 @@ describe("buildGroupingPipeline", () => { expect(pipeline[3]).toHaveProperty("$sort"); }); - it("returns 4 pipeline stages for submissionId grouping", () => { + it("returns 4 pipeline stages for experimentId grouping", () => { + const pipeline = buildGroupingPipeline("experimentId"); + expect(pipeline).toHaveLength(4); + }); + + it("preserves submissionId grouping", () => { const pipeline = buildGroupingPipeline("submissionId"); expect(pipeline).toHaveLength(4); + expect( + (pipeline[0].$addFields as Record)._groupKey, + ).toBeDefined(); }); it("$sort stage orders by key ascending", () => { @@ -32,8 +40,8 @@ describe("buildGroupingPipeline", () => { expect(addFields._groupKey).toBe("$taskPromptId"); }); - it("groups by submissionId for submissionId grouping", () => { - const pipeline = buildGroupingPipeline("submissionId"); + it("groups by experimentId for experimentId grouping", () => { + const pipeline = buildGroupingPipeline("experimentId"); const addFields = pipeline[0].$addFields as Record; expect(addFields._groupKey).toBeDefined(); }); @@ -75,6 +83,7 @@ describe("buildGroupingPipeline", () => { expect(group._platforms).toBeDefined(); expect(group._statuses).toBeDefined(); expect(group._submissionIds).toBeDefined(); + expect(group._experimentIds).toBeDefined(); expect(group._tasks).toBeDefined(); expect(group._mcpKeys).toBeDefined(); expect(group._skillKeys).toBeDefined(); diff --git a/apps/api/src/grouping.ts b/apps/api/src/grouping.ts index 3cc34cb18..307e2bedc 100644 --- a/apps/api/src/grouping.ts +++ b/apps/api/src/grouping.ts @@ -12,17 +12,27 @@ import type { GroupByKey, RunGroup, AggregateStats, GroupUniformValues } from "s * The pipeline is appended after a $match stage with the caller's filter. */ export function buildGroupingPipeline( - groupBy: "task" | "submissionId" | "profile", + groupBy: "task" | "submissionId" | "experimentId" | "profile", ): Document[] { const groupField = - groupBy === "task" ? "$taskPromptId" : groupBy === "profile" ? "$profileId" : "$submissionId"; + groupBy === "task" + ? "$taskPromptId" + : groupBy === "profile" + ? "$profileId" + : groupBy === "submissionId" + ? "$submissionId" + : "$experimentId"; const fallbackGroupField = - groupBy === "profile" ? { $literal: "no-profile" } : { $literal: "no-submission" }; + groupBy === "profile" + ? { $literal: "no-profile" } + : groupBy === "submissionId" + ? { $literal: "no-submission" } + : { $literal: "no-experiment" }; // `task` groups purely by the always-present `taskPromptId` — it is // materialized on every run-creation path (findOrCreate → taskPrompt._id) and // backfilled onto legacy docs by migration 001, so the group key is always a // real id the member-fetch filter (`{ taskPromptId: key }`) can match. Only - // `profile`/`submission` fall back to a fixed literal sentinel for runs that + // `profile`/`experiment` fall back to a fixed literal sentinel for runs that // genuinely lack the field. const groupKeyExpr = groupBy === "task" ? groupField : { $ifNull: [groupField, fallbackGroupField] }; @@ -132,6 +142,7 @@ export function buildGroupingPipeline( _platforms: { $addToSet: { $ifNull: ["$run.os.platform", ""] } }, _statuses: { $addToSet: "$run.status" }, _submissionIds: { $addToSet: { $ifNull: ["$submissionId", ""] } }, + _experimentIds: { $addToSet: { $ifNull: ["$experimentId", ""] } }, _tasks: { $addToSet: { $ifNull: ["$scenario.task", ""] } }, _mcpKeys: { $addToSet: "$_mcpKey" }, _skillKeys: { $addToSet: "$_skillKey" }, @@ -160,7 +171,7 @@ export function buildGroupingPipeline( label: { $switch: { branches: [ - { case: { $eq: ["$_id", "no-submission"] }, then: "No submission ID" }, + { case: { $eq: ["$_id", "no-experiment"] }, then: "No experiment ID" }, { case: { $eq: ["$_id", "no-profile"] }, then: "No profile" }, ], default: { $ifNull: ["$_firstTask", "$_id"] }, @@ -184,6 +195,7 @@ export function buildGroupingPipeline( platform: { $cond: { if: { $and: [{ $eq: [{ $size: "$_platforms" }, 1] }, { $ne: [{ $arrayElemAt: ["$_platforms", 0] }, ""] }] }, then: { $arrayElemAt: ["$_platforms", 0] }, else: "$$REMOVE" } }, status: { $cond: { if: { $eq: [{ $size: "$_statuses" }, 1] }, then: { $arrayElemAt: ["$_statuses", 0] }, else: "$$REMOVE" } }, submissionId: { $cond: { if: { $and: [{ $eq: [{ $size: "$_submissionIds" }, 1] }, { $ne: [{ $arrayElemAt: ["$_submissionIds", 0] }, ""] }] }, then: { $arrayElemAt: ["$_submissionIds", 0] }, else: "$$REMOVE" } }, + experimentId: { $cond: { if: { $and: [{ $eq: [{ $size: "$_experimentIds" }, 1] }, { $ne: [{ $arrayElemAt: ["$_experimentIds", 0] }, ""] }] }, then: { $arrayElemAt: ["$_experimentIds", 0] }, else: "$$REMOVE" } }, task: { $cond: { if: { $and: [{ $eq: [{ $size: "$_tasks" }, 1] }, { $ne: [{ $arrayElemAt: ["$_tasks", 0] }, ""] }] }, then: { $arrayElemAt: ["$_tasks", 0] }, else: "$$REMOVE" } }, mcpServers: { $cond: { if: { $and: [{ $eq: [{ $size: "$_mcpKeys" }, 1] }, { $ne: [{ $arrayElemAt: ["$_mcpKeys", 0] }, ""] }] }, then: "$_firstMcpServers", else: "$$REMOVE" } }, skillRevisions: { $cond: { if: { $and: [{ $eq: [{ $size: "$_skillKeys" }, 1] }, { $ne: [{ $arrayElemAt: ["$_skillKeys", 0] }, ""] }] }, then: "$_firstSkillRevisions", else: "$$REMOVE" } }, diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 9803a0581..0fcf64fe8 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -10,6 +10,7 @@ import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; import dotenv from "dotenv"; import { TaskPromptStore, SkillRevisionStore, SkillResolver, CodebaseStore, CodebaseRevisionStore, CodebaseResolver, ResourceStore, ResourceRevisionStore, ResourceResolver, McpSecretClient, McpSecretUnavailableError, BlobStorage, RedisHeartbeatStore, ProjectStore, loadAuthConfigFromEnv } from "shared"; +import type { ExperimentDocument, ExperimentExecutionHistoryDocument } from "shared"; import { initTelemetry } from "telemetry"; import type { TaskPromptDocument, SkillDocument, SkillRevisionDocument, CodebaseDocument, CodebaseRevisionDocument, ResourceDocument, ResourceRevisionDocument, ProfileDocument, ProfileVersionDocument, ProjectDocument, HeartbeatStore, AuthProvider, ProfileEnricher, UserDocument } from "shared"; import { UserStore } from "./auth/user-store.js"; @@ -47,9 +48,11 @@ import { registerInsightsRoutes } from "./routes/insights.js"; import { registerSecretsRoutes } from "./routes/secrets.js"; import { registerProfilesRoutes } from "./routes/profiles.js"; import { registerProjectsRoutes } from "./routes/projects.js"; +import { registerExperimentsRoutes } from "./routes/experiments.js"; import { ProjectScopeError } from "./utils/project-scope.js"; import { registerUsersRoutes } from "./routes/users.js"; import { shutdownPortalCopilotClientPool } from "./copilot-client-pool.js"; +import { ExperimentDispatcher } from "./experiment-dispatcher.js"; import type { RouteContext } from "./route-context.js"; import type { CriteriaDocument, @@ -88,6 +91,10 @@ const mongoCollection = process.env.MONGO_COLLECTION || "requests"; const storageAccountName = process.env.AZURE_STORAGE_ACCOUNT_NAME || ""; const storageConnectionString = process.env.STORAGE_CONNECTION_STRING || process.env.AZURE_STORAGE_CONNECTION_STRING || ""; const queueReport = process.env.AZURE_STORAGE_QUEUE_REPORT || "report-queue"; +const experimentPollIntervalMs = parseInt( + process.env.EXPERIMENT_POLL_INTERVAL_MS || "30000", + 10, +); const strictAgentCapabilities = process.env.SCOPE_STRICT_AGENT_CAPABILITIES?.toLowerCase() === "true"; const port = parseInt(process.env.PORT || "3000", 10); @@ -97,6 +104,9 @@ let mongoClient: MongoClient; let db: Db; let collection: Collection; let runsCol: Collection; +let experimentCollection: Collection; +let experimentExecutionsCollection: Collection; +let experimentChunksCollection: Collection; let projectCollection: Collection; let projectStore: ProjectStore; let criteriaCollection: Collection; @@ -136,6 +146,16 @@ let resourceResolver: ResourceResolver; let blobStorage: BlobStorage; let heartbeatStore: HeartbeatStore; let reportQueueClient: QueueClient; +let experimentDispatcher: ExperimentDispatcher; + +function createQueueClient(queueName: string): QueueClient { + if (storageConnectionString) { + return new QueueClient(storageConnectionString, queueName); + } + const credential = new DefaultAzureCredential(); + const queueUrl = `https://${storageAccountName}.queue.core.windows.net`; + return new QueueClient(`${queueUrl}/${queueName}`, credential); +} async function initializeClients(): Promise { // Connect to MongoDB @@ -144,6 +164,10 @@ async function initializeClients(): Promise { db = mongoClient.db(mongoDatabase); collection = db.collection(mongoCollection); runsCol = db.collection("runs"); + experimentCollection = db.collection("experiments"); + experimentExecutionsCollection = + db.collection("experiment-executions"); + experimentChunksCollection = db.collection("experiment-chunks"); projectCollection = db.collection("projects"); projectStore = new ProjectStore(projectCollection); criteriaCollection = db.collection("criteria"); @@ -238,15 +262,7 @@ async function initializeClients(): Promise { console.log(`Connected to MongoDB: ${mongoUri.replace(/\/\/[^:]+:[^@]+@/, "//***:***@")}`); - // Coding-agent queues are initialized by the scheduler from registry version - // manifests. The API only owns the report-generation queue. - if (storageConnectionString) { - reportQueueClient = new QueueClient(storageConnectionString, queueReport); - } else { - const credential = new DefaultAzureCredential(); - const queueUrl = `https://${storageAccountName}.queue.core.windows.net`; - reportQueueClient = new QueueClient(`${queueUrl}/${queueReport}`, credential); - } + reportQueueClient = createQueueClient(queueReport); await reportQueueClient.createIfNotExists(); console.log(`Ensured queue exists: ${queueReport}`); @@ -265,6 +281,16 @@ async function initializeClients(): Promise { redisPort: parseInt(process.env.REDIS_PORT || "6300", 10), redisPassword: process.env.REDIS_PASSWORD || "", }); + + experimentDispatcher = new ExperimentDispatcher( + collection, + experimentCollection, + experimentExecutionsCollection, + experimentChunksCollection, + db, + createQueueClient, + experimentPollIntervalMs, + ); } // --- OpenAPI documentation (lazy — Swagger UI mounted in main() after all routes register) --- @@ -277,6 +303,9 @@ const routeCtx: RouteContext = { get db() { return db; }, get requestCollection() { return collection; }, get runsCollection() { return runsCol; }, + get experimentCollection() { return experimentCollection; }, + get experimentExecutionsCollection() { return experimentExecutionsCollection; }, + get experimentChunksCollection() { return experimentChunksCollection; }, get projectCollection() { return projectCollection; }, get criteriaCollection() { return criteriaCollection; }, get promptFeatureCollection() { return promptFeatureCollection; }, @@ -338,6 +367,7 @@ app.use(createUserAccessMiddleware(() => userAccessResolver)); registerSecretsRoutes(routeCtx); registerSystemRoutes(routeCtx); registerProjectsRoutes(routeCtx); +registerExperimentsRoutes(routeCtx); registerRequestsRoutes(routeCtx); registerRequestsCancelRoutes(routeCtx); registerRequestsLogsRoutes(routeCtx); @@ -380,6 +410,10 @@ app.use((err: Error, _req: Request, res: Response, _next: NextFunction) => { async function main(): Promise { await initializeClients(); + experimentDispatcher.start(); + console.log( + `Experiment coordinator started (poll=${experimentPollIntervalMs}ms)`, + ); // Mount OpenAPI docs (after all routes are registered) const openapiDocument = generateOpenAPIDocument(); @@ -401,6 +435,7 @@ async function main(): Promise { process.exitCode = 1; } void Promise.all([ + experimentDispatcher.stop(), shutdownPortalCopilotClientPool(), userAccessCache?.close(), heartbeatStore.close(), @@ -424,6 +459,9 @@ export interface TestDependencies { db?: Db; collection?: Collection; runsCollection?: Collection; + experimentCollection?: Collection; + experimentExecutionsCollection?: Collection; + experimentChunksCollection?: Collection; criteriaCollection?: Collection; promptFeatureCollection?: Collection; reportCollection?: Collection; @@ -465,6 +503,9 @@ export function _injectTestDependencies(deps: TestDependencies): void { if (deps.db) db = deps.db; if (deps.collection) collection = deps.collection; if (deps.runsCollection) runsCol = deps.runsCollection; + if (deps.experimentCollection) experimentCollection = deps.experimentCollection; + if (deps.experimentExecutionsCollection) experimentExecutionsCollection = deps.experimentExecutionsCollection; + if (deps.experimentChunksCollection) experimentChunksCollection = deps.experimentChunksCollection; if (deps.criteriaCollection) criteriaCollection = deps.criteriaCollection; if (deps.promptFeatureCollection) promptFeatureCollection = deps.promptFeatureCollection; if (deps.reportCollection) reportCollection = deps.reportCollection; diff --git a/apps/api/src/openapi/openapi.test.ts b/apps/api/src/openapi/openapi.test.ts index 87426de43..6dbfa9152 100644 --- a/apps/api/src/openapi/openapi.test.ts +++ b/apps/api/src/openapi/openapi.test.ts @@ -466,7 +466,7 @@ apiRoute(testApp, registry, { apiRoute(testApp, registry, { method: "get", path: "/api/v1/analysis", tags: ["Requests"], summary: "Compute pass@k / success@T metrics", - query: z.object({ worker: z.string().optional(), taskPromptId: z.string().optional(), criteria: z.string().optional(), submissionId: z.string().optional(), k: z.string().optional() }), + query: z.object({ worker: z.string().optional(), taskPromptId: z.string().optional(), criteria: z.string().optional(), experimentId: z.string().optional(), k: z.string().optional() }), response: z.object({}).passthrough().describe("Analysis metrics"), handler: noop, }); apiRoute(testApp, registry, { diff --git a/apps/api/src/route-context.ts b/apps/api/src/route-context.ts index 580eae160..2037c04f0 100644 --- a/apps/api/src/route-context.ts +++ b/apps/api/src/route-context.ts @@ -52,6 +52,8 @@ import type { ReportTemplateResponseSchema, RequestResponseSchema, RunHistoryDocumentSchema, + ExperimentDocument, + ExperimentExecutionHistoryDocument, } from "shared"; // ─── Document types (inferred from Zod schemas) ───────────────────────────── @@ -74,7 +76,6 @@ export type ReportTrigger = z.infer; export type ReportTemplateDocument = Omit, "_id">; export type RequestDocument = z.infer; export type RunHistoryDocument = z.infer; - export type WorkerType = string; // ─── RouteContext ──────────────────────────────────────────────────────────── @@ -92,6 +93,9 @@ export interface RouteContext { db: Db; requestCollection: Collection; runsCollection: Collection; + experimentCollection: Collection; + experimentExecutionsCollection: Collection; + experimentChunksCollection: Collection; projectCollection: Collection; criteriaCollection: Collection; promptFeatureCollection: Collection; diff --git a/apps/api/src/routes/experiments.test.ts b/apps/api/src/routes/experiments.test.ts new file mode 100644 index 000000000..c3ff88c3d --- /dev/null +++ b/apps/api/src/routes/experiments.test.ts @@ -0,0 +1,300 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import express from "express"; +import request from "supertest"; +import { + OpenAPIRegistry, + OpenApiGeneratorV31, + extendZodWithOpenApi, +} from "@asteasolutions/zod-to-openapi"; +import { describe, expect, it, vi } from "vitest"; +import { z } from "zod"; +import type { RouteContext } from "../route-context.js"; +import type { ExperimentDocument } from "shared"; +import { registerExperimentsRoutes } from "./experiments.js"; + +extendZodWithOpenApi(z); + +function harness(overrides: Partial = {}) { + const app = express(); + app.use(express.json()); + const registry = new OpenAPIRegistry(); + registerExperimentsRoutes({ + app, + registry, + ...overrides, + } as unknown as RouteContext); + return { app, registry }; +} + +function emptyExperimentCollection() { + let document: ExperimentDocument | undefined; + return { + updateOne: vi.fn( + async ( + _filter: object, + update: { $setOnInsert?: ExperimentDocument }, + ) => { + document ??= structuredClone(update.$setOnInsert); + return { acknowledged: true, matchedCount: document ? 1 : 0 }; + }, + ), + findOne: vi.fn(async () => + document ? structuredClone(document) : null, + ), + findOneAndUpdate: vi.fn( + async ( + _filter: object, + update: { + $set?: Partial; + $inc?: { stateVersion?: number }; + }, + ) => { + if (!document) return null; + document = { + ...document, + ...update.$set, + stateVersion: + document.stateVersion + (update.$inc?.stateVersion ?? 0), + }; + return structuredClone(document); + }, + ), + }; +} + +function generateDocument(registry: OpenAPIRegistry) { + return new OpenApiGeneratorV31(registry.definitions).generateDocument({ + openapi: "3.1.0", + info: { title: "Experiment contract", version: "1.0.0" }, + }); +} + +describe("Experiment API contract", () => { + it("requires project scope on root routes", async () => { + const { app } = harness(); + + const create = await request(app) + .post("/api/v1/experiments") + .send({ + mode: "empty", + metadata: { name: "Empty experiment" }, + requests: [], + }); + const list = await request(app).get("/api/v1/experiments"); + const get = await request(app).get("/api/v1/experiments/experiment-1"); + const retry = await request(app) + .post("/api/v1/experiments/experiment-1/retry") + .send({ experimentExecutionId: "execution-1" }); + + expect(create.status).toBe(400); + expect(list.status).toBe(400); + expect(get.status).toBe(400); + expect(retry.status).toBe(400); + }); + + it("requires and verifies project scope on execution routes", async () => { + const findOne = vi.fn().mockResolvedValue(null); + const { app } = harness({ + experimentCollection: { findOne }, + } as unknown as Partial); + + const historyWithoutProject = await request(app).get( + "/api/v1/experiments/experiment-1/executions", + ); + const exactWithoutProject = await request(app).get( + "/api/v1/experiments/experiment-1/executions/execution-1", + ); + const wrongProject = await request(app).get( + "/api/v1/experiments/experiment-1/executions?projectId=wrong-project", + ); + const wrongProjectExact = await request(app).get( + "/api/v1/experiments/experiment-1/executions/execution-1?projectId=wrong-project", + ); + + expect(historyWithoutProject.status).toBe(400); + expect(exactWithoutProject.status).toBe(400); + expect(wrongProject.status).toBe(404); + expect(wrongProjectExact.status).toBe(404); + expect(findOne).toHaveBeenCalledWith({ + _id: "experiment-1", + projectId: "wrong-project", + finalizedAt: { $exists: true }, + }); + }); + + it("enforces explicit empty and compose modes", async () => { + const { app } = harness(); + + const emptyWithMembers = await request(app) + .post("/api/v1/experiments?projectId=project-1") + .send({ + mode: "empty", + metadata: { name: "Invalid empty experiment" }, + requests: [{ scenario: { task: "task" }, workerType: "worker" }], + }); + const composeWithoutMembers = await request(app) + .post("/api/v1/experiments?projectId=project-1") + .send({ + mode: "compose", + metadata: { name: "Invalid composed experiment" }, + requests: [], + }); + + expect(emptyWithMembers.status).toBe(400); + expect(composeWithoutMembers.status).toBe(400); + }); + + it("rejects removed Experiment metadata fields", async () => { + const { app } = harness(); + + const response = await request(app) + .post("/api/v1/experiments?projectId=project-1") + .send({ + mode: "empty", + metadata: { + name: "Tagged experiment", + tags: ["nightly"], + }, + requests: [], + }); + + expect(response.status).toBe(400); + }); + + it("creates an empty, visible, non-dispatchable experiment", async () => { + const experimentCollection = emptyExperimentCollection(); + const { app } = harness({ + experimentCollection, + experimentExecutionsCollection: {}, + experimentChunksCollection: {}, + requestCollection: { + insertMany: vi.fn(), + updateMany: vi.fn(), + }, + } as unknown as Partial); + + const response = await request(app) + .post("/api/v1/experiments?projectId=project-1") + .send({ + mode: "empty", + metadata: { name: "Empty experiment" }, + requests: [], + }); + + expect(response.status).toBe(201); + expect(response.body.requestIds).toEqual([]); + expect(response.body.experiment).toMatchObject({ + projectId: "project-1", + metadata: { name: "Empty experiment" }, + revision: { memberCount: 0 }, + currentExecution: { + readiness: "empty", + blockers: [], + progress: {}, + }, + }); + expect(JSON.stringify(response.body)).not.toMatch( + /stateVersion|claimToken|vectorFingerprint|storageProfile|rollover/, + ); + }); + + it("inserts composed requests before publishing their exact revision", async () => { + const experimentCollection = emptyExperimentCollection(); + const insertMany = vi.fn().mockResolvedValue({ + acknowledged: true, + insertedCount: 1, + }); + const updateMany = vi.fn().mockResolvedValue({ + acknowledged: true, + matchedCount: 1, + modifiedCount: 1, + }); + const { app } = harness({ + experimentCollection, + experimentExecutionsCollection: {}, + experimentChunksCollection: {}, + requestCollection: { insertMany, updateMany }, + taskPromptStore: { + findOrCreate: vi.fn().mockResolvedValue({ _id: "prompt-1" }), + }, + blobStorage: { + getLogsBlobUrl: vi.fn( + (path: string) => `https://storage.invalid/${path}`, + ), + }, + } as unknown as Partial); + + const response = await request(app) + .post("/api/v1/experiments?projectId=project-1") + .send({ + mode: "compose", + metadata: { name: "One request" }, + requests: [ + { + scenario: { task: "Build it", criteria: ["works"] }, + workerType: "coder", + }, + ], + }); + + expect(response.status).toBe(201); + expect(response.body.requestIds).toHaveLength(1); + const inserted = insertMany.mock.calls[0]?.[0]?.[0]; + expect(inserted).toMatchObject({ + projectId: "project-1", + experimentId: response.body.experiment.experimentId, + experimentExpectedRequestCount: 1, + experimentRequestIndex: 0, + experimentInitialRunId: expect.any(String), + run: { + _id: expect.any(String), + status: "pending", + }, + }); + expect(updateMany).toHaveBeenCalledWith( + expect.objectContaining({ + projectId: "project-1", + experimentId: response.body.experiment.experimentId, + }), + { $set: { experimentFinalizedAt: expect.any(Date) } }, + ); + }); + + it("publishes only the canonical routes and public execution fields", () => { + const { registry } = harness(); + const document = generateDocument(registry); + + expect(Object.keys(document.paths ?? {}).sort()).toEqual([ + "/api/v1/experiments", + "/api/v1/experiments/{id}", + "/api/v1/experiments/{id}/executions", + "/api/v1/experiments/{id}/executions/{experimentExecutionId}", + "/api/v1/experiments/{id}/retry", + ]); + expect(document.paths).not.toHaveProperty("/api/v1/submissions"); + + const executionSchema = document.components?.schemas + ?.ExperimentExecutionResponse; + expect( + JSON.stringify(document.components?.schemas?.CreateExperimentInput), + ).not.toContain('"tags"'); + expect( + JSON.stringify(document.components?.schemas?.ExperimentResponse), + ).not.toContain('"tags"'); + expect(executionSchema).toMatchObject({ + type: "object", + properties: { + experimentExecutionId: { type: "string" }, + readiness: expect.any(Object), + blockers: expect.any(Object), + progress: expect.any(Object), + supersededByExperimentExecutionId: { type: "string" }, + }, + }); + expect(JSON.stringify(executionSchema)).not.toMatch( + /claimToken|stateVersion|vectorFingerprint|storageProfile/, + ); + }); +}); diff --git a/apps/api/src/routes/experiments.ts b/apps/api/src/routes/experiments.ts new file mode 100644 index 000000000..1c43fd8bf --- /dev/null +++ b/apps/api/src/routes/experiments.ts @@ -0,0 +1,699 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { v4 as uuidv4 } from "uuid"; +import { z } from "zod"; +import type { Document } from "mongodb"; +import { + CreateRequestInputSchema, + ExperimentStore, + fingerprintExperimentPayload, + type ExperimentDocument, + type ExperimentExecution, + type ExperimentExecutionHistoryDocument, + type ExperimentMetadata, + type ExperimentRequestState, +} from "shared"; +import type { RequestDocument, RouteContext } from "../route-context.js"; +import { apiRoute } from "../openapi/api-route.js"; +import { + ProjectIdQuerySchema, + getQueryProjectId, +} from "../utils/project-scope.js"; + +const CursorSchema = z.string().min(1).optional(); +const PageQuerySchema = ProjectIdQuerySchema.extend({ + limit: z.coerce.number().int().min(1).max(100).default(50), + after: CursorSchema, +}); + +const ExperimentMetadataInputSchema = z.object({ + name: z.string().trim().min(1).max(256), + description: z.string().max(4_096).optional(), +}).strict(); + +const ExperimentRequestInputSchema = CreateRequestInputSchema.omit({ + profileVariations: true, +}).extend({ + workerType: z.string().trim().min(1), +}); + +const CreateExperimentInputSchema = z + .object({ + mode: z.enum(["compose", "empty"]), + metadata: ExperimentMetadataInputSchema, + requests: z.array(ExperimentRequestInputSchema).max(250_000).default([]), + }) + .superRefine((value, ctx) => { + if (value.mode === "empty" && value.requests.length !== 0) { + ctx.addIssue({ + code: "custom", + path: ["requests"], + message: "empty mode requires an empty requests array", + }); + } + if (value.mode === "compose" && value.requests.length === 0) { + ctx.addIssue({ + code: "custom", + path: ["requests"], + message: "compose mode requires at least one request", + }); + } + }) + .openapi("CreateExperimentInput"); + +const HandlerProgressSchema = z.record( + z.string(), + z.object({ + status: z.enum(["queued", "processing", "done", "failed"]), + updatedAt: z.coerce.date(), + error: z.string().optional(), + }), +); + +const ReadinessSchema = z.enum([ + "empty", + "waiting", + "blocked", + "ready", + "processing", + "done", + "failed", +]); + +const ExperimentResponseSchema = z + .object({ + experimentId: z.string(), + projectId: z.string(), + metadata: ExperimentMetadataInputSchema.extend({ + createdBy: z.string().optional(), + }), + revision: z.object({ + experimentRevisionId: z.string(), + revisionNumber: z.number().int(), + memberCount: z.number().int(), + createdAt: z.coerce.date(), + }), + currentExecution: z.object({ + experimentExecutionId: z.string(), + attemptNumber: z.number().int(), + readiness: ReadinessSchema, + blockers: z.array( + z.object({ + requestId: z.string().optional(), + runId: z.string().optional(), + handlerId: z.string(), + reason: z.string(), + error: z.string().optional(), + }), + ), + progress: HandlerProgressSchema, + createdAt: z.coerce.date(), + updatedAt: z.coerce.date(), + startedAt: z.coerce.date().optional(), + finishedAt: z.coerce.date().optional(), + discardedAt: z.coerce.date().optional(), + discardReason: z.string().optional(), + report: z + .object({ + reportId: z.string().optional(), + url: z.string().optional(), + updatedAt: z.coerce.date().optional(), + }) + .optional(), + }), + createdAt: z.coerce.date(), + updatedAt: z.coerce.date(), + }) + .openapi("ExperimentResponse"); + +const ExperimentListResponseSchema = z + .object({ + data: z.array(ExperimentResponseSchema), + nextCursor: z.string().optional(), + }) + .openapi("ExperimentListResponse"); + +const ExecutionResponseSchema = z + .object({ + experimentExecutionId: z.string(), + attemptNumber: z.number().int(), + readiness: ReadinessSchema, + blockers: z.array(z.object({ + requestId: z.string().optional(), + runId: z.string().optional(), + handlerId: z.string(), + reason: z.string(), + error: z.string().optional(), + })), + progress: HandlerProgressSchema, + createdAt: z.coerce.date(), + updatedAt: z.coerce.date(), + startedAt: z.coerce.date().optional(), + finishedAt: z.coerce.date().optional(), + discardedAt: z.coerce.date().optional(), + discardReason: z.string().optional(), + supersededAt: z.coerce.date().optional(), + supersededByExperimentExecutionId: z.string().optional(), + supersededReason: z.enum(["rollover", "retry"]).optional(), + }) + .openapi("ExperimentExecutionResponse"); + +const ExecutionListResponseSchema = z + .object({ + data: z.array(ExecutionResponseSchema), + nextCursor: z.string().optional(), + }) + .openapi("ExperimentExecutionListResponse"); + +const CreateExperimentResponseSchema = z + .object({ + experiment: ExperimentResponseSchema, + requestIds: z.array(z.string()), + }) + .openapi("CreateExperimentResponse"); + +const RetryExperimentResponseSchema = z + .object({ + experimentId: z.string(), + experimentExecutionId: z.string(), + attemptNumber: z.number().int(), + readiness: ReadinessSchema, + blockers: z.array(z.object({ + requestId: z.string().optional(), + runId: z.string().optional(), + handlerId: z.string(), + reason: z.string(), + error: z.string().optional(), + })), + }) + .openapi("RetryExperimentResponse"); + +type Readiness = z.infer; + +function store(ctx: RouteContext): ExperimentStore { + return new ExperimentStore({ + experiments: ctx.experimentCollection, + executions: ctx.experimentExecutionsCollection, + chunks: ctx.experimentChunksCollection, + }); +} + +function readiness(execution: ExperimentExecution): Readiness { + if (execution.requests.entryCount === 0) return "empty"; + if (execution.discardedAt) return "failed"; + if (execution.blocking.some((blocker) => blocker.reason === "dependency-failed")) { + return "failed"; + } + if ( + execution.blocking.length > 0 && + execution.blocking.every( + (blocker) => blocker.reason === "dependency-not-ready", + ) + ) { + return "waiting"; + } + if (execution.blocking.length > 0) return "blocked"; + const statuses = Object.values(execution.handlerSummary); + if (statuses.some((status) => status.status === "failed")) return "failed"; + if (statuses.some((status) => status.status === "processing" || status.status === "queued")) { + return "processing"; + } + if (statuses.length > 0 && statuses.every((status) => status.status === "done")) { + return "done"; + } + return "ready"; +} + +function executionResponse(execution: ExperimentExecution) { + return { + experimentExecutionId: execution.experimentExecutionId, + attemptNumber: execution.attemptNumber, + readiness: readiness(execution), + blockers: execution.blocking, + progress: Object.fromEntries( + Object.entries(execution.handlerSummary).map(([handlerId, status]) => [ + handlerId, + { + status: status.status, + updatedAt: status.updatedAt, + ...(status.error ? { error: status.error } : {}), + }, + ]), + ), + createdAt: execution.createdAt, + updatedAt: execution.updatedAt, + ...(execution.startedAt ? { startedAt: execution.startedAt } : {}), + ...(execution.finishedAt ? { finishedAt: execution.finishedAt } : {}), + ...(execution.discardedAt ? { discardedAt: execution.discardedAt } : {}), + ...(execution.discardReason ? { discardReason: execution.discardReason } : {}), + ...(execution.supersededAt ? { supersededAt: execution.supersededAt } : {}), + ...(execution.supersededByExperimentExecutionId + ? { + supersededByExperimentExecutionId: + execution.supersededByExperimentExecutionId, + } + : {}), + ...(execution.supersededReason + ? { supersededReason: execution.supersededReason } + : {}), + }; +} + +function experimentResponse(experiment: ExperimentDocument) { + return { + experimentId: experiment.experimentId, + projectId: experiment.projectId, + metadata: { + name: experiment.metadata.name, + description: experiment.metadata.description, + createdBy: experiment.metadata.createdBy, + }, + revision: { + experimentRevisionId: experiment.revision.experimentRevisionId, + revisionNumber: experiment.revision.revisionNumber, + memberCount: experiment.revision.memberCount, + createdAt: experiment.revision.createdAt, + }, + currentExecution: executionResponse(experiment.execution), + createdAt: experiment.createdAt, + updatedAt: experiment.updatedAt, + }; +} + +interface ExecutionCursor { + attemptNumber: number; + experimentExecutionId: string; +} + +function encodeExecutionCursor(cursor: ExecutionCursor): string { + return Buffer.from(JSON.stringify({ v: 1, ...cursor }), "utf8").toString( + "base64url", + ); +} + +function decodeExecutionCursor( + cursor: string | undefined, +): ExecutionCursor | undefined { + if (!cursor) return undefined; + try { + const parsed: unknown = JSON.parse( + Buffer.from(cursor, "base64url").toString("utf8"), + ); + const result = z + .object({ + v: z.literal(1), + attemptNumber: z.number().int().positive(), + experimentExecutionId: z.string().min(1), + }) + .strict() + .safeParse(parsed); + if (result.success) { + return { + attemptNumber: result.data.attemptNumber, + experimentExecutionId: result.data.experimentExecutionId, + }; + } + } catch { + // Converted to the route's standard 400 below. + } + throw new Error("Invalid experiment cursor"); +} + +function createRequestDocument(input: { + requestId: string; + runId: string; + experimentId: string; + expectedCount: number; + index: number; + projectId: string; + body: z.infer; + taskPromptId: string; + logsUrl: string; + now: Date; +}): RequestDocument { + const { body } = input; + return { + _id: input.requestId, + projectId: input.projectId, + scenario: body.scenario, + workerType: body.workerType, + createdAt: input.now, + taskPromptId: input.taskPromptId, + priority: body.priority ?? 0, + ...(body.model ? { model: body.model } : {}), + ...(body.reasoningEffort ? { reasoningEffort: body.reasoningEffort } : {}), + ...(body.maxIterations ? { maxIterations: body.maxIterations } : {}), + ...(body.personaInstructions + ? { personaInstructions: body.personaInstructions } + : {}), + ...(body.persona ? { persona: body.persona } : {}), + ...(body.mcpServers ? { mcpServers: body.mcpServers } : {}), + ...(body.skillRevisions ? { skillRevisions: body.skillRevisions } : {}), + ...(body.codebaseRevisionId + ? { codebaseRevisionId: body.codebaseRevisionId } + : {}), + ...(body.extensions ? { extensions: body.extensions } : {}), + ...(body.profileId ? { profileId: body.profileId } : {}), + ...(body.gates ? { gates: body.gates } : {}), + ...(body.observations ? { observations: body.observations } : {}), + experimentId: input.experimentId, + experimentExpectedRequestCount: input.expectedCount, + experimentRequestIndex: input.index, + experimentInitialRunId: input.runId, + run: { + _id: input.runId, + attemptNumber: 1, + status: "pending", + logsUrl: input.logsUrl, + }, + }; +} + +export function registerExperimentsRoutes(ctx: RouteContext): void { + apiRoute(ctx.app, ctx.registry, { + method: "post", + path: "/api/v1/experiments", + tags: ["Experiments"], + summary: "Create an experiment and zero or more new requests", + description: + "Canonical composition endpoint. Request and run IDs are minted before insertion; " + + "the exact revision/execution is then finalized idempotently.", + query: ProjectIdQuerySchema, + body: CreateExperimentInputSchema, + response: CreateExperimentResponseSchema, + errorResponses: { + 409: { description: "Experiment identity conflict" }, + }, + handler: async (req, res) => { + const projectId = getQueryProjectId(req); + const experimentId = uuidv4(); + const experimentExecutionId = uuidv4(); + const experimentRevisionId = uuidv4(); + const now = new Date(); + const identities = req.body.requests.map(() => ({ + requestId: uuidv4(), + runId: uuidv4(), + })); + const requestDocs = await Promise.all( + req.body.requests.map(async (body, index) => { + const identity = identities[index]!; + const taskPrompt = await ctx.taskPromptStore.findOrCreate( + projectId, + body.scenario.task, + ); + return createRequestDocument({ + ...identity, + experimentId, + expectedCount: req.body.requests.length, + index, + projectId, + body, + taskPromptId: taskPrompt._id, + logsUrl: ctx.blobStorage.getLogsBlobUrl( + `${identity.requestId}/runs/${identity.runId}/run.jsonl`, + ), + now, + }); + }), + ); + if (requestDocs.length > 0) { + await ctx.requestCollection.insertMany(requestDocs); + } + const requestStates: ExperimentRequestState[] = requestDocs.map((doc) => ({ + requestId: doc._id, + runId: doc.run!._id, + dependencies: [], + blockers: [], + })); + const metadata: ExperimentMetadata = req.body.metadata; + const experimentStore = store(ctx); + const pending = await experimentStore.createPending({ + experimentId, + projectId, + metadata, + revision: { + experimentRevisionId, + revisionNumber: 1, + memberCount: requestStates.length, + membershipFingerprint: fingerprintExperimentPayload( + requestStates.map(({ requestId, runId }) => ({ requestId, runId })), + ), + definitionFingerprint: fingerprintExperimentPayload({ + metadata, + requests: req.body.requests, + }), + createdAt: now, + }, + experimentExecutionId, + requests: requestStates, + now, + }); + const finalized = await experimentStore.finalizeCreation({ + projectId, + experimentId, + expectedExperimentExecutionId: experimentExecutionId, + expectedStateVersion: pending.stateVersion, + now, + }); + if (!finalized) { + res.status(409).json({ error: "Experiment finalization contention" }); + return; + } + if (requestDocs.length > 0) { + await ctx.requestCollection.updateMany( + { + _id: { $in: requestDocs.map((doc) => doc._id) }, + projectId, + experimentId, + experimentInitialRunId: { + $in: requestDocs.map((doc) => doc.run!._id), + }, + }, + { $set: { experimentFinalizedAt: now } }, + ); + } + res.status(201).json({ + experiment: experimentResponse(finalized), + requestIds: requestDocs.map((doc) => doc._id), + }); + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/experiments", + tags: ["Experiments"], + summary: "List experiments", + query: PageQuerySchema, + response: ExperimentListResponseSchema, + handler: async (req, res) => { + const page = await store(ctx).listPage({ + projectId: getQueryProjectId(req), + limit: req.query.limit, + afterExperimentId: req.query.after, + }); + res.json({ + data: page.experiments.map(experimentResponse), + ...(page.nextCursor ? { nextCursor: page.nextCursor } : {}), + }); + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/experiments/:id", + tags: ["Experiments"], + summary: "Get an experiment", + params: z.object({ id: z.string().min(1) }), + query: ProjectIdQuerySchema, + response: ExperimentResponseSchema, + errorResponses: { 404: { description: "Experiment not found" } }, + handler: async (req, res) => { + const experiment = await store(ctx).get( + getQueryProjectId(req), + req.params.id, + ); + if (!experiment) { + res.status(404).json({ error: "Experiment not found" }); + return; + } + res.json(experimentResponse(experiment)); + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/experiments/:id/executions", + tags: ["Experiments"], + summary: "List current and historical experiment executions", + params: z.object({ id: z.string().min(1) }), + query: ProjectIdQuerySchema.extend({ + limit: z.coerce.number().int().min(1).max(100).default(50), + after: CursorSchema, + }), + response: ExecutionListResponseSchema, + errorResponses: { 404: { description: "Experiment not found" } }, + handler: async (req, res) => { + const projectId = getQueryProjectId(req); + const experiment = await ctx.experimentCollection.findOne({ + _id: req.params.id, + projectId, + finalizedAt: { $exists: true }, + }); + if (!experiment) { + res.status(404).json({ error: "Experiment not found" }); + return; + } + const after = decodeExecutionCursor(req.query.after); + const historyFilter: Document = { + projectId: experiment.projectId, + experimentId: req.params.id, + }; + if (after) { + historyFilter.$or = [ + { attemptNumber: { $lt: after.attemptNumber } }, + { + attemptNumber: after.attemptNumber, + experimentExecutionId: { + $gt: after.experimentExecutionId, + }, + }, + ]; + } + const raw = await ctx.experimentExecutionsCollection + .find(historyFilter) + .sort({ attemptNumber: -1, experimentExecutionId: 1 }) + .limit(req.query.limit + 1) + .toArray(); + const executions: ExperimentExecution[] = + after ? raw : [experiment.execution, ...raw]; + const data = executions.slice(0, req.query.limit).map(executionResponse); + const hasMore = executions.length > req.query.limit; + const last = executions[Math.min(data.length, executions.length) - 1]; + res.json({ + data, + ...(hasMore && last + ? { + nextCursor: encodeExecutionCursor({ + attemptNumber: last.attemptNumber, + experimentExecutionId: last.experimentExecutionId, + }), + } + : {}), + }); + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/experiments/:id/executions/:experimentExecutionId", + tags: ["Experiments"], + summary: "Get one exact experiment execution", + params: z.object({ + id: z.string().min(1), + experimentExecutionId: z.string().min(1), + }), + query: ProjectIdQuerySchema, + response: ExecutionResponseSchema, + errorResponses: { 404: { description: "Experiment execution not found" } }, + handler: async (req, res) => { + const projectId = getQueryProjectId(req); + const experiment = await ctx.experimentCollection.findOne({ + _id: req.params.id, + projectId, + finalizedAt: { $exists: true }, + }); + if (!experiment) { + res.status(404).json({ error: "Experiment execution not found" }); + return; + } + const execution = await store(ctx).getExecution({ + projectId: experiment.projectId, + experimentId: req.params.id, + experimentExecutionId: req.params.experimentExecutionId, + }); + if (!execution) { + res.status(404).json({ error: "Experiment execution not found" }); + return; + } + res.json(executionResponse(execution)); + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "post", + path: "/api/v1/experiments/:id/retry", + tags: ["Experiments"], + summary: "Retry current experiment processing", + params: z.object({ id: z.string().min(1) }), + query: ProjectIdQuerySchema, + body: z.object({ experimentExecutionId: z.string().min(1) }), + response: RetryExperimentResponseSchema, + errorResponses: { + 404: { description: "Experiment not found" }, + 409: { description: "Execution is stale or cannot be retried" }, + }, + handler: async (req, res) => { + const projectId = getQueryProjectId(req); + const experimentStore = store(ctx); + const current = await experimentStore.get(projectId, req.params.id); + if (!current) { + res.status(404).json({ error: "Experiment not found" }); + return; + } + if ( + current.execution.experimentExecutionId !== + req.body.experimentExecutionId + ) { + res.status(409).json({ + error: "Experiment execution is stale", + experimentExecutionId: + current.execution.experimentExecutionId, + blockers: current.execution.blocking, + }); + return; + } + const frozen = await experimentStore.freezeCurrent({ + projectId, + experimentId: req.params.id, + experimentExecutionId: req.body.experimentExecutionId, + expectedExecutionVersion: current.execution.version, + }); + if (!frozen) { + res.status(409).json({ + error: "Experiment retry contention", + experimentExecutionId: + current.execution.experimentExecutionId, + blockers: current.execution.blocking, + }); + return; + } + const retried = await experimentStore.retryCurrent({ + projectId, + experimentId: req.params.id, + expectedExperimentExecutionId: + req.body.experimentExecutionId, + expectedExecutionVersion: frozen.execution.version, + nextExperimentExecutionId: uuidv4(), + }); + if (!retried) { + res.status(409).json({ + error: "Experiment retry could not be published", + experimentExecutionId: + current.execution.experimentExecutionId, + blockers: current.execution.blocking, + }); + return; + } + res.status(201).json({ + experimentId: retried.experimentId, + experimentExecutionId: + retried.execution.experimentExecutionId, + attemptNumber: retried.execution.attemptNumber, + readiness: readiness(retried.execution), + blockers: retried.execution.blocking, + }); + }, + }); +} diff --git a/apps/api/src/routes/profiles.ts b/apps/api/src/routes/profiles.ts index b61c0b808..c8da480c6 100644 --- a/apps/api/src/routes/profiles.ts +++ b/apps/api/src/routes/profiles.ts @@ -107,7 +107,7 @@ apiRoute(ctx.app, ctx.registry, { const profileId = uuidv4(); const versionRef = `${profileId}@1`; - // Resolve extension versions (same pattern as run submission) + // Resolve extension versions (same pattern as request creation) let resolvedExtensions: string[] | undefined; if (extensions && extensions.length > 0) { const extensionClient = new ExtensionClient(""); diff --git a/apps/api/src/routes/requests/archive.test.ts b/apps/api/src/routes/requests/archive.test.ts index e6bc8a176..bdb1fce40 100644 --- a/apps/api/src/routes/requests/archive.test.ts +++ b/apps/api/src/routes/requests/archive.test.ts @@ -956,7 +956,7 @@ describe("run import/export — round-trip (export → import)", () => { describe("run import — task-prompt entity creation (#832)", () => { // Task-prompt entities back features, report triggers, group-by-task, // the runs-list task filter and the criteria/MDP analysis. The submission - // flow (POST /api/v1/requests) materializes them via + // request flow (POST /api/v1/requests) materializes them via // taskPromptStore.findOrCreate(scenario.task). Imported runs used to skip // this step entirely \u2014 `taskPromptId` was preserved from `run.yaml` but // no row was ever created, leaving a dangling reference. @@ -1003,6 +1003,7 @@ describe("run import — task-prompt entity creation (#832)", () => { // The inserted request points at it. const inserted = reqCollection.docs.get(id); expect(inserted.taskPromptId).toBe(expectedId); + expect(inserted.submissionId).toEqual(expect.any(String)); }); it("reuses an existing task prompt and does not create a duplicate", async () => { diff --git a/apps/api/src/routes/requests/index.ts b/apps/api/src/routes/requests/index.ts index ffb48667d..0b5d2df81 100644 --- a/apps/api/src/routes/requests/index.ts +++ b/apps/api/src/routes/requests/index.ts @@ -37,7 +37,9 @@ import { isCriterionCompatibleWithGate, orderGates, computeTaskPromptId, + LEGACY_REQUEST_COMPOSITION_DEPRECATION_ID, } from "shared"; +import { trackMetric } from "telemetry"; import type { ProfileDocument, ProfileVersionDocument, @@ -235,6 +237,25 @@ const validatePersistedRequestTarget = (request: RequestDocument) => /** Maximum number of profile variations (including the base profile) allowed in one submit. */ const MAX_PROFILE_VARIATIONS = 25; +function emitLegacyCompositionHeaders( + res: { setHeader(name: string, value: string): void }, + properties: readonly string[], +): void { + if (properties.length === 0) return; + res.setHeader("Deprecation", "true"); + res.setHeader( + "Link", + '; rel="successor-version"; title="POST /api/v1/experiments"', + ); + trackMetric({ + name: "api.deprecated_input_used", + value: 1, + properties: { + identifier: LEGACY_REQUEST_COMPOSITION_DEPRECATION_ID, + properties: [...properties].sort().join(","), + }, + }); +} /** * Server-side validation of a request's gate configuration (docs/design/gates.md @@ -478,7 +499,16 @@ apiRoute(ctx.app, ctx.registry, { tags: ["Requests"], summary: "Submit request(s)", body: CreateRequestInputSchema.extend({ - count: z.number().min(1).max(10).default(1), + count: z + .number() + .min(1) + .max(10) + .default(1) + .openapi({ + deprecated: true, + description: + "Deprecated for composition. Use POST /api/v1/experiments and its requests array.", + }), skills: z.array(z.string()).optional(), agentVersion: z.string().optional(), codebase: z.string().optional(), @@ -553,6 +583,10 @@ apiRoute(ctx.app, ctx.registry, { ? profileVariations.filter((v: unknown): v is string => typeof v === "string") : []; const isVariationSubmit = typedProfileVariations.length > 0; + emitLegacyCompositionHeaders(res, [ + ...(count > 1 ? ["count"] : []), + ...(isVariationSubmit ? ["profileVariations"] : []), + ]); if (isVariationSubmit) { if (!requestedProfileSpec) { @@ -638,7 +672,7 @@ apiRoute(ctx.app, ctx.registry, { const taskPromptId = taskPrompt._id; // Gates are the shared evaluation harness for the whole comparative - // submission, not a per-variation controlled field: every variation + // request batch, not a per-variation controlled field: every variation // (including the base) runs the same gate configuration. Resolve, validate // and canonicalize them up front so an invalid config fails the whole // submit before any inserts (docs/design/gates.md §4.3), mirroring the @@ -835,9 +869,8 @@ apiRoute(ctx.app, ctx.registry, { }); } - // Pass 2: build documents and insert them under a single shared submissionId. - // All variations belong to the same comparative submission so consumers can - // group them via `submissionId` (matches docs/architecture/app-design.md). + // Preserve the legacy request-batch correlation value. Experiments remain + // explicit resources created only by POST /experiments. const submissionId = uuidv4(); const allNewIds: string[] = []; const newDocs: RequestDocument[] = []; @@ -1261,7 +1294,7 @@ apiRoute(ctx.app, ctx.registry, { const taskPrompt = await ctx.taskPromptStore.findOrCreate(projectId, scenario.task); const taskPromptId = taskPrompt._id; - // Resolve AGENTS.md lineage once for this submission (shared across count>1). + // Resolve AGENTS.md lineage once for this request batch (shared across count>1). const agentsMdPromptId = await resolveAgentsMdPromptId(); const agentsMdFields = buildAgentsMdFields(agentsMdPromptId); @@ -1274,12 +1307,10 @@ apiRoute(ctx.app, ctx.registry, { : undefined; // Persisted observation-criteria selection (issue #1156); deduped, evaluated - // by the pp-taxonomy post-processor. Absent/empty = no observations evaluated. + // by the pp-observations post-processor. Absent/empty = no observations evaluated. const persistedObservations: string[] | undefined = observationsProvided ? [...new Set(resolvedObservations)] : undefined; - - // Generate a submission ID to group all runs from this request const submissionId = uuidv4(); // Handle multiple runs (count > 1) @@ -1477,13 +1508,19 @@ apiRoute(ctx.app, ctx.registry, { const taskPromptIdFilter = req.query.taskPromptId as string; const criteriaFilter = req.query.criteria as string; const submissionIdFilter = req.query.submissionId as string; + const experimentIdFilter = req.query.experimentId as string; const searchFilter = req.query.search as string | undefined; const turnsFilterRaw = req.query.turns as string | undefined; const turnsOpFilter = (req.query.turnsOp as string | undefined) ?? "eq"; const maxIterationsFilterRaw = req.query.maxIterations as string | undefined; const maxIterationsOpFilter = (req.query.maxIterationsOp as string | undefined) ?? "eq"; const includeDeleted = req.query.includeDeleted === "true"; - const groupByParam = req.query.groupBy as "task" | "submissionId" | "profile" | undefined; + const groupByParam = req.query.groupBy as + | "task" + | "submissionId" + | "experimentId" + | "profile" + | undefined; const limit = Math.min(Math.max(Number(req.query.limit) || 10, 1), 100); const afterParam = req.query.after as string | undefined; const beforeParam = req.query.before as string | undefined; @@ -1551,8 +1588,13 @@ apiRoute(ctx.app, ctx.registry, { pushClause(buildMultiClause("agentVersion", agentVersionValues)); if (submissionIdFilter) { - // Prefix-based matching: allow filtering by partial submission ID. - flat.submissionId = { $regex: `^${submissionIdFilter.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}` }; + flat.submissionId = { + $regex: `^${submissionIdFilter.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}`, + }; + } + if (experimentIdFilter) { + // Prefix-based matching: allow filtering by partial experiment ID. + flat.experimentId = { $regex: `^${experimentIdFilter.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}` }; } if (!includeDeleted) { flat.deletedAt = { $exists: false }; @@ -1649,7 +1691,12 @@ apiRoute(ctx.app, ctx.registry, { // Grouped mode: paginated RunGroup[] via two-phase aggregation if (groupByParam) { - const groupByField = groupByParam === "task" ? "taskPromptId" : groupByParam === "profile" ? "profileId" : "submissionId"; + const groupByField = + groupByParam === "task" + ? "taskPromptId" + : groupByParam === "profile" + ? "profileId" + : groupByParam; const groupByAggField = `$${groupByField}`; // Decode group cursor @@ -1883,6 +1930,7 @@ apiRoute(ctx.app, ctx.registry, { criteria: z.string().optional(), features: z.string().optional(), submissionId: z.string().optional(), + experimentId: z.string().optional(), k: z.string().optional(), }).merge(ProjectIdQuerySchema), response: z.object({}).passthrough().describe("Analysis metrics"), @@ -2618,7 +2666,7 @@ async function finalizePendingRun( const topChat = run.artifacts.get("run.chat-export.json"); if (topChat) runState.rawChatUrl = topChat; - // Materialize the task-prompt entity for this run. The submission flow + // Materialize the task-prompt entity for this run. The request creation flow // (POST /api/v1/requests) goes through `taskPromptStore.findOrCreate` so // every run has a row in the `task-prompts` collection that backs // features, report triggers, group-by-task, the runs-list task filter and diff --git a/apps/api/src/test-helpers.ts b/apps/api/src/test-helpers.ts index 023c46b73..64229a113 100644 --- a/apps/api/src/test-helpers.ts +++ b/apps/api/src/test-helpers.ts @@ -49,11 +49,29 @@ export function createMockCollection(docs: T[] = insertedCount: 1, acknowledged: true, }), - updateOne: vi.fn().mockResolvedValue({ - matchedCount: 1, - modifiedCount: 1, - acknowledged: true, - }), + updateOne: vi.fn().mockImplementation( + async ( + filter: Record, + update: { $setOnInsert?: T }, + options?: { upsert?: boolean }, + ) => { + const existing = data.find((doc) => doc._id === filter._id); + if (!existing && options?.upsert && update.$setOnInsert) { + data.push({ ...update.$setOnInsert }); + return { + matchedCount: 0, + modifiedCount: 0, + upsertedCount: 1, + acknowledged: true, + }; + } + return { + matchedCount: 1, + modifiedCount: 1, + acknowledged: true, + }; + }, + ), updateMany: vi.fn().mockResolvedValue({ matchedCount: 1, modifiedCount: 1, @@ -204,6 +222,9 @@ export function createAllMockDependencies() { const db = createMockDb(); const collection = createMockCollection(); const runsCollection = createMockCollection(); + const experimentCollection = createMockCollection(); + const experimentExecutionsCollection = createMockCollection(); + const experimentChunksCollection = createMockCollection(); const criteriaCollection = createMockCollection(); const promptFeatureCollection = createMockCollection(); const reportCollection = createMockCollection(); @@ -232,6 +253,9 @@ export function createAllMockDependencies() { db, collection, runsCollection, + experimentCollection, + experimentExecutionsCollection, + experimentChunksCollection, criteriaCollection, promptFeatureCollection, reportCollection, @@ -262,6 +286,9 @@ export function createAllMockDependencies() { db: ReturnType; collection: Collection; runsCollection: Collection; + experimentCollection: Collection; + experimentExecutionsCollection: Collection; + experimentChunksCollection: Collection; criteriaCollection: Collection; promptFeatureCollection: Collection; reportCollection: Collection; diff --git a/apps/api/src/utils/agent-helpers.ts b/apps/api/src/utils/agent-helpers.ts index 768a635cf..95f130278 100644 --- a/apps/api/src/utils/agent-helpers.ts +++ b/apps/api/src/utils/agent-helpers.ts @@ -41,7 +41,7 @@ export interface ValidateAgentTargetOptions { } /** - * Resolve a submission target entirely from the agent registry. + * Resolve a request target entirely from the agent registry. * * Existence, deletion, explicit availability, active version, and advertised * queue are always enforced. Capability flags are enforced only while the diff --git a/apps/api/src/utils/codebase-helpers.ts b/apps/api/src/utils/codebase-helpers.ts index 1cb5d504d..fd9d69f0f 100644 --- a/apps/api/src/utils/codebase-helpers.ts +++ b/apps/api/src/utils/codebase-helpers.ts @@ -84,7 +84,7 @@ export interface CodebaseSpecResolveContext { } /** - * Resolve a codebase spec given at run-submission time into a concrete revision id. + * Resolve a codebase spec given at request creation time into a concrete revision id. * * Accepted forms: * - a raw revision UUID (`_id`) → validated as-is. diff --git a/apps/cli/src/__snapshots__/cli-commands.test.ts.snap b/apps/cli/src/__snapshots__/cli-commands.test.ts.snap index f972e2107..907676b78 100644 --- a/apps/cli/src/__snapshots__/cli-commands.test.ts.snap +++ b/apps/cli/src/__snapshots__/cli-commands.test.ts.snap @@ -200,6 +200,76 @@ exports[`CLI command registration > full command tree matches snapshot 1`] = ` }, }, }, + "experiment": { + "subcommands": { + "create": { + "options": [ + "--agent-version", + "--agents-md", + "--codebase", + "--count", + "--criteria", + "--description", + "--empty", + "--extensions", + "--gates", + "--max-iterations", + "--mcp-servers", + "--message", + "--model", + "--name", + "--observations", + "--output", + "--persona", + "--preview", + "--profile", + "--profile-variations-file", + "--project", + "--reasoning-effort", + "--resource-param", + "--resources", + "--scenario", + "--skills", + "--tags", + "--traits", + "--url", + "--worker", + ], + }, + "executions": { + "options": [ + "--cursor", + "--limit", + "--output", + "--project", + "--url", + ], + }, + "get": { + "options": [ + "--output", + "--project", + "--url", + ], + }, + "list": { + "options": [ + "--cursor", + "--limit", + "--output", + "--project", + "--url", + ], + }, + "retry": { + "options": [ + "--execution", + "--project", + "--url", + ], + }, + }, + }, "extension": { "subcommands": { "delete": { @@ -764,11 +834,11 @@ exports[`CLI command registration > full command tree matches snapshot 1`] = ` "download-batch": { "options": [ "--dir", + "--experiment-id", "--extract", "--ids", "--output", "--project", - "--submission-id", "--url", ], }, @@ -785,6 +855,7 @@ exports[`CLI command registration > full command tree matches snapshot 1`] = ` "--created-after", "--created-before", "--criteria", + "--experiment-id", "--include-deleted", "--max-iterations", "--model", @@ -798,7 +869,6 @@ exports[`CLI command registration > full command tree matches snapshot 1`] = ` "--sort-by", "--sort-dir", "--status", - "--submission-id", "--task", "--turns", "--url", diff --git a/apps/cli/src/bundle.integration.test.ts b/apps/cli/src/bundle.integration.test.ts index 11d8e0714..c1ca6b791 100644 --- a/apps/cli/src/bundle.integration.test.ts +++ b/apps/cli/src/bundle.integration.test.ts @@ -26,7 +26,7 @@ const MOCK_RESPONSES: Record = { id: "req-test-001", workerType: "coder-acp-copilot", run: { status: "done", outcome: "succeeded" }, - submissionId: "sub-abc123", + experimentId: "experiment-abc123", }, ], pageInfo: { hasNextPage: false, hasPreviousPage: false }, diff --git a/apps/cli/src/commands/experiment.test.ts b/apps/cli/src/commands/experiment.test.ts new file mode 100644 index 000000000..60e55c8c1 --- /dev/null +++ b/apps/cli/src/commands/experiment.test.ts @@ -0,0 +1,215 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { Command } from "commander"; +import { registerExperimentCommands } from "./experiment.js"; + +interface CapturedRequest { + url: string; + method: string; + body: string; +} + +const experiment = { + experimentId: "experiment-1", + projectId: "project-1", + metadata: { name: "Experiment 1" }, + revision: { + experimentRevisionId: "revision-1", + revisionNumber: 1, + memberCount: 0, + }, + execution: { + experimentExecutionId: "execution-1", + attemptNumber: 1, + readiness: "ready", + requests: [], + handlerProgress: {}, + blocking: [], + createdAt: "2026-01-01T00:00:00.000Z", + }, + readiness: "ready", + blocking: [], + createdRequestIds: [], + createdAt: "2026-01-01T00:00:00.000Z", +}; + +function makeProgram(): Command { + const program = new Command(); + registerExperimentCommands(program); + return program; +} + +function mockFetchWith( + body: unknown, +): { requests: CapturedRequest[] } { + const requests: CapturedRequest[] = []; + vi.stubGlobal( + "fetch", + vi.fn(async (request: Request) => { + requests.push({ + url: request.url, + method: request.method, + body: await request.text(), + }); + return new Response(JSON.stringify(body), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + }), + ); + return { requests }; +} + +describe("experiment commands", () => { + beforeEach(() => { + vi.restoreAllMocks(); + process.exitCode = undefined; + }); + + afterEach(() => { + vi.restoreAllMocks(); + process.exitCode = undefined; + }); + + it("does not expose removed Experiment tags", () => { + const experimentCommand = makeProgram().commands.find( + (command) => command.name() === "experiment", + ); + const createCommand = experimentCommand?.commands.find( + (command) => command.name() === "create", + ); + + expect(createCommand).toBeDefined(); + expect(createCommand?.options.map((option) => option.long)).not.toContain( + "--tags", + ); + }); + + it("previews composition without calling the API or warning", async () => { + const fetchSpy = vi.fn(); + vi.stubGlobal("fetch", fetchSpy); + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + + await makeProgram().parseAsync( + [ + "experiment", + "create", + "--name", + "Nightly", + "--message", + "Run tests", + "--count", + "2", + "--preview", + "--output", + "json", + "--project", + "project-1", + "--url", + "http://localhost:3100", + ], + { from: "user" }, + ); + + expect(fetchSpy).not.toHaveBeenCalled(); + expect(warn).not.toHaveBeenCalled(); + expect(log.mock.calls.flat().join("\n")).toContain('"expandedRunCount": 2'); + expect(log.mock.calls.flat().join("\n")).toContain('"count": 2'); + }); + + it("creates an explicitly empty project-scoped experiment", async () => { + const { requests } = mockFetchWith({ + experiment, + requestIds: [], + }); + vi.spyOn(console, "log").mockImplementation(() => {}); + + await makeProgram().parseAsync( + [ + "experiment", + "create", + "--name", + "Empty", + "--empty", + "--project", + "project-1", + "--url", + "http://localhost:3100", + ], + { from: "user" }, + ); + + expect(requests).toHaveLength(1); + expect(requests[0]).toMatchObject({ + method: "POST", + body: JSON.stringify({ + mode: "empty", + metadata: { name: "Empty" }, + requests: [], + }), + }); + expect(requests[0]?.url).toBe( + "http://localhost:3100/api/v1/experiments?projectId=project-1", + ); + }); + + it("gets an exact project-scoped execution", async () => { + const { requests } = mockFetchWith(experiment.execution); + vi.spyOn(console, "log").mockImplementation(() => {}); + + await makeProgram().parseAsync( + [ + "experiment", + "executions", + "experiment-1", + "execution-1", + "--project", + "project-1", + "--url", + "http://localhost:3100", + ], + { from: "user" }, + ); + + expect(requests[0]?.url).toBe( + "http://localhost:3100/api/v1/experiments/experiment-1/executions/execution-1?projectId=project-1", + ); + }); + + it("retries with the current execution identifier", async () => { + const { requests } = mockFetchWith({ + experimentId: "experiment-1", + experimentExecutionId: "execution-2", + attemptNumber: 2, + readiness: "ready", + blockers: [], + }); + vi.spyOn(console, "log").mockImplementation(() => {}); + + await makeProgram().parseAsync( + [ + "experiment", + "retry", + "experiment-1", + "--execution", + "execution-1", + "--project", + "project-1", + "--url", + "http://localhost:3100", + ], + { from: "user" }, + ); + + expect(requests[0]).toMatchObject({ + method: "POST", + body: JSON.stringify({ experimentExecutionId: "execution-1" }), + }); + expect(requests[0]?.url).toBe( + "http://localhost:3100/api/v1/experiments/experiment-1/retry?projectId=project-1", + ); + }); +}); diff --git a/apps/cli/src/commands/experiment.ts b/apps/cli/src/commands/experiment.ts new file mode 100644 index 000000000..a6c8041bc --- /dev/null +++ b/apps/cli/src/commands/experiment.ts @@ -0,0 +1,880 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { Command } from "commander"; +import { stringify as stringifyYaml } from "yaml"; +import { resolveScenarioAndPersona } from "../config-loader.js"; +import { apiFetch } from "../utils/api-client.js"; +import { requireProjectId } from "../utils/config.js"; +import { formatData, isMachineReadable } from "../utils/formatters.js"; +import { parseGatesOption } from "../utils/gates.js"; +import { configureHelp } from "../utils/helpFormatter.js"; +import { buildResourceBindingSpecs, collectRepeatable } from "../utils/resources.js"; +import { + getDefaultApiUrl, + withOutputOption, + withProjectOption, +} from "../utils/shared.js"; +import { errorText, label, successText, value } from "../utils/style.js"; +import type { DisplayField, OutputFormat } from "../utils/types.js"; + +interface ExperimentMetadata { + name: string; + description?: string; +} + +interface ExperimentExecution { + experimentExecutionId: string; + attemptNumber: number; + version?: number; + readiness?: string; + requestCount: number; + requests: ExperimentRequestSummary[]; + handlerProgress: JsonRecord; + blocking: ExperimentBlocker[]; + currentReport?: { + reportId?: string; + url?: string; + updatedAt?: string; + }; + createdAt: string; + updatedAt?: string; + startedAt?: string; + finishedAt?: string; + discardedAt?: string; + discardReason?: string; + supersededAt?: string; + supersededByExperimentExecutionId?: string; + supersededReason?: "rollover" | "retry"; + archivedAt?: string; +} + +interface ExperimentBlocker { + handlerId: string; + reason: string; + requestId?: string; + runId?: string; + error?: string; +} + +interface ExperimentRequestSummary { + requestId: string; + runId: string; + task?: string; + status?: string; + outcome?: string; + workerType?: string; + model?: string; + profileId?: string; +} + +interface ExperimentView { + experimentId: string; + projectId: string; + metadata: ExperimentMetadata; + revision: { + experimentRevisionId: string; + revisionNumber: number; + memberCount: number; + membershipFingerprint?: string; + definitionFingerprint?: string; + createdAt?: string; + }; + execution: ExperimentExecution; + readiness?: string; + blocking: ExperimentBlocker[]; + createdRequestIds: string[]; + createdAt: string; + updatedAt?: string; + finalizedAt?: string; +} + +interface ExperimentPage { + items: ExperimentView[]; + nextCursor?: string; +} + +interface ExperimentExecutionPage { + items: ExperimentExecution[]; + nextCursor?: string; +} + +type JsonRecord = Record; + +function asRecord(value: unknown, name: string): JsonRecord { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`Invalid ${name} response from Scope`); + } + return value as JsonRecord; +} + +function stringField( + value: JsonRecord, + key: string, + name: string, + optional = false, +): string | undefined { + const field = value[key]; + if (field === undefined && optional) return undefined; + if (typeof field !== "string" || field.length === 0) { + throw new Error(`Invalid ${name}.${key} response from Scope`); + } + return field; +} + +function numberField(value: JsonRecord, key: string, name: string): number { + const field = value[key]; + if (typeof field !== "number" || !Number.isFinite(field)) { + throw new Error(`Invalid ${name}.${key} response from Scope`); + } + return field; +} + +function parseBlocker(value: unknown): ExperimentBlocker { + const blocker = asRecord(value, "experiment blocker"); + return { + handlerId: stringField(blocker, "handlerId", "experiment blocker")!, + reason: stringField(blocker, "reason", "experiment blocker")!, + requestId: stringField(blocker, "requestId", "experiment blocker", true), + runId: stringField(blocker, "runId", "experiment blocker", true), + error: stringField(blocker, "error", "experiment blocker", true), + }; +} + +function parseRequestSummary(value: unknown): ExperimentRequestSummary { + const request = asRecord(value, "experiment request"); + return { + requestId: stringField(request, "requestId", "experiment request")!, + runId: stringField(request, "runId", "experiment request")!, + task: stringField(request, "task", "experiment request", true), + status: stringField(request, "status", "experiment request", true), + outcome: stringField(request, "outcome", "experiment request", true), + workerType: stringField(request, "workerType", "experiment request", true), + model: stringField(request, "model", "experiment request", true), + profileId: stringField(request, "profileId", "experiment request", true), + }; +} + +function parseExecution(value: unknown): ExperimentExecution { + const execution = asRecord(value, "experiment execution"); + const requests = execution.requests; + const requestVector = Array.isArray(requests) + ? { requestCount: requests.length, requests: requests.map(parseRequestSummary) } + : (() => { + const payload = requests === undefined + ? {} + : asRecord(requests, "experiment requests"); + const entries = Array.isArray(payload.entries) + ? payload.entries.map(parseRequestSummary) + : []; + return { + requestCount: + typeof payload.entryCount === "number" + ? payload.entryCount + : entries.length, + requests: entries, + }; + })(); + const blockingValue = execution.blockers ?? execution.blocking; + const blocking = Array.isArray(blockingValue) + ? blockingValue.map(parseBlocker) + : []; + const progress = asRecord( + execution.progress ?? + execution.handlerProgress ?? + execution.handlerSummary ?? + {}, + "experiment handler progress", + ); + const report = execution.currentReport === undefined + ? undefined + : asRecord(execution.currentReport, "experiment report"); + return { + experimentExecutionId: stringField( + execution, + "experimentExecutionId", + "experiment execution", + )!, + attemptNumber: numberField( + execution, + "attemptNumber", + "experiment execution", + ), + readiness: stringField( + execution, + "readiness", + "experiment execution", + true, + ), + version: + typeof execution.version === "number" ? execution.version : undefined, + requestCount: requestVector.requestCount, + requests: requestVector.requests, + handlerProgress: progress, + blocking, + currentReport: report + ? { + reportId: stringField(report, "reportId", "experiment report", true), + url: stringField(report, "url", "experiment report", true), + updatedAt: stringField(report, "updatedAt", "experiment report", true), + } + : undefined, + createdAt: stringField( + execution, + "createdAt", + "experiment execution", + )!, + updatedAt: stringField(execution, "updatedAt", "experiment execution", true), + startedAt: stringField(execution, "startedAt", "experiment execution", true), + finishedAt: stringField(execution, "finishedAt", "experiment execution", true), + discardedAt: stringField(execution, "discardedAt", "experiment execution", true), + discardReason: stringField(execution, "discardReason", "experiment execution", true), + supersededAt: stringField(execution, "supersededAt", "experiment execution", true), + supersededByExperimentExecutionId: stringField( + execution, + "supersededByExperimentExecutionId", + "experiment execution", + true, + ), + supersededReason: + execution.supersededReason === "rollover" || + execution.supersededReason === "retry" + ? execution.supersededReason + : undefined, + archivedAt: stringField(execution, "archivedAt", "experiment execution", true), + }; +} + +function parseExperiment(value: unknown): ExperimentView { + const experiment = asRecord(value, "experiment"); + const metadata = asRecord(experiment.metadata, "experiment metadata"); + const revision = asRecord(experiment.revision, "experiment revision"); + const execution = parseExecution( + experiment.execution ?? experiment.currentExecution, + ); + const blockingValue = experiment.blocking ?? execution.blocking; + const blocking = Array.isArray(blockingValue) + ? blockingValue.map(parseBlocker) + : []; + const createdRequestIds = Array.isArray(experiment.createdRequestIds) + ? experiment.createdRequestIds.map((id) => { + if (typeof id !== "string") { + throw new Error("Invalid experiment.createdRequestIds response from Scope"); + } + return id; + }) + : []; + return { + experimentId: stringField(experiment, "experimentId", "experiment")!, + projectId: stringField(experiment, "projectId", "experiment")!, + metadata: { + name: stringField(metadata, "name", "experiment metadata")!, + description: stringField( + metadata, + "description", + "experiment metadata", + true, + ), + }, + revision: { + experimentRevisionId: stringField( + revision, + "experimentRevisionId", + "experiment revision", + )!, + revisionNumber: numberField( + revision, + "revisionNumber", + "experiment revision", + ), + memberCount: + typeof revision.memberCount === "number" + ? revision.memberCount + : execution.requestCount, + membershipFingerprint: stringField( + revision, + "membershipFingerprint", + "experiment revision", + true, + ), + definitionFingerprint: stringField( + revision, + "definitionFingerprint", + "experiment revision", + true, + ), + createdAt: stringField(revision, "createdAt", "experiment revision", true), + }, + execution, + readiness: + stringField(experiment, "readiness", "experiment", true) ?? + execution.readiness, + blocking, + createdRequestIds, + createdAt: stringField(experiment, "createdAt", "experiment")!, + updatedAt: stringField(experiment, "updatedAt", "experiment", true), + finalizedAt: stringField(experiment, "finalizedAt", "experiment", true), + }; +} + +function parsePage(value: unknown): ExperimentPage { + const page = asRecord(value, "experiment page"); + const items = page.data ?? page.items; + if (!Array.isArray(items)) { + throw new Error("Invalid experiment page items response from Scope"); + } + return { + items: items.map(parseExperiment), + nextCursor: stringField(page, "nextCursor", "experiment page", true), + }; +} + +function parseExecutionPage(value: unknown): ExperimentExecutionPage { + const page = asRecord(value, "experiment execution page"); + const items = page.data ?? page.items; + if (!Array.isArray(items)) { + throw new Error("Invalid experiment execution page items response from Scope"); + } + return { + items: items.map(parseExecution), + nextCursor: stringField( + page, + "nextCursor", + "experiment execution page", + true, + ), + }; +} + +async function json(response: Response): Promise { + return response.json() as Promise; +} + +async function throwApiError(response: Response): Promise { + const value = await json(response); + const body = asRecord(value, "error"); + throw new Error( + typeof body.error === "string" ? body.error : `HTTP ${response.status}`, + ); +} + +function resolveTextOrFile(input: string): string { + return input.startsWith("@") + ? readFileSync(resolve(input.slice(1)), "utf8") + : input; +} + +const fields: DisplayField[] = [ + { key: "experimentId", label: "Experiment" }, + { + key: "metadata", + label: "Name", + formatter: (row: JsonRecord) => + String((row.metadata as ExperimentMetadata | undefined)?.name ?? ""), + }, + { + key: "execution", + label: "Attempt", + formatter: (row: JsonRecord) => + String((row.execution as ExperimentExecution | undefined)?.attemptNumber ?? ""), + }, + { key: "readiness", label: "Readiness" }, + { + key: "revision", + label: "Runs", + formatter: (row: JsonRecord) => + String( + (row.revision as ExperimentView["revision"] | undefined)?.memberCount ?? 0, + ), + }, + { key: "createdAt", label: "Created" }, +]; + +function addRunConfigurationOptions(command: Command): Command { + return command + .option("-s, --scenario ", "Scenario YAML file") + .option("-p, --persona ", "Persona YAML file") + .option("-t, --traits ", "Traits YAML file") + .option("-m, --message ", "Task prompt") + .option("-w, --worker ", "Registered worker ID") + .option("-c, --criteria ", "Evaluation criteria") + .option("--max-iterations ", "Max judge iterations", Number.parseInt) + .option("--model ", "Coding-agent model") + .option("--reasoning-effort ", "Reasoning effort") + .option("--mcp-servers ", "MCP server slugs") + .option("--skills ", "Skill slugs") + .option("--codebase ", "Codebase revision reference") + .option("--resources ", "Resource binding specs") + .option( + "--resource-param :=", + "Resource parameter override (repeatable)", + collectRepeatable, + [], + ) + .option("--extensions ", "VS Code extension IDs") + .option("--agent-version ", "Agent version") + .option("--profile ", "Base profile ID or version ref") + .option( + "--profile-variations-file ", + "JSON file containing profile variation refs", + ) + .option( + "--agents-md ", + "AGENTS.md content; prefix a path with @", + ) + .option("--gates ", "Gate configuration JSON or file") + .option("--observations ", "Observation criterion IDs") + .option( + "--count ", + "Occurrences per configuration (1-10)", + (value: string) => Number.parseInt(value, 10), + ); +} + +function buildRequest(options: Record): { + worker?: string; + request: JsonRecord; + profileCount: number; + occurrences: number; +} { + let message = + typeof options.message === "string" ? options.message : undefined; + let criteria = Array.isArray(options.criteria) + ? options.criteria.filter((item): item is string => typeof item === "string") + : undefined; + let personaInstructions: string | undefined; + let persona: Record | undefined; + if (typeof options.scenario === "string") { + const resolved = resolveScenarioAndPersona( + options.scenario, + typeof options.persona === "string" ? options.persona : undefined, + typeof options.traits === "string" ? options.traits : undefined, + ); + message ??= resolved.task; + if (!criteria?.length) criteria = resolved.criteria; + personaInstructions = resolved.personaInstructions; + persona = resolved.persona; + } + if (!message?.trim()) { + throw new Error("--message or --scenario is required unless --empty is used"); + } + + const occurrences = + typeof options.count === "number" ? options.count : 1; + if (!Number.isInteger(occurrences) || occurrences < 1 || occurrences > 10) { + throw new Error("--count must be an integer between 1 and 10"); + } + + const request: JsonRecord = { + scenario: { task: message.trim(), criteria: criteria ?? [] }, + }; + const copyString = (source: string, target = source): void => { + if (typeof options[source] === "string" && options[source]) { + request[target] = options[source]; + } + }; + copyString("model"); + copyString("reasoningEffort"); + copyString("agentVersion"); + copyString("profile", "profileId"); + copyString("codebase"); + if (typeof options.maxIterations === "number") { + request.maxIterations = options.maxIterations; + } + if (personaInstructions) request.personaInstructions = personaInstructions; + if (persona) request.persona = persona; + if (occurrences > 1) request.count = occurrences; + for (const [source, target] of [ + ["mcpServers", "mcpServers"], + ["skills", "skills"], + ["extensions", "extensions"], + ["observations", "observations"], + ] as const) { + if (Array.isArray(options[source]) && options[source].length > 0) { + request[target] = options[source]; + } + } + const resources = buildResourceBindingSpecs( + Array.isArray(options.resources) + ? options.resources.filter( + (item): item is string => typeof item === "string", + ) + : undefined, + Array.isArray(options.resourceParam) + ? options.resourceParam.filter( + (item): item is string => typeof item === "string", + ) + : [], + ); + if (resources) request.resources = resources; + if (typeof options.agentsMd === "string") { + const content = resolveTextOrFile(options.agentsMd); + if (content.trim()) request.agentsMd = content; + } + if (typeof options.gates === "string") { + request.gates = parseGatesOption( + options.gates, + typeof options.maxIterations === "number" + ? options.maxIterations + : undefined, + ); + } + + let profileCount = typeof options.profile === "string" ? 1 : 0; + if (typeof options.profileVariationsFile === "string") { + if (typeof options.profile !== "string") { + throw new Error( + "--profile is required with --profile-variations-file", + ); + } + const parsed: unknown = JSON.parse( + readFileSync(resolve(options.profileVariationsFile), "utf8"), + ); + if (!Array.isArray(parsed) || !parsed.every((item) => typeof item === "string")) { + throw new Error("profile variations file must be a JSON string array"); + } + request.profileVariations = parsed; + profileCount += parsed.length; + } + if (profileCount === 0) profileCount = 1; + + return { + worker: typeof options.worker === "string" ? options.worker : undefined, + request, + profileCount, + occurrences, + }; +} + +export function registerExperimentCommands(program: Command): void { + const experiment = program + .command("experiment") + .description("Create and inspect benchmark experiments") + .action(() => experiment.help()); + configureHelp(experiment); + + const create = addRunConfigurationOptions( + withProjectOption( + withOutputOption( + experiment + .command("create") + .description("Create an experiment from a run configuration") + .requiredOption("--name ", "Experiment name") + .option("--description ", "Experiment description") + .option("--empty", "Create an explicit empty experiment") + .option( + "--preview", + "Print the composition and request body without creating", + ) + .option("-u, --url ", "API base URL", getDefaultApiUrl()), + ), + ), + ); + + create.action(async (options) => { + try { + const projectId = requireProjectId(options.project); + const metadata: ExperimentMetadata = { + name: options.name, + ...(options.description ? { description: options.description } : {}), + }; + if (options.empty) { + const conflicting = [ + "scenario", + "message", + "worker", + "criteria", + "profile", + "profileVariationsFile", + "count", + ].filter((key) => options[key] !== undefined); + if (conflicting.length > 0) { + throw new Error( + `--empty cannot be combined with run configuration options: ${conflicting.join(", ")}`, + ); + } + } + const built = options.empty ? undefined : buildRequest(options); + const body = options.empty + ? { mode: "empty" as const, metadata, requests: [] } + : { + mode: "compose" as const, + metadata, + requests: [built!.request], + }; + const expandedRunCount = built + ? built.profileCount * built.occurrences + : 0; + if (options.preview) { + const preview = { + expandedRunCount, + body, + }; + const format = (options.output ?? "json") as OutputFormat; + if (format === "json") { + console.log(JSON.stringify(preview, null, 2)); + return; + } + if (format === "yaml") { + console.log(stringifyYaml(preview).trimEnd()); + return; + } + console.log( + formatData( + [preview], + [ + { key: "expandedRunCount", label: "Runs" }, + { + key: "body", + label: "Create body", + formatter: () => JSON.stringify(body), + }, + ], + format, + ), + ); + return; + } + const workerQuery = built?.worker + ? `?worker=${encodeURIComponent(built.worker)}` + : ""; + const response = await apiFetch( + options.url, + `/experiments${workerQuery}`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + projectId, + }, + ); + if (!response.ok) await throwApiError(response); + const createResponse = asRecord(await json(response), "experiment create"); + const created = parseExperiment(createResponse.experiment); + const format = (options.output ?? "table") as OutputFormat; + if (isMachineReadable(format)) { + console.log(formatData([created], fields, format)); + return; + } + console.log( + `${successText("Experiment created:")} ${value(created.experimentId)}`, + ); + console.log(`${label("Name:")} ${value(created.metadata.name)}`); + console.log( + `${label("Runs:")} ${value(String(created.revision.memberCount))}`, + ); + console.log( + `${label("Execution:")} ${value(created.execution.experimentExecutionId)}`, + ); + } catch (error) { + console.error( + errorText("Error:"), + error instanceof Error ? error.message : error, + ); + process.exitCode = 1; + } + }); + + withProjectOption( + withOutputOption( + experiment + .command("list") + .description("List experiments") + .option("--cursor ", "Continue from a list cursor") + .option("--limit ", "Maximum items", Number.parseInt) + .option("-u, --url ", "API base URL", getDefaultApiUrl()), + ), + ).action(async (options) => { + try { + const projectId = requireProjectId(options.project); + const params = new URLSearchParams(); + if (options.cursor) params.set("after", options.cursor); + if (options.limit) params.set("limit", String(options.limit)); + const response = await apiFetch( + options.url, + `/experiments${params.size ? `?${params}` : ""}`, + { projectId }, + ); + if (!response.ok) await throwApiError(response); + const page = parsePage(await json(response)); + console.log( + formatData( + page.items, + fields, + (options.output ?? "table") as OutputFormat, + ), + ); + if (page.nextCursor && !isMachineReadable(options.output ?? "table")) { + console.log(`${label("Next cursor:")} ${value(page.nextCursor)}`); + } + } catch (error) { + console.error( + errorText("Error:"), + error instanceof Error ? error.message : error, + ); + process.exitCode = 1; + } + }); + + withProjectOption( + withOutputOption( + experiment + .command("get") + .description("Get an experiment and its current execution") + .argument("", "Experiment ID") + .option("-u, --url ", "API base URL", getDefaultApiUrl()), + ), + ).action(async (id: string, options) => { + try { + const projectId = requireProjectId(options.project); + const response = await apiFetch( + options.url, + `/experiments/${encodeURIComponent(id)}`, + { projectId }, + ); + if (!response.ok) await throwApiError(response); + const item = parseExperiment(await json(response)); + const format = (options.output ?? "table") as OutputFormat; + if (isMachineReadable(format)) { + console.log(formatData([item], fields, format)); + return; + } + console.log(`${label("Experiment:")} ${value(item.experimentId)}`); + console.log(`${label("Name:")} ${value(item.metadata.name)}`); + console.log( + `${label("Execution:")} ${value(item.execution.experimentExecutionId)}`, + ); + console.log( + `${label("Attempt:")} ${value(String(item.execution.attemptNumber))}`, + ); + console.log( + `${label("Readiness:")} ${value(item.readiness ?? "unknown")}`, + ); + console.log( + `${label("Runs:")} ${value(String(item.revision.memberCount))}`, + ); + } catch (error) { + console.error( + errorText("Error:"), + error instanceof Error ? error.message : error, + ); + process.exitCode = 1; + } + }); + + withProjectOption( + withOutputOption( + experiment + .command("executions") + .description("List current and historical experiment executions") + .argument("", "Experiment ID") + .argument("[executionId]", "Get one exact execution") + .option("--cursor ", "Continue from a list cursor") + .option("--limit ", "Maximum items", Number.parseInt) + .option("-u, --url ", "API base URL", getDefaultApiUrl()), + ), + ).action(async (id: string, executionId: string | undefined, options) => { + try { + const projectId = requireProjectId(options.project); + if (executionId) { + const response = await apiFetch( + options.url, + `/experiments/${encodeURIComponent(id)}/executions/${encodeURIComponent(executionId)}`, + { projectId }, + ); + if (!response.ok) await throwApiError(response); + const item = parseExecution(await json(response)); + console.log( + formatData( + [item], + [ + { key: "experimentExecutionId", label: "Execution" }, + { key: "attemptNumber", label: "Attempt" }, + { key: "readiness", label: "Readiness" }, + { key: "requestCount", label: "Runs" }, + { key: "supersededReason", label: "Superseded" }, + { key: "createdAt", label: "Created" }, + ], + (options.output ?? "table") as OutputFormat, + ), + ); + return; + } + const params = new URLSearchParams(); + if (options.cursor) params.set("after", options.cursor); + if (options.limit) params.set("limit", String(options.limit)); + const response = await apiFetch( + options.url, + `/experiments/${encodeURIComponent(id)}/executions${ + params.size ? `?${params}` : "" + }`, + { projectId }, + ); + if (!response.ok) await throwApiError(response); + const page = parseExecutionPage(await json(response)); + console.log( + formatData( + page.items, + [ + { key: "experimentExecutionId", label: "Execution" }, + { key: "attemptNumber", label: "Attempt" }, + { key: "readiness", label: "Readiness" }, + { key: "requestCount", label: "Runs" }, + { key: "supersededReason", label: "Superseded" }, + { key: "createdAt", label: "Created" }, + ], + (options.output ?? "table") as OutputFormat, + ), + ); + } catch (error) { + console.error( + errorText("Error:"), + error instanceof Error ? error.message : error, + ); + process.exitCode = 1; + } + }); + + withProjectOption( + experiment + .command("retry") + .description("Retry the current experiment execution") + .argument("", "Experiment ID") + .requiredOption( + "--execution ", + "Current experiment execution ID", + ) + .option("-u, --url ", "API base URL", getDefaultApiUrl()), + ).action(async (id: string, options) => { + try { + const projectId = requireProjectId(options.project); + const response = await apiFetch( + options.url, + `/experiments/${encodeURIComponent(id)}/retry`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + experimentExecutionId: options.execution, + }), + projectId, + }, + ); + if (!response.ok) await throwApiError(response); + const item = asRecord(await json(response), "experiment retry"); + console.log( + successText( + `Experiment retry started: attempt ${numberField(item, "attemptNumber", "experiment retry")} (${stringField(item, "experimentExecutionId", "experiment retry")})`, + ), + ); + } catch (error) { + console.error( + errorText("Error:"), + error instanceof Error ? error.message : error, + ); + process.exitCode = 1; + } + }); +} diff --git a/apps/cli/src/commands/run.test.ts b/apps/cli/src/commands/run.test.ts index b26334bab..bc40eb363 100644 --- a/apps/cli/src/commands/run.test.ts +++ b/apps/cli/src/commands/run.test.ts @@ -75,7 +75,7 @@ describe("run list", () => { id: "req-paginated-1", workerType: "coder-acp-copilot", run: { status: "done", outcome: "succeeded" }, - submissionId: "submission-12345678", + experimentId: "experiment-12345678", }, ], pageInfo: { @@ -99,7 +99,7 @@ describe("run list", () => { id: "req-array-1", workerType: "coder-acp-claude-code", run: { status: "running", outcome: "finished" }, - submissionId: "submission-abcdef12", + experimentId: "experiment-abcdef12", }, ]); @@ -117,7 +117,7 @@ describe("run list", () => { id: "req-json-1", workerType: "coder-acp-copilot", run: { status: "done", outcome: "succeeded" }, - submissionId: "submission-feedbeef", + experimentId: "experiment-feedbeef", }, ], pageInfo: { @@ -139,7 +139,7 @@ describe("run list", () => { id: "req-tsv-1", workerType: "coder-acp-copilot", run: { status: "done", outcome: "succeeded" }, - submissionId: "submission-11223344", + experimentId: "experiment-11223344", }, ], pageInfo: { @@ -149,7 +149,7 @@ describe("run list", () => { }); const output = await runListAndCaptureOutput(["-o", "tsv"]); - expect(output).toContain("req-tsv-1\tcoder-acp-copilot\tdone\tsubmissi"); + expect(output).toContain("req-tsv-1\tcoder-acp-copilot\tdone\texperime"); expect(output).not.toContain("Found 1 request(s):"); }); @@ -160,7 +160,7 @@ describe("run list", () => { id: "req-yaml-1", workerType: "coder-acp-copilot", run: { status: "done", outcome: "succeeded" }, - submissionId: "submission-a1b2c3d4", + experimentId: "experiment-a1b2c3d4", }, ], pageInfo: { @@ -236,6 +236,59 @@ describe("run submit", () => { const { body } = captureSubmit(); expect(body).not.toHaveProperty("agentsMd"); }); + + it("warns on deprecated multi-request composition", async () => { + mockFetchWith({ + ids: ["req-submit-3", "req-submit-4"], + count: 2, + status: "queued", + }); + vi.spyOn(console, "log").mockImplementation(() => {}); + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + + await makeProgram().parseAsync( + [ + "run", "submit", + "-m", "repeat task", + "--worker", "dynamic-worker", + "--count", "2", + "--no-stream", + "-u", "http://localhost:3100", + "--project", "proj-test", + ], + { from: "user" }, + ); + + expect(warnSpy).toHaveBeenCalledOnce(); + expect(warnSpy.mock.calls.flat().join(" ")).toContain( + "legacy-request-composition", + ); + }); + + it("does not warn for an ordinary single request", async () => { + mockFetchWith({ + id: "req-submit-5", + workerType: "coder-acp-copilot", + status: "queued", + }); + vi.spyOn(console, "log").mockImplementation(() => {}); + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + + await makeProgram().parseAsync( + [ + "run", "submit", + "-m", "single task", + "--worker", "dynamic-worker", + "--count", "1", + "--no-stream", + "-u", "http://localhost:3100", + "--project", "proj-test", + ], + { from: "user" }, + ); + + expect(warnSpy).not.toHaveBeenCalled(); + }); }); describe("run retry", () => { diff --git a/apps/cli/src/commands/run.ts b/apps/cli/src/commands/run.ts index ab2f7c15d..0c559a223 100644 --- a/apps/cli/src/commands/run.ts +++ b/apps/cli/src/commands/run.ts @@ -21,6 +21,9 @@ import { apiFetch, getApiBasePath } from "../utils/api-client.js"; import { parseGatesOption } from "../utils/gates.js"; import { buildResourceBindingSpecs, collectRepeatable } from "../utils/resources.js"; +const LEGACY_REQUEST_COMPOSITION_DEPRECATION = + "legacy-request-composition" as const; + /** * Resolve a CLI option that may be either a literal string or a `@path` * reference to a file whose contents should be read. Used for flags like @@ -213,6 +216,17 @@ run body.profileVariations = parsed; } + if ( + (typeof body.count === "number" && body.count > 1) || + (Array.isArray(body.profileVariations) && body.profileVariations.length > 0) + ) { + console.warn( + `${label("Deprecation:")} [${LEGACY_REQUEST_COMPOSITION_DEPRECATION}] ` + + "Use `scope experiment create` for composed benchmarks. " + + "`scope run submit` remains permanent for a single request.", + ); + } + // In variation mode the API derives the worker per-variation from each // profile's workerType, and explicitly rejects `?worker=`. Skip the // query param so the request isn't 400'd, and warn if --worker was @@ -250,13 +264,13 @@ run const result = await response.json(); // A variation submit creates one request per profile and returns - // { ids, variations, submissionId, ... } with no `id` or `workerType`. + // { ids, variations, experimentId, ... } with no `id` or `workerType`. // Printing it through the single-request path rendered `undefined` and - // then crashed in chalk, so the whole submission looked like it failed + // then crashed in chalk, so the whole request expansion looked like it failed // when in fact every request had been created. if (Array.isArray(result.ids) && result.ids.length > 0 && !result.id) { console.log(`${successText('Submitted:')} ${value(String(result.count ?? result.ids.length))} request(s) across ${value(String(result.variationCount ?? result.ids.length))} profile(s)`); - if (result.submissionId) console.log(`${label('Submission:')} ${value(result.submissionId)}`); + if (result.experimentId) console.log(`${label('Experiment:')} ${value(result.experimentId)}`); for (const variation of (result.variations ?? [])) { const ids: string[] = variation.ids ?? (variation.id ? [variation.id] : []); const name = variation.label ?? variation.profileName ?? variation.profileId ?? 'variation'; @@ -266,13 +280,13 @@ run for (const warning of (Array.isArray(result.warnings) ? result.warnings : [])) { console.log(`${errorText('⚠ Warning:')} ${warning}`); } - // Streaming follows a single request; a submission has several. + // Streaming follows a single request; an expanded composition has several. printFollowUpCommands(result.ids[0]); return; } console.log(`${successText('Request submitted:')} ${value(result.id)}`); - if (result.submissionId) console.log(`${label('Submission:')} ${value(result.submissionId)}`); + if (result.experimentId) console.log(`${label('Experiment:')} ${value(result.experimentId)}`); console.log(`${label('Worker:')} ${value(result.workerType)}`); if (result.model) console.log(`${label('Model:')} ${value(result.model)}`); if (result.reasoningEffort) console.log(`${label('Reasoning Effort:')} ${value(result.reasoningEffort)}`); @@ -509,7 +523,7 @@ run .option("--search ", "Free-text search over id, task, model, and worker") .option("--created-after ", "Only runs created at/after this ISO-8601 datetime") .option("--created-before ", "Only runs created at/before this ISO-8601 datetime") - .option("--submission-id ", "Filter by submission ID") + .option("--experiment-id ", "Filter by experiment ID") .option("--turns ", "Filter by actual turns (e.g. '>=5', '<=10', '=3')") .option("--max-iterations ", "Filter by configured maxIterations (e.g. '>=5', '<=10', '=3')") .option("--sort-by ", "Sort field: created, updated, priority, worker, status, id, duration") @@ -550,8 +564,8 @@ run if (options.createdBefore) { params.set("createdBefore", options.createdBefore); } - if (options.submissionId) { - params.set("submissionId", options.submissionId); + if (options.experimentId) { + params.set("experimentId", options.experimentId); } const parseIterExpr = (raw: string, name: string): { op: "eq" | "gte" | "lte"; value: number } => { const m = /^(>=|<=|=)?\s*(\d+)$/.exec(String(raw).trim()); @@ -623,8 +637,8 @@ run return o === 'succeeded' ? successText(s) : o === 'failed' || o === 'finished' ? errorText(s) : value(s); }, }, - { key: 'submissionId', label: 'Submission', - formatter: (req: any) => req.submissionId ? req.submissionId.substring(0, 8) : '–', + { key: 'experimentId', label: 'Experiment', + formatter: (req: Record) => typeof req.experimentId === "string" ? req.experimentId.substring(0, 8) : '–', }, { key: 'createdAt', label: 'Created', formatter: (req: any) => formatDate(req.createdAt), @@ -792,24 +806,24 @@ run .command("download-batch") .description("Download multiple runs at once into a single batch archive") .option("-i, --ids ", "One or more run IDs") - .option("--submission-id ", "Download all runs for a submission") + .option("--experiment-id ", "Download all runs for an experiment") .option("-o, --output ", "Output file path (default: batch-.tar.gz)") .option("-e, --extract", "Extract the archive after downloading") .option("-d, --dir ", "Extraction directory (implies --extract)") .option("-u, --url ", "API base URL", getDefaultApiUrl()) - .option("--project ", "Project ID for scoped operations (required with --submission-id)") + .option("--project ", "Project ID for scoped operations (required with --experiment-id)") .action(async (options) => { const { url } = options; const shouldExtract = options.extract || !!options.dir; try { - // Resolve IDs: either from --ids or by querying --submission-id + // Resolve IDs: either from --ids or by querying --experiment-id let ids: string[] = options.ids || []; - if (options.submissionId) { - console.log(`${label('Fetching runs for submission')} ${value(options.submissionId)}...`); + if (options.experimentId) { + console.log(`${label('Fetching runs for experiment')} ${value(options.experimentId)}...`); const projectId = requireProjectId(options.project); - const listResp = await apiFetch(url, `/requests?submissionId=${encodeURIComponent(options.submissionId)}&limit=1000`, { projectId }); + const listResp = await apiFetch(url, `/requests?experimentId=${encodeURIComponent(options.experimentId)}&limit=1000`, { projectId }); if (!listResp.ok) { const error = await listResp.json(); console.error(errorText("Error fetching runs:"), error); @@ -818,7 +832,7 @@ run const listData = await listResp.json(); const runs = listData.data || listData; if (!Array.isArray(runs) || runs.length === 0) { - console.error(errorText("No runs found for this submission")); + console.error(errorText("No runs found for this experiment")); process.exit(1); } ids = runs.map((r: { _id: string }) => r._id); @@ -826,7 +840,7 @@ run } if (ids.length === 0) { - console.error(errorText("Error: provide --ids or --submission-id")); + console.error(errorText("Error: provide --ids or --experiment-id")); process.exit(1); } diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts index 399f12266..a9278391a 100644 --- a/apps/cli/src/index.ts +++ b/apps/cli/src/index.ts @@ -25,6 +25,7 @@ import { registerTaskPromptCommands } from "./commands/task-prompt.js"; import { registerProfileCommands } from "./commands/profile.js"; import { registerProjectCommands } from "./commands/project.js"; import { registerSecretCommands } from "./commands/secret.js"; +import { registerExperimentCommands } from "./commands/experiment.js"; import { registerUpdateCommand } from "./commands/update.js"; import { checkForUpdates } from "./utils/update-check.js"; import { errorText } from "./utils/style.js"; @@ -70,6 +71,7 @@ configureHelp(program); registerSecretCommands(program); registerProjectCommands(program); registerRunCommands(program); +registerExperimentCommands(program); registerCriteriaCommands(program); registerPromptFeatureCommands(program); registerReportCommands(program); diff --git a/apps/portal/src/App.tsx b/apps/portal/src/App.tsx index 26d5f1d74..195a75400 100644 --- a/apps/portal/src/App.tsx +++ b/apps/portal/src/App.tsx @@ -6,6 +6,8 @@ import { Layout } from "@/components/Layout"; import { RunsList } from "@/pages/RunsList"; import { RunDetail } from "@/pages/RunDetail"; import { SubmitRun } from "@/pages/SubmitRun"; +import { ExperimentDetail } from "@/pages/ExperimentDetail"; +import { ExperimentsList } from "@/pages/ExperimentsList"; import { CriteriaList } from "@/pages/CriteriaList"; import { CriterionDetail } from "@/pages/CriterionDetail"; import { CriterionPreviewPanel } from "@/pages/CriterionPreviewPanel"; @@ -85,6 +87,9 @@ export function App() { } /> } /> + } /> + } /> + } /> }> } /> diff --git a/apps/portal/src/components/ExperimentStatusCard.stories.tsx b/apps/portal/src/components/ExperimentStatusCard.stories.tsx new file mode 100644 index 000000000..9cae899e0 --- /dev/null +++ b/apps/portal/src/components/ExperimentStatusCard.stories.tsx @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { Meta, StoryObj } from "@storybook/react-vite"; +import type { ExperimentView } from "@/lib/experiments"; +import { ExperimentStatusCard } from "./ExperimentStatusCard"; + +const experiment: ExperimentView = { + experimentId: "experiment-123", + projectId: "project-123", + metadata: { name: "Profile comparison" }, + revision: { + experimentRevisionId: "revision-1", + revisionNumber: 1, + memberCount: 2, + }, + createdAt: "2026-10-15T00:00:00.000Z", + createdRequestIds: ["request-1", "request-2"], + readiness: "processing", + execution: { + experimentExecutionId: "execution-1", + attemptNumber: 2, + requestCount: 2, + createdAt: "2026-10-15T00:00:00.000Z", + requests: [], + handlerProgress: { + observations: { status: "done" }, + report: { status: "processing" }, + }, + blocking: [], + }, + blocking: [], +}; + +const meta = { + title: "Experiments/ExperimentStatusCard", + component: ExperimentStatusCard, + args: { experiment }, +} satisfies Meta; + +export default meta; +type Story = StoryObj; + +export const Processing: Story = {}; + +export const Blocked: Story = { + args: { + experiment: { + ...experiment, + readiness: "blocked", + blocking: [ + { + handlerId: "observations", + reason: "dependency-failed", + }, + ], + }, + }, +}; diff --git a/apps/portal/src/components/ExperimentStatusCard.test.tsx b/apps/portal/src/components/ExperimentStatusCard.test.tsx new file mode 100644 index 000000000..e01e90b3c --- /dev/null +++ b/apps/portal/src/components/ExperimentStatusCard.test.tsx @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// @vitest-environment happy-dom +import { cleanup, render, screen } from "@testing-library/react"; +import { afterEach, describe, expect, it } from "vitest"; +import type { ExperimentView } from "@/lib/experiments"; +import { ExperimentStatusCard } from "./ExperimentStatusCard"; + +afterEach(cleanup); + +describe("ExperimentStatusCard", () => { + it("renders readiness and exact handler progress", () => { + const experiment: ExperimentView = { + experimentId: "experiment-1", + projectId: "project-1", + metadata: { name: "Compiler comparison" }, + revision: { + experimentRevisionId: "revision-1", + revisionNumber: 1, + memberCount: 2, + }, + createdAt: "2026-10-15T00:00:00.000Z", + createdRequestIds: ["request-1", "request-2"], + readiness: "blocked", + execution: { + experimentExecutionId: "execution-1", + attemptNumber: 3, + requestCount: 2, + createdAt: "2026-10-15T00:00:00.000Z", + requests: [], + handlerProgress: { + observations: { status: "done" }, + report: { status: "processing" }, + }, + blocking: [], + }, + blocking: [ + { + handlerId: "observations", + reason: "dependency-failed", + }, + ], + }; + + render(); + + expect(screen.getByText("blocked")).toBeDefined(); + expect(screen.getByText("3")).toBeDefined(); + expect(screen.getByText("1/2")).toBeDefined(); + }); +}); diff --git a/apps/portal/src/components/ExperimentStatusCard.tsx b/apps/portal/src/components/ExperimentStatusCard.tsx new file mode 100644 index 000000000..5128838cc --- /dev/null +++ b/apps/portal/src/components/ExperimentStatusCard.tsx @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { Badge } from "@/components/ui/badge"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import type { ExperimentView } from "@/lib/experiments"; + +export function ExperimentStatusCard({ + experiment, +}: { + experiment: ExperimentView; +}) { + const readiness = experiment.readiness ?? "unknown"; + const variant = + readiness === "failed" || readiness === "blocked" + ? "destructive" + : readiness === "processing" || readiness === "done" + ? "default" + : readiness === "ready" + ? "outline" + : "secondary"; + const handlers = Object.values(experiment.execution.handlerProgress); + const completedHandlers = handlers.filter( + (handler) => + handler && + typeof handler === "object" && + "status" in handler && + handler.status === "done", + ).length; + return ( + + + Processing + {readiness} + + +
+
Attempt
+
+ {experiment.execution.attemptNumber} +
+
+
+
Handlers complete
+
+ {completedHandlers}/{handlers.length} +
+
+
+
Blocking runs
+
{experiment.blocking.length}
+
+ + + ); +} diff --git a/apps/portal/src/components/Layout.stories.tsx b/apps/portal/src/components/Layout.stories.tsx index fe59b9f18..e06bfafa8 100644 --- a/apps/portal/src/components/Layout.stories.tsx +++ b/apps/portal/src/components/Layout.stories.tsx @@ -51,6 +51,14 @@ export const Expanded: Story = { await expect(canvas.getByRole("button", { name: "Collapse sidebar" })).toBeVisible(); // With a project selected, scoped nav is present. await expect(canvas.getByRole("link", { name: "Runs" })).toBeVisible(); + const experimentsLink = canvas.getByRole("link", { name: "Experiments" }); + const runsLink = canvas.getByRole("link", { name: "Runs" }); + const criteriaLink = canvas.getByRole("link", { name: "Criteria" }); + await expect(experimentsLink.querySelector(".lucide-atom")).not.toBeNull(); + await expect( + criteriaLink.querySelector(".lucide-flask-conical"), + ).not.toBeNull(); + await expect(experimentsLink.nextElementSibling).toBe(runsLink); }, }; diff --git a/apps/portal/src/components/Layout.test.tsx b/apps/portal/src/components/Layout.test.tsx index 755a9cdf9..fb768d7b5 100644 --- a/apps/portal/src/components/Layout.test.tsx +++ b/apps/portal/src/components/Layout.test.tsx @@ -49,6 +49,18 @@ function renderLayout( }> Home page} /> + Experiments page} + /> + New experiment page} + /> + Experiment detail page} + /> Runs page} /> @@ -99,6 +111,16 @@ describe("Layout", () => { expect(screen.getByRole("main").className).not.toContain("px-6"); }); + it.each(["/experiments/new", "/experiments/experiment-1"])( + "keeps the conventional padded shell for %s", + (route) => { + renderLayout(route); + + expect(screen.getByRole("main").className).toContain("px-6"); + expect(screen.getByRole("main").className).toContain("py-6"); + }, + ); + it.each(["/secrets/keys/new", "/secrets/keys/demo-key"])( "preserves page padding for %s", (path) => { @@ -120,6 +142,17 @@ describe("Layout", () => { ).toBeTruthy(); }); + it("uses the Runs full-bleed shell for Experiments", () => { + const runs = renderLayout("/runs"); + const runsClasses = screen.getByRole("main").className; + runs.unmount(); + + renderLayout("/experiments"); + + expect(screen.getByRole("main").className).toBe(runsClasses); + expect(screen.getByRole("main").className).not.toContain("px-6"); + }); + it("expands the desktop sidebar to show navigation labels", () => { renderLayout(); @@ -168,6 +201,17 @@ describe("Layout", () => { expect(within(sidebar).getByRole("link", { name: "New Run" })).toBeTruthy(); expect(within(sidebar).getByText("Activity")).toBeTruthy(); expect(within(sidebar).getByRole("link", { name: "Runs" })).toBeTruthy(); + const experimentsLink = within(sidebar).getByRole("link", { + name: "Experiments", + }); + const criteriaLink = within(sidebar).getByRole("link", { + name: "Criteria", + }); + expect(experimentsLink.querySelector(".lucide-atom")).toBeTruthy(); + expect(criteriaLink.querySelector(".lucide-flask-conical")).toBeTruthy(); + expect(experimentsLink.nextElementSibling).toBe( + within(sidebar).getByRole("link", { name: "Runs" }), + ); expect(within(sidebar).getByRole("link", { name: "MCP" })).toBeTruthy(); // The Integrations group holds Resources alongside MCP and Extensions; the // group and its Resources item deliberately no longer share a label. diff --git a/apps/portal/src/components/Layout.tsx b/apps/portal/src/components/Layout.tsx index 2a08d36cd..ca6f47dc8 100644 --- a/apps/portal/src/components/Layout.tsx +++ b/apps/portal/src/components/Layout.tsx @@ -6,6 +6,7 @@ import { Link, useLocation, Outlet, matchPath } from "react-router-dom"; import { cn } from "@/lib/utils"; import { Activity, + Atom, Plus, List, FlaskConical, @@ -94,6 +95,7 @@ const navGroups: NavGroup[] = [ label: "Activity", scoped: true, items: [ + { to: "/experiments", label: "Experiments", icon: Atom }, { to: "/runs", label: "Runs", icon: List }, { to: "/statistics", label: "Statistics", icon: BarChart3 }, { to: "/reports", label: "Reports", icon: FileText }, @@ -155,6 +157,7 @@ const FULL_BLEED_ROUTE_PATTERNS = [ "/agents/:id", "/models", "/models/:id", + "/experiments", "/runs", "/runs/:id/preview", "/task-prompts", diff --git a/apps/portal/src/components/list-layout/EmbeddedTableBlock.tsx b/apps/portal/src/components/list-layout/EmbeddedTableBlock.tsx new file mode 100644 index 000000000..3f4c09022 --- /dev/null +++ b/apps/portal/src/components/list-layout/EmbeddedTableBlock.tsx @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { ReactNode } from "react"; +import { cn } from "@/lib/utils"; + +export interface EmbeddedTableBlockProps { + title: ReactNode; + description?: ReactNode; + actions?: ReactNode; + children: ReactNode; + className?: string; +} + +export function EmbeddedTableBlock({ + title, + description, + actions, + children, + className, +}: EmbeddedTableBlockProps) { + return ( +
+
+
+

{title}

+ {description && ( +

+ {description} +

+ )} +
+ {actions} +
+
{children}
+
+ ); +} diff --git a/apps/portal/src/components/list-layout/index.ts b/apps/portal/src/components/list-layout/index.ts index b83fd5492..6cfb44d41 100644 --- a/apps/portal/src/components/list-layout/index.ts +++ b/apps/portal/src/components/list-layout/index.ts @@ -11,6 +11,8 @@ export { CheckboxFilterGroup } from "./CheckboxFilterGroup"; export type { CheckboxFilterGroupProps, CheckboxFilterOption } from "./CheckboxFilterGroup"; export { DataTable } from "./DataTable"; export type { DataTableProps, DataTableColumn, DataTableSelection } from "./DataTable"; +export { EmbeddedTableBlock } from "./EmbeddedTableBlock"; +export type { EmbeddedTableBlockProps } from "./EmbeddedTableBlock"; export { Pagination } from "./Pagination"; export type { PaginationProps } from "./Pagination"; export { DetailPanel } from "./DetailPanel"; diff --git a/apps/portal/src/lib/api.ts b/apps/portal/src/lib/api.ts index 32e1d4a7d..b8787adf1 100644 --- a/apps/portal/src/lib/api.ts +++ b/apps/portal/src/lib/api.ts @@ -9,6 +9,17 @@ import { recordServerDate } from "./serverClock"; import { apiClient } from "./api-client"; import { getSelectedProjectId, ProjectRequiredError } from "./project-scope"; import { MAX_ARCHIVE_UPLOAD_LABEL } from "./codebaseUpload"; +import { + parseExperiment, + parseExperimentExecution, + parseExperimentExecutionPage, + parseExperimentPage, + type ExperimentExecutionPageView, + type ExperimentExecutionView, + type ExperimentMetadataView, + type ExperimentPageView, + type ExperimentView, +} from "./experiments"; const BASE = "/api/v1"; @@ -85,7 +96,7 @@ export interface RunFilterParams { profileId?: string | string[]; taskPromptId?: string; criteria?: string; - submissionId?: string; + experimentId?: string; search?: string; createdAfter?: string; createdBefore?: string; @@ -108,7 +119,7 @@ function runFilterQs(f: RunFilterParams): Record => @@ -178,8 +219,8 @@ export const api = { })}`, undefined, { scoped: true }); }, - /** List runs grouped by task/profile/submissionId, with cursor pagination and server-side filtering */ - listRunGroups: (opts: RunFilterParams & { groupBy: "task" | "submissionId" | "profile"; sortBy?: RunSortField; sortDir?: RunSortDir; limit?: number; after?: string; before?: string; last?: boolean }): Promise> => { + /** List runs grouped by task/profile/experimentId, with cursor pagination and server-side filtering */ + listRunGroups: (opts: RunFilterParams & { groupBy: "task" | "experimentId" | "profile"; sortBy?: RunSortField; sortDir?: RunSortDir; limit?: number; after?: string; before?: string; last?: boolean }): Promise> => { return request(`/requests${qs({ groupBy: opts.groupBy, ...runFilterQs(opts), @@ -209,29 +250,19 @@ export const api = { }, /** Submit a new run (or multiple runs if count > 1) */ - submitRun: (body: { - scenario: { task: string; criteria: string[]; version?: "v1" | "v2" }; - observations?: string[]; - worker?: string; - model?: string; - reasoningEffort?: string; - maxIterations?: number; - personaInstructions?: string; - persona?: { personality: string; experience: string; verbosity: string; type: string }; - count?: number; - mcpServers?: string[]; - skills?: string[]; - extensions?: string[]; - resources?: ResourceBindingSpec[]; - agentVersion?: string; - profileId?: string; - profileVariations?: string[]; - agentsMd?: string; - agentsMdParentIds?: string[]; - gates?: GateConfig[]; - codebase?: string; - codebaseRevisionId?: string; - }): Promise<(Run & { message: string }) | { ids: string[]; count: number; message: string }> => { + submitRun: (body: RunCreateInput): Promise< + (Run & { + message: string; + experimentId?: string; + submissionId?: string; + }) | { + ids: string[]; + count: number; + message: string; + experimentId?: string; + submissionId?: string; + } + > => { const { worker, ...payload } = body; const url = worker ? `/requests?worker=${encodeURIComponent(worker)}` : `/requests`; return request(url, { @@ -240,6 +271,124 @@ export const api = { }, { scoped: true }); }, + createExperiment: async ( + body: { metadata: ExperimentMetadataView } & ( + | { request: RunCreateInput; empty?: never } + | { empty: true; request?: never } + ), + ): Promise => { + const composedRequest = "request" in body ? body.request : undefined; + const worker = composedRequest?.worker; + const requestBody = composedRequest + ? { ...composedRequest, worker: undefined } + : undefined; + const path = worker + ? `/experiments?worker=${encodeURIComponent(worker)}` + : "/experiments"; + const value = await request( + path, + { + method: "POST", + body: JSON.stringify({ + mode: "empty" in body ? "empty" : "compose", + metadata: body.metadata, + requests: requestBody ? [requestBody] : [], + }), + }, + { scoped: true }, + ); + if (!value || typeof value !== "object" || !("experiment" in value)) { + throw new Error("Invalid experiment create response from Scope"); + } + return parseExperiment(value.experiment); + }, + + listExperiments: async (opts?: { + cursor?: string; + limit?: number; + }): Promise => { + const value = await request( + `/experiments${qs({ + after: opts?.cursor, + limit: opts?.limit ? String(opts.limit) : undefined, + })}`, + undefined, + { scoped: true }, + ); + return parseExperimentPage(value); + }, + + getExperiment: async (id: string): Promise => { + const value = await request( + `/experiments/${encodeURIComponent(id)}`, + undefined, + { scoped: true }, + ); + return parseExperiment(value); + }, + + listExperimentExecutions: async ( + id: string, + opts?: { cursor?: string; limit?: number }, + ): Promise => { + const value = await request( + `/experiments/${encodeURIComponent(id)}/executions${qs({ + after: opts?.cursor, + limit: opts?.limit ? String(opts.limit) : undefined, + })}`, + undefined, + { scoped: true }, + ); + return parseExperimentExecutionPage(value); + }, + + getExperimentExecution: async ( + id: string, + experimentExecutionId: string, + ): Promise => { + const value = await request( + `/experiments/${encodeURIComponent(id)}/executions/${encodeURIComponent(experimentExecutionId)}`, + undefined, + { scoped: true }, + ); + return parseExperimentExecution(value); + }, + + retryExperiment: async ( + id: string, + experimentExecutionId: string, + ): Promise<{ + experimentId: string; + experimentExecutionId: string; + attemptNumber: number; + }> => { + const value = await request( + `/experiments/${encodeURIComponent(id)}/retry`, + { + method: "POST", + body: JSON.stringify({ experimentExecutionId }), + }, + { scoped: true }, + ); + if ( + !value || + typeof value !== "object" || + !("experimentId" in value) || + typeof value.experimentId !== "string" || + !("experimentExecutionId" in value) || + typeof value.experimentExecutionId !== "string" || + !("attemptNumber" in value) || + typeof value.attemptNumber !== "number" + ) { + throw new Error("Invalid experiment retry response from Scope"); + } + return { + experimentId: value.experimentId, + experimentExecutionId: value.experimentExecutionId, + attemptNumber: value.attemptNumber, + }; + }, + /** Soft-delete a run */ deleteRun: (id: string): Promise<{ id: string; deleted: boolean }> => { return request(`/requests/${id}`, { method: "DELETE" }); diff --git a/apps/portal/src/lib/enrichment.test.ts b/apps/portal/src/lib/enrichment.test.ts index 879bc5b11..9e4b19024 100644 --- a/apps/portal/src/lib/enrichment.test.ts +++ b/apps/portal/src/lib/enrichment.test.ts @@ -33,59 +33,59 @@ describe("deriveEnrichmentStatus", () => { expect(deriveEnrichmentStatus(run, false)).toBe("done"); }); - it("does NOT flip to done after pp-atif while pp-taxonomy is still pending (the bug)", () => { - // Observations configured, pp-atif done, pp-taxonomy not yet dispatched. + it("does NOT flip to done after pp-atif while pp-observations is still pending (the bug)", () => { + // Observations configured, pp-atif done, pp-observations not yet dispatched. const beforeDispatch = makeRun({ postProcessorStatus: "done", // legacy scalar already flipped — must be ignored handlerStatus: { "pp-atif": { status: "done" } }, }); expect(deriveEnrichmentStatus(beforeDispatch, true)).toBe("processing"); - // pp-taxonomy queued. + // pp-observations queued. const queued = makeRun({ postProcessorStatus: "done", - handlerStatus: { "pp-atif": { status: "done" }, "pp-taxonomy": { status: "queued" } }, + handlerStatus: { "pp-atif": { status: "done" }, "pp-observations": { status: "queued" } }, }); expect(deriveEnrichmentStatus(queued, true)).toBe("processing"); - // pp-taxonomy processing. + // pp-observations processing. const processing = makeRun({ postProcessorStatus: "done", - handlerStatus: { "pp-atif": { status: "done" }, "pp-taxonomy": { status: "processing" } }, + handlerStatus: { "pp-atif": { status: "done" }, "pp-observations": { status: "processing" } }, }); expect(deriveEnrichmentStatus(processing, true)).toBe("processing"); }); - it("flips to done only once both pp-atif and pp-taxonomy are done", () => { + it("flips to done only once both pp-atif and pp-observations are done", () => { const run = makeRun({ - handlerStatus: { "pp-atif": { status: "done" }, "pp-taxonomy": { status: "done" } }, + handlerStatus: { "pp-atif": { status: "done" }, "pp-observations": { status: "done" } }, }); expect(deriveEnrichmentStatus(run, true)).toBe("done"); }); it("reports failed when a handler failed and nothing is still in flight", () => { const run = makeRun({ - handlerStatus: { "pp-atif": { status: "done" }, "pp-taxonomy": { status: "failed" } }, + handlerStatus: { "pp-atif": { status: "done" }, "pp-observations": { status: "failed" } }, }); expect(deriveEnrichmentStatus(run, true)).toBe("failed"); }); it("prefers processing over failed while another handler is still running", () => { const run = makeRun({ - handlerStatus: { "pp-atif": { status: "processing" }, "pp-taxonomy": { status: "failed" } }, + handlerStatus: { "pp-atif": { status: "processing" }, "pp-observations": { status: "failed" } }, }); expect(deriveEnrichmentStatus(run, true)).toBe("processing"); }); - it("treats a no-op pp-taxonomy (no observations) as part of the DAG", () => { - // No observations configured, but pp-taxonomy still runs as a no-op handler. + it("treats a no-op pp-observations (no observations) as part of the DAG", () => { + // No observations configured, but pp-observations still runs as a no-op handler. const stillRunning = makeRun({ - handlerStatus: { "pp-atif": { status: "done" }, "pp-taxonomy": { status: "processing" } }, + handlerStatus: { "pp-atif": { status: "done" }, "pp-observations": { status: "processing" } }, }); expect(deriveEnrichmentStatus(stillRunning, false)).toBe("processing"); const allDone = makeRun({ - handlerStatus: { "pp-atif": { status: "done" }, "pp-taxonomy": { status: "done" } }, + handlerStatus: { "pp-atif": { status: "done" }, "pp-observations": { status: "done" } }, }); expect(deriveEnrichmentStatus(allDone, false)).toBe("done"); }); diff --git a/apps/portal/src/lib/enrichment.ts b/apps/portal/src/lib/enrichment.ts index 99f5bfebc..756a510d9 100644 --- a/apps/portal/src/lib/enrichment.ts +++ b/apps/portal/src/lib/enrichment.ts @@ -12,14 +12,14 @@ export type PostProcessorStatus = "queued" | "processing" | "done" | "failed"; * * Each post-processor handler overwrites that scalar on its own completion, so * it flips to "done" as soon as the FIRST handler (pp-atif) finishes — even - * while pp-taxonomy is still computing observations. The "Enriched" badge (and + * while pp-observations is still computing observations. The "Enriched" badge (and * the run-detail poller) must instead stay "Enriching…" until every handler in * the DAG is terminal. * - * Known topology: pp-atif always runs; pp-taxonomy is expected whenever the run + * Known topology: pp-atif always runs; pp-observations is expected whenever the run * has configured observations. We treat taxonomy as still-pending until - * pp-taxonomy reaches a terminal state, which also covers the brief window - * after pp-atif finishes but before the scheduler has dispatched pp-taxonomy + * pp-observations reaches a terminal state, which also covers the brief window + * after pp-atif finishes but before the scheduler has dispatched pp-observations * (so it isn't yet present in `handlerStatus`). * * Falls back to the legacy `postProcessorStatus` scalar for runs that predate @@ -36,7 +36,7 @@ export function deriveEnrichmentStatus( return run.postProcessorStatus; } const statuses = Object.values(handlerStatus).map((h) => h?.status); - const taxonomyStatus = handlerStatus["pp-taxonomy"]?.status; + const taxonomyStatus = handlerStatus["pp-observations"]?.status; const taxonomyPending = hasConfiguredObservations && taxonomyStatus !== "done" && taxonomyStatus !== "failed"; if (taxonomyPending || statuses.some((s) => s === "queued" || s === "processing")) { diff --git a/apps/portal/src/lib/experiments.ts b/apps/portal/src/lib/experiments.ts new file mode 100644 index 000000000..e5c8be80d --- /dev/null +++ b/apps/portal/src/lib/experiments.ts @@ -0,0 +1,319 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +export const LEGACY_REQUEST_COMPOSITION_DEPRECATION = + "legacy-request-composition" as const; + +export interface ExperimentMetadataView { + name: string; + description?: string; + createdBy?: string; +} + +export interface ExperimentRevisionView { + experimentRevisionId: string; + revisionNumber: number; + memberCount: number; + membershipFingerprint?: string; + definitionFingerprint?: string; + createdAt?: string; +} + +export interface ExperimentBlockerView { + handlerId: string; + reason: string; + requestId?: string; + runId?: string; + error?: string; +} + +export interface ExperimentRequestSummaryView { + requestId: string; + runId: string; + task?: string; + status?: string; + outcome?: string; + workerType?: string; + model?: string; + profileId?: string; +} + +export interface ExperimentExecutionView { + experimentExecutionId: string; + attemptNumber: number; + version?: number; + readiness?: string; + requestCount: number; + requests: ExperimentRequestSummaryView[]; + handlerProgress: Record; + blocking: ExperimentBlockerView[]; + currentReport?: { reportId?: string; url?: string; updatedAt?: string }; + createdAt: string; + updatedAt?: string; + startedAt?: string; + finishedAt?: string; + discardedAt?: string; + discardReason?: string; + supersededAt?: string; + supersededByExperimentExecutionId?: string; + supersededReason?: "rollover" | "retry"; + archivedAt?: string; +} + +export interface ExperimentView { + experimentId: string; + projectId: string; + metadata: ExperimentMetadataView; + revision: ExperimentRevisionView; + execution: ExperimentExecutionView; + readiness?: string; + blocking: ExperimentBlockerView[]; + createdRequestIds: string[]; + createdAt: string; + updatedAt?: string; + finalizedAt?: string; +} + +export interface ExperimentPageView { + items: ExperimentView[]; + nextCursor?: string; +} + +export interface ExperimentExecutionPageView { + items: ExperimentExecutionView[]; + nextCursor?: string; +} + +function record(value: unknown, name: string): Record { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`Invalid ${name} response from Scope`); + } + return value as Record; +} + +function requiredString( + value: Record, + key: string, + name: string, +): string { + const candidate = value[key]; + if (typeof candidate !== "string" || candidate.length === 0) { + throw new Error(`Invalid ${name}.${key} response from Scope`); + } + return candidate; +} + +function optionalString( + value: Record, + key: string, +): string | undefined { + const candidate = value[key]; + if (candidate === undefined || candidate === null) return undefined; + if (typeof candidate !== "string") { + throw new Error(`Invalid ${key} response from Scope`); + } + return candidate; +} + +function requiredNumber( + value: Record, + key: string, + name: string, +): number { + const candidate = value[key]; + if (typeof candidate !== "number" || !Number.isFinite(candidate)) { + throw new Error(`Invalid ${name}.${key} response from Scope`); + } + return candidate; +} + +function parseBlocker(value: unknown): ExperimentBlockerView { + const item = record(value, "experiment blocker"); + return { + handlerId: requiredString(item, "handlerId", "experiment blocker"), + reason: requiredString(item, "reason", "experiment blocker"), + requestId: optionalString(item, "requestId"), + runId: optionalString(item, "runId"), + error: optionalString(item, "error"), + }; +} + +function parseRequestSummary(value: unknown): ExperimentRequestSummaryView { + const item = record(value, "experiment request"); + return { + requestId: requiredString(item, "requestId", "experiment request"), + runId: requiredString(item, "runId", "experiment request"), + task: optionalString(item, "task"), + status: optionalString(item, "status"), + outcome: optionalString(item, "outcome"), + workerType: optionalString(item, "workerType"), + model: optionalString(item, "model"), + profileId: optionalString(item, "profileId"), + }; +} + +function parseRequestVector( + value: unknown, +): { requestCount: number; requests: ExperimentRequestSummaryView[] } { + if (Array.isArray(value)) { + const requests = value.map(parseRequestSummary); + return { requestCount: requests.length, requests }; + } + const payload = record(value, "experiment request vector"); + const entries = Array.isArray(payload.entries) + ? payload.entries.map(parseRequestSummary) + : []; + const entryCount = + typeof payload.entryCount === "number" ? payload.entryCount : entries.length; + return { requestCount: entryCount, requests: entries }; +} + +export function parseExperimentExecution( + value: unknown, +): ExperimentExecutionView { + const execution = record(value, "experiment execution"); + const vector = parseRequestVector(execution.requests ?? []); + const blockingValue = execution.blockers ?? execution.blocking; + const blocking = Array.isArray(blockingValue) + ? blockingValue.map(parseBlocker) + : []; + const progressValue = + execution.progress ?? + execution.handlerProgress ?? + execution.handlerSummary ?? + {}; + const handlerProgress = record(progressValue, "experiment handler progress"); + const reportValue = execution.currentReport; + const currentReport = + reportValue === undefined + ? undefined + : (() => { + const report = record(reportValue, "experiment report"); + return { + reportId: optionalString(report, "reportId"), + url: optionalString(report, "url"), + updatedAt: optionalString(report, "updatedAt"), + }; + })(); + + return { + experimentExecutionId: requiredString( + execution, + "experimentExecutionId", + "experiment execution", + ), + attemptNumber: requiredNumber( + execution, + "attemptNumber", + "experiment execution", + ), + version: + typeof execution.version === "number" ? execution.version : undefined, + readiness: optionalString(execution, "readiness"), + requestCount: vector.requestCount, + requests: vector.requests, + handlerProgress, + blocking, + currentReport, + createdAt: requiredString(execution, "createdAt", "experiment execution"), + updatedAt: optionalString(execution, "updatedAt"), + startedAt: optionalString(execution, "startedAt"), + finishedAt: optionalString(execution, "finishedAt"), + discardedAt: optionalString(execution, "discardedAt"), + discardReason: optionalString(execution, "discardReason"), + supersededAt: optionalString(execution, "supersededAt"), + supersededByExperimentExecutionId: optionalString( + execution, + "supersededByExperimentExecutionId", + ), + supersededReason: + execution.supersededReason === "rollover" || + execution.supersededReason === "retry" + ? execution.supersededReason + : undefined, + archivedAt: optionalString(execution, "archivedAt"), + }; +} + +export function parseExperiment(value: unknown): ExperimentView { + const experiment = record(value, "experiment"); + const metadata = record(experiment.metadata, "experiment metadata"); + const revision = record(experiment.revision, "experiment revision"); + const executionValue = experiment.execution ?? experiment.currentExecution; + const execution = parseExperimentExecution(executionValue); + const blockingValue = experiment.blocking ?? execution.blocking; + const blocking = Array.isArray(blockingValue) + ? blockingValue.map(parseBlocker) + : []; + const createdRequestIds = Array.isArray(experiment.createdRequestIds) + ? experiment.createdRequestIds.map((id) => { + if (typeof id !== "string") { + throw new Error("Invalid experiment.createdRequestIds response from Scope"); + } + return id; + }) + : []; + + return { + experimentId: requiredString(experiment, "experimentId", "experiment"), + projectId: requiredString(experiment, "projectId", "experiment"), + metadata: { + name: requiredString(metadata, "name", "experiment metadata"), + description: optionalString(metadata, "description"), + createdBy: optionalString(metadata, "createdBy"), + }, + revision: { + experimentRevisionId: requiredString( + revision, + "experimentRevisionId", + "experiment revision", + ), + revisionNumber: requiredNumber( + revision, + "revisionNumber", + "experiment revision", + ), + memberCount: + typeof revision.memberCount === "number" + ? revision.memberCount + : execution.requestCount, + membershipFingerprint: optionalString(revision, "membershipFingerprint"), + definitionFingerprint: optionalString(revision, "definitionFingerprint"), + createdAt: optionalString(revision, "createdAt"), + }, + execution, + readiness: + optionalString(experiment, "readiness") ?? execution.readiness, + blocking, + createdRequestIds, + createdAt: requiredString(experiment, "createdAt", "experiment"), + updatedAt: optionalString(experiment, "updatedAt"), + finalizedAt: optionalString(experiment, "finalizedAt"), + }; +} + +export function parseExperimentPage(value: unknown): ExperimentPageView { + const page = record(value, "experiment page"); + const items = page.data ?? page.items; + if (!Array.isArray(items)) { + throw new Error("Invalid experiment page items response from Scope"); + } + return { + items: items.map(parseExperiment), + nextCursor: optionalString(page, "nextCursor"), + }; +} + +export function parseExperimentExecutionPage( + value: unknown, +): ExperimentExecutionPageView { + const page = record(value, "experiment execution page"); + const items = page.data ?? page.items; + if (!Array.isArray(items)) { + throw new Error("Invalid experiment execution page items response from Scope"); + } + return { + items: items.map(parseExperimentExecution), + nextCursor: optionalString(page, "nextCursor"), + }; +} diff --git a/apps/portal/src/pages/CriteriaList.tsx b/apps/portal/src/pages/CriteriaList.tsx index c5133763e..ce2da1c24 100644 --- a/apps/portal/src/pages/CriteriaList.tsx +++ b/apps/portal/src/pages/CriteriaList.tsx @@ -35,7 +35,11 @@ import { type CustomizeColumnsOption, } from "@/components/list-layout"; import { HelpTooltip } from "@/components/HelpTooltip"; -import { CriteriaKindBadge, CriteriaSubjectBadge } from "@/components/CriteriaBadge"; +import { + CriteriaBadge, + CriteriaKindBadge, + CriteriaSubjectBadge, +} from "@/components/CriteriaBadge"; import { TAXONOMY_ELEMENT_METADATA } from "@/types"; const FILTER_KEYS = ["gate", "kind"] as const; diff --git a/apps/portal/src/pages/ExperimentDetail.play.test.tsx b/apps/portal/src/pages/ExperimentDetail.play.test.tsx new file mode 100644 index 000000000..ebda77fc6 --- /dev/null +++ b/apps/portal/src/pages/ExperimentDetail.play.test.tsx @@ -0,0 +1,130 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// @vitest-environment happy-dom +import { render, within } from "@testing-library/react"; +import { beforeAll, describe, it, vi } from "vitest"; +import type { + ExperimentExecutionView, + ExperimentView, +} from "@/lib/experiments"; + +vi.mock("@/lib/api", async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + api: { + ...original.api, + getExperiment: vi.fn(), + listExperimentExecutions: vi.fn(), + getExperimentExecution: vi.fn(), + retryExperiment: vi.fn(), + }, + }; +}); + +import { api } from "@/lib/api"; +import experimentDetailMeta, { + AttemptSelectionAndRunsTable, +} from "./ExperimentDetail.stories"; + +class ResizeObserverMock { + observe = vi.fn(); + unobserve = vi.fn(); + disconnect = vi.fn(); +} + +class IntersectionObserverMock { + observe = vi.fn(); + unobserve = vi.fn(); + disconnect = vi.fn(); +} + +beforeAll(() => { + Object.defineProperty(globalThis, "ResizeObserver", { + configurable: true, + value: ResizeObserverMock, + }); + Object.defineProperty(globalThis, "IntersectionObserver", { + configurable: true, + value: IntersectionObserverMock, + }); + + const currentExecution: ExperimentExecutionView = { + experimentExecutionId: "execution-current", + attemptNumber: 2, + readiness: "done", + requestCount: 1, + requests: [ + { + requestId: "request-current", + runId: "run-current", + task: "Implement the current benchmark task", + status: "done", + outcome: "success", + }, + ], + handlerProgress: {}, + blocking: [], + createdAt: "2026-10-16T10:00:00.000Z", + }; + const historicalExecution: ExperimentExecutionView = { + experimentExecutionId: "execution-history", + attemptNumber: 1, + readiness: "failed", + requestCount: 1, + requests: [ + { + requestId: "request-history", + runId: "run-history", + task: "Inspect the historical benchmark task", + status: "done", + outcome: "failure", + }, + ], + handlerProgress: {}, + blocking: [], + createdAt: "2026-10-15T10:00:00.000Z", + }; + const experiment: ExperimentView = { + experimentId: "experiment-demo", + projectId: "demo-project", + metadata: { name: "Profile comparison" }, + revision: { + experimentRevisionId: "revision-demo", + revisionNumber: 1, + memberCount: 1, + }, + execution: currentExecution, + readiness: "done", + blocking: [], + createdRequestIds: ["request-current"], + createdAt: "2026-10-16T09:00:00.000Z", + }; + + vi.mocked(api.getExperiment).mockResolvedValue(experiment); + vi.mocked(api.listExperimentExecutions).mockResolvedValue({ + items: [currentExecution, historicalExecution], + }); + vi.mocked(api.getExperimentExecution).mockResolvedValue( + historicalExecution, + ); +}); + +describe("ExperimentDetail stories", () => { + it("runs the attempt-selection interaction", async () => { + const StoryRender = experimentDetailMeta.render; + if (!StoryRender) throw new Error("Experiment detail story is missing render"); + const { container } = render(); + const play = AttemptSelectionAndRunsTable.play as unknown as + | ((context: { + canvas: ReturnType; + canvasElement: HTMLElement; + }) => Promise) + | undefined; + await play?.({ + canvas: within(container), + canvasElement: container, + }); + }); +}); diff --git a/apps/portal/src/pages/ExperimentDetail.stories.tsx b/apps/portal/src/pages/ExperimentDetail.stories.tsx new file mode 100644 index 000000000..35330ccff --- /dev/null +++ b/apps/portal/src/pages/ExperimentDetail.stories.tsx @@ -0,0 +1,161 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { Meta, StoryObj } from "@storybook/react-vite"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { expect, screen, userEvent } from "storybook/test"; +import { HttpResponse, http } from "msw"; +import { MemoryRouter, Navigate, Route, Routes } from "react-router-dom"; +import { ExperimentDetail } from "./ExperimentDetail"; + +const currentExecution = { + experimentExecutionId: "execution-current", + attemptNumber: 2, + version: 1, + readiness: "done", + requests: [ + { + requestId: "request-current", + runId: "run-current", + task: "Implement the current benchmark task", + status: "done", + outcome: "success", + workerType: "coder-acp-copilot", + model: "gpt-5.6-sol", + }, + ], + handlerProgress: {}, + blockers: [], + createdAt: "2026-10-16T10:00:00.000Z", +}; + +const historicalExecution = { + experimentExecutionId: "execution-history", + attemptNumber: 1, + version: 1, + readiness: "failed", + requests: [ + { + requestId: "request-history", + runId: "run-history", + task: "Inspect the historical benchmark task", + status: "done", + outcome: "failure", + workerType: "coder-acp-claude-code", + model: "claude-opus-4.6", + }, + ], + handlerProgress: {}, + blockers: [], + createdAt: "2026-10-15T10:00:00.000Z", + supersededReason: "retry", +}; + +const experiment = { + experimentId: "experiment-demo", + projectId: "demo-project", + metadata: { + name: "Profile comparison", + description: "Compare two agent profiles over the same benchmark task.", + }, + revision: { + experimentRevisionId: "revision-demo", + revisionNumber: 1, + memberCount: 1, + }, + execution: currentExecution, + readiness: "done", + blocking: [], + createdRequestIds: ["request-current"], + createdAt: "2026-10-16T09:00:00.000Z", +}; + +function ExperimentDetailStory() { + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); + return ( + + + + } + /> + } /> + + + + ); +} + +const meta = { + component: ExperimentDetail, + parameters: { + layout: "fullscreen", + msw: { + handlers: [ + http.get("/api/v1/experiments/experiment-demo", () => + HttpResponse.json(experiment), + ), + http.get( + "/api/v1/experiments/experiment-demo/executions", + () => + HttpResponse.json({ + data: [currentExecution, historicalExecution], + }), + ), + http.get( + "/api/v1/experiments/experiment-demo/executions/execution-history", + () => HttpResponse.json(historicalExecution), + ), + ], + }, + }, + render: () => , +} satisfies Meta; + +export default meta; +type Story = StoryObj; + +export const AttemptSelectionAndRunsTable: Story = { + play: async ({ canvas, canvasElement }) => { + await expect( + await screen.findByText("Runs in attempt 2"), + ).toBeVisible(); + await expect( + screen.getAllByText("Implement the current benchmark task")[0], + ).toBeVisible(); + const embeddedTable = canvasElement.querySelector( + '[data-slot="embedded-table-block"]', + ); + if (!embeddedTable) throw new Error("Embedded runs table block is missing"); + await expect(embeddedTable).toHaveClass("space-y-3"); + await expect(embeddedTable.parentElement).toHaveClass("p-6"); + + await userEvent.click( + canvas.getByRole("combobox", { name: "Experiment attempt" }), + ); + await userEvent.click( + await screen.findByRole("option", { + name: "Attempt 1 · historical", + }), + ); + + await expect( + await screen.findByText(/Viewing processing attempt #1 \(historical\)/), + ).toBeVisible(); + await expect( + screen.getAllByText("Inspect the historical benchmark task")[0], + ).toBeVisible(); + await expect(screen.queryAllByText("Implement the current benchmark task")) + .toHaveLength(0); + + await userEvent.click( + canvas.getByRole("button", { name: "Customize Columns" }), + ); + await expect( + canvas.getByRole("complementary", { name: "Customize columns" }), + ).toBeVisible(); + }, +}; diff --git a/apps/portal/src/pages/ExperimentDetail.test.tsx b/apps/portal/src/pages/ExperimentDetail.test.tsx new file mode 100644 index 000000000..c53653b77 --- /dev/null +++ b/apps/portal/src/pages/ExperimentDetail.test.tsx @@ -0,0 +1,223 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// @vitest-environment happy-dom +import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { + MemoryRouter, + Route, + Routes, + useLocation, +} from "react-router-dom"; +import type { + ExperimentExecutionView, + ExperimentView, +} from "@/lib/experiments"; + +vi.mock("@/lib/api", async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + api: { + ...original.api, + getExperiment: vi.fn(), + listExperimentExecutions: vi.fn(), + getExperimentExecution: vi.fn(), + retryExperiment: vi.fn(), + }, + }; +}); + +import { api } from "@/lib/api"; +import { ExperimentDetail } from "./ExperimentDetail"; + +class ResizeObserverMock { + observe = vi.fn(); + unobserve = vi.fn(); + disconnect = vi.fn(); +} + +class IntersectionObserverMock { + observe = vi.fn(); + unobserve = vi.fn(); + disconnect = vi.fn(); +} + +const currentRequests = Array.from({ length: 11 }, (_, index) => ({ + requestId: `request-${String(index + 1).padStart(2, "0")}`, + runId: `run-current-${String(index + 1).padStart(2, "0")}`, + task: `Current task ${index + 1}`, + status: index === 0 ? "done" : "pending", + outcome: index === 0 ? "success" : undefined, + workerType: "coder-acp-copilot", + model: "gpt-5.6-sol", +})); + +const currentExecution: ExperimentExecutionView = { + experimentExecutionId: "execution-current", + attemptNumber: 2, + readiness: "done", + requestCount: currentRequests.length, + requests: currentRequests, + handlerProgress: {}, + blocking: [], + createdAt: "2026-10-16T10:00:00.000Z", +}; + +const historicalExecution: ExperimentExecutionView = { + experimentExecutionId: "execution-history", + attemptNumber: 1, + readiness: "failed", + requestCount: 1, + requests: [ + { + requestId: "request-history", + runId: "run-history", + task: "Historical task", + status: "done", + outcome: "failure", + workerType: "coder-acp-claude-code", + model: "claude-opus-4.6", + }, + ], + handlerProgress: {}, + blocking: [], + createdAt: "2026-10-15T10:00:00.000Z", + supersededReason: "retry", +}; + +const experiment: ExperimentView = { + experimentId: "experiment-1", + projectId: "project-1", + metadata: { name: "Profile comparison" }, + revision: { + experimentRevisionId: "revision-1", + revisionNumber: 1, + memberCount: 11, + }, + execution: currentExecution, + readiness: "done", + blocking: [], + createdRequestIds: currentRequests.map((request) => request.requestId), + createdAt: "2026-10-16T09:00:00.000Z", +}; + +function LocationProbe() { + const location = useLocation(); + return
{location.pathname}{location.search}
; +} + +function renderDetail(initialEntry = "/experiments/experiment-1") { + vi.mocked(api.getExperiment).mockResolvedValue(experiment); + vi.mocked(api.listExperimentExecutions).mockResolvedValue({ + items: [currentExecution, historicalExecution], + }); + vi.mocked(api.getExperimentExecution).mockImplementation( + async (_experimentId, executionId) => { + if (executionId === historicalExecution.experimentExecutionId) { + return historicalExecution; + } + return currentExecution; + }, + ); + + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); + return render( + + + + } /> + } /> + + + , + ); +} + +beforeAll(() => { + Object.defineProperty(globalThis, "ResizeObserver", { + configurable: true, + value: ResizeObserverMock, + }); + Object.defineProperty(globalThis, "IntersectionObserver", { + configurable: true, + value: IntersectionObserverMock, + }); +}); + +afterEach(() => { + cleanup(); + localStorage.clear(); + vi.clearAllMocks(); +}); + +describe("ExperimentDetail", () => { + it("defaults to the current attempt and paginates its shared runs table", async () => { + renderDetail(); + + expect(await screen.findByText("Runs in attempt 2")).toBeTruthy(); + expect(screen.getAllByText("Current task 1").length).toBeGreaterThan(0); + expect(screen.queryAllByText("Historical task")).toHaveLength(0); + expect(screen.getByText("1–10 of 11 runs")).toBeTruthy(); + const embeddedTable = document.querySelector( + '[data-slot="embedded-table-block"]', + ); + if (!embeddedTable) throw new Error("Embedded runs table block is missing"); + expect(embeddedTable.className).toContain("space-y-3"); + expect(embeddedTable.parentElement?.className).toContain("p-6"); + + fireEvent.click(screen.getByRole("button", { name: "Next page" })); + expect((await screen.findAllByText("Current task 11")).length).toBeGreaterThan(0); + expect(screen.queryAllByText("Current task 1")).toHaveLength(0); + + fireEvent.click( + screen.getByRole("button", { name: "Customize Columns" }), + ); + expect(screen.getByRole("complementary", { + name: "Customize columns", + })).toBeTruthy(); + }); + + it("deep-links one historical attempt and preserves exact run navigation", async () => { + renderDetail( + "/experiments/experiment-1?executionId=execution-history", + ); + + expect( + await screen.findByText(/Viewing processing attempt #1 \(historical\)/), + ).toBeTruthy(); + expect(screen.getByText("Runs in attempt 1")).toBeTruthy(); + expect(screen.getAllByText("Historical task").length).toBeGreaterThan(0); + expect(screen.queryAllByText("Current task 1")).toHaveLength(0); + expect( + screen.getByRole("button", { name: "Retry processing" }).hasAttribute( + "disabled", + ), + ).toBe(true); + + fireEvent.click(screen.getAllByText("Historical task")[0]); + expect((await screen.findByTestId("location")).textContent).toBe( + "/runs/request-history?runId=run-history", + ); + }); + + it("returns from a historical attempt to the current attempt", async () => { + renderDetail( + "/experiments/experiment-1?executionId=execution-history", + ); + + fireEvent.click(await screen.findByRole("button", { + name: "View current attempt", + })); + + await waitFor(() => { + expect(screen.getByText("Runs in attempt 2")).toBeTruthy(); + }); + expect(screen.getAllByText("Current task 1").length).toBeGreaterThan(0); + expect(screen.queryAllByText("Historical task")).toHaveLength(0); + }); +}); diff --git a/apps/portal/src/pages/ExperimentDetail.tsx b/apps/portal/src/pages/ExperimentDetail.tsx new file mode 100644 index 000000000..2fb41eab7 --- /dev/null +++ b/apps/portal/src/pages/ExperimentDetail.tsx @@ -0,0 +1,679 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { useEffect, useMemo, useState } from "react"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { ArrowLeft, Clock, RotateCcw } from "lucide-react"; +import { + Link, + useNavigate, + useParams, + useSearchParams, +} from "react-router-dom"; +import { toast } from "sonner"; +import { ExperimentStatusCard } from "@/components/ExperimentStatusCard"; +import { ShortId } from "@/components/ShortId"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { + CustomizeColumnsLink, + CustomizeColumnsPanel, + DataTable, + EmbeddedTableBlock, + Pagination, + useColumnOrder, + useHiddenColumns, + type CustomizeColumnsOption, + type DataTableColumn, +} from "@/components/list-layout"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { ApiError, api } from "@/lib/api"; +import type { + ExperimentExecutionView, + ExperimentRequestSummaryView, + ExperimentView, +} from "@/lib/experiments"; +import { formatDate, truncate } from "@/lib/utils"; + +const RUN_COLUMN_OPTIONS: CustomizeColumnsOption[] = [ + { id: "requestId", label: "ID", required: true }, + { id: "status", label: "Status" }, + { id: "outcome", label: "Outcome" }, + { id: "task", label: "Task" }, + { id: "worker", label: "Worker" }, + { id: "model", label: "Model" }, + { id: "profile", label: "Profile" }, + { id: "runId", label: "Run ID" }, +]; +const RUN_COLUMN_IDS = RUN_COLUMN_OPTIONS.map((column) => column.id); +const DEFAULT_PAGE_SIZE = 10; + +type RunSortColumn = (typeof RUN_COLUMN_IDS)[number]; +type SortDirection = "asc" | "desc"; + +function requestSortValue( + request: ExperimentRequestSummaryView, + column: RunSortColumn, +): string { + switch (column) { + case "requestId": + return request.requestId; + case "runId": + return request.runId; + case "task": + return request.task ?? ""; + case "status": + return request.status ?? ""; + case "outcome": + return request.outcome ?? ""; + case "worker": + return request.workerType ?? ""; + case "model": + return request.model ?? ""; + case "profile": + return request.profileId ?? ""; + default: + return ""; + } +} + +function ExecutionSummary({ + execution, + current, +}: { + execution: ExperimentExecutionView; + current: boolean; +}) { + return ( + + + + Execution attempt {execution.attemptNumber} + + + {current ? "Current" : "Historical"} + + + +
+
Execution ID
+ +
+
+
Runs
+
{execution.requestCount}
+
+
+
Readiness
+
{execution.readiness ?? "unknown"}
+
+
+
Created
+
{formatDate(execution.createdAt)}
+
+ + + ); +} + +function selectedExperimentView( + experiment: ExperimentView, + execution: ExperimentExecutionView, +): ExperimentView { + const current = + execution.experimentExecutionId === + experiment.execution.experimentExecutionId; + return { + ...experiment, + execution, + readiness: current + ? experiment.readiness + : (execution.readiness ?? "unknown"), + blocking: current ? experiment.blocking : execution.blocking, + }; +} + +export function ExperimentDetail() { + const { id } = useParams<{ id: string }>(); + const navigate = useNavigate(); + const queryClient = useQueryClient(); + const [searchParams, setSearchParams] = useSearchParams(); + const selectedExecutionId = searchParams.get("executionId") ?? undefined; + const [customizingColumns, setCustomizingColumns] = useState(false); + const [page, setPage] = useState(1); + const [pageSize, setPageSize] = useState(DEFAULT_PAGE_SIZE); + const [sortColumn, setSortColumn] = + useState("requestId"); + const [sortDirection, setSortDirection] = + useState("asc"); + const columnVisibility = useHiddenColumns({ + storageKey: "experiment-runs", + defaultHidden: ["profile", "runId"], + }); + const columnOrder = useColumnOrder({ + storageKey: "experiment-runs", + columnIds: RUN_COLUMN_IDS, + }); + + const experimentQuery = useQuery({ + queryKey: ["experiment", id], + queryFn: () => api.getExperiment(id!), + enabled: Boolean(id), + refetchInterval: (query) => { + const readiness = query.state.data?.readiness; + return readiness === "done" || + readiness === "failed" || + readiness === "blocked" + ? false + : 5_000; + }, + }); + const executionsQuery = useQuery({ + queryKey: ["experiment-executions", id], + queryFn: () => api.listExperimentExecutions(id!, { limit: 100 }), + enabled: Boolean(id) && experimentQuery.isSuccess, + }); + const currentExecutionId = + experimentQuery.data?.execution.experimentExecutionId; + const isViewingHistorical = + Boolean(selectedExecutionId) && + selectedExecutionId !== currentExecutionId; + const selectedExecutionQuery = useQuery({ + queryKey: ["experiment-execution", id, selectedExecutionId], + queryFn: () => api.getExperimentExecution(id!, selectedExecutionId!), + enabled: Boolean(id && selectedExecutionId && isViewingHistorical), + }); + const retry = useMutation({ + mutationFn: () => + api.retryExperiment( + experimentQuery.data!.experimentId, + experimentQuery.data!.execution.experimentExecutionId, + ), + onSuccess: () => { + toast.success("Experiment processing restarted"); + void queryClient.invalidateQueries({ queryKey: ["experiment", id] }); + void queryClient.invalidateQueries({ + queryKey: ["experiment-executions", id], + }); + }, + onError: (error) => { + toast.error(error instanceof Error ? error.message : "Retry failed"); + }, + }); + + const attemptOptions = useMemo(() => { + const byId = new Map(); + if (experimentQuery.data?.execution) { + byId.set( + experimentQuery.data.execution.experimentExecutionId, + experimentQuery.data.execution, + ); + } + for (const execution of executionsQuery.data?.items ?? []) { + byId.set(execution.experimentExecutionId, execution); + } + if (selectedExecutionQuery.data) { + byId.set( + selectedExecutionQuery.data.experimentExecutionId, + selectedExecutionQuery.data, + ); + } + return [...byId.values()].sort( + (left, right) => right.attemptNumber - left.attemptNumber, + ); + }, [ + executionsQuery.data?.items, + experimentQuery.data?.execution, + selectedExecutionQuery.data, + ]); + + const activeExecution = isViewingHistorical + ? selectedExecutionQuery.data + : experimentQuery.data?.execution; + const activeExperiment = + experimentQuery.data && activeExecution + ? selectedExperimentView(experimentQuery.data, activeExecution) + : undefined; + + const sortedRequests = useMemo(() => { + if (!activeExecution) return []; + const direction = sortDirection === "asc" ? 1 : -1; + return [...activeExecution.requests].sort((left, right) => { + const compared = requestSortValue(left, sortColumn).localeCompare( + requestSortValue(right, sortColumn), + undefined, + { numeric: true, sensitivity: "base" }, + ); + if (compared !== 0) return compared * direction; + return left.requestId.localeCompare(right.requestId) * direction; + }); + }, [activeExecution, sortColumn, sortDirection]); + + useEffect(() => { + setPage(1); + }, [activeExecution?.experimentExecutionId, pageSize, sortColumn, sortDirection]); + + const pagedRequests = useMemo(() => { + const start = (page - 1) * pageSize; + return sortedRequests.slice(start, start + pageSize); + }, [page, pageSize, sortedRequests]); + + const columns = useMemo[]>(() => { + const naturalColumns: DataTableColumn[] = [ + { + id: "requestId", + header: "ID", + sortable: true, + width: "120px", + sticky: "left", + cell: (request) => ( + + ), + }, + { + id: "status", + header: "Status", + sortable: true, + width: "110px", + hidden: columnVisibility.isHidden("status"), + cell: (request) => + request.status ? ( + {request.status} + ) : ( + — + ), + }, + { + id: "outcome", + header: "Outcome", + sortable: true, + width: "110px", + hidden: columnVisibility.isHidden("outcome"), + cell: (request) => + request.outcome ? ( + {request.outcome} + ) : ( + — + ), + }, + { + id: "task", + header: "Task", + sortable: true, + hidden: columnVisibility.isHidden("task"), + cell: (request) => ( + + {request.task ? truncate(request.task, 80) : "—"} + + ), + }, + { + id: "worker", + header: "Worker", + sortable: true, + width: "180px", + hidden: columnVisibility.isHidden("worker"), + cell: (request) => ( + + {request.workerType ?? "—"} + + ), + }, + { + id: "model", + header: "Model", + sortable: true, + width: "180px", + hidden: columnVisibility.isHidden("model"), + cell: (request) => ( + {request.model ?? "—"} + ), + }, + { + id: "profile", + header: "Profile", + sortable: true, + width: "150px", + hidden: columnVisibility.isHidden("profile"), + cell: (request) => ( + + {request.profileId ?? "—"} + + ), + }, + { + id: "runId", + header: "Run ID", + sortable: true, + width: "120px", + hidden: columnVisibility.isHidden("runId"), + cell: (request) => , + }, + ]; + const byId = new Map(naturalColumns.map((column) => [column.id, column])); + return columnOrder.order + .map((columnId) => byId.get(columnId)) + .filter( + ( + column, + ): column is DataTableColumn => + Boolean(column), + ); + }, [columnOrder.order, columnVisibility]); + + const selectExecution = (executionId: string) => { + const next = new URLSearchParams(searchParams); + if (executionId === currentExecutionId) { + next.delete("executionId"); + } else { + next.set("executionId", executionId); + } + setSearchParams(next, { replace: true }); + }; + + const changeSort = (column: string) => { + if (!RUN_COLUMN_IDS.includes(column)) return; + const nextColumn = column as RunSortColumn; + if (nextColumn === sortColumn) { + setSortDirection((current) => (current === "asc" ? "desc" : "asc")); + return; + } + setSortColumn(nextColumn); + setSortDirection("asc"); + }; + + if (experimentQuery.isLoading) { + return ( +
+ Loading experiment… +
+ ); + } + if (experimentQuery.isError || !experimentQuery.data) { + const correlationOnly = + experimentQuery.error instanceof ApiError && + experimentQuery.error.status === 404; + return ( +
+ + + + + {correlationOnly + ? "Historical experiment correlation" + : "Experiment could not be loaded"} + + + + {correlationOnly ? ( + <> +

+ {id} is + attached to historical runs but has no canonical Experiment + document. +

+

+ This correlation is view-only. It cannot be retried or used to + trigger processing. +

+ + ) : ( +

+ {experimentQuery.error instanceof Error + ? experimentQuery.error.message + : "The experiment request failed."} +

+ )} + + +
+ ); + } + + const experiment = experimentQuery.data; + const canRetry = !["waiting", "processing"].includes( + experiment.readiness ?? "", + ); + const selectedValue = + activeExecution?.experimentExecutionId ?? + selectedExecutionId ?? + experiment.execution.experimentExecutionId; + + return ( +
+
+
+ +
+

+ {experiment.metadata.name} +

+ +
+

+ Created {formatDate(experiment.createdAt)} +

+
+
+
+ + +
+ +
+
+ + {experiment.metadata.description && ( +

+ {experiment.metadata.description} +

+ )} + + {isViewingHistorical && ( +
+ + + Viewing processing attempt #{activeExecution?.attemptNumber ?? "…"}{" "} + (historical).{" "} + + +
+ )} + + {selectedExecutionQuery.isLoading && isViewingHistorical && ( +

+ Loading processing attempt… +

+ )} + {selectedExecutionQuery.isError && isViewingHistorical && ( + + + {selectedExecutionQuery.error instanceof Error + ? selectedExecutionQuery.error.message + : "The selected processing attempt could not be loaded."} + + + )} + + {activeExperiment && activeExecution && ( + <> + + + + {activeExperiment.blocking.length > 0 && ( + + + + Blocking dependencies + + + + {activeExperiment.blocking.map((blocking, index) => ( +
+ {blocking.requestId && ( + + {blocking.requestId.slice(0, 8)} + + )} + {blocking.reason} + + {blocking.handlerId} + + {blocking.error && {blocking.error}} +
+ ))} + + + )} + + + + setCustomizingColumns((open) => !open)} + /> + } + > +
+ {customizingColumns && ( +
+ + + + + )} +
+ ); +} diff --git a/apps/portal/src/pages/ExperimentsList.play.test.tsx b/apps/portal/src/pages/ExperimentsList.play.test.tsx new file mode 100644 index 000000000..9167338be --- /dev/null +++ b/apps/portal/src/pages/ExperimentsList.play.test.tsx @@ -0,0 +1,124 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// @vitest-environment happy-dom +import { render, within } from "@testing-library/react"; +import { beforeAll, describe, it, vi } from "vitest"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { MemoryRouter } from "react-router-dom"; +import { api } from "@/lib/api"; +import experimentsListMeta, { + SharedListInteractions, +} from "./ExperimentsList.stories"; + +vi.mock("@/lib/api", async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + api: { + ...original.api, + listExperiments: vi.fn(), + }, + }; +}); + +class ResizeObserverMock { + observe = vi.fn(); + unobserve = vi.fn(); + disconnect = vi.fn(); +} + +class IntersectionObserverMock { + observe = vi.fn(); + unobserve = vi.fn(); + disconnect = vi.fn(); +} + +beforeAll(() => { + Object.defineProperty(globalThis, "ResizeObserver", { + configurable: true, + value: ResizeObserverMock, + }); + Object.defineProperty(globalThis, "IntersectionObserver", { + configurable: true, + value: IntersectionObserverMock, + }); + vi.mocked(api.listExperiments).mockResolvedValue({ + items: [ + { + experimentId: "experiment-alpha", + projectId: "demo-project", + metadata: { name: "Alpha benchmark" }, + revision: { + experimentRevisionId: "revision-alpha", + revisionNumber: 1, + memberCount: 4, + }, + execution: { + experimentExecutionId: "execution-alpha", + attemptNumber: 2, + requestCount: 4, + requests: [], + handlerProgress: {}, + blocking: [], + createdAt: "2026-10-16T10:00:00.000Z", + }, + readiness: "done", + blocking: [], + createdRequestIds: [], + createdAt: "2026-10-16T09:00:00.000Z", + }, + { + experimentId: "experiment-beta", + projectId: "demo-project", + metadata: { name: "Beta benchmark" }, + revision: { + experimentRevisionId: "revision-beta", + revisionNumber: 1, + memberCount: 4, + }, + execution: { + experimentExecutionId: "execution-beta", + attemptNumber: 2, + requestCount: 4, + requests: [], + handlerProgress: {}, + blocking: [], + createdAt: "2026-10-16T10:00:00.000Z", + }, + readiness: "blocked", + blocking: [], + createdRequestIds: [], + createdAt: "2026-10-16T09:00:00.000Z", + }, + ], + }); +}); + +describe("ExperimentsList stories", () => { + it("runs the shared-list interaction", async () => { + const StoryRender = experimentsListMeta.component; + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); + const { container } = render( + + +
+ +
+ + , + ); + const play = SharedListInteractions.play as unknown as + | ((context: { + canvas: ReturnType; + canvasElement: HTMLElement; + }) => Promise) + | undefined; + await play?.({ + canvas: within(container), + canvasElement: container, + }); + }); +}); diff --git a/apps/portal/src/pages/ExperimentsList.stories.tsx b/apps/portal/src/pages/ExperimentsList.stories.tsx new file mode 100644 index 000000000..c506335cc --- /dev/null +++ b/apps/portal/src/pages/ExperimentsList.stories.tsx @@ -0,0 +1,106 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { Meta, StoryObj } from "@storybook/react-vite"; +import { + expect, + fireEvent, + screen, + userEvent, + waitFor, +} from "storybook/test"; +import { HttpResponse, http } from "msw"; +import { ExperimentsList } from "./ExperimentsList"; + +const experiment = ( + experimentId: string, + name: string, + readiness: string, +) => ({ + experimentId, + projectId: "demo-project", + metadata: { + name, + description: `${name} compares pinned agent configurations.`, + }, + revision: { + experimentRevisionId: `revision-${experimentId}`, + revisionNumber: 1, + memberCount: 4, + }, + execution: { + experimentExecutionId: `execution-${experimentId}`, + attemptNumber: 2, + version: 1, + readiness, + requests: [], + handlerProgress: {}, + blockers: [], + createdAt: "2026-10-16T10:00:00.000Z", + }, + readiness, + blocking: [], + createdRequestIds: [], + createdAt: "2026-10-16T09:00:00.000Z", +}); + +const meta = { + component: ExperimentsList, + parameters: { + layout: "fullscreen", + msw: { + handlers: [ + http.get("/api/v1/experiments", () => + HttpResponse.json({ + data: [ + experiment("experiment-alpha", "Alpha benchmark", "done"), + experiment("experiment-beta", "Beta benchmark", "blocked"), + ], + }), + ), + ], + }, + }, + decorators: [ + (Story) => ( +
+ +
+ ), + ], +} satisfies Meta; + +export default meta; +type Story = StoryObj; + +export const SharedListInteractions: Story = { + play: async ({ canvas }) => { + await expect( + (await screen.findAllByText("Alpha benchmark"))[0], + ).toBeVisible(); + await expect( + canvas.getByRole("complementary", { name: "Filters" }), + ).toBeVisible(); + const listShell = canvas + .getByRole("heading", { name: "Experiments" }) + .closest("section")?.parentElement; + if (!listShell) throw new Error("Experiments list shell is missing"); + await expect(listShell).toHaveClass("h-full"); + await expect(listShell).not.toHaveClass("px-6"); + + fireEvent.change(canvas.getByPlaceholderText("Search experiments…"), { + target: { value: "Beta" }, + }); + await waitFor(() => { + expect(screen.queryAllByText("Alpha benchmark")).toHaveLength(0); + }); + await expect(screen.getAllByText("Beta benchmark")[0]).toBeVisible(); + + await userEvent.click( + canvas.getByRole("button", { name: "Customize Columns" }), + ); + await expect( + canvas.getByRole("complementary", { name: "Customize columns" }), + ).toBeVisible(); + }, +}; diff --git a/apps/portal/src/pages/ExperimentsList.test.tsx b/apps/portal/src/pages/ExperimentsList.test.tsx new file mode 100644 index 000000000..68e6e6a22 --- /dev/null +++ b/apps/portal/src/pages/ExperimentsList.test.tsx @@ -0,0 +1,172 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// @vitest-environment happy-dom +import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { + MemoryRouter, + Route, + Routes, + useLocation, +} from "react-router-dom"; +import type { ExperimentView } from "@/lib/experiments"; + +vi.mock("@/lib/api", async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + api: { + ...original.api, + listExperiments: vi.fn(), + }, + }; +}); + +import { api } from "@/lib/api"; +import { ExperimentsList } from "./ExperimentsList"; + +class ResizeObserverMock { + observe = vi.fn(); + unobserve = vi.fn(); + disconnect = vi.fn(); +} + +class IntersectionObserverMock { + observe = vi.fn(); + unobserve = vi.fn(); + disconnect = vi.fn(); +} + +function experiment( + experimentId: string, + name: string, + readiness: ExperimentView["readiness"], +): ExperimentView { + return { + experimentId, + projectId: "project-1", + metadata: { + name, + description: `${name} description`, + }, + revision: { + experimentRevisionId: `revision-${experimentId}`, + revisionNumber: 1, + memberCount: 2, + }, + execution: { + experimentExecutionId: `execution-${experimentId}`, + attemptNumber: 1, + requestCount: 2, + requests: [], + handlerProgress: {}, + blocking: [], + createdAt: "2026-10-16T10:00:00.000Z", + }, + readiness, + blocking: [], + createdRequestIds: [], + createdAt: "2026-10-16T09:00:00.000Z", + }; +} + +const firstPage = [ + experiment("experiment-alpha", "Alpha benchmark", "done"), + experiment("experiment-beta", "Beta benchmark", "blocked"), +]; +const secondPage = [ + experiment("experiment-gamma", "Gamma benchmark", "processing"), +]; + +function LocationProbe() { + const location = useLocation(); + return
{location.pathname}
; +} + +function renderList() { + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); + return render( + + + + } /> + } /> + + + , + ); +} + +beforeAll(() => { + Object.defineProperty(globalThis, "ResizeObserver", { + configurable: true, + value: ResizeObserverMock, + }); + Object.defineProperty(globalThis, "IntersectionObserver", { + configurable: true, + value: IntersectionObserverMock, + }); +}); + +afterEach(() => { + cleanup(); + localStorage.clear(); + vi.clearAllMocks(); +}); + +describe("ExperimentsList", () => { + it("uses the shared list controls for filtering and customization", async () => { + vi.mocked(api.listExperiments).mockResolvedValue({ + items: firstPage, + nextCursor: "next-page", + }); + renderList(); + + expect((await screen.findAllByText("Alpha benchmark")).length).toBeGreaterThan(0); + expect(screen.getByRole("complementary", { name: "Filters" })).toBeTruthy(); + const listShell = screen.getByRole("heading", { + name: "Experiments", + }).closest("section")?.parentElement; + expect(listShell?.className).toContain("h-full"); + expect(listShell?.className).not.toContain("px-6"); + + fireEvent.change(screen.getByPlaceholderText("Search experiments…"), { + target: { value: "Beta" }, + }); + expect(screen.queryAllByText("Alpha benchmark")).toHaveLength(0); + expect(screen.getAllByText("Beta benchmark").length).toBeGreaterThan(0); + + fireEvent.click(screen.getByRole("button", { name: "Customize Columns" })); + expect( + screen.getByRole("complementary", { name: "Customize columns" }), + ).toBeTruthy(); + }); + + it("keeps cursor pagination and row navigation", async () => { + vi.mocked(api.listExperiments).mockImplementation(async ({ cursor } = {}) => + cursor === "next-page" + ? { items: secondPage } + : { items: firstPage, nextCursor: "next-page" }, + ); + renderList(); + + expect((await screen.findAllByText("Alpha benchmark")).length).toBeGreaterThan(0); + fireEvent.click(screen.getByRole("button", { name: "Next page" })); + expect((await screen.findAllByText("Gamma benchmark")).length).toBeGreaterThan(0); + expect(api.listExperiments).toHaveBeenLastCalledWith({ + cursor: "next-page", + limit: 25, + }); + + fireEvent.click(screen.getAllByText("Gamma benchmark")[0]); + await waitFor(() => { + expect(screen.getByTestId("location").textContent).toBe( + "/experiments/experiment-gamma", + ); + }); + }); +}); diff --git a/apps/portal/src/pages/ExperimentsList.tsx b/apps/portal/src/pages/ExperimentsList.tsx new file mode 100644 index 000000000..6c13e4c4e --- /dev/null +++ b/apps/portal/src/pages/ExperimentsList.tsx @@ -0,0 +1,401 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { useEffect, useMemo, useState } from "react"; +import { keepPreviousData, useQuery } from "@tanstack/react-query"; +import { Atom, Plus } from "lucide-react"; +import { Link, useNavigate } from "react-router-dom"; +import { + CheckboxFilterGroup, + ClearFiltersLink, + CustomizeColumnsLink, + CustomizeColumnsPanel, + DataTable, + FilterRail, + FilterSection, + ListLayout, + Pagination, + useColumnOrder, + useHiddenColumns, + useListUrlState, + type CustomizeColumnsOption, + type DataTableColumn, +} from "@/components/list-layout"; +import { ShortId } from "@/components/ShortId"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { api } from "@/lib/api"; +import type { ExperimentView } from "@/lib/experiments"; +import { formatDate } from "@/lib/utils"; + +const FILTER_KEYS = ["readiness"] as const; +const READINESS_VALUES = [ + "waiting", + "processing", + "done", + "blocked", + "failed", +] as const; +const COLUMN_OPTIONS: CustomizeColumnsOption[] = [ + { id: "name", label: "Experiment", required: true }, + { id: "readiness", label: "Status" }, + { id: "runs", label: "Runs" }, + { id: "revision", label: "Revision" }, + { id: "attempt", label: "Attempt" }, + { id: "created", label: "Created" }, + { id: "id", label: "ID" }, +]; +const COLUMN_IDS = COLUMN_OPTIONS.map((column) => column.id); + +type ExperimentSortColumn = (typeof COLUMN_IDS)[number]; + +function readinessVariant(readiness?: string) { + return readiness === "failed" || readiness === "blocked" + ? "destructive" + : "secondary"; +} + +function sortValue( + experiment: ExperimentView, + column: ExperimentSortColumn, +): string | number { + switch (column) { + case "name": + return experiment.metadata.name; + case "readiness": + return experiment.readiness ?? ""; + case "runs": + return experiment.revision.memberCount; + case "revision": + return experiment.revision.revisionNumber; + case "attempt": + return experiment.execution.attemptNumber; + case "created": + return new Date(experiment.createdAt).getTime(); + case "id": + return experiment.experimentId; + default: + return ""; + } +} + +export function ExperimentsList() { + const navigate = useNavigate(); + const state = useListUrlState({ + defaultPageSize: 25, + filterKeys: FILTER_KEYS, + }); + const [customizeOpen, setCustomizeOpen] = useState(false); + const [cursorStack, setCursorStack] = useState>([ + undefined, + ]); + const columnVisibility = useHiddenColumns({ + storageKey: "experiments", + defaultHidden: ["id"], + }); + const columnOrder = useColumnOrder({ + storageKey: "experiments", + columnIds: COLUMN_IDS, + }); + const cursor = cursorStack[state.page - 1]; + const experimentsQuery = useQuery({ + queryKey: ["experiments", cursor, state.pageSize], + queryFn: () => + api.listExperiments({ cursor, limit: state.pageSize }), + placeholderData: keepPreviousData, + }); + + useEffect(() => { + setCursorStack([undefined]); + }, [state.pageSize]); + + const readinessOptions = useMemo( + () => + READINESS_VALUES.map((readiness) => ({ + value: readiness, + label: readiness, + count: + experimentsQuery.data?.items.filter( + (experiment) => experiment.readiness === readiness, + ).length ?? 0, + })), + [experimentsQuery.data?.items], + ); + + const filteredExperiments = useMemo(() => { + const selectedReadiness = state.getFilterList("readiness"); + const search = state.search.trim().toLocaleLowerCase(); + return (experimentsQuery.data?.items ?? []).filter((experiment) => { + if ( + selectedReadiness.length > 0 && + !selectedReadiness.includes(experiment.readiness ?? "unknown") + ) { + return false; + } + if (!search) return true; + return [ + experiment.metadata.name, + experiment.metadata.description, + experiment.experimentId, + ] + .filter(Boolean) + .join(" ") + .toLocaleLowerCase() + .includes(search); + }); + }, [experimentsQuery.data?.items, state]); + + const sortedExperiments = useMemo(() => { + if (!state.sort || !COLUMN_IDS.includes(state.sort)) { + return filteredExperiments; + } + const column = state.sort as ExperimentSortColumn; + const direction = state.sortDir === "asc" ? 1 : -1; + return [...filteredExperiments].sort((left, right) => { + const leftValue = sortValue(left, column); + const rightValue = sortValue(right, column); + const compared = + typeof leftValue === "number" && typeof rightValue === "number" + ? leftValue - rightValue + : String(leftValue).localeCompare(String(rightValue), undefined, { + numeric: true, + sensitivity: "base", + }); + if (compared !== 0) return compared * direction; + return left.experimentId.localeCompare(right.experimentId) * direction; + }); + }, [filteredExperiments, state.sort, state.sortDir]); + + const columns = useMemo[]>(() => { + const naturalColumns: DataTableColumn[] = [ + { + id: "name", + header: "Experiment", + sortable: true, + sticky: "left", + cell: (experiment) => ( +
+ +
+
+ {experiment.metadata.name} +
+ {experiment.metadata.description && ( +
+ {experiment.metadata.description} +
+ )} +
+
+ ), + }, + { + id: "readiness", + header: "Status", + sortable: true, + width: "120px", + hidden: columnVisibility.isHidden("readiness"), + cell: (experiment) => ( + + {experiment.readiness ?? "unknown"} + + ), + }, + { + id: "runs", + header: "Runs", + sortable: true, + width: "90px", + align: "right", + hidden: columnVisibility.isHidden("runs"), + cell: (experiment) => experiment.revision.memberCount, + }, + { + id: "revision", + header: "Revision", + sortable: true, + width: "100px", + align: "right", + hidden: columnVisibility.isHidden("revision"), + cell: (experiment) => experiment.revision.revisionNumber, + }, + { + id: "attempt", + header: "Attempt", + sortable: true, + width: "100px", + align: "right", + hidden: columnVisibility.isHidden("attempt"), + cell: (experiment) => experiment.execution.attemptNumber, + }, + { + id: "created", + header: "Created", + sortable: true, + width: "170px", + hidden: columnVisibility.isHidden("created"), + cell: (experiment) => ( + + {formatDate(experiment.createdAt)} + + ), + }, + { + id: "id", + header: "ID", + sortable: true, + width: "120px", + hidden: columnVisibility.isHidden("id"), + cell: (experiment) => ( + + ), + }, + ]; + const byId = new Map(naturalColumns.map((column) => [column.id, column])); + return columnOrder.order + .map((columnId) => byId.get(columnId)) + .filter( + (column): column is DataTableColumn => Boolean(column), + ); + }, [columnOrder.order, columnVisibility]); + + const handlePageChange = (nextPage: number) => { + if (nextPage === state.page) return; + if (nextPage === 1) { + state.setPage(1); + return; + } + if (nextPage === state.page + 1) { + const nextCursor = experimentsQuery.data?.nextCursor; + if (!nextCursor) return; + setCursorStack((current) => { + const next = [...current]; + next[nextPage - 1] = nextCursor; + return next; + }); + state.setPage(nextPage); + return; + } + if (nextPage === state.page - 1 && cursorStack[nextPage - 1] !== undefined) { + state.setPage(nextPage); + } + }; + + const emptyState = state.hasActiveFilters ? ( + "No experiments on this page match the current filters." + ) : ( +
+ +
+

No experiments yet

+

+ Create one from a scenario, or leave the scenario empty to create a + durable placeholder. +

+
+
+ ); + + return ( + + + + New Experiment + + + } + filterRail={ + + + setCustomizeOpen(true)} /> + + } + > + + + state.toggleFilterValue("readiness", value) + } + /> + + + } + secondaryPanel={ + customizeOpen ? ( + + ); +} diff --git a/apps/portal/src/pages/RunDetail.tsx b/apps/portal/src/pages/RunDetail.tsx index 8cd9ab00e..81d3429bb 100644 --- a/apps/portal/src/pages/RunDetail.tsx +++ b/apps/portal/src/pages/RunDetail.tsx @@ -531,7 +531,7 @@ export function RunDetail() { (activeRun?.observationResults?.length ?? 0) > 0; const hasConfiguredObservations = (run?.observations?.length ?? 0) > 0; // Aggregate enrichment status across the whole post-processing DAG so the - // "Enriched" badge waits for pp-taxonomy instead of flipping after pp-atif. + // "Enriched" badge waits for pp-observations instead of flipping after pp-atif. const enrichmentStatus = useMemo( () => deriveEnrichmentStatus(activeRun, hasConfiguredObservations), [activeRun, hasConfiguredObservations], @@ -905,6 +905,19 @@ export function RunDetail() { /> )} + {run.experimentId && ( + + {run.experimentId.slice(0, 8)} + + )} + /> + )} {run.maxIterations && } {(() => { const totalDuration = activeRun?.turns?.reduce((sum, t) => sum + (t.durationMs ?? 0), 0); diff --git a/apps/portal/src/pages/RunsList.tsx b/apps/portal/src/pages/RunsList.tsx index 27b685df5..31007a293 100644 --- a/apps/portal/src/pages/RunsList.tsx +++ b/apps/portal/src/pages/RunsList.tsx @@ -56,7 +56,7 @@ import { isAgentAvailable, isAgentVersionAvailable, STATUS_LIST, OUTCOME_LIST } import type { Run, RunStatus, RunOutcome, IterationOp, BulkResubmitOverrides, RunSortField, RunFacetBucket } from "@/types"; import { buildRunWorkerFilterOptions } from "./run-worker-filter-options"; -const FILTER_KEYS = ["worker", "status", "outcome", "taskPromptId", "submissionId", "criteria", "model", "profile", "os", "priority", "version", "dateFrom", "dateTo", "groupBy", "turns", "turnsOp", "maxIter", "maxIterOp"] as const; +const FILTER_KEYS = ["worker", "status", "outcome", "taskPromptId", "experimentId", "criteria", "model", "profile", "os", "priority", "version", "dateFrom", "dateTo", "groupBy", "turns", "turnsOp", "maxIter", "maxIterOp"] as const; /** * Describes how to fetch a given (1-based) page from the cursor-paginated @@ -132,18 +132,18 @@ const GROUP_MEMBER_PAGE = 100; /** * Map a server group key (issue #1138 server-side grouping) to the extra * server-side filter that selects that group's member runs. `task` groups by - * `taskPromptId`, `profile` by `profileId`, `submissionId` by `submissionId` - * (the literal `no-submission`/`no-profile` keys mean "missing", expressed as + * `taskPromptId`, `profile` by `profileId`, `experimentId` by `experimentId` + * (the literal `no-experiment`/`no-profile` keys mean "missing", expressed as * the `__empty__` sentinel so the API matches rows lacking the field). */ function groupKeyExtraFilter( - groupBy: "task" | "submissionId" | "profile", + groupBy: "task" | "experimentId" | "profile", key: string, -): { taskPromptId?: string; submissionId?: string; profileId?: string[] } { +): { taskPromptId?: string; experimentId?: string; profileId?: string[] } { if (groupBy === "task") return { taskPromptId: key }; if (groupBy === "profile") return { profileId: [key === "no-profile" ? EMPTY_FILTER_VALUE : key] }; - return { submissionId: key === "no-submission" ? EMPTY_FILTER_VALUE : key }; + return { experimentId: key === "no-experiment" ? EMPTY_FILTER_VALUE : key }; } /** Compact inline badges with overflow (+N) dropdown for dense table cells. */ @@ -313,7 +313,7 @@ const COLUMN_OPTIONS: CustomizeColumnsOption[] = [ // growth-ecosystems/scope-project#127). { id: "status", label: "Status" }, { id: "outcome", label: "Outcome" }, - { id: "submission", label: "Submission" }, + { id: "experiment", label: "Experiment" }, { id: "task", label: "Task" }, { id: "criteria", label: "Criteria" }, { id: "worker", label: "Worker" }, @@ -339,18 +339,18 @@ const COLUMN_IDS = COLUMN_OPTIONS.map((o) => o.id); // Segmented toggle used in the list header for the Group By control. Kept // inline because it is only used here. -const GROUP_BY_OPTIONS: ReadonlyArray<{ value: "profile" | "task" | "submissionId"; label: string }> = [ +const GROUP_BY_OPTIONS: ReadonlyArray<{ value: "profile" | "task" | "experimentId"; label: string }> = [ { value: "profile", label: "Profile" }, { value: "task", label: "Task" }, - { value: "submissionId", label: "Submission" }, + { value: "experimentId", label: "Experiment" }, ]; function GroupByToggle({ value, onChange, }: { - value: "none" | "profile" | "task" | "submissionId"; - onChange: (next: "none" | "profile" | "task" | "submissionId") => void; + value: "none" | "profile" | "task" | "experimentId"; + onChange: (next: "none" | "profile" | "task" | "experimentId") => void; }) { return ( ); }, }} diff --git a/apps/portal/src/pages/SubmitRun.stories.tsx b/apps/portal/src/pages/SubmitRun.stories.tsx new file mode 100644 index 000000000..a8962419a --- /dev/null +++ b/apps/portal/src/pages/SubmitRun.stories.tsx @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { Meta, StoryObj } from "@storybook/react-vite"; +import { expect, userEvent, within } from "storybook/test"; +import { http, HttpResponse } from "msw"; +import { SubmitRun } from "./SubmitRun"; + +const meta = { + title: "Pages/Submit Run", + component: SubmitRun, + parameters: { + layout: "fullscreen", + msw: { + handlers: [ + http.get("/api/v1/agents", () => + HttpResponse.json([ + { + _id: "coder-acp-copilot", + id: "coder-acp-copilot", + name: "GitHub Copilot CLI", + defaultModel: "gpt-5.6-sol", + versions: [], + capabilities: {}, + createdAt: "2026-01-01T00:00:00.000Z", + }, + ]), + ), + http.get("/api/v1/*", () => HttpResponse.json([])), + ], + }, + }, +} satisfies Meta; + +export default meta; +type Story = StoryObj; + +export const ExperimentProgressiveDisclosure: Story = { + args: { + mode: "experiment", + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + + await expect( + canvas.getByRole("button", { name: /Create empty experiment/ }), + ).toBeInTheDocument(); + await expect(canvas.queryByLabelText("Tags")).not.toBeInTheDocument(); + await expect(canvas.queryByText("Gate pipeline")).not.toBeInTheDocument(); + await expect( + canvas.queryByRole("switch", { name: "Toggle advanced options" }), + ).not.toBeInTheDocument(); + + await userEvent.type( + canvas.getByLabelText(/Task/), + "Implement the benchmark task", + ); + + await expect( + await canvas.findByText("Gate pipeline"), + ).toBeInTheDocument(); + await expect( + canvas.getByRole("switch", { name: "Toggle advanced options" }), + ).toBeInTheDocument(); + await expect( + canvas.getByRole("button", { + name: /Create experiment with 5 runs/, + }), + ).toBeInTheDocument(); + }, +}; diff --git a/apps/portal/src/pages/SubmitRun.test.tsx b/apps/portal/src/pages/SubmitRun.test.tsx new file mode 100644 index 000000000..feca6e36c --- /dev/null +++ b/apps/portal/src/pages/SubmitRun.test.tsx @@ -0,0 +1,154 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// @vitest-environment happy-dom +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + cleanup, + fireEvent, + render, + screen, + waitFor, +} from "@testing-library/react"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { MemoryRouter } from "react-router-dom"; + +vi.mock("@/lib/api", () => ({ + api: { + listAgents: vi.fn(async () => []), + listMcpServers: vi.fn(async () => []), + listProfiles: vi.fn(async () => []), + listProfileVersions: vi.fn(async () => []), + listRuns: vi.fn(async () => ({ data: [] })), + listModels: vi.fn(async () => []), + createExperiment: vi.fn(async () => ({ experimentId: "experiment-1" })), + }, +})); + +vi.mock("@/hooks/useVisibleGates", () => ({ + useVisibleGates: () => [], +})); +vi.mock("@/hooks/useStrictAgentCapabilities", () => ({ + useStrictAgentCapabilities: () => false, +})); +vi.mock("@/hooks/useAdvancedMode", () => ({ + useAdvancedMode: () => [false, vi.fn()], +})); +vi.mock("@/hooks/useCommandEnter", () => ({ + useCommandEnter: vi.fn(), + createCommandEnterKeyDown: () => () => undefined, + isMac: false, +})); +vi.mock("@/components/ReasoningEffortSelect", () => ({ + ModelSelectItems: () => null, + ReasoningEffortSelect: () => null, + useModelCapabilities: () => ({ + capabilitiesMap: new Map(), + activeModelIds: [], + capabilitiesLoaded: true, + }), + useReasoningEffort: () => ({ + supportedEfforts: [], + workerEffortWarning: undefined, + }), +})); +vi.mock("@/components/TaskPromptPicker", () => ({ + TaskPromptPicker: () => null, +})); +vi.mock("@/components/CriteriaPicker", () => ({ + CriteriaPicker: () => null, +})); +vi.mock("@/components/CreateCriterionDialog", () => ({ + CreateCriterionDialog: () => null, +})); +vi.mock("@/components/SkillPicker", () => ({ + SkillPicker: () => null, +})); +vi.mock("@/components/CodebasePicker", () => ({ + CodebasePicker: () => null, +})); +vi.mock("@/components/ResourcePicker", () => ({ + ResourcePicker: () => null, +})); +vi.mock("@/components/ExtensionPicker", () => ({ + ExtensionPicker: () => null, +})); +vi.mock("@/components/ProfilePicker", () => ({ + ProfilePicker: () => null, +})); +vi.mock("@/components/ProfileCreateForm", () => ({ + ProfileCreateForm: () => null, +})); +vi.mock("@/components/AgentBadge", () => ({ + AgentBadge: () => null, +})); + +import { SubmitRun } from "./SubmitRun"; +import { api } from "@/lib/api"; + +afterEach(() => { + cleanup(); + vi.clearAllMocks(); +}); + +function renderExperimentForm() { + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); + + return render( + + + + + , + ); +} + +describe("SubmitRun Experiment composition", () => { + it("infers an empty Experiment until a scenario is entered", async () => { + renderExperimentForm(); + + expect( + await screen.findByRole("button", { name: /Create empty experiment/ }), + ).toBeTruthy(); + expect(screen.queryByLabelText("Tags")).toBeNull(); + expect(screen.queryByText("Gate pipeline")).toBeNull(); + expect( + screen.queryByRole("switch", { name: "Toggle advanced options" }), + ).toBeNull(); + expect(screen.getAllByText(/No scenario selected/).length).toBeGreaterThan(0); + + fireEvent.change(screen.getByLabelText(/Task/), { + target: { value: "Implement the benchmark task" }, + }); + + expect(screen.getByText("Gate pipeline")).toBeTruthy(); + expect( + screen.getByRole("switch", { name: "Toggle advanced options" }), + ).toBeTruthy(); + expect( + screen.getByRole("button", { + name: /Create experiment with 5 runs/, + }), + ).toBeTruthy(); + }); + + it("submits an explicit empty payload when no scenario is selected", async () => { + renderExperimentForm(); + + fireEvent.change(screen.getByLabelText("Name *"), { + target: { value: "Reserved benchmark" }, + }); + fireEvent.click( + screen.getByRole("button", { name: /Create empty experiment/ }), + ); + + await waitFor(() => { + expect(api.createExperiment).toHaveBeenCalledWith({ + metadata: { name: "Reserved benchmark" }, + empty: true, + }); + }); + }); +}); diff --git a/apps/portal/src/pages/SubmitRun.tsx b/apps/portal/src/pages/SubmitRun.tsx index ec6b639ea..d4e56990e 100644 --- a/apps/portal/src/pages/SubmitRun.tsx +++ b/apps/portal/src/pages/SubmitRun.tsx @@ -2,9 +2,13 @@ // Licensed under the MIT License. import { useState, useEffect, useCallback, useMemo } from "react"; -import { useNavigate } from "react-router-dom"; +import { Link, useNavigate } from "react-router-dom"; import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; -import { api } from "@/lib/api"; +import { api, type RunCreateInput } from "@/lib/api"; +import { + LEGACY_REQUEST_COMPOSITION_DEPRECATION, + type ExperimentView, +} from "@/lib/experiments"; import { Button } from "@/components/ui/button"; import { Card, CardContent, CardHeader, CardTitle, CardDescription } from "@/components/ui/card"; import { Input } from "@/components/ui/input"; @@ -184,7 +188,11 @@ function CollapsibleCard({ icon: Icon, title, summary, open, onOpenChange, disab // SubmitRun // ──────────────────────────────────────────────────────────────────────────── -export function SubmitRun() { +export function SubmitRun({ + mode = "run", +}: { + mode?: "run" | "experiment"; +}) { const navigate = useNavigate(); const queryClient = useQueryClient(); const strictAgentCapabilities = useStrictAgentCapabilities(); @@ -215,6 +223,8 @@ export function SubmitRun() { const [occurrences, setOccurrences] = useState(5); const [priority, setPriority] = useState(0); const [agentsMd, setAgentsMd] = useState(""); + const [experimentName, setExperimentName] = useState(""); + const [experimentDescription, setExperimentDescription] = useState(""); // Optional add-ons const [selectedMcpServers, setSelectedMcpServers] = useState([]); @@ -285,6 +295,8 @@ export function SubmitRun() { queryFn: () => api.listRuns({ limit: 3 }), }); const recentRuns: Run[] = recentRunsResp?.data ?? []; + const hasScenario = task.trim().length > 0; + const showRunComposition = mode === "run" || hasScenario; // ─── Derived ──────────────────────────────────────────────────────────── const activeMcpServers = mcpServers.filter((s: McpServerDocument) => !s.deletedAt); @@ -664,20 +676,48 @@ export function SubmitRun() { : []; const gateErrors = [...gateValidationErrors, ...promptValidationErrors]; - const submitMutation = useMutation({ - mutationFn: api.submitRun, + type SubmitResult = + | ExperimentView + | Awaited>; + + const submitMutation = useMutation< + SubmitResult, + Error, + { request?: RunCreateInput } + >({ + mutationFn: ({ request }: { request?: RunCreateInput }) => + mode === "experiment" + ? api.createExperiment({ + metadata: { + name: experimentName.trim(), + ...(experimentDescription.trim() + ? { description: experimentDescription.trim() } + : {}), + }, + ...(request ? { request } : { empty: true }), + }) + : api.submitRun(request!), onSuccess: (data) => { - if ("ids" in data && data.ids.length > 1) { - navigate("/runs"); - } else if ("id" in data) { + if (mode === "experiment" && "experimentId" in data) { + navigate(`/experiments/${data.experimentId}`); + return; + } + if ("id" in data && typeof data.id === "string") { navigate(`/runs/${data.id}`); - } else { - navigate("/runs"); + return; + } + if ("ids" in data && Array.isArray(data.ids) && data.ids[0]) { + navigate(`/runs/${data.ids[0]}`); } }, }); const doSubmit = () => { + if (mode === "experiment" && !hasScenario) { + if (!experimentName.trim()) return; + submitMutation.mutate({}); + return; + } if (!task.trim()) return; const normalizedVariationDrafts = variationDrafts.filter((v) => v.profileId.trim().length > 0); const profileVariations: string[] = normalizedVariationDrafts.map((v) => @@ -688,7 +728,7 @@ export function SubmitRun() { ? (selectedProfileVersion ? `${selectedProfileId}@${selectedProfileVersion}` : selectedProfileId) : null; - submitMutation.mutate({ + const request: RunCreateInput = { scenario: { task: task.trim(), criteria: pickedCriteria }, ...(pickedObservations.length > 0 ? { observations: pickedObservations } : {}), ...(inVariationMode ? {} : { ...(worker ? { worker } : {}) }), @@ -713,7 +753,8 @@ export function SubmitRun() { : { ...(baseProfileSpec ? { profileId: baseProfileSpec } : {}), }), - }); + }; + submitMutation.mutate({ request }); }; const handleSubmit = (e: React.FormEvent) => { @@ -734,20 +775,25 @@ export function SubmitRun() { }); const canSubmit = - !!task.trim() && - !!selectedAgent && - isAgentAvailable(selectedAgent) && - !selectedVersionIsHistorical && - selectedVariationTargetsAvailable && - !submitMutation.isPending && - !(selectedAgent && activeModelIds.length > 0 && !model) && - !(maxIterations !== 1 && pickedCriteria.length === 0) && - gateErrors.length === 0; + mode === "experiment" && !hasScenario + ? !!experimentName.trim() && !submitMutation.isPending + : !!task.trim() && + (mode === "run" || !!experimentName.trim()) && + !!selectedAgent && + isAgentAvailable(selectedAgent) && + !selectedVersionIsHistorical && + selectedVariationTargetsAvailable && + !submitMutation.isPending && + !(selectedAgent && activeModelIds.length > 0 && !model) && + !(maxIterations !== 1 && pickedCriteria.length === 0) && + gateErrors.length === 0; const selectedVariationCount = variationDrafts.filter((v) => v.profileId.trim().length > 0).length; const compositionProfileCount = selectedProfileId ? 1 + selectedVariationCount : 0; const expandedRunCount = compositionProfileCount > 0 ? compositionProfileCount * occurrences : occurrences; const submitRunCount = expandedRunCount; + const usesLegacyComposition = + mode === "run" && (occurrences > 1 || selectedVariationCount > 0); const lastVariationDraft = variationDrafts.length > 0 ? variationDrafts[variationDrafts.length - 1] : null; const lastVariationName = lastVariationDraft @@ -804,7 +850,7 @@ export function SubmitRun() { useCommandEnter(doSubmit, canSubmit); // ─── Render helpers ───────────────────────────────────────────────────── - const summaryChips: string[] = [ + const configuredSummaryChips: string[] = [ `${maxIterations} iteration${maxIterations === 1 ? "" : "s"}`, `${pickedCriteria.length} requirements criteri${pickedCriteria.length === 1 ? "on" : "a"}`, pickedObservations.length > 0 ? `${pickedObservations.length} observation${pickedObservations.length === 1 ? "" : "s"}` : "", @@ -823,6 +869,10 @@ export function SubmitRun() { agentsMd.trim() ? "AGENTS.md" : "", selectedExtensions.length > 0 ? `${selectedExtensions.length} ext` : "", ].filter(Boolean); + const summaryChips = + mode === "experiment" && !hasScenario + ? ["No scenario selected", "0 runs"] + : configuredSummaryChips; return (
@@ -832,28 +882,89 @@ export function SubmitRun() { type="button" variant="ghost" size="icon" - onClick={() => navigate("/runs")} - aria-label="Back to runs" + onClick={() => navigate(mode === "experiment" ? "/experiments" : "/runs")} + aria-label={mode === "experiment" ? "Back to experiments" : "Back to runs"} >
-

New Run

-

Submit a benchmark run to a coding agent worker

+

+ {mode === "experiment" ? "New Experiment" : "New Run"} +

+

+ {mode === "experiment" + ? "Create a named benchmark composition and its first execution" + : "Submit a benchmark run to a coding agent worker"} +

- + + {showRunComposition && ( + + )} + {mode === "experiment" && ( + + + Experiment metadata + + Name this reproducible composition. Add a scenario below to + configure runs, or create it empty. + + + +
+ + setExperimentName(event.target.value)} + placeholder="Copilot versus Claude on authentication" + /> +
+
+ +