Add reusable experiment support across runs

apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap

@@ -912,6 +912,10 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = `
           "codebaseRevisionId": {
             "type": "string",
           },
+          "experimentId": {
+            "minLength": 1,
+            "type": "string",
+          },
           "extensions": {
             "items": {
               "type": "string",
@@ -2662,6 +2666,9 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = `
               "null",
             ],
           },
+          "experimentId": {
+            "type": "string",
+          },
           "extensions": {
             "items": {
               "type": "string",
@@ -8707,6 +8714,24 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = `
               "type": "string",
             },
           },
+          {
+            "in": "query",
+            "name": "experimentId",
+            "required": false,
+            "schema": {
+              "anyOf": [
+                {
+                  "type": "string",
+                },
+                {
+                  "items": {
+                    "type": "string",
+                  },
+                  "type": "array",
+                },
+              ],
+            },
+          },
           {
             "in": "query",
             "name": "profileId",
@@ -8850,6 +8875,7 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = `
                 "task",
                 "submissionId",
                 "profile",
+                "experiment",
               ],
               "type": "string",
             },

apps/api/src/endpoints.test.ts

@@ -721,6 +721,50 @@ describe("API Endpoints", () => {
       expect(res.body.data[0]).toHaveProperty("key", "sub-1");
     });
 
+    it("returns grouped results when groupBy=experiment", async () => {
+      const groupedDocs = [
+        { key: "exp-1", label: "exp-1", runIds: ["r1"], aggregates: { count: 1, turns: null, duration: null, promptTokens: null, completionTokens: null, llmCalls: null, statusCounts: {}, outcomeCounts: {} }, uniform: {} },
+      ];
+      (mocks.collection.aggregate as any)
+        .mockReturnValueOnce({ toArray: vi.fn().mockResolvedValue([{ _id: "exp-1" }]) })
+        .mockReturnValueOnce({ toArray: vi.fn().mockResolvedValue(groupedDocs) })
+        .mockReturnValueOnce({ toArray: vi.fn().mockResolvedValue([]) })
+        .mockReturnValueOnce({ toArray: vi.fn().mockResolvedValue([]) });
+
+      const res = await request(testServer()).get(`/api/v1/requests?groupBy=experiment&projectId=${TEST_PROJECT_ID}`);
+
+      expect(res.status).toBe(200);
+      expect(res.body.data[0]).toHaveProperty("key", "exp-1");
+    });
+
+    it("fetches members with missing experiment ids for the no-experiment group", async () => {
+      (mocks.collection.aggregate as any)
+        .mockReturnValueOnce({ toArray: vi.fn().mockResolvedValue([{ _id: "no-experiment" }]) })
+        .mockReturnValueOnce({ toArray: vi.fn().mockResolvedValue([]) })
+        .mockReturnValueOnce({ toArray: vi.fn().mockResolvedValue([]) })
+        .mockReturnValueOnce({ toArray: vi.fn().mockResolvedValue([]) });
+
+      const res = await request(testServer()).get(
+        `/api/v1/requests?groupBy=experiment&projectId=${TEST_PROJECT_ID}`,
+      );
+
+      expect(res.status).toBe(200);
+      const phase2Pipeline = (mocks.collection.aggregate as any).mock.calls[1][0];
+      expect(phase2Pipeline[0]).toEqual({
+        $match: {
+          $and: [
+            expect.any(Object),
+            {
+              $or: [
+                { experimentId: { $exists: false } },
+                { experimentId: null },
+              ],
+            },
+          ],
+        },
+      });
+    });
+
     it("calls aggregate pipeline when groupBy is provided", async () => {
       (mocks.collection.aggregate as any)
         .mockReturnValueOnce({ toArray: vi.fn().mockResolvedValue([]) }); // keys (empty)
@@ -761,6 +805,22 @@ describe("API Endpoints", () => {
       );
     });
 
+    it("passes experimentId filter to the find query", async () => {
+      (mocks.collection.find as any).mockReturnValue({
+        sort: vi.fn().mockReturnValue({
+          limit: vi.fn().mockReturnValue({
+            toArray: vi.fn().mockResolvedValue([]),
+          }),
+        }),
+      });
+
+      await request(testServer()).get(`/api/v1/requests?experimentId=exp-42&projectId=${TEST_PROJECT_ID}`);
+
+      expect(mocks.collection.find).toHaveBeenCalledWith(
+        expect.objectContaining({ experimentId: "exp-42" }),
+      );
+    });
+
     it("passes outcome filter to find query", async () => {
       (mocks.collection.find as any).mockReturnValue({
         sort: vi.fn().mockReturnValue({
@@ -2028,6 +2088,23 @@ describe("API Endpoints", () => {
       expect(doc).toHaveProperty("agentsMdParentIds", ["agents-p1", "agents-p2"]);
     });
 
+    it("persists experimentId on the request", async () => {
+      setupCopilotAgent();
+
+      const res = await request(testServer())
+        .post(`/api/v1/requests?worker=coder-acp-copilot&projectId=${TEST_PROJECT_ID}`)
+        .send({
+          scenario: { task: "Build something", criteria: ["works"] },
+          model: "claude-haiku-4.5",
+          experimentId: "exp-42",
+        });
+
+      expect(res.status).toBe(201);
+      const doc = (mocks.collection.insertOne as any).mock.calls[0][0];
+      expect(doc).toHaveProperty("experimentId", "exp-42");
+      expect(res.body).toHaveProperty("experimentId", "exp-42");
+    });
+
     it("omits AGENTS.md fields when not provided", async () => {
       setupCopilotAgent();
 
@@ -2320,7 +2397,7 @@ describe("API Endpoints", () => {
   // ===================================================================
 
   describe("POST /api/v1/requests/bulk-resubmit", () => {
-    it("preserves agentVersion and taskPromptId from original run", async () => {
+    it("preserves agentVersion, taskPromptId, and experimentId from original run", async () => {
       const originalRun = {
         _id: "run-original",
         scenario: { task: "Build a form", criteria: ["has_react"] },
@@ -2329,6 +2406,7 @@ describe("API Endpoints", () => {
         model: "gpt-4o",
         agentVersion: "copilot-0.0.415",
         taskPromptId: "tp-123",
+        experimentId: "exp-42",
         personaInstructions: "Be helpful",
         createdAt: new Date(),
         maxIterations: 5,
@@ -2364,6 +2442,7 @@ describe("API Endpoints", () => {
       expect(insertCall).toHaveLength(1);
       expect(insertCall[0]).toHaveProperty("agentVersion", "copilot-0.0.420");
       expect(insertCall[0]).toHaveProperty("taskPromptId", "tp-123");
+      expect(insertCall[0]).toHaveProperty("experimentId", "exp-42");
       expect(insertCall[0]).toHaveProperty("model", "gpt-4o");
       expect(insertCall[0]).toHaveProperty("maxIterations", 5);
     });

apps/api/src/grouping.test.ts

@@ -38,6 +38,14 @@ describe("buildGroupingPipeline", () => {
     expect(addFields._groupKey).toBeDefined();
   });
 
+  it("groups missing experiment ids under a stable sentinel", () => {
+    const pipeline = buildGroupingPipeline("experiment");
+    const addFields = pipeline[0].$addFields as Record<string, unknown>;
+    expect(addFields._groupKey).toEqual({
+      $ifNull: ["$experimentId", { $literal: "no-experiment" }],
+    });
+  });
+
   it("$group stage uses _groupKey as _id", () => {
     const pipeline = buildGroupingPipeline("task");
     const group = pipeline[1].$group as Record<string, unknown>;

apps/api/src/grouping.ts

@@ -12,18 +12,28 @@ import type { GroupByKey, RunGroup, AggregateStats, GroupUniformValues } from "s
  * The pipeline is appended after a $match stage with the caller's filter.
  */
 export function buildGroupingPipeline(
-  groupBy: "task" | "submissionId" | "profile",
+  groupBy: "task" | "submissionId" | "profile" | "experiment",
 ): Document[] {
   const groupField =
-    groupBy === "task" ? "$taskPromptId" : groupBy === "profile" ? "$profileId" : "$submissionId";
+    groupBy === "task"
+      ? "$taskPromptId"
+      : groupBy === "profile"
+        ? "$profileId"
+        : groupBy === "experiment"
+          ? "$experimentId"
+          : "$submissionId";
   const fallbackGroupField =
-    groupBy === "profile" ? { $literal: "no-profile" } : { $literal: "no-submission" };
+    groupBy === "profile"
+      ? { $literal: "no-profile" }
+      : groupBy === "experiment"
+        ? { $literal: "no-experiment" }
+        : { $literal: "no-submission" };
   // `task` groups purely by the always-present `taskPromptId` — it is
   // materialized on every run-creation path (findOrCreate → taskPrompt._id) and
   // backfilled onto legacy docs by migration 001, so the group key is always a
   // real id the member-fetch filter (`{ taskPromptId: key }`) can match. Only
-  // `profile`/`submission` fall back to a fixed literal sentinel for runs that
-  // genuinely lack the field.
+  // `profile`/`experiment`/`submission` fall back to a fixed literal sentinel
+  // for runs that genuinely lack the field.
   const groupKeyExpr =
     groupBy === "task" ? groupField : { $ifNull: [groupField, fallbackGroupField] };
 
@@ -162,6 +172,7 @@ export function buildGroupingPipeline(
             branches: [
               { case: { $eq: ["$_id", "no-submission"] }, then: "No submission ID" },
               { case: { $eq: ["$_id", "no-profile"] }, then: "No profile" },
+              { case: { $eq: ["$_id", "no-experiment"] }, then: "No experiment ID" },
             ],
             default: { $ifNull: ["$_firstTask", "$_id"] },
           },