feat: DAG-based post-processing + taxonomy readout post-processor

AGENTS.md

@@ -19,6 +19,7 @@ apps/
     coder-acp-copilot/          # GitHub Copilot worker (ACP SDK)
     coder-acp-claude-code/      # Claude Code worker (ACP SDK)
     report-generator/           # Post-run report generation
+    taxonomy/                   # Post-run taxonomy generation
   model-scanners/               # Feature detection for Copilot and Anthropic models
   version-checkers/             # Poll for new agent/tool releases
   key-updaters/                 # GitHub auth cookie management for VS Code Web
@@ -69,6 +70,7 @@ Each worker implements the same queue-processor interface but adapts a different
 | `coder-acp-copilot` | GitHub Copilot | ACP SDK v0.14.1 |
 | `coder-acp-claude-code` | Claude Code | ACP SDK v0.13.1 |
 | `report-generator` | — | Copilot SDK |
+| `taxonomy` | — | Copilot SDK |
 
 Workers consume tasks from Azure Storage Queues (named `queue-<worker-name>`) and write results to MongoDB and Blob Storage. Each has its own Dockerfile and docker-compose profile.
 

ENV_VARIABLES.md

@@ -889,6 +889,34 @@ Timeout for the Copilot SDK session used by the report-generator worker. If the
 
 Git commit hash embedded in reporter metadata. Automatically set during CI/CD builds. Used to track which version of the report-generator produced a given report.
 
+## Taxonomy Handler Configuration
+
+### AZURE_STORAGE_QUEUE_TAXONOMY
+**Default:** `pp-taxonomy-queue`
+**Type:** string
+
+Azure Storage Queue name for taxonomy-generation jobs. The scheduler enqueues `{ type: "taxonomy", requestId, runId }` messages here after the `pp-atif` dependency completes, and the taxonomy worker polls this queue.
+
+### TAXONOMY_MODEL
+**Default:** `gpt-5.4`
+**Type:** string
+
+The LLM model used by the taxonomy worker (via the Copilot SDK) to generate structured taxonomy JSON from run metadata and ATIF trajectories.
+
+> **Model choice matters for taxonomy quality.** A/B testing across nine models on a multi-iteration run with a real defect (agent lost context in iteration 2) showed the verdict ranged from 72 to 100 depending on the model. `gpt-4.1` (the previous default) missed action items and, on harder runs, produced the thinnest output; weaker models rubber-stamped a buggy run with a perfect score. `gpt-5.4` was the best value: it correctly caught the critical defect while using the fewest tokens and lowest cost of the models tested. Stronger alternatives (e.g. `claude-opus-4-7`) yield richer evidence at higher token cost; avoid `gpt-5.3-codex` as a default (≈1M tokens / ~7 min per run).
+
+### SCOPE_MT_API_URL
+**Default:** `http://localhost:3001` (local), `http://api:80` (Docker)
+**Type:** URL string
+
+Base URL of the Scope API. The taxonomy worker calls this to fetch the request document and active run state through its Copilot tools.
+
+### SESSION_TIMEOUT_MS
+**Default:** `300000` (5 minutes)
+**Type:** integer (milliseconds)
+
+Timeout for the Copilot SDK session used by the taxonomy worker. If the model does not finish generating a valid taxonomy JSON document within this window, the worker marks `run.handlerStatus.pp-taxonomy` as failed.
+
 ## Scheduler Configuration
 
 ### SCHEDULER_POLL_INTERVAL_MS

apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap

@@ -3267,6 +3267,9 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = `
             ],
             "type": "string",
           },
+          "taxonomyUrl": {
+            "type": "string",
+          },
           "tokenUsage": {
             "$ref": "#/components/schemas/TokenUsage",
           },
@@ -10309,6 +10312,40 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = `
         ],
       },
     },
+    "/api/v1/requests/{id}/runs/{runId}/taxonomy": {
+      "get": {
+        "parameters": [
+          {
+            "in": "path",
+            "name": "id",
+            "required": true,
+            "schema": {
+              "type": "string",
+            },
+          },
+          {
+            "in": "path",
+            "name": "runId",
+            "required": true,
+            "schema": {
+              "type": "string",
+            },
+          },
+        ],
+        "responses": {
+          "200": {
+            "description": "Taxonomy v1 JSON document (scorecard, behavior analysis, action list)",
+          },
+          "404": {
+            "description": "Not found — taxonomy not yet generated",
+          },
+        },
+        "summary": "Download taxonomy JSON for a specific attempt",
+        "tags": [
+          "Requests",
+        ],
+      },
+    },
     "/api/v1/requests/{id}/runs/{runId}/tool-calls": {
       "get": {
         "parameters": [
@@ -10411,6 +10448,32 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = `
         ],
       },
     },
+    "/api/v1/requests/{id}/taxonomy": {
+      "get": {
+        "parameters": [
+          {
+            "in": "path",
+            "name": "id",
+            "required": true,
+            "schema": {
+              "type": "string",
+            },
+          },
+        ],
+        "responses": {
+          "200": {
+            "description": "Taxonomy v1 JSON document (scorecard, behavior analysis, action list)",
+          },
+          "404": {
+            "description": "Not found — taxonomy not yet generated",
+          },
+        },
+        "summary": "Download taxonomy JSON for the run",
+        "tags": [
+          "Requests",
+        ],
+      },
+    },
     "/api/v1/requests/{id}/tool-calls": {
       "get": {
         "parameters": [

apps/api/src/archive-har.test.ts

@@ -99,6 +99,17 @@ describe("rewriteHarUrlsForArchive", () => {
     expect(result.run!.turns![1].toolCallsUrl).toBe("iteration-2.tool-calls.jsonl");
     expect(result.run!.turns![2].toolCallsUrl).toBeUndefined();
   });
+
+  it("rewrites run.taxonomyUrl to taxonomy.json", () => {
+    const resource = {
+      run: {
+        taxonomyUrl: "https://storage.blob.core.windows.net/snapshots/abc123/runs/attempt-1/taxonomy.json",
+        turns: [],
+      },
+    };
+    const result = rewriteHarUrlsForArchive(resource);
+    expect(result.run!.taxonomyUrl).toBe("taxonomy.json");
+  });
 });
 
 // --- blobNameFromSnapshotsUrl ---
@@ -902,7 +913,36 @@ describe("packRunIntoTar", () => {
     await packRunIntoTar(p, run, container, "run-011", () => true);
     p.finalize();
     await entriesPromise;
-
     expect(run.codebase!.archiveUrl).toBe(blobUrl);
   });
+
+  it("bundles run-level taxonomy.json and rewrites taxonomyUrl", async () => {
+    const { pack } = await import("tar-stream");
+    const p = pack();
+    const taxonomyData = Buffer.from('{"topics":["agent"]}');
+    const container = makeMockBlobContainer({
+      "run-012/runs/attempt-1/taxonomy.json": { body: taxonomyData, length: taxonomyData.length },
+    });
+
+    const run: ArchivableRun = {
+      _id: "run-012",
+      run: {
+        taxonomyUrl: "https://storage.blob.core.windows.net/snapshots/run-012/runs/attempt-1/taxonomy.json",
+        turns: [],
+      },
+    };
+
+    const entriesPromise = collectPackEntries(p);
+    await packRunIntoTar(p, run, container, "run-012", () => true);
+    p.finalize();
+
+    const entries = await entriesPromise;
+    const names = entries.map(e => e.name);
+    expect(names).toContain("run-012/taxonomy.json");
+    expect(entries.find(e => e.name === "run-012/taxonomy.json")!.data).toEqual(taxonomyData);
+
+    const yaml = entries.find(e => e.name === "run-012/run.yaml")!.data.toString();
+    expect(yaml).toContain("taxonomy.json");
+    expect(yaml).not.toContain("blob.core.windows.net");
+  });
 });

apps/api/src/archive-har.ts

@@ -41,11 +41,12 @@ export function blobNameFromLogsUrl(url: string): string | null {
 
 /**
  * Deep-clone a run resource and rewrite `harUrl`, `rawChatUrl`,
- * `chatResultUrl`, and `toolCallsUrl` fields to relative archive paths.
+ * `taxonomyUrl`, `chatResultUrl`, and `toolCallsUrl` fields to relative archive paths.
  *
  * - `run.harUrl`              → `"run.har"`
  * - Per-turn `harUrl`          → `"iteration-{N}.har"`
  * - `run.rawChatUrl`           → `"run.chat-export.json"`
+ * - `run.taxonomyUrl`          → `"taxonomy.json"`
  * - Per-turn `rawChatUrl`      → `"iteration-{N}.chat-export.json"`
  * - Per-turn `chatResultUrl`   → `"iteration-{N}.chat-result.json"`
  * - Per-turn `toolCallsUrl`    → `"iteration-{N}.tool-calls.jsonl"`
@@ -54,6 +55,7 @@ export function rewriteHarUrlsForArchive<T extends {
   run?: {
     harUrl?: string;
     rawChatUrl?: string;
+    taxonomyUrl?: string;
     turns?: Array<{ iteration: number; harUrl?: string; rawChatUrl?: string; chatResultUrl?: string; toolCallsUrl?: string; atifUrl?: string; [key: string]: unknown }>;
     [key: string]: unknown;
   };
@@ -66,6 +68,9 @@ export function rewriteHarUrlsForArchive<T extends {
     if (copy.run.rawChatUrl) {
       copy.run.rawChatUrl = "run.chat-export.json";
     }
+    if (copy.run.taxonomyUrl) {
+      copy.run.taxonomyUrl = "taxonomy.json";
+    }
     if (copy.run.turns) {
       for (const turn of copy.run.turns) {
         if (turn.harUrl) {
@@ -383,6 +388,7 @@ export interface ArchivableRun {
     logsUrl?: string;
     harUrl?: string;
     rawChatUrl?: string;
+    taxonomyUrl?: string;
     turns?: Array<{
       iteration: number;
       snapshotUrl?: string;
@@ -441,6 +447,7 @@ export async function packRunIntoTar(
   const turns = resource.run?.turns ?? [];
   const topHarUrl = resource.run?.harUrl;
   const topRawChatUrl = resource.run?.rawChatUrl;
+  const topTaxonomyUrl = resource.run?.taxonomyUrl;
 
   // Entries 2..N: iteration snapshots as-is (.tar.gz blobs)
   for (const turn of turns) {
@@ -594,6 +601,26 @@ export async function packRunIntoTar(
     }
   }
 
+  // Bundle run-level taxonomy JSON into the archive.
+  if (topTaxonomyUrl) {
+    try {
+      const blobName = blobNameFromSnapshotsUrl(topTaxonomyUrl);
+      if (blobName) {
+        const blobClient = container.getBlockBlobClient(blobName);
+        const downloadResponse = await blobClient.download();
+        if (downloadResponse.readableStreamBody && downloadResponse.contentLength) {
+          const entry = pack.entry({
+            name: `${id}/taxonomy.json`,
+            size: downloadResponse.contentLength,
+          });
+          await pipeline(downloadResponse.readableStreamBody, entry);
+        }
+      }
+    } catch (blobError) {
+      if (!isBlobNotFound(blobError)) throw blobError;
+    }
+  }
+
   // Bundle log events (logs.jsonl) from the logs container.
   // Logs are stored as AppendBlobs, so use getBlobClient (type-agnostic) rather
   // than getBlockBlobClient — the latter returns contentLength: undefined for

apps/api/src/index.ts

@@ -27,6 +27,7 @@ import { registerRequestsCancelRoutes } from "./routes/requests/cancel.js";
 import { registerRequestsLogsRoutes } from "./routes/requests/logs.js";
 import { registerRequestsHarRoutes } from "./routes/requests/har.js";
 import { registerRequestsAtifRoutes } from "./routes/requests/atif.js";
+import { registerRequestsTaxonomyRoutes } from "./routes/requests/taxonomy.js";
 import { registerRequestsVideoRoutes } from "./routes/requests/video.js";
 import { registerRequestsToolCallsRoutes } from "./routes/requests/tool-calls.js";
 import { registerRequestsSnapshotsRoutes } from "./routes/requests/snapshots.js";
@@ -343,6 +344,7 @@ registerRequestsCancelRoutes(routeCtx);
 registerRequestsLogsRoutes(routeCtx);
 registerRequestsHarRoutes(routeCtx);
 registerRequestsAtifRoutes(routeCtx);
+registerRequestsTaxonomyRoutes(routeCtx);
 registerRequestsVideoRoutes(routeCtx);
 registerRequestsToolCallsRoutes(routeCtx);
 registerRequestsSnapshotsRoutes(routeCtx);

apps/api/src/routes/requests/taxonomy.test.ts

@@ -0,0 +1,107 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+/**
+ * Unit tests for taxonomy download endpoints:
+ *   - GET /api/v1/requests/:id/taxonomy
+ *   - GET /api/v1/requests/:id/runs/:runId/taxonomy
+ */
+import { describe, it, expect, beforeAll, beforeEach, vi } from "vitest";
+import supertest from "supertest";
+import { app, _injectTestDependencies } from "../../index.js";
+import { createAllMockDependencies } from "../../test-helpers.js";
+import { readableFrom, rawParser, blobUrl, VARIANTS, RUN_LEVEL, rewireBlobMocks } from "./test-blob-helpers.js";
+
+vi.mock("db-migrations/check-migrations", () => ({
+  checkMigrations: vi.fn().mockResolvedValue({ ready: true, applied: ["001", "002"], pending: [] }),
+}));
+
+vi.hoisted(() => {
+  process.env.STORAGE_CONNECTION_STRING = "DefaultEndpointsProtocol=https;AccountName=mockaccount;AccountKey=bW9jaw==;EndpointSuffix=core.windows.net";
+});
+
+vi.mock("../../llm.js", () => ({ isLlmAvailable: vi.fn().mockReturnValue(false) }));
+vi.mock("../../prompt-feature-llm.js", () => ({ isLlmAvailable: vi.fn().mockReturnValue(false) }));
+vi.mock("../../task-prompt-llm.js", () => ({ isTaskPromptLlmAvailable: vi.fn().mockReturnValue(false) }));
+
+const { mockDownload, mockGetProperties, mockGetBlockBlobClient, mockGetBlobClient, mockGetContainerClient } = vi.hoisted(() => {
+  const mockDownload = vi.fn();
+  const mockGetProperties = vi.fn();
+  const mockGetBlockBlobClient = vi.fn().mockReturnValue({ download: mockDownload, getProperties: mockGetProperties });
+  const mockGetBlobClient = vi.fn().mockReturnValue({ download: mockDownload, getProperties: mockGetProperties });
+  const mockGetContainerClient = vi.fn().mockReturnValue({ getBlockBlobClient: mockGetBlockBlobClient, getBlobClient: mockGetBlobClient });
+  return { mockDownload, mockGetProperties, mockGetBlockBlobClient, mockGetBlobClient, mockGetContainerClient };
+});
+
+vi.mock("@azure/storage-blob", async (importOriginal) => {
+  const original = await importOriginal<typeof import("@azure/storage-blob")>();
+  return {
+    ...original,
+    BlobServiceClient: { fromConnectionString: vi.fn().mockReturnValue({ getContainerClient: mockGetContainerClient }) },
+  };
+});
+
+describe("taxonomy endpoints", () => {
+  let mocks: ReturnType<typeof createAllMockDependencies>;
+
+  beforeAll(() => {
+    mocks = createAllMockDependencies();
+    _injectTestDependencies(mocks);
+  });
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    mocks = createAllMockDependencies();
+    _injectTestDependencies(mocks);
+    rewireBlobMocks({ mockDownload, mockGetProperties, mockGetBlockBlobClient, mockGetBlobClient, mockGetContainerClient });
+    mockDownload.mockResolvedValue({ readableStreamBody: readableFrom('{"status":"ok"}'), contentLength: 15 });
+    mockGetProperties.mockResolvedValue({ contentLength: 15 });
+  });
+
+  describe.each(VARIANTS)("$label", (variant) => {
+    const taxonomyUrl = (id: string) => variant.url(id) + "taxonomy";
+
+    it("returns 404 when request not found", async () => {
+      (mocks.collection.findOne as ReturnType<typeof vi.fn>).mockResolvedValue(null);
+      const res = await supertest(app).get(taxonomyUrl("missing"));
+      expect(res.status).toBe(404);
+      expect(res.body).toMatchObject({ error: "Request not found" });
+    });
+
+    it("returns 404 when taxonomyUrl is not set", async () => {
+      variant.mockRequest(mocks, "req-1", { _id: "run-1", attemptNumber: 1, status: "done" });
+      const res = await supertest(app).get(taxonomyUrl("req-1"));
+      expect(res.status).toBe(404);
+      expect(res.body).toMatchObject({ error: "Taxonomy not yet generated for this run" });
+    });
+
+    it("returns taxonomy JSON when taxonomyUrl is set", async () => {
+      variant.mockRequest(mocks, "req-1", {
+        _id: "run-1",
+        attemptNumber: 1,
+        status: "done",
+        taxonomyUrl: blobUrl("req-1/taxonomy.json"),
+      });
+
+      const res = await supertest(app).get(taxonomyUrl("req-1")).buffer(true).parse(rawParser);
+      expect(res.status).toBe(200);
+      expect(res.headers["content-type"]).toMatch("application/json");
+      expect(res.headers["content-disposition"]).toContain("req-1-taxonomy.json");
+      expect(mockGetBlockBlobClient).toHaveBeenCalledWith("req-1/taxonomy.json");
+    });
+  });
+
+  it("works with per-run variant", async () => {
+    RUN_LEVEL.mockRequest(mocks, "req-1", {
+      _id: "run-1",
+      attemptNumber: 1,
+      status: "done",
+      taxonomyUrl: blobUrl("req-1/runs/run-1/taxonomy.json"),
+    });
+
+    const res = await supertest(app).get(RUN_LEVEL.url("req-1") + "taxonomy").buffer(true).parse(rawParser);
+    expect(res.status).toBe(200);
+    expect(res.headers["content-disposition"]).toContain("req-1-taxonomy.json");
+    expect(mockGetBlockBlobClient).toHaveBeenCalledWith("req-1/runs/run-1/taxonomy.json");
+  });
+});