feat: observation criteria with taxonomy and run/iteration subject (#1156)

ENV_VARIABLES.md

@@ -936,7 +936,13 @@ TTL for the DAG dispatcher's in-memory cache of handler topology (the `services`
 **Default:** `post-processor-queue`
 **Type:** string
 
-Azure Storage Queue name used by both the scheduler (to enqueue post-processing work) and the post-processor worker (to dequeue). Must match between the two services.
+Azure Storage Queue name used by both the scheduler (to enqueue post-processing work) and the post-processor worker (to dequeue). Must match between the two services. The `pp-taxonomy` observation handler shares this queue.
+
+### JUDGE_URL
+**Default:** falls back to `JUDGE_SERVICE_URL`, then `http://localhost:3002`
+**Type:** URL string
+
+Base URL of the judge service used by the `pp-taxonomy` post-process handler to evaluate observation criteria per iteration (snapshot + ATIF trajectory → boolean + evidence). The handler reuses the shared `JudgeClient`, so `JUDGE_CLIENT_TIMEOUT` / `JUDGE_CLIENT_RETRIES` apply. In K8s the post-processor reads `JUDGE_SERVICE_URL` from the `worker-config` ConfigMap (`envFrom`); in docker-compose the `post-processor` service sets `JUDGE_SERVICE_URL: http://judge:80` explicitly (it uses the `common-env` anchor, which — unlike `worker-env` — does not carry it). See issue #1156 and [docs/architecture/post-processing.md](docs/architecture/post-processing.md).
 
 ### SCHEDULER_URL
 **Default:** (not set)

apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap

@@ -539,6 +539,12 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = `
           "judgeFeedback": {
             "type": "string",
           },
+          "observationResults": {
+            "items": {
+              "$ref": "#/components/schemas/CriterionResult",
+            },
+            "type": "array",
+          },
           "passed": {
             "type": "boolean",
           },
@@ -682,9 +688,32 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = `
             "pattern": "^[a-z][a-z0-9_]*$",
             "type": "string",
           },
+          "kind": {
+            "default": "gate",
+            "enum": [
+              "gate",
+              "observation",
+            ],
+            "type": "string",
+          },
           "prompt": {
             "type": "string",
           },
+          "subject": {
+            "enum": [
+              "run",
+              "iteration",
+            ],
+            "type": "string",
+          },
+          "taxonomyElementId": {
+            "enum": [
+              "dimension:idiomatic-use",
+              "dimension:dependency-currency",
+              "dimension:configuration-correctness",
+            ],
+            "type": "string",
+          },
         },
         "required": [
           "id",
@@ -938,6 +967,12 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = `
           "model": {
             "type": "string",
           },
+          "observations": {
+            "items": {
+              "type": "string",
+            },
+            "type": "array",
+          },
           "persona": {
             "$ref": "#/components/schemas/Persona",
           },
@@ -1156,9 +1191,31 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = `
           "id": {
             "type": "string",
           },
+          "kind": {
+            "enum": [
+              "gate",
+              "observation",
+            ],
+            "type": "string",
+          },
           "prompt": {
             "type": "string",
           },
+          "subject": {
+            "enum": [
+              "run",
+              "iteration",
+            ],
+            "type": "string",
+          },
+          "taxonomyElementId": {
+            "enum": [
+              "dimension:idiomatic-use",
+              "dimension:dependency-currency",
+              "dimension:configuration-correctness",
+            ],
+            "type": "string",
+          },
         },
         "required": [
           "id",
@@ -1204,12 +1261,34 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = `
           "id": {
             "type": "string",
           },
+          "kind": {
+            "enum": [
+              "gate",
+              "observation",
+            ],
+            "type": "string",
+          },
           "projectId": {
             "type": "string",
           },
           "prompt": {
             "type": "string",
           },
+          "subject": {
+            "enum": [
+              "run",
+              "iteration",
+            ],
+            "type": "string",
+          },
+          "taxonomyElementId": {
+            "enum": [
+              "dimension:idiomatic-use",
+              "dimension:dependency-currency",
+              "dimension:configuration-correctness",
+            ],
+            "type": "string",
+          },
           "updatedAt": {
             "format": "date-time",
             "type": [
@@ -2692,6 +2771,12 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = `
           "model": {
             "type": "string",
           },
+          "observations": {
+            "items": {
+              "type": "string",
+            },
+            "type": "array",
+          },
           "persona": {
             "$ref": "#/components/schemas/Persona",
           },
@@ -3185,6 +3270,12 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = `
           "mcpRegistered": {
             "type": "boolean",
           },
+          "observationResults": {
+            "items": {
+              "$ref": "#/components/schemas/CriterionResult",
+            },
+            "type": "array",
+          },
           "os": {
             "properties": {
               "arch": {
@@ -3734,9 +3825,31 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = `
             },
             "type": "array",
           },
+          "kind": {
+            "enum": [
+              "gate",
+              "observation",
+            ],
+            "type": "string",
+          },
           "prompt": {
             "type": "string",
           },
+          "subject": {
+            "enum": [
+              "run",
+              "iteration",
+            ],
+            "type": "string",
+          },
+          "taxonomyElementId": {
+            "enum": [
+              "dimension:idiomatic-use",
+              "dimension:dependency-currency",
+              "dimension:configuration-correctness",
+            ],
+            "type": "string",
+          },
         },
         "type": "object",
       },
@@ -4925,6 +5038,20 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = `
               "type": "string",
             },
           },
+          {
+            "description": "Filter by criterion kind",
+            "in": "query",
+            "name": "kind",
+            "required": false,
+            "schema": {
+              "description": "Filter by criterion kind",
+              "enum": [
+                "gate",
+                "observation",
+              ],
+              "type": "string",
+            },
+          },
           {
             "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.",
             "in": "query",

apps/api/src/endpoints.test.ts

@@ -269,6 +269,66 @@
 
       expect(res.status).toBe(400);
     });
+
+    it("creates an observation criterion with a taxonomy element", async () => {
+      (mocks.criteriaCollection.findOne as any).mockResolvedValue(null);
+
+      const res = await request(app)
+        .post("/api/v1/criteria")
+        .send({
+          id: "obs_crit",
+          prompt: "Did the agent use current deps?",
+          dependsOn: [],
+          kind: "observation",
+          taxonomyElementId: "dimension:dependency-currency",
+        });
+
+      expect(res.status).toBe(201);
+      expect(res.body).toHaveProperty("kind", "observation");
+      expect(res.body).toHaveProperty("taxonomyElementId", "dimension:dependency-currency");
+    });
+
+    it("rejects an unknown taxonomyElementId", async () => {
+      (mocks.criteriaCollection.findOne as any).mockResolvedValue(null);
+
+      const res = await request(app)
+        .post("/api/v1/criteria")
+        .send({ id: "bad_tax", prompt: "x", dependsOn: [], kind: "observation", taxonomyElementId: "dimension:nope" });
+
+      expect(res.status).toBe(400);
+    });
+
+    it("defaults an observation's subject to 'run'", async () => {
+      (mocks.criteriaCollection.findOne as any).mockResolvedValue(null);
+
+      const res = await request(app)
+        .post("/api/v1/criteria")
+        .send({ id: "obs_subj", prompt: "Did the agent add then remove a dep?", dependsOn: [], kind: "observation" });
+
+      expect(res.status).toBe(201);
+      expect(res.body).toHaveProperty("subject", "run");
+    });
+
+    it("creates an observation with an explicit subject:'iteration' override", async () => {
+      (mocks.criteriaCollection.findOne as any).mockResolvedValue(null);
+
+      const res = await request(app)
+        .post("/api/v1/criteria")
+        .send({ id: "obs_iter", prompt: "x", dependsOn: [], kind: "observation", subject: "iteration" });
+
+      expect(res.status).toBe(201);
+      expect(res.body).toHaveProperty("subject", "iteration");
+    });
+
+    it("rejects subject:'run' on a gate criterion", async () => {
+      (mocks.criteriaCollection.findOne as any).mockResolvedValue(null);
+
+      const res = await request(app)
+        .post("/api/v1/criteria")
+        .send({ id: "gate_run", prompt: "x", dependsOn: [], kind: "gate", subject: "run" });
+
+      expect(res.status).toBe(400);
+    });
   });
 
   describe("POST /api/v1/criteria/seed", () => {

apps/api/src/llm.ts

@@ -3,6 +3,7 @@
 
 import {
   gatesSatisfyInvariant,
+  type CriterionSubject,
   type GateId,
 } from "shared";
 import {
@@ -31,6 +32,11 @@ Given a natural-language description of a behavior or pattern to detect, you mus
    - Contain only lowercase letters, digits, and underscores
    - Be concise but descriptive (e.g., has_unit_tests, uses_typescript, build_succeeds)
 
+3. Classify the evaluation SUBJECT — whether the behavior must be judged across the WHOLE run or within a SINGLE iteration in isolation:
+   - "run": the behavior spans multiple steps/iterations or describes a transition over the course of the session — phrasing like "at any point", "ever", "then later", "added then removed", "during the run", or anything only confirmable by viewing the entire run together.
+   - "iteration": the behavior is a single end-state fact or a per-step property judgeable from one iteration alone (e.g., "uses TypeScript", "has a Dockerfile", "ran the test suite this turn").
+   When unsure, prefer "run".
+
 Here are examples of good criteria prompts:
 - Codebase evidence: "The project uses the React framework. Look for a react dependency in package.json and .jsx or .tsx files containing React components."
 - Codebase evidence: "The project uses Azure Bicep for infrastructure as code. Look for *.bicep files, bicepconfig.json, or a main.bicep entry point."
@@ -39,7 +45,7 @@ Here are examples of good criteria prompts:
 - Tool-output evidence (non-gate command): "The agent verified the running server responds. The captured tool-call history contains a request to the local endpoint (e.g. a curl to http://localhost) that returned a 2xx status."
 
 Respond with ONLY a JSON object in this exact format (no markdown, no code fences):
-{"prompt": "your evaluation prompt here", "suggestedId": "your_suggested_id"}`;
+{"prompt": "your evaluation prompt here", "suggestedId": "your_suggested_id", "suggestedSubject": "run"}`;
 
 /**
  * Per-gate description of the evidence a criterion compatible with that gate is
@@ -165,6 +171,10 @@ export interface GenerateResult {
   suggestedId: string;
   suggestedParents: string[];
   suggestedChildren: string[];
+  /** Suggested evaluation granularity for observation criteria. Defaults to
+   *  "run"; "iteration" only when the model clearly detects an end-state /
+   *  per-step property. Ignored by gate criteria (always per-iteration). #1156. */
+  suggestedSubject: CriterionSubject;
 }
 
 export interface CriteriaPromptRequest {
@@ -192,6 +202,12 @@ function sanitizeId(suggestedId: unknown): string {
   return sanitized || "new_criterion";
 }
 
+/** Coerce a model-suggested subject. Defaults to "run" — "iteration" is only
+ *  honored when the model returns it exactly (matching "prefer run when unsure"). */
+function sanitizeSubject(value: unknown): CriterionSubject {
+  return value === "iteration" ? "iteration" : "run";
+}
+
 function parseJson(content: string): any {
   const cleaned = content.replace(/```json\s*|```\s*/g, "").trim();
   try {
@@ -222,7 +238,7 @@ export function buildCriteriaAuthoringRequest(
 
 export function parseCriteriaAuthoringResponse(
   content: string,
-): { prompt: string; suggestedId: string } {
+): { prompt: string; suggestedId: string; suggestedSubject: CriterionSubject } {
   const parsed: unknown = parseJson(content);
   if (
     !parsed ||
@@ -236,6 +252,9 @@ export function parseCriteriaAuthoringResponse(
   return {
     prompt: String(value.prompt).trim(),
     suggestedId: sanitizeId(value.suggestedId),
+    suggestedSubject: sanitizeSubject(
+      (parsed as { suggestedSubject?: unknown }).suggestedSubject,
+    ),
   };
 }
 
@@ -306,7 +325,7 @@ async function author(
   behavior: string,
   gates?: GateId[],
   explicitModel?: string,
-): Promise<{ prompt: string; suggestedId: string }> {
+): Promise<{ prompt: string; suggestedId: string; suggestedSubject: CriterionSubject }> {
   const content = await chat(
     buildCriteriaAuthoringRequest(behavior, gates, model),
     explicitModel,
@@ -431,5 +450,6 @@ export async function generateCriteriaPrompt(
     suggestedId: authored.suggestedId,
     suggestedParents,
     suggestedChildren,
+    suggestedSubject: authored.suggestedSubject,
   };
 }