feat: GEPA prompt optimizer with Scope adapter (AGENTS.md optimization)

.github/workflows/ci.yml

@@ -668,6 +668,32 @@ jobs:
           name: ctrf-test-results-gateway
           path: ctrf/
 
+  gepa:
+    name: "GEPA Optimizer: test"
+    needs: [detect-changes]
+    if: >
+      github.event_name == 'workflow_dispatch' ||
+      needs.detect-changes.outputs.gepa == 'true'
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+          cache-dependency-glob: apps/gepa-optimizer/uv.lock
+
+      - name: Install locked dependencies
+        working-directory: apps/gepa-optimizer
+        run: uv sync --locked
+
+      - name: Run tests
+        working-directory: apps/gepa-optimizer
+        run: uv run pytest
+
   notice-check:
     name: Third-party NOTICE
     # Runs on every PR with no detect-changes gate: the NOTICE / NOTICE-REVIEW
@@ -782,6 +808,7 @@ jobs:
       db-migrations: ${{ steps.changes.outputs.db-migrations }}
       shared: ${{ steps.changes.outputs.shared }}
       gateway: ${{ steps.changes.outputs.gateway }}
+      gepa: ${{ steps.changes.outputs.gepa }}
       coder-acp-copilot-windows: ${{ steps.changes.outputs.coder-acp-copilot-windows }}
       model-scanning: ${{ steps.changes.outputs.model-scanning }}
       typescript: ${{ steps.changes.outputs.typescript }}
@@ -834,6 +861,10 @@ jobs:
               - 'packages/db-migrations/**'
             gateway:
               - 'apps/gateway/**'
+            gepa:
+              - 'apps/gepa-optimizer/**'
+              - 'package.json'
+              - '.github/workflows/ci.yml'
             coder-acp-copilot-windows:
               - 'apps/workers/coder-acp-copilot-windows/**'
               - 'apps/workers/coder-acp-copilot/versions.env'
@@ -874,7 +905,7 @@ jobs:
   ci-summary:
     name: CI Summary
     runs-on: ubuntu-latest
-    needs: [test, integration-test, integration-test-queue, windows-build, lint, build, cli-bundle-test, gateway, llm-evals, notice-check, license-headers]
+    needs: [test, integration-test, integration-test-queue, windows-build, lint, build, cli-bundle-test, gateway, gepa, llm-evals, notice-check, license-headers]
     if: always()
 
     steps:
@@ -890,7 +921,8 @@ jobs:
              [ "${{ needs.llm-evals.result }}" = "success" -o "${{ needs.llm-evals.result }}" = "skipped" ] && \
              [ "${{ needs.notice-check.result }}" = "success" -o "${{ needs.notice-check.result }}" = "skipped" ] && \
              [ "${{ needs.license-headers.result }}" = "success" ] && \
-             [ "${{ needs.gateway.result }}" = "success" -o "${{ needs.gateway.result }}" = "skipped" ]; then
+             [ "${{ needs.gateway.result }}" = "success" -o "${{ needs.gateway.result }}" = "skipped" ] && \
+             [ "${{ needs.gepa.result }}" = "success" -o "${{ needs.gepa.result }}" = "skipped" ]; then
             echo "✅ All checks passed!"
             exit 0
           else
@@ -906,5 +938,6 @@ jobs:
             echo "  Third-party NOTICE: ${{ needs.notice-check.result }}"
             echo "  License Headers: ${{ needs.license-headers.result }}"
             echo "  Gateway: ${{ needs.gateway.result }}"
+            echo "  GEPA Optimizer: ${{ needs.gepa.result }}"
             exit 1
           fi

AGENTS.md

@@ -23,6 +23,7 @@ apps/
   model-scanners/               # Feature detection for Copilot and Anthropic models
   version-checkers/             # Poll for new agent/tool releases
   key-updaters/                 # GitHub auth cookie management for VS Code Web
+  gepa-optimizer/               # Python+uv GEPA driver — optimizes AGENTS.md via Scope runs
 packages/
   shared/                       # Monorepo foundation — types, DB models, queue/blob/redis clients
   db-migrations/                # MongoDB migration framework (mongo-migrate-ts)
@@ -141,6 +142,20 @@ Express service for centralized token storage, validation, and round-robin distr
 
 - Architecture: [docs/architecture/token-manager.md](docs/architecture/token-manager.md)
 
+### GEPA Optimizer (`apps/gepa-optimizer/`)
+
+Python + uv app (outside the pnpm workspace, like the Rust `apps/gateway`) that
+runs the [GEPA](https://github.com/gepa-ai/gepa) reflective prompt-optimization
+algorithm to optimize an **AGENTS.md** file. Each candidate is scored by a real
+Scope run (Copilot CLI worker + Claude Haiku 4.5) via a custom `ScopeAdapter`:
+submit standalone requests with `agentsMd`, add them through canonical Experiment
+membership revisions, poll exact requests/executions/reports, and score from the
+run's `criteriaResults`. Scoring is derived, never persisted. Run tests with
+`pnpm test:gepa` (`uv run pytest`); CI runs `uv sync --locked && uv run pytest`.
+
+- README: [apps/gepa-optimizer/README.md](apps/gepa-optimizer/README.md)
+- Experiment integration: [docs/architecture/experiments.md](docs/architecture/experiments.md)
+
 ### Shared Package (`packages/shared/`)
 
 Monorepo foundation. Exports types (runs, iterations, criteria, scenarios, personas), Mongoose models, and service clients (queue, blob, redis, config loader, criteria-store, criteria-provider). All apps and workers depend on it — breaking changes here affect everything.

ENV_VARIABLES.md

@@ -1369,6 +1369,73 @@ URL of the gateway/DevProxy REST API. Used to start/stop recording, check status
 
 Host port mapping for the Copilot DevProxy REST API in Docker Compose.
 
+## GEPA Optimizer Configuration (`apps/gepa-optimizer`)
+
+The Python + uv GEPA app optimizes an AGENTS.md file by scoring each candidate
+with real Scope runs. Configure it through `apps/gepa-optimizer/.env`; see the
+co-located `.env.example`.
+
+### SCOPE_API_URL
+**Default:** `http://localhost:3100`
+
+Base URL of the Scope REST API used to submit and poll candidate runs.
+
+### SCOPE_PROJECT_ID
+**Default:** none (required)
+
+Project that owns the criteria and candidate runs. Scope's catalog and run
+creation APIs are project-scoped, so the optimizer fails fast when this is not
+configured.
+
+### SCOPE_EXPERIMENT_ID / SCOPE_EXPERIMENT_NAME
+**Defaults:** empty / `GEPA AGENTS.md optimization <timestamp>`
+
+With no ID, the optimizer creates one empty canonical Experiment and uses its
+membership revisions for every evaluated candidate batch. Set
+`SCOPE_EXPERIMENT_ID` to target an existing Experiment; the optimizer reads its
+current revision before adding requests. `SCOPE_EXPERIMENT_NAME` controls the
+metadata name only when creating a new Experiment.
+
+### SCOPE_WORKER / SCOPE_MODEL
+**Defaults:** `coder-acp-copilot` / `claude-haiku-4.5`
+
+Worker and model used to evaluate each candidate.
+
+### SCOPE_MAX_CONCURRENCY
+**Default:** `10`
+**Type:** integer
+
+Maximum number of candidate runs kept in flight concurrently.
+
+### SCOPE_RUN_TIMEOUT_SECONDS / SCOPE_POLL_INTERVAL_SECONDS
+**Defaults:** `1800` / `15`
+
+Per-run timeout and polling interval, in seconds.
+
+### SCOPE_MAX_ITERATIONS
+**Default:** `10`
+
+Maximum agent/judge iterations for each candidate run.
+
+### SCOPE_SCORE_STRATEGY
+**Default:** `mean`
+**Options:** `mean` | `final` | `weighted`
+
+Aggregation applied to per-turn criteria pass fractions. `weighted` computes
+`0.8 * final + 0.2 * mean`.
+
+### GEPA_MAX_METRIC_CALLS / GEPA_REFLECTION_MINIBATCH_SIZE
+**Defaults:** `12` / `2`
+
+Optimizer budget. Each metric call is a complete Scope agent run.
+
+### GEPA_REFLECTION_LM
+**Default:** `azure/gpt-5.5`
+
+litellm model string for the reflection model that rewrites AGENTS.md.
+Provider-specific credentials such as `AZURE_API_KEY`, `AZURE_API_BASE`, and
+`AZURE_API_VERSION` are consumed by litellm.
+
 ## Observability
 
 ### OTEL_COLLECTOR_ENDPOINT

apps/cli/src/run-get-action.ts

@@ -5,7 +5,7 @@
  * Action handler for `run get` subcommand — extracted for testability.
  */
 import { colorLevel, dimTimestamp, errorText, successText, label, value, banner, warnBanner, criterionIcon } from "./utils/style.js";
-import { TAXONOMY_ELEMENT_METADATA } from "shared";
+import { TAXONOMY_ELEMENT_METADATA } from "shared/observation-taxonomy";
 import { GATE_METADATA, GATE_ORDER, type ConversationTurn, type GateId, type GateRunSummary, type RequestDocument, type TaxonomyElementId } from "shared/types";
 import { formatData, isMachineReadable } from "./utils/formatters.js";
 import type { OutputFormat, DisplayField } from "./utils/types.js";

apps/gepa-optimizer/.env.example

@@ -0,0 +1,41 @@
+# Scope REST API base URL (no trailing slash).
+SCOPE_API_URL=http://localhost:3100
+SCOPE_PROJECT_ID=
+
+# Leave empty to create one canonical Experiment for this optimization. Set an
+# existing empty/current Experiment id to append evaluation revisions there.
+SCOPE_EXPERIMENT_ID=
+SCOPE_EXPERIMENT_NAME=
+
+# Scope worker + model used to evaluate each AGENTS.md candidate.
+SCOPE_WORKER=coder-acp-copilot
+SCOPE_MODEL=claude-haiku-4.5
+
+# Max number of Scope runs kept in-flight at once. Defaults to 10 to match the
+# Linux coder-acp-copilot KEDA ceiling (maxReplicaCount: 10).
+SCOPE_MAX_CONCURRENCY=10
+
+# Per-run polling: how long to wait for a run to reach status=done, and how
+# often to poll (seconds).
+SCOPE_RUN_TIMEOUT_SECONDS=1800
+SCOPE_POLL_INTERVAL_SECONDS=15
+
+# Multi-turn iterations per Scope run.
+SCOPE_MAX_ITERATIONS=10
+
+# Scoring strategy over a run's turns: mean | final | weighted.
+SCOPE_SCORE_STRATEGY=mean
+
+# GEPA budget. Each metric call is ONE full (slow) Scope agent run, so keep
+# this small. See README for budgeting math.
+GEPA_MAX_METRIC_CALLS=12
+GEPA_REFLECTION_MINIBATCH_SIZE=2
+
+# Reflection LM passed to gepa.optimize (litellm model string). The reflection
+# LM rewrites the AGENTS.md; it is NOT the agent under test.
+GEPA_REFLECTION_LM=azure/gpt-5.5
+
+# Azure OpenAI credentials for the reflection LM (litellm reads these).
+AZURE_API_KEY=
+AZURE_API_BASE=
+AZURE_API_VERSION=2024-12-01-preview

apps/gepa-optimizer/.gitignore

@@ -0,0 +1,6 @@
+.venv/
+__pycache__/
+*.pyc
+.env
+runs/
+.pytest_cache/

apps/gepa-optimizer/.python-version

@@ -0,0 +1 @@
+3.12

apps/gepa-optimizer/README.md

@@ -0,0 +1,116 @@
+# GEPA × Scope Optimizer (`apps/gepa-optimizer`)
+
+Optimizes an **`AGENTS.md`** file with the [GEPA](https://github.com/gepa-ai/gepa)
+reflective prompt-optimization algorithm, where every candidate is scored by a
+**real Scope benchmark run** — the Copilot CLI worker (Linux) with the
+**Claude Haiku 4.5** model.
+
+This mirrors the reference experiment (`geo-experiments/gepa-prompt-optimization`)
+but, instead of optimizing a system prompt evaluated by a single LLM completion,
+it optimizes an `AGENTS.md` evaluated by a multi-turn Scope agent run and judged
+against a criteria DAG.
+
+```mermaid
+flowchart LR
+    Candidate[GEPA AGENTS.md candidate] --> Submit[Submit standalone Scope requests]
+    Submit --> Membership[Publish one Experiment membership revision]
+    Membership --> Runs[Poll exact request runs]
+    Runs --> Score[Derive criteria scores]
+    Membership --> Execution[Poll exact Experiment execution]
+    Execution --> Reports[Read exact execution reports]
+    Score --> Reflection[Build reflective evidence]
+    Reports --> Reflection
+    Reflection --> Candidate
+```
+
+One optimization owns one canonical Scope Experiment. By default the entry
+point creates an empty Experiment, then every `evaluate()` call adds its newly
+submitted requests through `POST /experiments/:id/requests`. That publishes a
+new immutable revision and execution, whose exact identities and reports are
+stored in the GEPA outputs and reflective trajectories. Set
+`SCOPE_EXPERIMENT_ID` to target an existing Experiment instead.
+
+## Polyglot exception
+
+This is a **Python** app (managed by [`uv`](https://docs.astral.sh/uv/)) living in
+a TypeScript/pnpm monorepo, exactly like the Rust `apps/gateway`. It is
+intentionally **outside** the pnpm workspace. `uv.lock` is committed and CI runs
+`uv sync --locked && uv run pytest`. Root scripts: `pnpm test:gepa`,
+`pnpm build:gepa`.
+
+## Setup
+
+```bash
+cd apps/gepa-optimizer
+uv sync                 # create .venv and install deps from uv.lock
+cp .env.example .env    # then edit SCOPE_API_URL, SCOPE_PROJECT_ID, criteria ids, reflection LM creds
+```
+
+## How scoring works
+
+The GEPA score is **derived, never persisted in Scope**. For a finished run:
+
+- The **denominator is the number of requested criteria** (not whatever Scope
+  returned), so a missing/skipped criterion always counts as failed.
+- A criterion only counts as passed when it was **evaluated and passed**;
+  `evaluated == false` (skipped because a DAG ancestor failed) counts as failed.
+- Per-turn score = `passed / requested`. The run score aggregates turns via
+  `SCOPE_SCORE_STRATEGY`:
+  - `mean` (default) — average across all turns;
+  - `final` — last turn only (closest to "task success");
+  - `weighted` — `0.8 * final + 0.2 * mean`.
+- A run that did not succeed, has no turns, or errored scores `0.0` (never `NaN`).
+
+## Budget
+
+GEPA counts **one metric call per dataset example**, and each metric call is a
+full (minutes-long) Scope agent run. Rough spend per optimization:
+
+```text
+len(valset)                      # seed validation
++ 2 * reflection_minibatch_size  # parent + child minibatch per proposal
++ len(valset)                    # full re-validation of each accepted candidate
+```
+
+Keep `valset` tiny and `GEPA_MAX_METRIC_CALLS` small.
+
+## Run
+
+Edit `src/gepa_optimizer/dataset.py` so the criterion ids exist in your Scope
+instance (`pnpm cli criteria list`), then:
+
+```bash
+uv run gepa-optimizer
+# or
+uv run python -m gepa_optimizer.main
+```
+
+Results (best candidate, Pareto front, and a best-effort lineage DAG
+reconstructed from GEPA's parent indices) are written to `runs/<timestamp>/results.json`.
+The artifact also records the canonical `experimentId` and final
+`experimentRevisionId`. Every evaluated request remains visible in the normal
+Scope Runs, Experiment Detail, CLI, and API surfaces.
+
+## Tests
+
+```bash
+uv run pytest            # or, from repo root: pnpm test:gepa
+```
+
+Unit tests cover scoring, reflective-dataset construction, lineage
+reconstruction, the REST client (via `httpx.MockTransport`), and the adapter
+(with a stubbed client). No live Scope instance is required.
+
+## Concurrency
+
+`SCOPE_MAX_CONCURRENCY` (default **10**) caps in-flight runs to match the Linux
+`coder-acp-copilot` KEDA ceiling (`maxReplicaCount: 10`). Submitting more is
+harmless — extras just queue — but yields no speedup.
+
+## Lineage
+
+Parent→child lineage is reconstructed **after** optimization from GEPA's public
+`result.parents` / `result.candidates` and saved into `results.json`
+(`root` / `mutation` / `merge`). Candidate evaluation requests are grouped by
+canonical Experiment membership revisions rather than a caller-provided
+correlation field.

apps/gepa-optimizer/pyproject.toml

@@ -0,0 +1,32 @@
+[project]
+name = "gepa-optimizer"
+version = "0.1.0"
+description = "GEPA prompt-optimization driver that optimizes AGENTS.md files evaluated by real Scope benchmark runs."
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "gepa>=0.1.1",
+    "httpx>=0.27",
+    "python-dotenv>=1.0",
+    "litellm>=1.0",
+]
+
+[project.scripts]
+gepa-optimizer = "gepa_optimizer.main:main"
+
+[dependency-groups]
+dev = [
+    "pytest>=8.0",
+    "pytest-asyncio>=0.23",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/gepa_optimizer"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+asyncio_mode = "auto"