[COVAL-5819] feat: add Python metric draft commands and audit request fields

README.md

@@ -65,6 +65,7 @@ coval simulated-conversations list --run-id <run_id>
 | `coval test-sets` | Manage test set collections |
 | `coval test-cases` | Manage individual test cases |
 | `coval personas` | Manage simulated personas |
+| `coval python-metric-drafts` | Create, preview, and publish Python metric drafts |
 | `coval metrics` | Manage evaluation metrics |
 | `coval models` | Inspect supported metric models |
 | `coval mutations` | Test agent variations with config overrides |
@@ -307,9 +308,28 @@ coval personas update <persona_id> --audio-degradation cell-handoff
 # Make the persona interrupt the agent more often (NONE, LOW, MEDIUM, or HIGH)
 coval personas update <persona_id> --interruption-rate HIGH
 
+# Enable interruption instructions
+coval personas update <persona_id> --interruption-instructions-enabled true
+
 # Clear a preset. A flag can only set a value, so clearing needs an explicit null.
 coval personas update <persona_id> --input-json '{"situate_speaker":null}'
 
+# Create an immutable Python metric draft and preview it on a simulation output
+coval python-metric-drafts create \
+  --display-name "Greeting quality" \
+  --source-file ./greeting_quality.py \
+  --output-type boolean \
+  --description "Checks whether the agent greets the caller" \
+  --require-audio false \
+  --require-traces false \
+  --frame-table turns
+coval python-metric-drafts previews <draft_id> create \
+  --simulation-output-ids <simulation_output_id>
+coval python-metric-drafts previews <draft_id> get <preview_id>
+coval python-metric-drafts publish <draft_id> \
+  --preview-ids <successful_preview_id> \
+  --fingerprint <draft_fingerprint>
+
 # Create a dashboard and make it the organization default
 coval dashboards create \
   --name "Production Metrics" \

api-coverage-report.md

@@ -13,12 +13,12 @@ only when coverage actually changes.
 | Metric | Value |
 | --- | ---: |
 | Reconciliation status | PASS |
-| Published operations | 186 |
-| First-class CLI operations | 119 |
-| Reviewed gaps | 67 |
-| Client operations | 143 |
-| Published request fields on covered operations | 451 |
-| Request fields modeled by the CLI | 451 |
+| Published operations | 199 |
+| First-class CLI operations | 125 |
+| Reviewed gaps | 74 |
+| Client operations | 149 |
+| Published request fields on covered operations | 472 |
+| Request fields modeled by the CLI | 472 |
 | Reviewed request-field gaps | 0 |
 
 Catalog: https://api.coval.dev/v1/openapi
@@ -67,6 +67,11 @@ Catalog: https://api.coval.dev/v1/openapi
 - `GET /alerts`
 - `GET /alerts/{alert_id}`
 - `GET /alerts/{alert_id}/events`
+- `GET /conversation-python-analyses/{analysis_id}`
+- `GET /conversations/simulated/{conversation_id}/comments`
+- `GET /conversations/simulated/{conversation_id}/comments/{comment_id}`
+- `GET /conversations/uploaded/{conversation_id}/comments`
+- `GET /conversations/uploaded/{conversation_id}/comments/{comment_id}`
 - `GET /integrations/slack`
 - `GET /metrics/flows`
 - `GET /metrics/recently-deleted`
@@ -104,6 +109,7 @@ Catalog: https://api.coval.dev/v1/openapi
 - `POST /alerts`
 - `POST /alerts/{alert_id}/test-evaluate`
 - `POST /audio:upload`
+- `POST /conversation-python-analyses`
 - `POST /conversations/simulated:rerunMetrics`
 - `POST /integrations/slack/connect`
 - `POST /metrics/flows`
@@ -113,6 +119,7 @@ Catalog: https://api.coval.dev/v1/openapi
 - `POST /metrics/{metric_id}/versions/{version_id}/revert`
 - `POST /personas/{persona_id}/duplicate`
 - `POST /personas/{persona_id}/versions/{version_id}/revert`
+- `POST /python-metrics:fromAnalysis`
 - `POST /review-annotations:withMetricOutputs`
 - `POST /review-projects/disagreement-state`
 - `POST /review-projects/{project_id}/complete-conversation`

api-coverage.toml

@@ -5,11 +5,11 @@
 
 [snapshot]
 catalog_url = "https://api.coval.dev/v1/openapi"
-reviewed_at = "2026-10-19"
-published_operations = 186
-cli_supported_operations = 119
-published_request_fields = 451
-cli_modeled_request_fields = 451
+reviewed_at = "2026-11-02"
+published_operations = 199
+cli_supported_operations = 125
+published_request_fields = 472
+cli_modeled_request_fields = 472
 
 [[allowed_extra]]
 operation = "POST /test-cases/{test_case_id}/media:upload-url"
@@ -438,3 +438,31 @@ reason = "Served but deliberately hidden from the published schema as an interna
 operation = "POST /conversations/uploaded:submit"
 field = "audio"
 reason = "Served but deliberately hidden from the published schema as an internal field; the API still accepts it."
+
+[[known_gap]]
+operation = "GET /conversations/simulated/{conversation_id}/comments"
+reason = "Transcript comments need a dedicated cursor-paginated CLI resource with comments:read and workspace selection; they are distinct from metric review annotations."
+
+[[known_gap]]
+operation = "GET /conversations/simulated/{conversation_id}/comments/{comment_id}"
+reason = "Transcript comment retrieval belongs to the dedicated comment-thread CLI resource, with conversation ownership and workspace access checks."
+
+[[known_gap]]
+operation = "GET /conversations/uploaded/{conversation_id}/comments"
+reason = "Transcript comments need a dedicated cursor-paginated CLI resource with comments:read and workspace selection; they are distinct from metric review annotations."
+
+[[known_gap]]
+operation = "GET /conversations/uploaded/{conversation_id}/comments/{comment_id}"
+reason = "Transcript comment retrieval belongs to the dedicated comment-thread CLI resource, with conversation ownership and workspace access checks."
+
+[[known_gap]]
+operation = "POST /conversation-python-analyses"
+reason = "Exploratory audio analysis needs a request-UUID recovery and asynchronous-result CLI workflow; the CLI currently supports the immutable metric-draft and preview workflow instead."
+
+[[known_gap]]
+operation = "GET /conversation-python-analyses/{analysis_id}"
+reason = "Analysis polling belongs to the dedicated exploratory-audio-analysis CLI workflow, including recovery and expired-result handling."
+
+[[known_gap]]
+operation = "POST /python-metrics:fromAnalysis"
+reason = "Publishing from an exploratory analysis needs its own explicit approval workflow; the CLI currently publishes immutable metric drafts using exact fingerprints and successful previews."

scripts/audit_api_coverage.py

@@ -261,9 +261,19 @@ def _request_body_types(client_method: str, block: str) -> set[str]:
     """Return the Rust types a client function serializes as its request body."""
 
     types: set[str] = set()
-    for match in re.finditer(
-        r"\.(?:post|patch|put)\(\s*url\s*,\s*&(?P<binding>[A-Za-z0-9_]+)\s*\)", block
-    ):
+    body_calls = re.findall(r"\.(?:post|patch|put)\(", block)
+    body_matches = list(
+        re.finditer(
+            r"\.(?:post|patch|put)\(\s*url\s*,\s*&(?P<binding>[A-Za-z0-9_]+)\s*\)",
+            block,
+        )
+    )
+    if len(body_calls) != len(body_matches):
+        raise RuntimeError(
+            "could not resolve every request-body call in client function "
+            f"{client_method}; use a local url binding"
+        )
+    for match in body_matches:
         binding = match.group("binding")
         if re.search(rf"\b{binding}\s*:\s*serde_json::Value\b", block):
             types.add(PASSTHROUGH_BODY)

scripts/test_audit_api_coverage.py

@@ -436,6 +436,44 @@ def test_rejects_an_unresolvable_body_binding(self):
             audit_api_coverage._request_body_types("agents.create", block)
 
 
+class RequestBodyAuditRegressionTests(unittest.TestCase):
+    def test_rejects_an_unrecognized_json_body_call(self):
+        block = """
+    pub async fn create(&self, request: models::CreateThingRequest) -> Result<T, E> {
+        self.0.post(self.0.url("/v1/things"), &request).await
+    }
+"""
+        with self.assertRaisesRegex(
+            RuntimeError, "could not resolve every request-body"
+        ):
+            audit_api_coverage._request_body_types("things.create", block)
+
+    def test_audits_all_python_metric_draft_request_fields(self):
+        operations, _ = audit_api_coverage._command_operations(
+            audit_api_coverage._client_operations()
+        )
+        fields = audit_api_coverage._cli_request_fields(
+            operations, audit_api_coverage._model_request_fields()
+        )
+        expected = {
+            "POST /python-metric-drafts": {
+                "display_name",
+                "source",
+                "output_type",
+                "target_metric_id",
+                "target_version_id",
+                "description",
+                "unit",
+                "requirements",
+            },
+            "POST /python-metric-drafts/{id}/previews": {"simulation_output_ids"},
+            "POST /python-metric-drafts/{id}/publish": {"preview_ids", "fingerprint"},
+        }
+        for operation, expected_fields in expected.items():
+            with self.subTest(operation=operation):
+                self.assertEqual(expected_fields, fields.get(operation))
+
+
 class ModelRequestFieldTests(unittest.TestCase):
     def _fields(self, source: str) -> dict[str, set[str]]:
         with tempfile.TemporaryDirectory() as directory:

src/agent_discovery.rs

@@ -453,6 +453,47 @@ const RESOURCE_SPECS: &[ResourceSpec] = &[
         }],
         pitfalls: &["Metric type controls required creation fields."],
     },
+    ResourceSpec {
+        name: "python-metric-drafts",
+        commands: &[
+            "context",
+            "create",
+            "get",
+            "publish",
+            "previews.create",
+            "previews.get",
+            "previews.cancel",
+        ],
+        description: "Python metric drafts are immutable candidate definitions that must be previewed successfully before publishing.",
+        id_name: "draft_id",
+        id_format: "Draft ID returned by the Python metric drafts API",
+        requires: &["simulated-conversations"],
+        optional: &[],
+        produces: &["Python metric"],
+        related: &["metrics", "simulated-conversations"],
+        workflows: &[
+            WorkflowSpec {
+                name: "Create an immutable Python metric draft",
+                argv: &["python-metric-drafts", "create"],
+            },
+            WorkflowSpec {
+                name: "Preview a Python metric draft",
+                argv: &[
+                    "python-metric-drafts",
+                    "previews",
+                    "<draft_id>",
+                    "create",
+                    "--simulation-output-ids",
+                    "<simulation_output_id>",
+                ],
+            },
+        ],
+        pitfalls: &[
+            "Source or input/output changes require a new draft and preview.",
+            "Preview execution is asynchronous; poll its preview ID until terminal.",
+            "Publish only with the exact draft fingerprint and successful preview IDs.",
+        ],
+    },
     ResourceSpec {
         name: "mutations",
         commands: &["context", "list", "get", "create", "update", "delete"],

src/cli.rs

@@ -95,6 +95,11 @@ pub enum Commands {
         #[command(subcommand)]
         command: commands::personas::PersonaCommands,
     },
+    #[command(name = "python-metric-drafts")]
+    PythonMetricDrafts {
+        #[command(subcommand)]
+        command: commands::python_metric_drafts::PythonMetricDraftCommands,
+    },
     Metrics {
         #[command(subcommand)]
         command: commands::metrics::MetricCommands,
@@ -172,6 +177,7 @@ impl Commands {
                 commands::personas::PersonaCommands::BackgroundSounds { .. } => "background-sounds",
                 _ => "personas",
             },
+            Self::PythonMetricDrafts { .. } => "python-metric-drafts",
             Self::Metrics { .. } => "metrics",
             Self::Models { .. } => "models",
             Self::Mutations { .. } => "mutations",
@@ -208,6 +214,7 @@ impl Commands {
             Self::TestSets { command } => command.operation(),
             Self::TestCases { command } => command.operation(),
             Self::Personas { command } => command.operation(),
+            Self::PythonMetricDrafts { command } => command.operation(),
             Self::Metrics { command } => command.operation(),
             Self::Models { command } => command.operation(),
             Self::Mutations { command } => command.operation(),
@@ -289,6 +296,9 @@ pub async fn run(cli: Cli, ctx: &OutputContext) -> anyhow::Result<()> {
         Commands::Personas {
             command: commands::personas::PersonaCommands::Context,
         } => commands::agent::resource_context("personas", ctx),
+        Commands::PythonMetricDrafts {
+            command: commands::python_metric_drafts::PythonMetricDraftCommands::Context,
+        } => commands::agent::resource_context("python-metric-drafts", ctx),
         Commands::Metrics {
             command: commands::metrics::MetricCommands::Context,
         } => commands::agent::resource_context("metrics", ctx),
@@ -360,6 +370,9 @@ pub async fn run(cli: Cli, ctx: &OutputContext) -> anyhow::Result<()> {
                 Commands::Personas { command } => {
                     commands::personas::execute(command, &client, ctx).await
                 }
+                Commands::PythonMetricDrafts { command } => {
+                    commands::python_metric_drafts::execute(command, &client, ctx).await
+                }
                 Commands::Metrics { command } => {
                     commands::metrics::execute(command, &client, ctx).await
                 }

src/client/mod.rs

@@ -178,6 +178,10 @@ impl CovalClient {
         MetricsClient(self)
     }
 
+    pub fn python_metric_drafts(&self) -> PythonMetricDraftsClient<'_> {
+        PythonMetricDraftsClient(self)
+    }
+
     pub fn mutations(&self, agent_id: &str) -> MutationsClient<'_> {
         MutationsClient {
             client: self,
@@ -247,6 +251,7 @@ pub struct TestSetsClient<'a>(&'a CovalClient);
 pub struct TestCasesClient<'a>(&'a CovalClient);
 pub struct PersonasClient<'a>(&'a CovalClient);
 pub struct MetricsClient<'a>(&'a CovalClient);
+pub struct PythonMetricDraftsClient<'a>(&'a CovalClient);
 pub struct MutationsClient<'a> {
     client: &'a CovalClient,
     agent_id: String,
@@ -1007,6 +1012,75 @@ impl MetricsClient<'_> {
     }
 }
 
+impl PythonMetricDraftsClient<'_> {
+    pub async fn create(
+        &self,
+        request: models::CreatePythonMetricDraftRequest,
+    ) -> Result<models::PythonMetricDraftResponse, ApiError> {
+        let url = self.0.url("/v1/python-metric-drafts");
+        self.0.post(url, &request).await
+    }
+
+    pub async fn get(&self, draft_id: &str) -> Result<models::PythonMetricDraftResponse, ApiError> {
+        let draft_id = encode_path_segment(draft_id);
+        self.0
+            .get(self.0.url(&format!("/v1/python-metric-drafts/{draft_id}")))
+            .await
+    }
+
+    pub async fn create_preview(
+        &self,
+        draft_id: &str,
+        request: models::PreviewPythonMetricRequest,
+    ) -> Result<models::PythonMetricPreviewsResponse, ApiError> {
+        let draft_id = encode_path_segment(draft_id);
+        let url = self
+            .0
+            .url(&format!("/v1/python-metric-drafts/{draft_id}/previews"));
+        self.0.post(url, &request).await
+    }
+
+    pub async fn get_preview(
+        &self,
+        draft_id: &str,
+        preview_id: &str,
+    ) -> Result<models::PythonMetricPreviewsResponse, ApiError> {
+        let draft_id = encode_path_segment(draft_id);
+        let preview_id = encode_path_segment(preview_id);
+        self.0
+            .get(self.0.url(&format!(
+                "/v1/python-metric-drafts/{draft_id}/previews/{preview_id}"
+            )))
+            .await
+    }
+
+    pub async fn cancel_preview(
+        &self,
+        draft_id: &str,
+        preview_id: &str,
+    ) -> Result<models::PythonMetricPreviewsResponse, ApiError> {
+        let draft_id = encode_path_segment(draft_id);
+        let preview_id = encode_path_segment(preview_id);
+        self.0
+            .post_empty(self.0.url(&format!(
+                "/v1/python-metric-drafts/{draft_id}/previews/{preview_id}/cancel"
+            )))
+            .await
+    }
+
+    pub async fn publish(
+        &self,
+        draft_id: &str,
+        request: models::PublishPythonMetricRequest,
+    ) -> Result<models::PublishPythonMetricResponse, ApiError> {
+        let draft_id = encode_path_segment(draft_id);
+        let url = self
+            .0
+            .url(&format!("/v1/python-metric-drafts/{draft_id}/publish"));
+        self.0.post(url, &request).await
+    }
+}
+
 impl MutationsClient<'_> {
     pub async fn list(
         &self,