From 9b73868dc72c0917aef60437461a4baa08425bf2 Mon Sep 17 00:00:00 2001 From: callumreid <75899979+callumreid@users.noreply.github.com> Date: Mon, 26 Oct 2026 10:10:52 +0000 Subject: [PATCH 1/7] chore(cli): refresh API parity report --- api-coverage-report.md | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/api-coverage-report.md b/api-coverage-report.md index 61743b0..56f54b4 100644 --- a/api-coverage-report.md +++ b/api-coverage-report.md @@ -12,12 +12,12 @@ only when coverage actually changes. | Metric | Value | | --- | ---: | -| Reconciliation status | PASS | -| Published operations | 186 | +| Reconciliation status | ACTION REQUIRED | +| Published operations | 192 | | First-class CLI operations | 119 | | Reviewed gaps | 67 | | Client operations | 143 | -| Published request fields on covered operations | 451 | +| Published request fields on covered operations | 453 | | Request fields modeled by the CLI | 451 | | Reviewed request-field gaps | 0 | @@ -25,7 +25,12 @@ Catalog: https://api.coval.dev/v1/openapi ## New published operations without CLI commands -- None. +- `GET /python-metric-drafts/{draft_id}` +- `GET /python-metric-drafts/{draft_id}/previews/{preview_id}` +- `POST /python-metric-drafts` +- `POST /python-metric-drafts/{draft_id}/previews` +- `POST /python-metric-drafts/{draft_id}/previews/{preview_id}/cancel` +- `POST /python-metric-drafts/{draft_id}/publish` ## Reviewed gaps no longer present @@ -49,7 +54,8 @@ Catalog: https://api.coval.dev/v1/openapi ## Coverage snapshot mismatches -- None. +- `published_operations: recorded 186, current 192` +- `published_request_fields: recorded 451, current 453` ## Client-only operations @@ -79,6 +85,8 @@ Catalog: https://api.coval.dev/v1/openapi - `GET /organization/users` - `GET /personas/tags` - `GET /personas/{persona_id}/versions` +- `GET /python-metric-drafts/{draft_id}` +- `GET /python-metric-drafts/{draft_id}/previews/{preview_id}` - `GET /review-annotations/metric-health-stats` - `GET /review-projects/{project_id}/insights` - `GET /review-projects/{project_id}/metric-agreement` @@ -113,6 +121,10 @@ Catalog: https://api.coval.dev/v1/openapi - `POST /metrics/{metric_id}/versions/{version_id}/revert` - `POST /personas/{persona_id}/duplicate` - `POST /personas/{persona_id}/versions/{version_id}/revert` +- `POST /python-metric-drafts` +- `POST /python-metric-drafts/{draft_id}/previews` +- `POST /python-metric-drafts/{draft_id}/previews/{preview_id}/cancel` +- `POST /python-metric-drafts/{draft_id}/publish` - `POST /review-annotations:withMetricOutputs` - `POST /review-projects/disagreement-state` - `POST /review-projects/{project_id}/complete-conversation` @@ -127,7 +139,8 @@ Catalog: https://api.coval.dev/v1/openapi ## New published request fields the CLI drops -- None. +- `PATCH /personas/{id} interruption_instructions_enabled` +- `POST /personas interruption_instructions_enabled` ## Reviewed request-field gaps no longer present @@ -143,4 +156,5 @@ Catalog: https://api.coval.dev/v1/openapi ## All current request-field gaps -- None. +- `PATCH /personas/{id} interruption_instructions_enabled` +- `POST /personas interruption_instructions_enabled` From 22f2052bc4aeb28bb31e6b67e6d00c5a72a69f2f Mon Sep 17 00:00:00 2001 From: Callum Reid Date: Mon, 26 Oct 2026 15:29:21 +0000 Subject: [PATCH 2/7] feat: add Python metric draft API coverage Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- Cargo.lock | 2 +- Cargo.toml | 2 +- README.md | 20 ++ api-coverage-report.md | 30 +- api-coverage.toml | 8 +- src/agent_discovery.rs | 41 +++ src/cli.rs | 13 + src/client/mod.rs | 74 +++++ src/client/models/mod.rs | 2 + src/client/models/persona.rs | 6 + src/client/models/python_metric_draft.rs | 94 ++++++ src/commands/mod.rs | 1 + src/commands/personas.rs | 16 + src/commands/python_metric_drafts.rs | 345 +++++++++++++++++++ tests/cli_tests.rs | 400 ++++++++++++++++++++++- 15 files changed, 1025 insertions(+), 29 deletions(-) create mode 100644 src/client/models/python_metric_draft.rs create mode 100644 src/commands/python_metric_drafts.rs diff --git a/Cargo.lock b/Cargo.lock index d17faf7..31c083e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -278,7 +278,7 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "coval" -version = "0.9.2" +version = "0.10.0" dependencies = [ "anyhow", "assert_cmd", diff --git a/Cargo.toml b/Cargo.toml index 87e8773..03192ec 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "coval" -version = "0.9.2" +version = "0.10.0" edition = "2021" description = "CLI for Coval AI agent evaluation platform" license = "MIT" diff --git a/README.md b/README.md index ddcccea..04e4ed3 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,7 @@ coval simulated-conversations list --run-id | `coval test-sets` | Manage test set collections | | `coval test-cases` | Manage individual test cases | | `coval personas` | Manage simulated personas | +| `coval python-metric-drafts` | Create, preview, and publish Python metric drafts | | `coval metrics` | Manage evaluation metrics | | `coval models` | Inspect supported metric models | | `coval mutations` | Test agent variations with config overrides | @@ -307,9 +308,28 @@ coval personas update --audio-degradation cell-handoff # Make the persona interrupt the agent more often (NONE, LOW, MEDIUM, or HIGH) coval personas update --interruption-rate HIGH +# Enable interruption instructions +coval personas update --interruption-instructions-enabled true + # Clear a preset. A flag can only set a value, so clearing needs an explicit null. coval personas update --input-json '{"situate_speaker":null}' +# Create an immutable Python metric draft and preview it on a simulation output +coval python-metric-drafts create \ + --display-name "Greeting quality" \ + --source-file ./greeting_quality.py \ + --output-type boolean \ + --description "Checks whether the agent greets the caller" \ + --require-audio false \ + --require-traces false \ + --frame-table turns +coval python-metric-drafts previews create \ + --simulation-output-ids +coval python-metric-drafts previews get +coval python-metric-drafts publish \ + --preview-ids \ + --fingerprint + # Create a dashboard and make it the organization default coval dashboards create \ --name "Production Metrics" \ diff --git a/api-coverage-report.md b/api-coverage-report.md index 56f54b4..b30fa90 100644 --- a/api-coverage-report.md +++ b/api-coverage-report.md @@ -12,25 +12,20 @@ only when coverage actually changes. | Metric | Value | | --- | ---: | -| Reconciliation status | ACTION REQUIRED | +| Reconciliation status | PASS | | Published operations | 192 | -| First-class CLI operations | 119 | +| First-class CLI operations | 125 | | Reviewed gaps | 67 | -| Client operations | 143 | +| Client operations | 149 | | Published request fields on covered operations | 453 | -| Request fields modeled by the CLI | 451 | +| Request fields modeled by the CLI | 453 | | Reviewed request-field gaps | 0 | Catalog: https://api.coval.dev/v1/openapi ## New published operations without CLI commands -- `GET /python-metric-drafts/{draft_id}` -- `GET /python-metric-drafts/{draft_id}/previews/{preview_id}` -- `POST /python-metric-drafts` -- `POST /python-metric-drafts/{draft_id}/previews` -- `POST /python-metric-drafts/{draft_id}/previews/{preview_id}/cancel` -- `POST /python-metric-drafts/{draft_id}/publish` +- None. ## Reviewed gaps no longer present @@ -54,8 +49,7 @@ Catalog: https://api.coval.dev/v1/openapi ## Coverage snapshot mismatches -- `published_operations: recorded 186, current 192` -- `published_request_fields: recorded 451, current 453` +- None. ## Client-only operations @@ -85,8 +79,6 @@ Catalog: https://api.coval.dev/v1/openapi - `GET /organization/users` - `GET /personas/tags` - `GET /personas/{persona_id}/versions` -- `GET /python-metric-drafts/{draft_id}` -- `GET /python-metric-drafts/{draft_id}/previews/{preview_id}` - `GET /review-annotations/metric-health-stats` - `GET /review-projects/{project_id}/insights` - `GET /review-projects/{project_id}/metric-agreement` @@ -121,10 +113,6 @@ Catalog: https://api.coval.dev/v1/openapi - `POST /metrics/{metric_id}/versions/{version_id}/revert` - `POST /personas/{persona_id}/duplicate` - `POST /personas/{persona_id}/versions/{version_id}/revert` -- `POST /python-metric-drafts` -- `POST /python-metric-drafts/{draft_id}/previews` -- `POST /python-metric-drafts/{draft_id}/previews/{preview_id}/cancel` -- `POST /python-metric-drafts/{draft_id}/publish` - `POST /review-annotations:withMetricOutputs` - `POST /review-projects/disagreement-state` - `POST /review-projects/{project_id}/complete-conversation` @@ -139,8 +127,7 @@ Catalog: https://api.coval.dev/v1/openapi ## New published request fields the CLI drops -- `PATCH /personas/{id} interruption_instructions_enabled` -- `POST /personas interruption_instructions_enabled` +- None. ## Reviewed request-field gaps no longer present @@ -156,5 +143,4 @@ Catalog: https://api.coval.dev/v1/openapi ## All current request-field gaps -- `PATCH /personas/{id} interruption_instructions_enabled` -- `POST /personas interruption_instructions_enabled` +- None. diff --git a/api-coverage.toml b/api-coverage.toml index 8fab441..8594f35 100644 --- a/api-coverage.toml +++ b/api-coverage.toml @@ -6,10 +6,10 @@ [snapshot] catalog_url = "https://api.coval.dev/v1/openapi" reviewed_at = "2026-10-19" -published_operations = 186 -cli_supported_operations = 119 -published_request_fields = 451 -cli_modeled_request_fields = 451 +published_operations = 192 +cli_supported_operations = 125 +published_request_fields = 453 +cli_modeled_request_fields = 453 [[allowed_extra]] operation = "POST /test-cases/{test_case_id}/media:upload-url" diff --git a/src/agent_discovery.rs b/src/agent_discovery.rs index bd68da8..3d58e8f 100644 --- a/src/agent_discovery.rs +++ b/src/agent_discovery.rs @@ -453,6 +453,47 @@ const RESOURCE_SPECS: &[ResourceSpec] = &[ }], pitfalls: &["Metric type controls required creation fields."], }, + ResourceSpec { + name: "python-metric-drafts", + commands: &[ + "context", + "create", + "get", + "publish", + "previews.create", + "previews.get", + "previews.cancel", + ], + description: "Python metric drafts are immutable candidate definitions that must be previewed successfully before publishing.", + id_name: "draft_id", + id_format: "Draft ID returned by the Python metric drafts API", + requires: &["simulated-conversations"], + optional: &[], + produces: &["Python metric"], + related: &["metrics", "simulated-conversations"], + workflows: &[ + WorkflowSpec { + name: "Create an immutable Python metric draft", + argv: &["python-metric-drafts", "create"], + }, + WorkflowSpec { + name: "Preview a Python metric draft", + argv: &[ + "python-metric-drafts", + "previews", + "", + "create", + "--simulation-output-ids", + "", + ], + }, + ], + pitfalls: &[ + "Source or input/output changes require a new draft and preview.", + "Preview execution is asynchronous; poll its preview ID until terminal.", + "Publish only with the exact draft fingerprint and successful preview IDs.", + ], + }, ResourceSpec { name: "mutations", commands: &["context", "list", "get", "create", "update", "delete"], diff --git a/src/cli.rs b/src/cli.rs index 94ce8e4..08df746 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -95,6 +95,11 @@ pub enum Commands { #[command(subcommand)] command: commands::personas::PersonaCommands, }, + #[command(name = "python-metric-drafts")] + PythonMetricDrafts { + #[command(subcommand)] + command: commands::python_metric_drafts::PythonMetricDraftCommands, + }, Metrics { #[command(subcommand)] command: commands::metrics::MetricCommands, @@ -172,6 +177,7 @@ impl Commands { commands::personas::PersonaCommands::BackgroundSounds { .. } => "background-sounds", _ => "personas", }, + Self::PythonMetricDrafts { .. } => "python-metric-drafts", Self::Metrics { .. } => "metrics", Self::Models { .. } => "models", Self::Mutations { .. } => "mutations", @@ -208,6 +214,7 @@ impl Commands { Self::TestSets { command } => command.operation(), Self::TestCases { command } => command.operation(), Self::Personas { command } => command.operation(), + Self::PythonMetricDrafts { command } => command.operation(), Self::Metrics { command } => command.operation(), Self::Models { command } => command.operation(), Self::Mutations { command } => command.operation(), @@ -289,6 +296,9 @@ pub async fn run(cli: Cli, ctx: &OutputContext) -> anyhow::Result<()> { Commands::Personas { command: commands::personas::PersonaCommands::Context, } => commands::agent::resource_context("personas", ctx), + Commands::PythonMetricDrafts { + command: commands::python_metric_drafts::PythonMetricDraftCommands::Context, + } => commands::agent::resource_context("python-metric-drafts", ctx), Commands::Metrics { command: commands::metrics::MetricCommands::Context, } => commands::agent::resource_context("metrics", ctx), @@ -360,6 +370,9 @@ pub async fn run(cli: Cli, ctx: &OutputContext) -> anyhow::Result<()> { Commands::Personas { command } => { commands::personas::execute(command, &client, ctx).await } + Commands::PythonMetricDrafts { command } => { + commands::python_metric_drafts::execute(command, &client, ctx).await + } Commands::Metrics { command } => { commands::metrics::execute(command, &client, ctx).await } diff --git a/src/client/mod.rs b/src/client/mod.rs index 0c9f1ad..da56eb0 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -178,6 +178,10 @@ impl CovalClient { MetricsClient(self) } + pub fn python_metric_drafts(&self) -> PythonMetricDraftsClient<'_> { + PythonMetricDraftsClient(self) + } + pub fn mutations(&self, agent_id: &str) -> MutationsClient<'_> { MutationsClient { client: self, @@ -247,6 +251,7 @@ pub struct TestSetsClient<'a>(&'a CovalClient); pub struct TestCasesClient<'a>(&'a CovalClient); pub struct PersonasClient<'a>(&'a CovalClient); pub struct MetricsClient<'a>(&'a CovalClient); +pub struct PythonMetricDraftsClient<'a>(&'a CovalClient); pub struct MutationsClient<'a> { client: &'a CovalClient, agent_id: String, @@ -1007,6 +1012,75 @@ impl MetricsClient<'_> { } } +impl PythonMetricDraftsClient<'_> { + pub async fn create( + &self, + request: models::CreatePythonMetricDraftRequest, + ) -> Result { + self.0 + .post(self.0.url("/v1/python-metric-drafts"), &request) + .await + } + + pub async fn get(&self, draft_id: &str) -> Result { + self.0 + .get(self.0.url(&format!("/v1/python-metric-drafts/{draft_id}"))) + .await + } + + pub async fn create_preview( + &self, + draft_id: &str, + request: models::PreviewPythonMetricRequest, + ) -> Result { + self.0 + .post( + self.0 + .url(&format!("/v1/python-metric-drafts/{draft_id}/previews")), + &request, + ) + .await + } + + pub async fn get_preview( + &self, + draft_id: &str, + preview_id: &str, + ) -> Result { + self.0 + .get(self.0.url(&format!( + "/v1/python-metric-drafts/{draft_id}/previews/{preview_id}" + ))) + .await + } + + pub async fn cancel_preview( + &self, + draft_id: &str, + preview_id: &str, + ) -> Result { + self.0 + .post_empty(self.0.url(&format!( + "/v1/python-metric-drafts/{draft_id}/previews/{preview_id}/cancel" + ))) + .await + } + + pub async fn publish( + &self, + draft_id: &str, + request: models::PublishPythonMetricRequest, + ) -> Result { + self.0 + .post( + self.0 + .url(&format!("/v1/python-metric-drafts/{draft_id}/publish")), + &request, + ) + .await + } +} + impl MutationsClient<'_> { pub async fn list( &self, diff --git a/src/client/models/mod.rs b/src/client/models/mod.rs index 41b6c14..04687af 100644 --- a/src/client/models/mod.rs +++ b/src/client/models/mod.rs @@ -10,6 +10,7 @@ mod metric_model; mod monitor; mod mutation; mod persona; +mod python_metric_draft; mod report; mod review_annotation; mod review_project; @@ -33,6 +34,7 @@ pub use metric_model::*; pub use monitor::*; pub use mutation::*; pub use persona::*; +pub use python_metric_draft::*; pub use report::*; pub use review_annotation::*; pub use review_project::*; diff --git a/src/client/models/persona.rs b/src/client/models/persona.rs index 3e87ed1..f3dc42f 100644 --- a/src/client/models/persona.rs +++ b/src/client/models/persona.rs @@ -25,6 +25,8 @@ pub struct Persona { #[serde(skip_serializing_if = "Option::is_none")] pub interruption_rate: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub interruption_instructions_enabled: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub multi_language_stt: Option, pub create_time: DateTime, #[serde(skip_serializing_if = "Option::is_none")] @@ -80,6 +82,8 @@ pub struct CreatePersonaRequest { pub conversation_initiation: Option, #[serde(skip_serializing_if = "Option::is_none")] pub interruption_rate: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub interruption_instructions_enabled: Option, #[serde(alias = "multiLanguageStt", skip_serializing_if = "Option::is_none")] pub multi_language_stt: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -132,6 +136,8 @@ pub struct UpdatePersonaRequest { pub conversation_initiation: Option, #[serde(skip_serializing_if = "Option::is_none")] pub interruption_rate: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub interruption_instructions_enabled: Option, #[serde(alias = "multiLanguageStt", skip_serializing_if = "Option::is_none")] pub multi_language_stt: Option, #[serde(skip_serializing_if = "Option::is_none")] diff --git a/src/client/models/python_metric_draft.rs b/src/client/models/python_metric_draft.rs new file mode 100644 index 0000000..fde542a --- /dev/null +++ b/src/client/models/python_metric_draft.rs @@ -0,0 +1,94 @@ +use clap::ValueEnum; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, Deserialize, Serialize, ValueEnum)] +#[serde(rename_all = "lowercase")] +pub enum PythonMetricOutputType { + Number, + Boolean, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct PythonMetricInputRequirements { + #[serde(skip_serializing_if = "Option::is_none")] + pub audio: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub traces: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub frame_tables: Option>, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct CreatePythonMetricDraftRequest { + pub display_name: String, + pub source: String, + pub output_type: PythonMetricOutputType, + #[serde(skip_serializing_if = "Option::is_none")] + pub target_metric_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub target_version_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub unit: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub requirements: Option, +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct PythonMetricDraft { + pub draft_id: String, + pub version_id: String, + pub display_name: String, + pub description: String, + pub source: String, + pub output_type: PythonMetricOutputType, + pub unit: Option, + pub requirements: PythonMetricInputRequirements, + pub fingerprint: String, + pub target_metric_id: Option, + pub target_version_id: Option, + pub published: bool, +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct PythonMetricDraftResponse { + pub draft: PythonMetricDraft, +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct PreviewPythonMetricRequest { + pub simulation_output_ids: Vec, +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct PublishPythonMetricRequest { + pub preview_ids: Vec, + pub fingerprint: String, +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct PythonMetricPreview { + pub preview_id: String, + pub simulation_output_id: String, + pub status: String, + #[serde(default)] + pub value: Option, + #[serde(default)] + pub explanation: Option, + #[serde(default)] + pub error: Option, +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct PythonMetricPreviewsResponse { + pub previews: Vec, +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct PublishPythonMetricResponse { + pub metric_id: String, + pub version_id: String, +} diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 13bfe9a..89b1ca3 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -10,6 +10,7 @@ pub mod models; pub mod monitors; pub mod mutations; pub mod personas; +pub mod python_metric_drafts; pub mod reports; pub mod review_annotations; pub mod review_projects; diff --git a/src/commands/personas.rs b/src/commands/personas.rs index 8e87ed5..f3e640a 100644 --- a/src/commands/personas.rs +++ b/src/commands/personas.rs @@ -115,6 +115,9 @@ pub struct CreateArgs { /// Timer-driven interruption of agent speech (NONE, LOW, MEDIUM, or HIGH) #[arg(long)] interruption_rate: Option, + /// Enable or disable interruption instructions + #[arg(long)] + interruption_instructions_enabled: Option, /// Background sound volume multiplier (>= 0.0) #[arg(long)] background_sound_volume: Option, @@ -167,6 +170,9 @@ pub struct UpdateArgs { /// Timer-driven interruption of agent speech (NONE, LOW, MEDIUM, or HIGH) #[arg(long)] interruption_rate: Option, + /// Enable or disable interruption instructions + #[arg(long)] + interruption_instructions_enabled: Option, /// Background sound volume multiplier (>= 0.0) #[arg(long)] background_sound_volume: Option, @@ -293,6 +299,11 @@ pub async fn execute( input_json::insert(&mut input, "wait_seconds", args.wait_seconds)?; input_json::insert(&mut input, "multi_language_stt", args.multi_language_stt)?; input_json::insert(&mut input, "interruption_rate", args.interruption_rate)?; + input_json::insert( + &mut input, + "interruption_instructions_enabled", + args.interruption_instructions_enabled, + )?; input_json::insert( &mut input, "background_sound_volume", @@ -333,6 +344,11 @@ pub async fn execute( input_json::insert(&mut input, "wait_seconds", args.wait_seconds)?; input_json::insert(&mut input, "multi_language_stt", args.multi_language_stt)?; input_json::insert(&mut input, "interruption_rate", args.interruption_rate)?; + input_json::insert( + &mut input, + "interruption_instructions_enabled", + args.interruption_instructions_enabled, + )?; input_json::insert( &mut input, "background_sound_volume", diff --git a/src/commands/python_metric_drafts.rs b/src/commands/python_metric_drafts.rs new file mode 100644 index 0000000..5f74ac5 --- /dev/null +++ b/src/commands/python_metric_drafts.rs @@ -0,0 +1,345 @@ +use std::path::PathBuf; + +use anyhow::{ensure, Context, Result}; +use clap::{Args, Subcommand}; + +use crate::client::models::{ + CreatePythonMetricDraftRequest, PreviewPythonMetricRequest, PublishPythonMetricRequest, + PublishPythonMetricResponse, PythonMetricDraftResponse, PythonMetricOutputType, + PythonMetricPreviewsResponse, +}; +use crate::client::CovalClient; +use crate::input_json::{self, InputJsonArg}; +use crate::next_actions; +use crate::output::{emit_one_with_actions, NextAction, OutputContext}; + +#[derive(Subcommand)] +pub enum PythonMetricDraftCommands { + Context, + Create(CreateArgs), + Get(DraftIdArgs), + Publish(PublishArgs), + Previews { + draft_id: String, + #[command(subcommand)] + command: PreviewCommands, + }, +} + +#[derive(Subcommand)] +pub enum PreviewCommands { + Create(PreviewCreateArgs), + Get(PreviewIdArgs), + Cancel(PreviewIdArgs), +} + +impl PythonMetricDraftCommands { + pub fn operation(&self) -> &'static str { + match self { + Self::Context => "context", + Self::Create(_) => "create", + Self::Get(_) => "get", + Self::Publish(_) => "publish", + Self::Previews { command, .. } => match command { + PreviewCommands::Create(_) => "previews-create", + PreviewCommands::Get(_) => "previews-get", + PreviewCommands::Cancel(_) => "previews-cancel", + }, + } + } +} + +#[derive(Args)] +pub struct CreateArgs { + #[command(flatten)] + input_json: InputJsonArg, + #[arg(long)] + display_name: Option, + #[arg(long)] + target_metric_id: Option, + #[arg(long)] + target_version_id: Option, + #[arg(long)] + description: Option, + #[arg(long, conflicts_with = "source_file")] + source: Option, + #[arg(long)] + source_file: Option, + #[arg(long, value_enum)] + output_type: Option, + #[arg(long)] + unit: Option, + #[arg(long)] + requirements: Option, + #[arg(long)] + require_audio: Option, + #[arg(long)] + require_traces: Option, + #[arg(long)] + frame_table: Vec, +} + +#[derive(Args)] +pub struct DraftIdArgs { + draft_id: String, +} + +#[derive(Args)] +pub struct PublishArgs { + draft_id: String, + #[arg(long, required = true, value_delimiter = ',')] + preview_ids: Vec, + #[arg(long, required = true)] + fingerprint: String, +} + +#[derive(Args)] +pub struct PreviewCreateArgs { + #[arg(long, required = true)] + simulation_output_ids: Vec, +} + +#[derive(Args)] +pub struct PreviewIdArgs { + preview_id: String, +} + +pub async fn execute( + command: PythonMetricDraftCommands, + client: &CovalClient, + ctx: &OutputContext, +) -> Result<()> { + match command { + PythonMetricDraftCommands::Context => { + crate::commands::agent::resource_context("python-metric-drafts", ctx) + } + PythonMetricDraftCommands::Create(args) => { + let mut input = args.input_json.object()?; + input_json::insert(&mut input, "display_name", args.display_name)?; + input_json::insert(&mut input, "target_metric_id", args.target_metric_id)?; + input_json::insert(&mut input, "target_version_id", args.target_version_id)?; + input_json::insert(&mut input, "description", args.description)?; + input_json::insert(&mut input, "source", args.source)?; + if let Some(path) = args.source_file { + let source = std::fs::read_to_string(&path) + .with_context(|| format!("failed to read --source-file: {}", path.display()))?; + input.insert("source".to_string(), serde_json::Value::String(source)); + } + input_json::insert(&mut input, "output_type", args.output_type)?; + input_json::insert(&mut input, "unit", args.unit)?; + if args.requirements.is_some() + || args.require_audio.is_some() + || args.require_traces.is_some() + || !args.frame_table.is_empty() + { + let mut requirements = match args.requirements { + Some(value) => { + serde_json::from_str::>(&value) + .context("--requirements must be a JSON object")? + } + None => input + .remove("requirements") + .map(|value| { + value.as_object().cloned().ok_or_else(|| { + anyhow::anyhow!( + "requirements from --input-json must be a JSON object" + ) + }) + }) + .transpose()? + .unwrap_or_default(), + }; + input_json::insert(&mut requirements, "audio", args.require_audio)?; + input_json::insert(&mut requirements, "traces", args.require_traces)?; + if !args.frame_table.is_empty() { + requirements.insert( + "frame_tables".to_string(), + serde_json::to_value(args.frame_table)?, + ); + } + input.insert( + "requirements".to_string(), + serde_json::Value::Object(requirements), + ); + } + let request: CreatePythonMetricDraftRequest = input_json::finish(input)?; + let response: PythonMetricDraftResponse = + client.python_metric_drafts().create(request).await?; + let draft = &response.draft; + emit_one_with_actions( + ctx, + "python-metric-drafts", + "create", + &response, + vec![ + NextAction::new( + "python-metric-drafts.previews.create", + "Create a preview for this draft", + next_actions::argv([ + "python-metric-drafts", + "previews", + &draft.draft_id, + "create", + "--simulation-output-ids", + "", + ]), + false, + ) + .primary(), + next_actions::context("python-metric-drafts"), + ], + ); + Ok(()) + } + PythonMetricDraftCommands::Get(args) => { + let response = client.python_metric_drafts().get(&args.draft_id).await?; + emit_one_with_actions( + ctx, + "python-metric-drafts", + "get", + &response, + vec![next_actions::context("python-metric-drafts")], + ); + Ok(()) + } + PythonMetricDraftCommands::Publish(args) => { + let request = PublishPythonMetricRequest { + preview_ids: args.preview_ids, + fingerprint: args.fingerprint, + }; + let response: PublishPythonMetricResponse = client + .python_metric_drafts() + .publish(&args.draft_id, request) + .await?; + emit_one_with_actions( + ctx, + "python-metric-drafts", + "publish", + &response, + vec![next_actions::context("python-metric-drafts")], + ); + Ok(()) + } + PythonMetricDraftCommands::Previews { draft_id, command } => match command { + PreviewCommands::Create(args) => { + ensure!( + args.simulation_output_ids.len() == 1, + "--simulation-output-ids requires exactly one value" + ); + let request = PreviewPythonMetricRequest { + simulation_output_ids: args.simulation_output_ids, + }; + let response: PythonMetricPreviewsResponse = client + .python_metric_drafts() + .create_preview(&draft_id, request) + .await?; + let actions = response + .previews + .first() + .map(|preview| { + vec![ + NextAction::new( + "python-metric-drafts.previews.get", + "Poll this preview", + next_actions::argv([ + "python-metric-drafts", + "previews", + &draft_id, + "get", + &preview.preview_id, + ]), + true, + ) + .primary() + .with_description("Preview evaluation is asynchronous; poll until it reaches a terminal state."), + next_actions::context("python-metric-drafts"), + ] + }) + .unwrap_or_else(|| vec![next_actions::context("python-metric-drafts")]); + emit_one_with_actions( + ctx, + "python-metric-drafts", + "previews-create", + &response, + actions, + ); + Ok(()) + } + PreviewCommands::Get(args) => { + let response = client + .python_metric_drafts() + .get_preview(&draft_id, &args.preview_id) + .await?; + let mut actions = vec![next_actions::context("python-metric-drafts")]; + if let Some(preview) = response.previews.first() { + if preview.status.eq_ignore_ascii_case("succeeded") { + actions.insert( + 0, + NextAction::new( + "python-metric-drafts.publish", + "Publish this approved draft", + next_actions::argv([ + "python-metric-drafts", + "publish", + &draft_id, + "--preview-ids", + &preview.preview_id, + "--fingerprint", + "", + ]), + false, + ) + .primary() + .with_description("Publish only after the preview succeeds and the exact draft fingerprint is approved."), + ); + } else if preview.error.is_none() + && !matches!( + preview.status.to_ascii_lowercase().as_str(), + "failed" | "cancelled" | "canceled" | "error" + ) + { + actions.insert( + 0, + NextAction::new( + "python-metric-drafts.previews.get", + "Poll this preview", + next_actions::argv([ + "python-metric-drafts", + "previews", + &draft_id, + "get", + &preview.preview_id, + ]), + true, + ) + .primary() + .with_description("Preview evaluation is asynchronous; poll until it reaches a terminal state."), + ); + } + } + emit_one_with_actions( + ctx, + "python-metric-drafts", + "previews-get", + &response, + actions, + ); + Ok(()) + } + PreviewCommands::Cancel(args) => { + let response = client + .python_metric_drafts() + .cancel_preview(&draft_id, &args.preview_id) + .await?; + emit_one_with_actions( + ctx, + "python-metric-drafts", + "previews-cancel", + &response, + vec![next_actions::context("python-metric-drafts")], + ); + Ok(()) + } + }, + } +} diff --git a/tests/cli_tests.rs b/tests/cli_tests.rs index f082860..01fafe0 100644 --- a/tests/cli_tests.rs +++ b/tests/cli_tests.rs @@ -5,7 +5,7 @@ use assert_cmd::Command; use predicates::prelude::*; use serde_json::{json, Value}; -use wiremock::matchers::{body_partial_json, header, method, path, query_param}; +use wiremock::matchers::{body_json, body_partial_json, header, method, path, query_param}; use wiremock::{Match, Mock, MockServer, Request, ResponseTemplate}; fn coval() -> Command { @@ -110,6 +110,7 @@ const AGENT_RESOURCES: &[&str] = &[ "test-sets", "test-cases", "personas", + "python-metric-drafts", "metrics", "mutations", "api-keys", @@ -136,6 +137,7 @@ const INPUT_JSON_HELP_COMMANDS: &[&[&str]] = &[ &["test-cases", "update", "--help"], &["personas", "create", "--help"], &["personas", "update", "--help"], + &["python-metric-drafts", "create", "--help"], &["personas", "background-sounds", "update", "--help"], &["metrics", "create", "--help"], &["metrics", "update", "--help"], @@ -846,6 +848,305 @@ async fn test_agent_doctor_with_connectivity() { assert_eq!(value["warnings"].as_array().unwrap().len(), 0); } +#[tokio::test] +async fn test_python_metric_draft_commands_send_specified_requests() { + let mock_server = MockServer::start().await; + let temp_dir = tempfile::tempdir().unwrap(); + let draft_response = json!({ + "draft": { + "draft_id": "draft_123", + "version_id": "version_123", + "display_name": "Metric", + "description": "Description", + "source": "return True", + "output_type": "boolean", + "unit": null, + "requirements": {"audio": false, "traces": false}, + "fingerprint": "a".repeat(64), + "published": false + } + }); + let preview_response = json!({ + "previews": [{ + "preview_id": "preview_123", + "simulation_output_id": "simulation_123", + "status": "SUCCEEDED", + "value": true, + "explanation": null, + "error": null + }] + }); + let preview_pending_response = json!({ + "previews": [{ + "preview_id": "preview_123", + "simulation_output_id": "simulation_123", + "status": "PENDING", + "value": null, + "explanation": null, + "error": null + }] + }); + + Mock::given(method("POST")) + .and(path("/v1/python-metric-drafts")) + .and(header("X-API-Key", "test_key")) + .and(body_json(json!({ + "display_name": "Metric", + "target_metric_id": "metric_123", + "target_version_id": "version_old", + "description": "Description", + "source": "return True", + "output_type": "boolean", + "unit": "score", + "requirements": { + "audio": true, + "traces": false, + "frame_tables": ["turns"] + } + }))) + .respond_with(ResponseTemplate::new(201).set_body_json(draft_response.clone())) + .mount(&mock_server) + .await; + Mock::given(method("GET")) + .and(path("/v1/python-metric-drafts/draft_123")) + .and(header("X-API-Key", "test_key")) + .respond_with(ResponseTemplate::new(200).set_body_json(draft_response)) + .mount(&mock_server) + .await; + Mock::given(method("POST")) + .and(path("/v1/python-metric-drafts/draft_123/previews")) + .and(body_json( + json!({"simulation_output_ids": ["simulation_123"]}), + )) + .respond_with(ResponseTemplate::new(200).set_body_json(preview_pending_response)) + .mount(&mock_server) + .await; + Mock::given(method("GET")) + .and(path( + "/v1/python-metric-drafts/draft_123/previews/preview_123", + )) + .respond_with(ResponseTemplate::new(200).set_body_json(preview_response.clone())) + .mount(&mock_server) + .await; + Mock::given(method("POST")) + .and(path( + "/v1/python-metric-drafts/draft_123/previews/preview_123/cancel", + )) + .respond_with(ResponseTemplate::new(200).set_body_json(preview_response)) + .mount(&mock_server) + .await; + Mock::given(method("POST")) + .and(path("/v1/python-metric-drafts/draft_123/publish")) + .and(body_json(json!({ + "preview_ids": ["preview_123"], + "fingerprint": "a".repeat(64) + }))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "metric_id": "metric_123", + "version_id": "version_new" + }))) + .mount(&mock_server) + .await; + + let mut create_command = coval_with_api(&mock_server); + let create = stdout_json( + create_command + .args(["--agent", "python-metric-drafts", "create"]) + .args([ + "--display-name", + "Metric", + "--target-metric-id", + "metric_123", + "--target-version-id", + "version_old", + "--description", + "Description", + "--source", + "return True", + "--output-type", + "boolean", + "--unit", + "score", + "--require-audio", + "true", + "--require-traces", + "false", + "--frame-table", + "turns", + ]) + .arg("--format") + .arg("json") + .env("HOME", temp_dir.path()) + .env("XDG_CONFIG_HOME", temp_dir.path()) + .assert() + .success(), + ); + assert_eq!(create["ok"], true); + assert_eq!( + create["next_actions"][0]["id"], + "python-metric-drafts.previews.create" + ); + + let fingerprint = "a".repeat(64); + for args in [ + vec!["python-metric-drafts", "get", "draft_123"], + vec![ + "python-metric-drafts", + "previews", + "draft_123", + "create", + "--simulation-output-ids", + "simulation_123", + ], + vec![ + "python-metric-drafts", + "previews", + "draft_123", + "get", + "preview_123", + ], + vec![ + "python-metric-drafts", + "previews", + "draft_123", + "cancel", + "preview_123", + ], + vec![ + "python-metric-drafts", + "publish", + "draft_123", + "--preview-ids", + "preview_123", + "--fingerprint", + fingerprint.as_str(), + ], + ] { + let command_operation = args.get(3).copied(); + let mut command = coval_with_api(&mock_server); + let value = stdout_json( + command + .arg("--agent") + .args(&args) + .arg("--format") + .arg("json") + .env("HOME", temp_dir.path()) + .env("XDG_CONFIG_HOME", temp_dir.path()) + .assert() + .success(), + ); + assert_eq!(value["ok"], true); + if command_operation == Some("create") { + assert_eq!( + value["next_actions"][0]["id"], + "python-metric-drafts.previews.get" + ); + } else if command_operation == Some("get") { + assert_eq!( + value["next_actions"][0]["id"], + "python-metric-drafts.publish" + ); + } + } + let invalid_preview = stdout_json( + coval_with_api(&mock_server) + .args([ + "--agent", + "python-metric-drafts", + "previews", + "draft_123", + "create", + "--simulation-output-ids", + "simulation_123", + "--simulation-output-ids", + "simulation_456", + "--format", + "json", + ]) + .env("HOME", temp_dir.path()) + .env("XDG_CONFIG_HOME", temp_dir.path()) + .assert() + .failure(), + ); + assert_eq!(invalid_preview["ok"], false); +} + +#[tokio::test] +async fn test_python_metric_draft_source_file_and_missing_file() { + let mock_server = MockServer::start().await; + let temp_dir = tempfile::tempdir().unwrap(); + let source_path = temp_dir.path().join("metric.py"); + std::fs::write(&source_path, "def evaluate(conversation):\n return 1\n").unwrap(); + Mock::given(method("POST")) + .and(path("/v1/python-metric-drafts")) + .and(body_json(json!({ + "display_name": "Metric", + "source": "def evaluate(conversation):\n return 1\n", + "output_type": "number" + }))) + .respond_with(ResponseTemplate::new(201).set_body_json(json!({ + "draft": { + "draft_id": "draft_123", + "version_id": "version_123", + "display_name": "Metric", + "description": "", + "source": "def evaluate(conversation):\n return 1\n", + "output_type": "number", + "unit": null, + "requirements": {}, + "fingerprint": "a".repeat(64), + "published": false + } + }))) + .mount(&mock_server) + .await; + + let mut source_command = coval_with_api(&mock_server); + assert_eq!( + stdout_json( + source_command + .arg("--agent") + .args([ + "python-metric-drafts", + "create", + "--display-name", + "Metric", + "--source-file", + ]) + .arg(&source_path) + .args(["--output-type", "number", "--format", "json"]) + .env("HOME", temp_dir.path()) + .env("XDG_CONFIG_HOME", temp_dir.path()) + .assert() + .success(), + )["ok"], + true + ); + let mut missing_command = coval_with_api(&mock_server); + let missing = stdout_json( + missing_command + .arg("--agent") + .args([ + "python-metric-drafts", + "create", + "--display-name", + "Metric", + "--source-file", + ]) + .arg(temp_dir.path().join("missing.py")) + .args(["--output-type", "number", "--format", "json"]) + .env("HOME", temp_dir.path()) + .env("XDG_CONFIG_HOME", temp_dir.path()) + .assert() + .failure(), + ); + assert_eq!(missing["ok"], false); + assert!(missing["error"]["message"] + .as_str() + .unwrap() + .contains("failed to read --source-file")); +} + #[test] fn test_missing_api_key() { let temp_dir = tempfile::tempdir().unwrap(); @@ -2100,6 +2401,103 @@ async fn test_personas_update_forwards_the_interruption_rate_flag() { assert_eq!(capture.take()["interruption_rate"], "HIGH"); } +#[tokio::test] +async fn test_personas_interruption_instructions_flag_preserves_true_false_and_omitted() { + let mock_server = MockServer::start().await; + let persona = json!({ + "persona": { + "resource_name": "personas/persona_123", + "id": "persona_123", + "name": "Test persona", + "create_time": "2025-01-15T10:30:00Z" + } + }); + Mock::given(method("POST")) + .and(path("/v1/personas")) + .and(body_partial_json(json!({ + "name": "Test persona", + "voice_name": "aria", + "language_code": "en-US", + "interruption_instructions_enabled": true + }))) + .respond_with(ResponseTemplate::new(201).set_body_json(persona.clone())) + .mount(&mock_server) + .await; + Mock::given(method("PATCH")) + .and(path("/v1/personas/persona_123")) + .and(body_json( + json!({"interruption_instructions_enabled": false}), + )) + .respond_with(ResponseTemplate::new(200).set_body_json(persona.clone())) + .mount(&mock_server) + .await; + Mock::given(method("PATCH")) + .and(path("/v1/personas/persona_123")) + .and(body_json(json!({}))) + .respond_with(ResponseTemplate::new(200).set_body_json(persona)) + .mount(&mock_server) + .await; + + let temp_dir = tempfile::tempdir().unwrap(); + assert_eq!( + stdout_json( + coval_with_api(&mock_server) + .args([ + "--agent", + "personas", + "create", + "--name", + "Test persona", + "--voice", + "aria", + "--language", + "en-US", + "--interruption-instructions-enabled", + "true", + "--format", + "json", + ]) + .env("HOME", temp_dir.path()) + .env("XDG_CONFIG_HOME", temp_dir.path()) + .assert() + .success(), + )["ok"], + true + ); + for args in [ + vec![ + "--agent", + "personas", + "update", + "persona_123", + "--interruption-instructions-enabled", + "false", + "--format", + "json", + ], + vec![ + "--agent", + "personas", + "update", + "persona_123", + "--format", + "json", + ], + ] { + assert_eq!( + stdout_json( + coval_with_api(&mock_server) + .args(args) + .env("HOME", temp_dir.path()) + .env("XDG_CONFIG_HOME", temp_dir.path()) + .assert() + .success(), + )["ok"], + true + ); + } +} + #[tokio::test] async fn test_personas_update_audio_degradation_flag_accepts_a_preset_id() { let mock_server = MockServer::start().await; From d834cdb398d32a2437f0148a1a611f9808dd1c2c Mon Sep 17 00:00:00 2001 From: Callum Reid Date: Mon, 26 Oct 2026 15:32:35 +0000 Subject: [PATCH 3/7] fix: validate and encode Python metric draft path IDs Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- src/client/mod.rs | 7 +++++++ tests/cli_tests.rs | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/src/client/mod.rs b/src/client/mod.rs index da56eb0..647252a 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -1023,6 +1023,7 @@ impl PythonMetricDraftsClient<'_> { } pub async fn get(&self, draft_id: &str) -> Result { + let draft_id = encode_path_segment(draft_id); self.0 .get(self.0.url(&format!("/v1/python-metric-drafts/{draft_id}"))) .await @@ -1033,6 +1034,7 @@ impl PythonMetricDraftsClient<'_> { draft_id: &str, request: models::PreviewPythonMetricRequest, ) -> Result { + let draft_id = encode_path_segment(draft_id); self.0 .post( self.0 @@ -1047,6 +1049,8 @@ impl PythonMetricDraftsClient<'_> { draft_id: &str, preview_id: &str, ) -> Result { + let draft_id = encode_path_segment(draft_id); + let preview_id = encode_path_segment(preview_id); self.0 .get(self.0.url(&format!( "/v1/python-metric-drafts/{draft_id}/previews/{preview_id}" @@ -1059,6 +1063,8 @@ impl PythonMetricDraftsClient<'_> { draft_id: &str, preview_id: &str, ) -> Result { + let draft_id = encode_path_segment(draft_id); + let preview_id = encode_path_segment(preview_id); self.0 .post_empty(self.0.url(&format!( "/v1/python-metric-drafts/{draft_id}/previews/{preview_id}/cancel" @@ -1071,6 +1077,7 @@ impl PythonMetricDraftsClient<'_> { draft_id: &str, request: models::PublishPythonMetricRequest, ) -> Result { + let draft_id = encode_path_segment(draft_id); self.0 .post( self.0 diff --git a/tests/cli_tests.rs b/tests/cli_tests.rs index 01fafe0..d93d92f 100644 --- a/tests/cli_tests.rs +++ b/tests/cli_tests.rs @@ -1147,6 +1147,39 @@ async fn test_python_metric_draft_source_file_and_missing_file() { .contains("failed to read --source-file")); } +#[tokio::test] +async fn test_python_metric_draft_ids_are_encoded_as_single_path_segments() { + let mock_server = MockServer::start().await; + let temp_dir = tempfile::tempdir().unwrap(); + Mock::given(method("GET")) + .and(path( + "/v1/python-metric-drafts/a%2F..%2Fb/previews/c%2F..%2Fd", + )) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({"previews": []}))) + .expect(1) + .mount(&mock_server) + .await; + + let response = stdout_json( + coval_with_api(&mock_server) + .args([ + "--agent", + "python-metric-drafts", + "previews", + "a/../b", + "get", + "c/../d", + "--format", + "json", + ]) + .env("HOME", temp_dir.path()) + .env("XDG_CONFIG_HOME", temp_dir.path()) + .assert() + .success(), + ); + assert_eq!(response["ok"], true); +} + #[test] fn test_missing_api_key() { let temp_dir = tempfile::tempdir().unwrap(); From 97eed40f52a4956b276fb5cae8c0671eac9f1f6b Mon Sep 17 00:00:00 2001 From: Callum Reid Date: Mon, 26 Oct 2026 09:16:25 -0700 Subject: [PATCH 4/7] fix: keep Python draft request fields in the parity audit --- Cargo.lock | 2 +- Cargo.toml | 2 +- api-coverage-report.md | 4 ++-- api-coverage.toml | 6 ++--- scripts/audit_api_coverage.py | 16 ++++++++++--- scripts/test_audit_api_coverage.py | 38 ++++++++++++++++++++++++++++++ src/client/mod.rs | 27 ++++++++------------- 7 files changed, 68 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 31c083e..d17faf7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -278,7 +278,7 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "coval" -version = "0.10.0" +version = "0.9.2" dependencies = [ "anyhow", "assert_cmd", diff --git a/Cargo.toml b/Cargo.toml index 03192ec..87e8773 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "coval" -version = "0.10.0" +version = "0.9.2" edition = "2021" description = "CLI for Coval AI agent evaluation platform" license = "MIT" diff --git a/api-coverage-report.md b/api-coverage-report.md index b30fa90..d00195f 100644 --- a/api-coverage-report.md +++ b/api-coverage-report.md @@ -17,8 +17,8 @@ only when coverage actually changes. | First-class CLI operations | 125 | | Reviewed gaps | 67 | | Client operations | 149 | -| Published request fields on covered operations | 453 | -| Request fields modeled by the CLI | 453 | +| Published request fields on covered operations | 464 | +| Request fields modeled by the CLI | 464 | | Reviewed request-field gaps | 0 | Catalog: https://api.coval.dev/v1/openapi diff --git a/api-coverage.toml b/api-coverage.toml index 8594f35..40f40df 100644 --- a/api-coverage.toml +++ b/api-coverage.toml @@ -5,11 +5,11 @@ [snapshot] catalog_url = "https://api.coval.dev/v1/openapi" -reviewed_at = "2026-10-19" +reviewed_at = "2026-10-26" published_operations = 192 cli_supported_operations = 125 -published_request_fields = 453 -cli_modeled_request_fields = 453 +published_request_fields = 464 +cli_modeled_request_fields = 464 [[allowed_extra]] operation = "POST /test-cases/{test_case_id}/media:upload-url" diff --git a/scripts/audit_api_coverage.py b/scripts/audit_api_coverage.py index c1820d7..8c6ab0d 100644 --- a/scripts/audit_api_coverage.py +++ b/scripts/audit_api_coverage.py @@ -261,9 +261,19 @@ def _request_body_types(client_method: str, block: str) -> set[str]: """Return the Rust types a client function serializes as its request body.""" types: set[str] = set() - for match in re.finditer( - r"\.(?:post|patch|put)\(\s*url\s*,\s*&(?P[A-Za-z0-9_]+)\s*\)", block - ): + body_calls = re.findall(r"\.(?:post|patch|put)\(", block) + body_matches = list( + re.finditer( + r"\.(?:post|patch|put)\(\s*url\s*,\s*&(?P[A-Za-z0-9_]+)\s*\)", + block, + ) + ) + if len(body_calls) != len(body_matches): + raise RuntimeError( + "could not resolve every request-body call in client function " + f"{client_method}; use a local url binding" + ) + for match in body_matches: binding = match.group("binding") if re.search(rf"\b{binding}\s*:\s*serde_json::Value\b", block): types.add(PASSTHROUGH_BODY) diff --git a/scripts/test_audit_api_coverage.py b/scripts/test_audit_api_coverage.py index 1440d61..d5d4c83 100644 --- a/scripts/test_audit_api_coverage.py +++ b/scripts/test_audit_api_coverage.py @@ -436,6 +436,44 @@ def test_rejects_an_unresolvable_body_binding(self): audit_api_coverage._request_body_types("agents.create", block) +class RequestBodyAuditRegressionTests(unittest.TestCase): + def test_rejects_an_unrecognized_json_body_call(self): + block = """ + pub async fn create(&self, request: models::CreateThingRequest) -> Result { + self.0.post(self.0.url("/v1/things"), &request).await + } +""" + with self.assertRaisesRegex( + RuntimeError, "could not resolve every request-body" + ): + audit_api_coverage._request_body_types("things.create", block) + + def test_audits_all_python_metric_draft_request_fields(self): + operations, _ = audit_api_coverage._command_operations( + audit_api_coverage._client_operations() + ) + fields = audit_api_coverage._cli_request_fields( + operations, audit_api_coverage._model_request_fields() + ) + expected = { + "POST /python-metric-drafts": { + "display_name", + "source", + "output_type", + "target_metric_id", + "target_version_id", + "description", + "unit", + "requirements", + }, + "POST /python-metric-drafts/{id}/previews": {"simulation_output_ids"}, + "POST /python-metric-drafts/{id}/publish": {"preview_ids", "fingerprint"}, + } + for operation, expected_fields in expected.items(): + with self.subTest(operation=operation): + self.assertEqual(expected_fields, fields.get(operation)) + + class ModelRequestFieldTests(unittest.TestCase): def _fields(self, source: str) -> dict[str, set[str]]: with tempfile.TemporaryDirectory() as directory: diff --git a/src/client/mod.rs b/src/client/mod.rs index 647252a..107f828 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -1017,9 +1017,8 @@ impl PythonMetricDraftsClient<'_> { &self, request: models::CreatePythonMetricDraftRequest, ) -> Result { - self.0 - .post(self.0.url("/v1/python-metric-drafts"), &request) - .await + let url = self.0.url("/v1/python-metric-drafts"); + self.0.post(url, &request).await } pub async fn get(&self, draft_id: &str) -> Result { @@ -1035,13 +1034,10 @@ impl PythonMetricDraftsClient<'_> { request: models::PreviewPythonMetricRequest, ) -> Result { let draft_id = encode_path_segment(draft_id); - self.0 - .post( - self.0 - .url(&format!("/v1/python-metric-drafts/{draft_id}/previews")), - &request, - ) - .await + let url = self + .0 + .url(&format!("/v1/python-metric-drafts/{draft_id}/previews")); + self.0.post(url, &request).await } pub async fn get_preview( @@ -1078,13 +1074,10 @@ impl PythonMetricDraftsClient<'_> { request: models::PublishPythonMetricRequest, ) -> Result { let draft_id = encode_path_segment(draft_id); - self.0 - .post( - self.0 - .url(&format!("/v1/python-metric-drafts/{draft_id}/publish")), - &request, - ) - .await + let url = self + .0 + .url(&format!("/v1/python-metric-drafts/{draft_id}/publish")); + self.0.post(url, &request).await } } From 4ac1efc5a5b3a6ae8459a82efe81745c2487faf8 Mon Sep 17 00:00:00 2001 From: Callum Reid Date: Mon, 2 Nov 2026 09:20:26 -0800 Subject: [PATCH 5/7] fix: preserve current uploaded conversation and review request fields --- api-coverage-report.md | 15 +- api-coverage.toml | 36 +++- src/client/models/common.rs | 8 + src/client/models/conversation.rs | 12 +- src/client/models/review_project.rs | 34 ++++ src/client/models/test_set.rs | 6 + src/commands/conversations.rs | 23 ++- src/commands/review_projects.rs | 31 +++- tests/cli_tests.rs | 256 ++++++++++++++++++++++++++++ 9 files changed, 405 insertions(+), 16 deletions(-) diff --git a/api-coverage-report.md b/api-coverage-report.md index d00195f..20c7824 100644 --- a/api-coverage-report.md +++ b/api-coverage-report.md @@ -13,12 +13,12 @@ only when coverage actually changes. | Metric | Value | | --- | ---: | | Reconciliation status | PASS | -| Published operations | 192 | +| Published operations | 199 | | First-class CLI operations | 125 | -| Reviewed gaps | 67 | +| Reviewed gaps | 74 | | Client operations | 149 | -| Published request fields on covered operations | 464 | -| Request fields modeled by the CLI | 464 | +| Published request fields on covered operations | 472 | +| Request fields modeled by the CLI | 472 | | Reviewed request-field gaps | 0 | Catalog: https://api.coval.dev/v1/openapi @@ -67,6 +67,11 @@ Catalog: https://api.coval.dev/v1/openapi - `GET /alerts` - `GET /alerts/{alert_id}` - `GET /alerts/{alert_id}/events` +- `GET /conversation-python-analyses/{analysis_id}` +- `GET /conversations/simulated/{conversation_id}/comments` +- `GET /conversations/simulated/{conversation_id}/comments/{comment_id}` +- `GET /conversations/uploaded/{conversation_id}/comments` +- `GET /conversations/uploaded/{conversation_id}/comments/{comment_id}` - `GET /integrations/slack` - `GET /metrics/flows` - `GET /metrics/recently-deleted` @@ -104,6 +109,7 @@ Catalog: https://api.coval.dev/v1/openapi - `POST /alerts` - `POST /alerts/{alert_id}/test-evaluate` - `POST /audio:upload` +- `POST /conversation-python-analyses` - `POST /conversations/simulated:rerunMetrics` - `POST /integrations/slack/connect` - `POST /metrics/flows` @@ -113,6 +119,7 @@ Catalog: https://api.coval.dev/v1/openapi - `POST /metrics/{metric_id}/versions/{version_id}/revert` - `POST /personas/{persona_id}/duplicate` - `POST /personas/{persona_id}/versions/{version_id}/revert` +- `POST /python-metrics:fromAnalysis` - `POST /review-annotations:withMetricOutputs` - `POST /review-projects/disagreement-state` - `POST /review-projects/{project_id}/complete-conversation` diff --git a/api-coverage.toml b/api-coverage.toml index 40f40df..0fda6ee 100644 --- a/api-coverage.toml +++ b/api-coverage.toml @@ -5,11 +5,11 @@ [snapshot] catalog_url = "https://api.coval.dev/v1/openapi" -reviewed_at = "2026-10-26" -published_operations = 192 +reviewed_at = "2026-11-02" +published_operations = 199 cli_supported_operations = 125 -published_request_fields = 464 -cli_modeled_request_fields = 464 +published_request_fields = 472 +cli_modeled_request_fields = 472 [[allowed_extra]] operation = "POST /test-cases/{test_case_id}/media:upload-url" @@ -438,3 +438,31 @@ reason = "Served but deliberately hidden from the published schema as an interna operation = "POST /conversations/uploaded:submit" field = "audio" reason = "Served but deliberately hidden from the published schema as an internal field; the API still accepts it." + +[[known_gap]] +operation = "GET /conversations/simulated/{conversation_id}/comments" +reason = "Transcript comments need a dedicated cursor-paginated CLI resource with comments:read and workspace selection; they are distinct from metric review annotations." + +[[known_gap]] +operation = "GET /conversations/simulated/{conversation_id}/comments/{comment_id}" +reason = "Transcript comment retrieval belongs to the dedicated comment-thread CLI resource, with conversation ownership and workspace access checks." + +[[known_gap]] +operation = "GET /conversations/uploaded/{conversation_id}/comments" +reason = "Transcript comments need a dedicated cursor-paginated CLI resource with comments:read and workspace selection; they are distinct from metric review annotations." + +[[known_gap]] +operation = "GET /conversations/uploaded/{conversation_id}/comments/{comment_id}" +reason = "Transcript comment retrieval belongs to the dedicated comment-thread CLI resource, with conversation ownership and workspace access checks." + +[[known_gap]] +operation = "POST /conversation-python-analyses" +reason = "Exploratory audio analysis needs a request-UUID recovery and asynchronous-result CLI workflow; the CLI currently supports the immutable metric-draft and preview workflow instead." + +[[known_gap]] +operation = "GET /conversation-python-analyses/{analysis_id}" +reason = "Analysis polling belongs to the dedicated exploratory-audio-analysis CLI workflow, including recovery and expired-result handling." + +[[known_gap]] +operation = "POST /python-metrics:fromAnalysis" +reason = "Publishing from an exploratory analysis needs its own explicit approval workflow; the CLI currently publishes immutable metric drafts using exact fingerprints and successful previews." diff --git a/src/client/models/common.rs b/src/client/models/common.rs index 3190348..99934fe 100644 --- a/src/client/models/common.rs +++ b/src/client/models/common.rs @@ -61,3 +61,11 @@ where { Option::::deserialize(deserializer).map(Some) } + +pub(crate) fn non_nullable_option<'de, D, T>(deserializer: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, + T: Deserialize<'de>, +{ + T::deserialize(deserializer).map(Some) +} diff --git a/src/client/models/conversation.rs b/src/client/models/conversation.rs index 9dfac17..e59674c 100644 --- a/src/client/models/conversation.rs +++ b/src/client/models/conversation.rs @@ -101,6 +101,12 @@ pub struct SubmitConversationRequest { pub occurred_at: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub agent_id: Option, + #[serde( + default, + deserialize_with = "super::explicit_option", + skip_serializing_if = "Option::is_none" + )] + pub agent_channel: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub tags: Option>, } @@ -135,14 +141,16 @@ pub struct ConversationAudioUrlResponse { #[derive(Debug, Serialize)] pub struct PatchConversationRequest { + #[serde(skip_serializing_if = "Option::is_none")] + pub agent_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub agent_channel: Option, #[serde(skip_serializing_if = "Option::is_none")] pub audio: Option, #[serde(skip_serializing_if = "Option::is_none")] pub audio_url: Option, #[serde(skip_serializing_if = "Option::is_none")] pub audio_reference: Option, - /// Additive only: the API rejects a key that already has a value. Exactly one - /// of the four fields on this request may be supplied. #[serde(skip_serializing_if = "Option::is_none")] pub metadata: Option>, } diff --git a/src/client/models/review_project.rs b/src/client/models/review_project.rs index b05574a..aabc26b 100644 --- a/src/client/models/review_project.rs +++ b/src/client/models/review_project.rs @@ -45,6 +45,16 @@ impl std::fmt::Display for ProjectType { } } +#[derive(Debug, Clone, Copy, Serialize, Deserialize, ValueEnum)] +pub enum BatchAssignmentMode { + #[serde(rename = "SELF")] + #[value(name = "SELF")] + SelfAssigned, + #[serde(rename = "MANAGER")] + #[value(name = "MANAGER")] + ManagerAssigned, +} + #[derive(Debug, Serialize, Deserialize)] pub struct CreateReviewProjectRequest { pub display_name: String, @@ -66,6 +76,18 @@ pub struct CreateReviewProjectRequest { pub blind_labeling_shown_metric_ids: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub enforced_collaboration: Option, + #[serde( + default, + deserialize_with = "super::explicit_option", + skip_serializing_if = "Option::is_none" + )] + pub batch_assignment_mode: Option>, + #[serde( + default, + deserialize_with = "super::explicit_option", + skip_serializing_if = "Option::is_none" + )] + pub batch_size: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub allow_metric_skipping: Option, #[serde( @@ -118,6 +140,18 @@ pub struct UpdateReviewProjectRequest { pub metric_addition_completion_action: Option, #[serde(skip_serializing_if = "Option::is_none")] pub enforced_collaboration: Option, + #[serde( + default, + deserialize_with = "super::explicit_option", + skip_serializing_if = "Option::is_none" + )] + pub batch_assignment_mode: Option>, + #[serde( + default, + deserialize_with = "super::explicit_option", + skip_serializing_if = "Option::is_none" + )] + pub batch_size: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub allow_metric_skipping: Option, #[serde( diff --git a/src/client/models/test_set.rs b/src/client/models/test_set.rs index 43cb62d..b87702e 100644 --- a/src/client/models/test_set.rs +++ b/src/client/models/test_set.rs @@ -28,6 +28,12 @@ pub struct TestSet { #[derive(Debug, Serialize, Deserialize)] pub struct CreateTestSetRequest { + #[serde( + default, + deserialize_with = "super::non_nullable_option", + skip_serializing_if = "Option::is_none" + )] + pub test_cases: Option>, pub display_name: String, #[serde(skip_serializing_if = "Option::is_none")] pub slug: Option, diff --git a/src/commands/conversations.rs b/src/commands/conversations.rs index 69a623b..b19e467 100644 --- a/src/commands/conversations.rs +++ b/src/commands/conversations.rs @@ -106,6 +106,8 @@ pub struct SubmitArgs { /// Agent ID to associate with this conversation (22-char ID) #[arg(long)] agent_id: Option, + #[arg(long, value_parser = clap::value_parser!(u8).range(0..=1), help = "Agent channel in stereo audio: 0 is left, 1 is right")] + agent_channel: Option, /// When the conversation occurred (ISO 8601, e.g. 2026-05-05T12:34:56Z) #[arg(long)] occurred_at: Option>, @@ -138,6 +140,10 @@ pub struct MetricDetailArgs { #[derive(Args)] pub struct PatchArgs { conversation_id: String, + #[arg(long, help = "Assign an agent to a conversation submitted without one")] + agent_id: Option, + #[arg(long, value_parser = clap::value_parser!(u8).range(0..=1), help = "Agent channel in stereo audio: 0 is left, 1 is right")] + agent_channel: Option, #[arg(long)] audio_url: Option, #[arg(long)] @@ -405,22 +411,26 @@ async fn execute_for( } ConversationCommands::Patch(args) => { use crate::client::models::PatchConversationRequest; - // The API patches audio and metadata separately so a rejected metadata - // key can never leave audio half-attached, and accepts exactly one - // target per call. + if args.agent_channel.is_some() && args.audio_file.is_none() && args.audio_url.is_none() + { + anyhow::bail!("--agent-channel requires audio via --audio-file or --audio-url"); + } let targets = [ args.audio_file.is_some(), args.audio_url.is_some(), !args.metadata.is_empty(), + args.agent_id.is_some(), ] .into_iter() .filter(|supplied| *supplied) .count(); if targets > 1 { - anyhow::bail!("--audio-file, --audio-url, and --metadata are mutually exclusive"); + anyhow::bail!( + "--audio-file, --audio-url, --metadata, and --agent-id are mutually exclusive" + ); } if targets == 0 { - anyhow::bail!("must provide exactly one of: --audio-file, --audio-url, --metadata"); + anyhow::bail!("must provide exactly one of: --audio-file, --audio-url, --metadata, --agent-id"); } let audio_b64 = match args.audio_file { Some(path) => { @@ -438,6 +448,8 @@ async fn execute_for( .collect() }); let req = PatchConversationRequest { + agent_id: args.agent_id, + agent_channel: args.agent_channel, audio: audio_b64, audio_url: args.audio_url, audio_reference: None, @@ -630,6 +642,7 @@ fn build_submit_request(args: SubmitArgs) -> Result { input_json::insert(&mut input, "external_conversation_id", args.external_id)?; input_json::insert(&mut input, "occurred_at", args.occurred_at)?; input_json::insert(&mut input, "agent_id", args.agent_id)?; + input_json::insert(&mut input, "agent_channel", args.agent_channel)?; input_json::insert( &mut input, "tags", diff --git a/src/commands/review_projects.rs b/src/commands/review_projects.rs index eebf7ef..c1401fd 100644 --- a/src/commands/review_projects.rs +++ b/src/commands/review_projects.rs @@ -2,7 +2,8 @@ use anyhow::Result; use clap::{Args, Subcommand}; use crate::client::models::{ - CreateReviewProjectRequest, ListParams, ProjectType, UpdateReviewProjectRequest, + BatchAssignmentMode, CreateReviewProjectRequest, ListParams, ProjectType, + UpdateReviewProjectRequest, }; use crate::client::CovalClient; use crate::input_json::{self, InputJsonArg}; @@ -91,6 +92,14 @@ pub struct CreateArgs { allow_metric_skipping: Option, #[arg(long, value_delimiter = ',')] skippable_metric_ids: Option>, + #[arg( + long, + value_enum, + help = "Who forms batches on an enforced collaborative project" + )] + batch_assignment_mode: Option, + #[arg(long, value_parser = clap::value_parser!(u8).range(1..=50), help = "Conversations per reviewer batch (1-50)")] + batch_size: Option, } #[derive(Args)] @@ -148,6 +157,14 @@ pub struct UpdateArgs { allow_metric_skipping: Option, #[arg(long, value_delimiter = ',')] skippable_metric_ids: Option>, + #[arg( + long, + value_enum, + help = "Who forms batches on an enforced collaborative project" + )] + batch_assignment_mode: Option, + #[arg(long, value_parser = clap::value_parser!(u8).range(1..=50), help = "Conversations per reviewer batch (1-50)")] + batch_size: Option, } #[derive(Args)] @@ -228,6 +245,12 @@ pub async fn execute( "skippable_metric_ids", args.skippable_metric_ids, )?; + input_json::insert( + &mut input, + "batch_assignment_mode", + args.batch_assignment_mode, + )?; + input_json::insert(&mut input, "batch_size", args.batch_size)?; let req: CreateReviewProjectRequest = input_json::finish(input)?; let project = client.review_projects().create(req).await?; emit_one_with_actions( @@ -289,6 +312,12 @@ pub async fn execute( "skippable_metric_ids", args.skippable_metric_ids, )?; + input_json::insert( + &mut input, + "batch_assignment_mode", + args.batch_assignment_mode, + )?; + input_json::insert(&mut input, "batch_size", args.batch_size)?; let req: UpdateReviewProjectRequest = input_json::finish(input)?; let project = client .review_projects() diff --git a/tests/cli_tests.rs b/tests/cli_tests.rs index d93d92f..5986df5 100644 --- a/tests/cli_tests.rs +++ b/tests/cli_tests.rs @@ -8645,3 +8645,259 @@ async fn test_review_annotations_create_forwards_both_reviewer_identifiers() { assert_eq!(body["assignee"], "reviewer@example.com"); assert_eq!(body["assignee_user_id"], REVIEWER_USER_ID); } + +#[tokio::test] +async fn test_november_parity_review_batch_settings_preserve_null_and_omission() { + for (verb, command, endpoint) in [ + ("POST", "create", "/v1/review-projects"), + ("PATCH", "update", "/v1/review-projects/proj456"), + ] { + for settings in [ + json!({"batch_assignment_mode": "SELF", "batch_size": 1}), + json!({"batch_assignment_mode": "MANAGER", "batch_size": 50}), + json!({"batch_assignment_mode": null, "batch_size": null}), + json!({}), + ] { + let server = MockServer::start().await; + let capture = BodyCapture::default(); + Mock::given(method(verb)) + .and(path(endpoint)) + .and(capture.clone()) + .respond_with(ResponseTemplate::new(200).set_body_json(review_project_response())) + .expect(1) + .mount(&server) + .await; + let mut input = settings.clone(); + if command == "create" { + input["display_name"] = json!("Review"); + input["assignees"] = json!(["reviewer@example.com"]); + input["linked_simulation_ids"] = json!(["sim1"]); + input["linked_metric_ids"] = json!(["met1"]); + } + let mut cli = coval_with_api(&server); + cli.args(["review-projects", command]); + if command == "update" { + cli.arg("proj456"); + } + cli.arg("--input-json") + .arg(input.to_string()) + .assert() + .success(); + let body = capture.take(); + for field in ["batch_assignment_mode", "batch_size"] { + assert_eq!(body.get(field), settings.get(field)); + } + } + } +} + +#[tokio::test] +async fn test_november_parity_review_batch_flags_override_json() { + let body = patch_body_for( + "/v1/review-projects/proj456", + review_project_response(), + &[ + "review-projects", + "update", + "proj456", + "--input-json", + r#"{"batch_assignment_mode":null,"batch_size":null}"#, + "--batch-assignment-mode", + "MANAGER", + "--batch-size", + "7", + ], + ) + .await; + assert_eq!(body["batch_assignment_mode"], "MANAGER"); + assert_eq!(body["batch_size"], 7); +} + +fn november_uploaded_response() -> Value { + json!({"uploaded_conversation": { + "name": "conversations/uploaded123", "conversation_id": "uploaded123", + "status": "COMPLETED", "create_time": "2026-11-02T10:00:00Z", "has_audio": true + }}) +} + +#[tokio::test] +async fn test_november_parity_uploaded_submit_preserves_channel_zero_and_one() { + for channel in [Some(json!(0)), Some(json!(1)), Some(Value::Null), None] { + let server = MockServer::start().await; + let capture = BodyCapture::default(); + Mock::given(method("POST")) + .and(path("/v1/conversations/uploaded:submit")) + .and(capture.clone()) + .respond_with(ResponseTemplate::new(200).set_body_json(november_uploaded_response())) + .expect(1) + .mount(&server) + .await; + let mut input = json!({"audio_url": "https://example.com/call.wav"}); + if let Some(value) = channel { + input["agent_channel"] = json!(value); + } + coval_with_api(&server) + .args(["uploaded-conversations", "submit", "--input-json"]) + .arg(input.to_string()) + .assert() + .success(); + assert_eq!( + capture.take().get("agent_channel"), + input.get("agent_channel") + ); + } +} + +#[tokio::test] +async fn test_november_parity_uploaded_patch_agent_and_channel_targets() { + let body = patch_body_for( + "/v1/conversations/uploaded/uploaded123", + november_uploaded_response(), + &[ + "uploaded-conversations", + "patch", + "uploaded123", + "--agent-id", + "fZSUMoKq67ZarVYCbugjsd", + ], + ) + .await; + assert_eq!(body, json!({"agent_id": "fZSUMoKq67ZarVYCbugjsd"})); + for channel in ["0", "1"] { + let body = patch_body_for( + "/v1/conversations/uploaded/uploaded123", + november_uploaded_response(), + &[ + "uploaded-conversations", + "patch", + "uploaded123", + "--audio-url", + "https://example.com/call.wav", + "--agent-channel", + channel, + ], + ) + .await; + assert_eq!( + body, + json!({"audio_url": "https://example.com/call.wav", "agent_channel": channel.parse::().unwrap()}) + ); + } + let server = MockServer::start().await; + for extra in [ + vec!["--metadata", "key=value"], + vec!["--audio-url", "https://example.com/call.wav"], + ] { + coval_with_api(&server) + .args([ + "uploaded-conversations", + "patch", + "uploaded123", + "--agent-id", + "fZSUMoKq67ZarVYCbugjsd", + ]) + .args(extra) + .assert() + .failure() + .stderr(predicate::str::contains("mutually exclusive")); + } + coval_with_api(&server) + .args([ + "uploaded-conversations", + "patch", + "uploaded123", + "--agent-channel", + "0", + ]) + .assert() + .failure() + .stderr(predicate::str::contains("requires audio")); +} + +#[tokio::test] +async fn test_november_parity_test_set_inline_cases_and_invalid_null() { + for cases in [ + Some( + json!([{"input_str":"Read the script", "input_type":"SCRIPT", "script_turns":["Hello"], "description":null, "expected_behaviors":["Greets"], "expected_output_json":{"category":"support"}}]), + ), + Some(json!([])), + None, + ] { + let server = MockServer::start().await; + let capture = BodyCapture::default(); + Mock::given(method("POST")) + .and(path("/v1/test-sets")) + .and(capture.clone()) + .respond_with(ResponseTemplate::new(200).set_body_json(test_set_patch_response())) + .expect(1) + .mount(&server) + .await; + let mut input = json!({"display_name":"Inline cases"}); + if let Some(value) = &cases { + input["test_cases"] = value.clone(); + } + coval_with_api(&server) + .args(["test-sets", "create", "--input-json"]) + .arg(input.to_string()) + .assert() + .success(); + let body = capture.take(); + assert_eq!(body.get("test_cases"), input.get("test_cases")); + } + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/test-sets")) + .respond_with(ResponseTemplate::new(200).set_body_json(test_set_patch_response())) + .expect(0) + .mount(&server) + .await; + coval_with_api(&server) + .args([ + "test-sets", + "create", + "--input-json", + r#"{"display_name":"Invalid","test_cases":null}"#, + ]) + .assert() + .failure(); +} + +#[tokio::test] +async fn test_november_parity_rejects_invalid_batch_mode_and_flag_ranges() { + let server = MockServer::start().await; + coval_with_api(&server) + .args([ + "review-projects", + "update", + "proj456", + "--input-json", + r#"{"batch_assignment_mode":"OTHER"}"#, + ]) + .assert() + .failure() + .stderr(predicate::str::contains("unknown variant")); + for value in ["0", "51"] { + coval_with_api(&server) + .args([ + "review-projects", + "update", + "proj456", + "--batch-size", + value, + ]) + .assert() + .failure(); + } + coval_with_api(&server) + .args([ + "uploaded-conversations", + "patch", + "uploaded123", + "--audio-url", + "https://example.com/call.wav", + "--agent-channel", + "2", + ]) + .assert() + .failure(); +} From ba2b63aef12947f2748d942d92db393cde31e659 Mon Sep 17 00:00:00 2001 From: Callum Reid Date: Mon, 2 Nov 2026 14:26:28 -0800 Subject: [PATCH 6/7] fix: recognize completed previews and keep polling queued ones --- src/commands/python_metric_drafts.rs | 14 ++++---- tests/cli_tests.rs | 49 +++++++++++++++++++++++++++- 2 files changed, 55 insertions(+), 8 deletions(-) diff --git a/src/commands/python_metric_drafts.rs b/src/commands/python_metric_drafts.rs index 5f74ac5..be6d355 100644 --- a/src/commands/python_metric_drafts.rs +++ b/src/commands/python_metric_drafts.rs @@ -272,7 +272,9 @@ pub async fn execute( .await?; let mut actions = vec![next_actions::context("python-metric-drafts")]; if let Some(preview) = response.previews.first() { - if preview.status.eq_ignore_ascii_case("succeeded") { + if preview.status.eq_ignore_ascii_case("completed") + || preview.status.eq_ignore_ascii_case("succeeded") + { actions.insert( 0, NextAction::new( @@ -292,12 +294,10 @@ pub async fn execute( .primary() .with_description("Publish only after the preview succeeds and the exact draft fingerprint is approved."), ); - } else if preview.error.is_none() - && !matches!( - preview.status.to_ascii_lowercase().as_str(), - "failed" | "cancelled" | "canceled" | "error" - ) - { + } else if !matches!( + preview.status.to_ascii_lowercase().as_str(), + "failed" | "cancelled" | "canceled" | "error" + ) { actions.insert( 0, NextAction::new( diff --git a/tests/cli_tests.rs b/tests/cli_tests.rs index 5986df5..0804c69 100644 --- a/tests/cli_tests.rs +++ b/tests/cli_tests.rs @@ -870,7 +870,7 @@ async fn test_python_metric_draft_commands_send_specified_requests() { "previews": [{ "preview_id": "preview_123", "simulation_output_id": "simulation_123", - "status": "SUCCEEDED", + "status": "COMPLETED", "value": true, "explanation": null, "error": null @@ -1071,6 +1071,53 @@ async fn test_python_metric_draft_commands_send_specified_requests() { assert_eq!(invalid_preview["ok"], false); } +#[tokio::test] +async fn test_python_metric_preview_queued_with_admission_warning_offers_polling() { + let mock_server = MockServer::start().await; + let temp_dir = tempfile::tempdir().unwrap(); + + Mock::given(method("GET")) + .and(path("/v1/python-metric-drafts/draft_123/previews/preview_123")) + .and(header("X-API-Key", "test_key")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "previews": [{ + "preview_id": "preview_123", + "simulation_output_id": "simulation_123", + "status": "QUEUED", + "value": null, + "explanation": null, + "error": "queue admission is unconfirmed; the submission may still run, retain and poll this preview id before retrying" + }] + }))) + .mount(&mock_server) + .await; + + let value = stdout_json( + coval_with_api(&mock_server) + .arg("--agent") + .args([ + "python-metric-drafts", + "previews", + "draft_123", + "get", + "preview_123", + ]) + .arg("--format") + .arg("json") + .env("HOME", temp_dir.path()) + .env("XDG_CONFIG_HOME", temp_dir.path()) + .assert() + .success(), + ); + + assert_eq!(value["ok"], true); + assert_eq!(value["data"]["previews"][0]["status"], "QUEUED"); + assert_eq!( + value["next_actions"][0]["id"], + "python-metric-drafts.previews.get" + ); +} + #[tokio::test] async fn test_python_metric_draft_source_file_and_missing_file() { let mock_server = MockServer::start().await; From 112ad1f9c025c9a8802b5b52161cdb9349198a72 Mon Sep 17 00:00:00 2001 From: Callum Reid Date: Mon, 2 Nov 2026 14:57:43 -0800 Subject: [PATCH 7/7] fix: stop polling skipped Python metric previews (COVAL-7661) --- src/commands/python_metric_drafts.rs | 2 +- tests/cli_tests.rs | 43 ++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/src/commands/python_metric_drafts.rs b/src/commands/python_metric_drafts.rs index be6d355..661c16b 100644 --- a/src/commands/python_metric_drafts.rs +++ b/src/commands/python_metric_drafts.rs @@ -296,7 +296,7 @@ pub async fn execute( ); } else if !matches!( preview.status.to_ascii_lowercase().as_str(), - "failed" | "cancelled" | "canceled" | "error" + "failed" | "cancelled" | "canceled" | "error" | "skipped" ) { actions.insert( 0, diff --git a/tests/cli_tests.rs b/tests/cli_tests.rs index 0804c69..64a75fa 100644 --- a/tests/cli_tests.rs +++ b/tests/cli_tests.rs @@ -8948,3 +8948,46 @@ async fn test_november_parity_rejects_invalid_batch_mode_and_flag_ranges() { .assert() .failure(); } + +#[tokio::test] +async fn test_python_metric_preview_unavailable_is_terminal() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path( + "/v1/python-metric-drafts/draft_123/previews/preview_123", + )) + .and(header("X-API-Key", "test_key")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "previews": [{ + "preview_id": "preview_123", "simulation_output_id": "simulation_123", + "status": "SKIPPED", "value": null, "explanation": null, + "error": "Required conversation audio is unavailable" + }] + }))) + .expect(1) + .mount(&server) + .await; + let value = stdout_json( + coval_with_api(&server) + .args([ + "--agent", + "python-metric-drafts", + "previews", + "draft_123", + "get", + "preview_123", + ]) + .assert() + .success(), + ); + assert_eq!(value["ok"], true); + assert_eq!(value["data"]["previews"][0]["status"], "SKIPPED"); + assert_eq!( + value["data"]["previews"][0]["error"], + "Required conversation audio is unavailable" + ); + for action in value["next_actions"].as_array().unwrap() { + assert_ne!(action["id"], "python-metric-drafts.previews.get"); + assert_ne!(action["id"], "python-metric-drafts.publish"); + } +}