From 37d54f6f1324dcbf50634e3816c19c4039c08b07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20=C5=A0=C4=87eki=C4=87?= Date: Fri, 7 Aug 2026 20:02:32 +0200 Subject: [PATCH 1/5] feat(shared): label picker triggers --- apps/web/src/components/shared/ModelCombobox.tsx | 7 +++++++ apps/web/src/components/shared/VariantCombobox.tsx | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/apps/web/src/components/shared/ModelCombobox.tsx b/apps/web/src/components/shared/ModelCombobox.tsx index 47bf490e695..f208a9cc43a 100644 --- a/apps/web/src/components/shared/ModelCombobox.tsx +++ b/apps/web/src/components/shared/ModelCombobox.tsx @@ -64,6 +64,8 @@ export type ModelComboboxProps = { * cannot be scrolled. */ modal?: boolean; + /** Optional aria-label for the trigger button, overriding the default accessible name */ + triggerAriaLabel?: string; }; export function ModelCombobox({ @@ -85,6 +87,7 @@ export function ModelCombobox({ disabled = false, pinnedModel, modal = false, + triggerAriaLabel, }: ModelComboboxProps) { const [open, setOpen] = useState(false); const triggerRef = useRef(null); @@ -128,6 +131,7 @@ export function ModelCombobox({ type="button" variant="outline" size="sm" + aria-label={triggerAriaLabel} className={cn('h-9 border-red-400/50 text-red-400', className)} disabled > @@ -159,6 +163,7 @@ export function ModelCombobox({ type="button" variant="outline" size="sm" + aria-label={triggerAriaLabel} className={cn('text-muted-foreground h-9', className)} disabled > @@ -194,6 +199,7 @@ export function ModelCombobox({ variant="outline" size="sm" role="combobox" + aria-label={triggerAriaLabel} aria-expanded={open} disabled={disabled} className={cn('h-9 justify-between gap-1.5', className)} @@ -256,6 +262,7 @@ export function ModelCombobox({ type="button" variant="outline" role="combobox" + aria-label={triggerAriaLabel} aria-expanded={open} disabled={disabled} className={cn('w-full justify-between', className)} diff --git a/apps/web/src/components/shared/VariantCombobox.tsx b/apps/web/src/components/shared/VariantCombobox.tsx index ebd76260854..20cd798f091 100644 --- a/apps/web/src/components/shared/VariantCombobox.tsx +++ b/apps/web/src/components/shared/VariantCombobox.tsx @@ -21,6 +21,8 @@ type VariantComboboxProps = { variant?: 'compact'; /** Optional className for the trigger button */ className?: string; + /** Optional aria-label for the trigger button, overriding the default accessible name */ + triggerAriaLabel?: string; }; export function VariantCombobox({ @@ -29,6 +31,7 @@ export function VariantCombobox({ onValueChange, disabled = false, className, + triggerAriaLabel, }: VariantComboboxProps) { const [open, setOpen] = useState(false); @@ -44,6 +47,7 @@ export function VariantCombobox({ variant="outline" size="sm" role="combobox" + aria-label={triggerAriaLabel} aria-expanded={open} disabled={disabled} className={cn('h-9 justify-between gap-1.5', className)} From da86a79ecd4520cbd5e13cd13ef83a77492bc726 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20=C5=A0=C4=87eki=C4=87?= Date: Fri, 7 Aug 2026 20:02:44 +0200 Subject: [PATCH 2/5] feat(benchmark): persist decider variants --- .../migrations/0009_cold_cobalt_man.sql | 1 + .../migrations/meta/0009_snapshot.json | 1067 +++++++++++++++++ .../migrations/meta/_journal.json | 7 + .../auto-routing-benchmark/src/admin.test.ts | 30 +- .../src/auto-decider-sync.test.ts | 4 +- .../auto-routing-benchmark/src/config.test.ts | 58 +- services/auto-routing-benchmark/src/config.ts | 31 +- .../auto-routing-benchmark/src/db-schema.ts | 2 + .../auto-routing-benchmark/src/db.test.ts | 59 + services/auto-routing-benchmark/src/db.ts | 9 +- .../src/profile-runs.test.ts | 39 +- .../src/routing-table-builder.test.ts | 41 +- .../src/routing-table-builder.ts | 11 +- .../src/run-process-job.test.ts | 114 ++ services/auto-routing-benchmark/src/run.ts | 23 +- 15 files changed, 1450 insertions(+), 46 deletions(-) create mode 100644 services/auto-routing-benchmark/migrations/0009_cold_cobalt_man.sql create mode 100644 services/auto-routing-benchmark/migrations/meta/0009_snapshot.json diff --git a/services/auto-routing-benchmark/migrations/0009_cold_cobalt_man.sql b/services/auto-routing-benchmark/migrations/0009_cold_cobalt_man.sql new file mode 100644 index 00000000000..d4f5ad2c34a --- /dev/null +++ b/services/auto-routing-benchmark/migrations/0009_cold_cobalt_man.sql @@ -0,0 +1 @@ +ALTER TABLE `config_decider_models` ADD `variant` text; \ No newline at end of file diff --git a/services/auto-routing-benchmark/migrations/meta/0009_snapshot.json b/services/auto-routing-benchmark/migrations/meta/0009_snapshot.json new file mode 100644 index 00000000000..41be6122774 --- /dev/null +++ b/services/auto-routing-benchmark/migrations/meta/0009_snapshot.json @@ -0,0 +1,1067 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "71e8de8c-6976-494d-8532-109016ea7f84", + "prevId": "c0afe0b5-ea9a-4206-a099-34099ac02ee9", + "tables": { + "benchmark_config": { + "name": "benchmark_config", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "min_accuracy": { + "name": "min_accuracy", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "switch_cost_factor": { + "name": "switch_cost_factor", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "best_accuracy_switch_threshold": { + "name": "best_accuracy_switch_threshold", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0.05 + }, + "max_concurrency": { + "name": "max_concurrency", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "benchmark_user_id": { + "name": "benchmark_user_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "benchmark_org_id": { + "name": "benchmark_org_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "classifier_repetitions": { + "name": "classifier_repetitions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "decider_repetitions": { + "name": "decider_repetitions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "classifier_max_p95_latency_ms": { + "name": "classifier_max_p95_latency_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "auto_decider_min_cost_usd": { + "name": "auto_decider_min_cost_usd", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 15 + }, + "auto_decider_max_cost_usd": { + "name": "auto_decider_max_cost_usd", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 25 + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "benchmark_profiles": { + "name": "benchmark_profiles", + "columns": { + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "engine_identity": { + "name": "engine_identity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "repetitions": { + "name": "repetitions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "requested_at": { + "name": "requested_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "completed_at": { + "name": "completed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "benchmark_profiles_model_variant_engine_identity_repetitions_pk": { + "columns": [ + "model", + "variant", + "engine_identity", + "repetitions" + ], + "name": "benchmark_profiles_model_variant_engine_identity_repetitions_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "benchmark_runs": { + "name": "benchmark_runs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "completed_at": { + "name": "completed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "min_accuracy": { + "name": "min_accuracy", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "switch_cost_factor": { + "name": "switch_cost_factor", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "best_accuracy_switch_threshold": { + "name": "best_accuracy_switch_threshold", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0.05 + }, + "max_concurrency": { + "name": "max_concurrency", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "benchmark_user_id": { + "name": "benchmark_user_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "benchmark_org_id": { + "name": "benchmark_org_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "repetitions": { + "name": "repetitions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "classifier_max_p95_latency_ms": { + "name": "classifier_max_p95_latency_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "engine_identity": { + "name": "engine_identity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "purpose": { + "name": "purpose", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'platform'" + } + }, + "indexes": { + "UQ_benchmark_runs_one_running_per_kind": { + "name": "UQ_benchmark_runs_one_running_per_kind", + "columns": [ + "kind" + ], + "isUnique": true, + "where": "\"benchmark_runs\".\"status\" = 'running'" + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "case_results": { + "name": "case_results", + "columns": { + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "case_id": { + "name": "case_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "route_key": { + "name": "route_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "score": { + "name": "score", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "latency_ms": { + "name": "latency_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cost_usd": { + "name": "cost_usd", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fallback_reason": { + "name": "fallback_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "retried": { + "name": "retried", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "exit_code": { + "name": "exit_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "output_prefix": { + "name": "output_prefix", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "event_count": { + "name": "event_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_event_types": { + "name": "last_event_types", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "rep": { + "name": "rep", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "timed_out": { + "name": "timed_out", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "case_results_run_id_model_variant_case_id_rep_pk": { + "columns": [ + "run_id", + "model", + "variant", + "case_id", + "rep" + ], + "name": "case_results_run_id_model_variant_case_id_rep_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "config_auto_decider_exclusions": { + "name": "config_auto_decider_exclusions", + "columns": { + "model": { + "name": "model", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "config_auto_decider_models": { + "name": "config_auto_decider_models", + "columns": { + "model": { + "name": "model", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "reasoning_effort": { + "name": "reasoning_effort", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "avg_attempt_cost_usd": { + "name": "avg_attempt_cost_usd", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "synced_at": { + "name": "synced_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "config_classifier_models": { + "name": "config_classifier_models", + "columns": { + "model": { + "name": "model", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "config_decider_models": { + "name": "config_decider_models", + "columns": { + "model": { + "name": "model", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reasoning_effort": { + "name": "reasoning_effort", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "model_summaries": { + "name": "model_summaries", + "columns": { + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "route_key": { + "name": "route_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accuracy": { + "name": "accuracy", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "avg_cost_usd": { + "name": "avg_cost_usd", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "avg_latency_ms": { + "name": "avg_latency_ms", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "p50_latency_ms": { + "name": "p50_latency_ms", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cases": { + "name": "cases", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "errors": { + "name": "errors", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "p95_latency_ms": { + "name": "p95_latency_ms", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "timeouts": { + "name": "timeouts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "carried": { + "name": "carried", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "model_summaries_run_id_model_variant_route_key_pk": { + "columns": [ + "run_id", + "model", + "variant", + "route_key" + ], + "name": "model_summaries_run_id_model_variant_route_key_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "profile_request_events": { + "name": "profile_request_events", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "owner_type": { + "name": "owner_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "owner_id": { + "name": "owner_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "engine_identity": { + "name": "engine_identity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "repetitions": { + "name": "repetitions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "admitted_at": { + "name": "admitted_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "IDX_profile_request_events_owner_admitted": { + "name": "IDX_profile_request_events_owner_admitted", + "columns": [ + "owner_type", + "owner_id", + "admitted_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "routing_table_candidates": { + "name": "routing_table_candidates", + "columns": { + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "route_key": { + "name": "route_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "rank": { + "name": "rank", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accuracy": { + "name": "accuracy", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "avg_cost_usd": { + "name": "avg_cost_usd", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "meets_threshold": { + "name": "meets_threshold", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reasoning_effort": { + "name": "reasoning_effort", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "routing_table_candidates_run_id_route_key_rank_pk": { + "columns": [ + "run_id", + "route_key", + "rank" + ], + "name": "routing_table_candidates_run_id_route_key_rank_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "routing_tables": { + "name": "routing_tables", + "columns": { + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "published_at": { + "name": "published_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "generated_at": { + "name": "generated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "min_accuracy": { + "name": "min_accuracy", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "switch_cost_factor": { + "name": "switch_cost_factor", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "best_accuracy_switch_threshold": { + "name": "best_accuracy_switch_threshold", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0.05 + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "run_lane_failures": { + "name": "run_lane_failures", + "columns": { + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "rep": { + "name": "rep", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "chunk": { + "name": "chunk", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "shard": { + "name": "shard", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "failed_at": { + "name": "failed_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "run_lane_failures_run_id_model_variant_rep_chunk_shard_pk": { + "columns": [ + "run_id", + "model", + "variant", + "rep", + "chunk", + "shard" + ], + "name": "run_lane_failures_run_id_model_variant_rep_chunk_shard_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "run_models": { + "name": "run_models", + "columns": { + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "enqueued": { + "name": "enqueued", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reasoning_effort": { + "name": "reasoning_effort", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "run_models_run_id_model_variant_pk": { + "columns": [ + "run_id", + "model", + "variant" + ], + "name": "run_models_run_id_model_variant_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/services/auto-routing-benchmark/migrations/meta/_journal.json b/services/auto-routing-benchmark/migrations/meta/_journal.json index 2fc3bbc764f..d78b15322ae 100644 --- a/services/auto-routing-benchmark/migrations/meta/_journal.json +++ b/services/auto-routing-benchmark/migrations/meta/_journal.json @@ -64,6 +64,13 @@ "when": 1786106641954, "tag": "0008_easy_doctor_faustus", "breakpoints": true + }, + { + "idx": 9, + "version": "6", + "when": 1786124177494, + "tag": "0009_cold_cobalt_man", + "breakpoints": true } ] } \ No newline at end of file diff --git a/services/auto-routing-benchmark/src/admin.test.ts b/services/auto-routing-benchmark/src/admin.test.ts index 9b88914fa8c..c4b04d9b9e1 100644 --- a/services/auto-routing-benchmark/src/admin.test.ts +++ b/services/auto-routing-benchmark/src/admin.test.ts @@ -69,6 +69,7 @@ const TEST_CONFIG_ROWS = { classifierModels: TEST_CONFIG.classifierModels, deciderModels: TEST_CONFIG.deciderModels.map(m => ({ model: m.id, + variant: null, reasoning_effort: m.reasoningEffort ?? null, })), autoDeciderModels: [], @@ -236,6 +237,7 @@ describe('GET /admin/config', () => { const classifierModels = ['some/model']; const deciderModels = TEST_CONFIG.deciderModels.map(m => ({ model: m.id, + variant: null, reasoning_effort: null, })); vi.mocked(getConfigRows).mockResolvedValueOnce({ @@ -349,7 +351,9 @@ describe('PUT /admin/config', () => { expect(configArg.auto_decider_max_cost_usd).toBe(25); expect(typeof configArg.updated_at).toBe('string'); expect(configArg.updated_by).toBe('igor@kilocode.ai'); - expect(deciderModelRows).toEqual([{ model: 'manual/model', reasoning_effort: 'low' }]); + expect(deciderModelRows).toEqual([ + { model: 'manual/model', variant: null, reasoning_effort: 'low' }, + ]); expect(excludedAutoDeciderModels).toEqual(['auto/excluded']); }); }); @@ -478,7 +482,7 @@ describe('POST /admin/runs', () => { benchmark_user_id: null, benchmark_org_id: null, }, - deciderModels: [{ model: 'vendor/a', reasoning_effort: null }], + deciderModels: [{ model: 'vendor/a', variant: null, reasoning_effort: null }], }); const res = await authedPost('/admin/runs', { kind: 'decider' }); @@ -494,8 +498,8 @@ describe('POST /admin/runs', () => { ...TEST_CONFIG_ROWS, config: { ...TEST_CONFIG_ROWS.config, benchmark_user_id: 'user-123' }, deciderModels: [ - { model: 'vendor/a', reasoning_effort: null }, - { model: 'vendor/b', reasoning_effort: null }, + { model: 'vendor/a', variant: null, reasoning_effort: null }, + { model: 'vendor/b', variant: null, reasoning_effort: null }, ], }); // vendor/a has a prior result measured under the current engine identity, @@ -527,7 +531,7 @@ describe('POST /admin/runs', () => { vi.mocked(getConfigRows).mockResolvedValue({ ...TEST_CONFIG_ROWS, config: { ...TEST_CONFIG_ROWS.config, benchmark_user_id: 'user-123' }, - deciderModels: [{ model: 'vendor/a', reasoning_effort: null }], + deciderModels: [{ model: 'vendor/a', variant: null, reasoning_effort: null }], }); // Prior result was measured at variant 'high'; current config runs it at // null, so the carry is invalidated and the model is re-enqueued. @@ -557,7 +561,7 @@ describe('POST /admin/runs', () => { vi.mocked(getConfigRows).mockResolvedValue({ ...TEST_CONFIG_ROWS, config: { ...TEST_CONFIG_ROWS.config, benchmark_user_id: 'user-123' }, - deciderModels: [{ model: 'vendor/a', reasoning_effort: 'high' }], + deciderModels: [{ model: 'vendor/a', variant: null, reasoning_effort: 'high' }], }); // Same model/engine/reps but prior was measured at a different variant. vi.mocked(getLatestSummariesByModel).mockResolvedValue( @@ -589,8 +593,8 @@ describe('POST /admin/runs', () => { // vendor/b has no prior → stays enqueued so we do not hit the all-carried // finalize path (which needs a real D1 client in unit tests). deciderModels: [ - { model: 'vendor/a', reasoning_effort: 'high' }, - { model: 'vendor/b', reasoning_effort: null }, + { model: 'vendor/a', variant: null, reasoning_effort: 'high' }, + { model: 'vendor/b', variant: null, reasoning_effort: null }, ], }); // Legacy rows store variant from reasoning_effort; exact pair high matches. @@ -627,7 +631,7 @@ describe('POST /admin/runs', () => { vi.mocked(getConfigRows).mockResolvedValue({ ...TEST_CONFIG_ROWS, config: { ...TEST_CONFIG_ROWS.config, benchmark_user_id: 'user-123' }, - deciderModels: manyModels.map(m => ({ model: m.id, reasoning_effort: null })), + deciderModels: manyModels.map(m => ({ model: m.id, variant: null, reasoning_effort: null })), }); const res = await authedPost('/admin/runs', { kind: 'decider' }); @@ -661,7 +665,7 @@ describe('POST /admin/runs', () => { benchmark_org_id: 'org-123', decider_repetitions: 3, }, - deciderModels: manyModels.map(m => ({ model: m.id, reasoning_effort: null })), + deciderModels: manyModels.map(m => ({ model: m.id, variant: null, reasoning_effort: null })), }); const res = await authedPost('/admin/runs', { kind: 'decider' }); @@ -693,7 +697,11 @@ describe('POST /admin/runs', () => { benchmark_org_id: 'org-123', decider_repetitions: 5, }, - deciderModels: tooManyModels.map(m => ({ model: m.id, reasoning_effort: null })), + deciderModels: tooManyModels.map(m => ({ + model: m.id, + variant: null, + reasoning_effort: null, + })), }); const res = await authedPost('/admin/runs', { kind: 'decider' }); diff --git a/services/auto-routing-benchmark/src/auto-decider-sync.test.ts b/services/auto-routing-benchmark/src/auto-decider-sync.test.ts index aebcb47b9a2..5aaa09695d0 100644 --- a/services/auto-routing-benchmark/src/auto-decider-sync.test.ts +++ b/services/auto-routing-benchmark/src/auto-decider-sync.test.ts @@ -79,7 +79,7 @@ describe('syncAutoDeciderModels', () => { vi.mocked(getConfigRows).mockResolvedValue({ config, classifierModels: ['classifier/model'], - deciderModels: [{ model: 'manual/model', reasoning_effort: null }], + deciderModels: [{ model: 'manual/model', variant: null, reasoning_effort: null }], autoDeciderModels: [ { model: 'auto/existing', @@ -167,7 +167,7 @@ describe('syncAutoDeciderModels', () => { vi.mocked(getConfigRows).mockResolvedValue({ config, classifierModels: ['classifier/model'], - deciderModels: [{ model: 'manual/model', reasoning_effort: null }], + deciderModels: [{ model: 'manual/model', variant: null, reasoning_effort: null }], autoDeciderModels: [ { model: 'auto/existing', diff --git a/services/auto-routing-benchmark/src/config.test.ts b/services/auto-routing-benchmark/src/config.test.ts index b12980d903a..6099ae2a092 100644 --- a/services/auto-routing-benchmark/src/config.test.ts +++ b/services/auto-routing-benchmark/src/config.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { mapConfigRows } from './config'; +import { mapConfigRows, toDeciderModelRows } from './config'; import type { ConfigDeciderModelRow } from './db'; const configRow = { @@ -22,6 +22,7 @@ const configRow = { const deciderRows: ConfigDeciderModelRow[] = [ { model: 'some/decider', + variant: null, reasoning_effort: 'high', }, ]; @@ -81,7 +82,7 @@ describe('mapConfigRows', () => { const result = mapConfigRows( configRow, ['some/model'], - [{ model: 'auto/model', reasoning_effort: 'medium' }], + [{ model: 'auto/model', variant: null, reasoning_effort: 'medium' }], autoRows, ['auto/model'] ); @@ -94,7 +95,7 @@ describe('mapConfigRows', () => { const result = mapConfigRows( configRow, ['some/model'], - [{ model: 'manual/thinking', reasoning_effort: 'thinking' }], + [{ model: 'manual/thinking', variant: null, reasoning_effort: 'thinking' }], [ { model: 'auto/none', @@ -115,4 +116,55 @@ describe('mapConfigRows', () => { { id: 'auto/none', reasoningEffort: null }, ]); }); + + it('maps a canonical variant row to variant with reasoningEffort null (never through the effort parser)', () => { + const result = mapConfigRows( + configRow, + ['some/model'], + [{ model: 'manual/max', variant: 'max', reasoning_effort: null }], + [], + [] + ); + + expect(result?.manualDeciderModels).toEqual([ + { id: 'manual/max', variant: 'max', reasoningEffort: null }, + ]); + expect(result?.deciderModels[0].variant).toBe('max'); + expect(result?.deciderModels[0].reasoningEffort).toBeNull(); + }); + + it('maps a legacy effort row to reasoningEffort with no variant key', () => { + const result = mapConfigRows( + configRow, + ['some/model'], + [{ model: 'manual/high', variant: null, reasoning_effort: 'high' }], + [], + [] + ); + + const manual = result?.manualDeciderModels?.[0]; + expect(manual).toEqual({ id: 'manual/high', reasoningEffort: 'high' }); + // Regression guard for run.ts: a legacy row must not carry `variant` at all; + // a present `variant: null` would be read as "no variant" and drop the effort. + expect('variant' in (manual ?? {})).toBe(false); + }); +}); + +describe('toDeciderModelRows', () => { + it('writes variant for a canonical model and reasoning_effort for a legacy model, never both', () => { + const rows = toDeciderModelRows([ + { id: 'canonical/max', variant: 'max', reasoningEffort: null }, + { id: 'legacy/high', reasoningEffort: 'high' }, + { id: 'plain/model', reasoningEffort: null }, + ]); + + expect(rows).toEqual([ + { model: 'canonical/max', variant: 'max', reasoning_effort: null }, + { model: 'legacy/high', variant: null, reasoning_effort: 'high' }, + { model: 'plain/model', variant: null, reasoning_effort: null }, + ]); + for (const row of rows) { + expect(row.variant === null || row.reasoning_effort === null).toBe(true); + } + }); }); diff --git a/services/auto-routing-benchmark/src/config.ts b/services/auto-routing-benchmark/src/config.ts index 8dc0fdbee7a..87e8a983b0d 100644 --- a/services/auto-routing-benchmark/src/config.ts +++ b/services/auto-routing-benchmark/src/config.ts @@ -1,4 +1,4 @@ -import type { BenchmarkConfig } from '@kilocode/auto-routing-contracts'; +import type { BenchmarkConfig, BenchmarkDeciderModel } from '@kilocode/auto-routing-contracts'; import { getConfigRows, replaceConfig, @@ -7,6 +7,18 @@ import { } from './db'; import { parsePersistedReasoningEffort } from './reasoning-effort'; +/** + * Config rows for the manual decider list. The contract already forbids both + * variant and reasoningEffort on one model, so each row writes at most one. + */ +export function toDeciderModelRows(models: BenchmarkDeciderModel[]): ConfigDeciderModelRow[] { + return models.map(m => ({ + model: m.id, + variant: m.variant ?? null, + reasoning_effort: m.reasoningEffort ?? null, + })); +} + // Maps the three normalized config tables to the BenchmarkConfig contract. // Null when no admin has saved a config yet — the worker never fabricates // one, and runs cannot start until a config exists. @@ -32,10 +44,14 @@ export function mapConfigRows( excludedAutoDeciderModels: string[] = [] ): BenchmarkConfig | null { const excludedAuto = new Set(excludedAutoDeciderModels); - const manualDeciderModels = deciderModelRows.map(r => ({ - id: r.model, - reasoningEffort: parsePersistedReasoningEffort(r.reasoning_effort), - })); + const manualDeciderModels: BenchmarkDeciderModel[] = deciderModelRows.map(r => + r.variant != null && r.variant !== '' + ? // Canonical row: variant only, never both (contract rejects both-set). + { id: r.model, variant: r.variant, reasoningEffort: null } + : // Legacy row: leave `variant` ABSENT. Emitting `variant: null` here would + // make run.ts treat the entry as "no variant" and drop the saved effort. + { id: r.model, reasoningEffort: parsePersistedReasoningEffort(r.reasoning_effort) } + ); const manualIds = new Set(manualDeciderModels.map(model => model.id)); const autoDeciderModels = autoDeciderModelRows.map(r => ({ id: r.model, @@ -95,10 +111,7 @@ export async function saveBenchmarkConfig( const stamped: BenchmarkConfig = { ...config, updatedAt, updatedBy }; const manualDeciderModels = config.manualDeciderModels ?? config.deciderModels; - const deciderModelRows: ConfigDeciderModelRow[] = manualDeciderModels.map(m => ({ - model: m.id, - reasoning_effort: m.reasoningEffort ?? null, - })); + const deciderModelRows = toDeciderModelRows(manualDeciderModels); await replaceConfig( db, diff --git a/services/auto-routing-benchmark/src/db-schema.ts b/services/auto-routing-benchmark/src/db-schema.ts index 4bffc594f3f..f73cbe45bed 100644 --- a/services/auto-routing-benchmark/src/db-schema.ts +++ b/services/auto-routing-benchmark/src/db-schema.ts @@ -40,6 +40,8 @@ export const configClassifierModels = sqliteTable('config_classifier_models', { export const configDeciderModels = sqliteTable('config_decider_models', { model: text('model').primaryKey(), + // Canonical catalog variant key. Null for legacy rows that hold reasoning_effort. + variant: text('variant'), reasoning_effort: text('reasoning_effort'), }); diff --git a/services/auto-routing-benchmark/src/db.test.ts b/services/auto-routing-benchmark/src/db.test.ts index b143a67fce3..06483b3945e 100644 --- a/services/auto-routing-benchmark/src/db.test.ts +++ b/services/auto-routing-benchmark/src/db.test.ts @@ -271,4 +271,63 @@ describe('rowsToRoutingTable', () => { expect(cand?.reasoningEffort).toBe('high'); expect(cand && 'variant' in cand ? cand.variant : undefined).toBeUndefined(); }); + + it('reads back a reasoning_effort candidate row as effort-only', () => { + const tableRow = { + run_id: 'run-effort', + published_at: '2026-06-01T11:00:00.000Z', + generated_at: '2026-06-01T10:00:00.000Z', + min_accuracy: 0.7, + switch_cost_factor: 3, + best_accuracy_switch_threshold: 0.05, + source: 'benchmark', + }; + const candidateRows = [ + { + run_id: 'run-effort', + route_key: 'implementation/code_generation', + rank: 0, + model: 'model-a', + accuracy: 0.9, + avg_cost_usd: 0.001, + meets_threshold: true, + reasoning_effort: 'high', + variant: 'high', + }, + ]; + const table = rowsToRoutingTable(tableRow, candidateRows); + const cand = table.routes['implementation/code_generation']?.[0]; + expect(cand?.reasoningEffort).toBe('high'); + expect(cand && 'variant' in cand ? cand.variant : undefined).toBeUndefined(); + }); + + it('reads back a variant-only candidate row as variant', () => { + const tableRow = { + run_id: 'run-variant', + published_at: '2026-06-01T11:00:00.000Z', + generated_at: '2026-06-01T10:00:00.000Z', + min_accuracy: 0.7, + switch_cost_factor: 3, + best_accuracy_switch_threshold: 0.05, + source: 'benchmark', + }; + const candidateRows = [ + { + run_id: 'run-variant', + route_key: 'implementation/code_generation', + rank: 0, + model: 'model-a', + accuracy: 0.9, + avg_cost_usd: 0.001, + meets_threshold: true, + reasoning_effort: null, + variant: 'max', + }, + ]; + const table = rowsToRoutingTable(tableRow, candidateRows); + const cand = table.routes['implementation/code_generation']?.[0]; + expect(cand?.variant).toBe('max'); + expect(cand?.reasoningEffort).toBeNull(); + expect(RoutingTableSchema.parse(table)).toEqual(table); + }); }); diff --git a/services/auto-routing-benchmark/src/db.ts b/services/auto-routing-benchmark/src/db.ts index 6c6a745e075..3c9c25eaafb 100644 --- a/services/auto-routing-benchmark/src/db.ts +++ b/services/auto-routing-benchmark/src/db.ts @@ -1015,14 +1015,17 @@ export function rowsToRoutingTable( }); for (const row of sorted) { routeMap[row.route_key] ??= []; - // Platform artifact compatibility: keep reading reasoning_effort exactly - // as today so published JSON shape stays effort-based during rolling deploys. + // Platform artifact compatibility: a row with a reasoning_effort keeps the + // exact current read shape. A variant-only row (non-enum key) returns variant. + const effort = parsePersistedReasoningEffort(row.reasoning_effort); + const variant = effort === null ? variantFromStorage(row.variant) : null; routeMap[row.route_key].push({ model: row.model, accuracy: row.accuracy, avgCostUsd: row.avg_cost_usd, meetsThreshold: row.meets_threshold, - reasoningEffort: parsePersistedReasoningEffort(row.reasoning_effort), + ...(variant !== null ? { variant } : {}), + reasoningEffort: effort, }); } return { diff --git a/services/auto-routing-benchmark/src/profile-runs.test.ts b/services/auto-routing-benchmark/src/profile-runs.test.ts index d66167db7a3..1962d5c7d6a 100644 --- a/services/auto-routing-benchmark/src/profile-runs.test.ts +++ b/services/auto-routing-benchmark/src/profile-runs.test.ts @@ -108,8 +108,8 @@ function mockConfig(overrides: Partial = {}) { // mapConfigRows requires non-empty classifier + decider lists. classifierModels: ['classifier/a'], deciderModels: [ - { model: 'platform/a', reasoning_effort: null }, - { model: 'platform/b', reasoning_effort: 'high' }, + { model: 'platform/a', variant: null, reasoning_effort: null }, + { model: 'platform/b', variant: null, reasoning_effort: 'high' }, ], autoDeciderModels: [], excludedAutoDeciderModels: [], @@ -191,6 +191,41 @@ describe('startRun — profile purpose', () => { expect(markProfilesRunningForRun).not.toHaveBeenCalled(); }); + it('platform startRun maps a saved canonical variant into the run_models row', async () => { + vi.mocked(getConfigRows).mockResolvedValue({ + config: configRow, + classifierModels: ['classifier/a'], + deciderModels: [{ model: 'platform/a', variant: 'max', reasoning_effort: null }], + autoDeciderModels: [], + excludedAutoDeciderModels: [], + }); + + const result = await startRun(env, 'decider'); + expect(result.runId).toMatch(/^decider-/); + const [, , modelRows] = vi.mocked(insertRun).mock.calls[0]; + expect(modelRows).toEqual([ + expect.objectContaining({ + model: 'platform/a', + variant: 'max', + enqueued: true, + }), + ]); + }); + + it("platform startRun keeps a legacy enum effort as today's exact row shape", async () => { + // mockConfig() default: platform/b holds reasoning_effort 'high' (legacy shape). + const result = await startRun(env, 'decider'); + const [, , modelRows] = vi.mocked(insertRun).mock.calls[0]; + const row = modelRows.find(m => m.model === 'platform/b'); + expect(row).toMatchObject({ + model: 'platform/b', + variant: 'high', + enqueued: true, + reasoning_effort: 'high', + }); + expect(result.enqueuedModels).toBe(2); + }); + it('rejects profile runs that are not decider or lack entries', async () => { await expect(startRun(env, 'classifier', { purpose: 'profile', entries: [] })).rejects.toThrow( /profile runs must be decider/ diff --git a/services/auto-routing-benchmark/src/routing-table-builder.test.ts b/services/auto-routing-benchmark/src/routing-table-builder.test.ts index 0716ebece2b..27133f82fd9 100644 --- a/services/auto-routing-benchmark/src/routing-table-builder.test.ts +++ b/services/auto-routing-benchmark/src/routing-table-builder.test.ts @@ -4,7 +4,7 @@ import type { BenchmarkModelSummary, TaxonomyRouteKey, } from '@kilocode/auto-routing-contracts'; -import { TAXONOMY_ROUTE_KEYS } from '@kilocode/auto-routing-contracts'; +import { RoutingTableSchema, TAXONOMY_ROUTE_KEYS } from '@kilocode/auto-routing-contracts'; import { buildRoutingTable } from './routing-table-builder'; const DECIDER_MODELS: BenchmarkDeciderModel[] = [ @@ -254,4 +254,43 @@ describe('buildRoutingTable', () => { expect(table.routes['implementation/code_generation']).toHaveLength(3); }); + + it('keeps the exact effort-only shape for an enum reasoningEffort snapshot', () => { + const table = buildRoutingTable({ + runId: 'test-run-enum-shape', + generatedAt: '2026-01-01T00:00:00.000Z', + minAccuracy: 0.7, + switchCostFactor: 3, + bestAccuracySwitchThreshold: 0.05, + deciderModels: [{ id: 'model/value', variant: null, reasoningEffort: 'medium' }], + summaries: TAXONOMY_ROUTE_KEYS.map(routeKey => ({ + ...summary('model/value', routeKey, 0.9, 0.002), + variant: 'medium', + })), + }); + const cand = table.routes['implementation/code_generation']?.[0]; + expect(cand?.reasoningEffort).toBe('medium'); + expect(cand && 'variant' in cand ? cand.variant : undefined).toBeUndefined(); + }); + + it('emits variant for a snapshot entry whose key is outside the effort enum', () => { + const table = buildRoutingTable({ + runId: 'test-run-non-enum-variant', + generatedAt: '2026-01-01T00:00:00.000Z', + minAccuracy: 0.7, + switchCostFactor: 3, + bestAccuracySwitchThreshold: 0.05, + deciderModels: [{ id: 'model/max', variant: 'max', reasoningEffort: null }], + summaries: TAXONOMY_ROUTE_KEYS.map(routeKey => ({ + ...summary('model/max', routeKey, 0.9, 0.002), + variant: 'max', + })), + }); + const cand = table.routes['implementation/code_generation']?.[0]; + expect(cand).toBeDefined(); + expect(cand?.variant).toBe('max'); + expect(cand?.reasoningEffort).toBeNull(); + // The published artifact must satisfy the contract schema (variant field allowed). + expect(RoutingTableSchema.parse(table)).toEqual(table); + }); }); diff --git a/services/auto-routing-benchmark/src/routing-table-builder.ts b/services/auto-routing-benchmark/src/routing-table-builder.ts index f2747a61945..57dd6eb2489 100644 --- a/services/auto-routing-benchmark/src/routing-table-builder.ts +++ b/services/auto-routing-benchmark/src/routing-table-builder.ts @@ -70,16 +70,15 @@ export function buildRoutingTable(params: { .filter(s => s.routeKey === routeKey && s.cases > 0 && s.avgCostUsd !== null) .map(s => { const cfg = findSnapshot(s.model, s.variant); - // Platform artifact: emit reasoningEffort from the run snapshot's - // effort key, NOT variant (custom sparse tables are a later slice). - const effort = - cfg?.reasoningEffort !== undefined && cfg.reasoningEffort !== null - ? cfg.reasoningEffort - : null; + const effort = cfg.reasoningEffort ?? null; + // Legacy enum efforts keep the exact current shape. A snapshot that only + // has a non-enum variant emits `variant` instead — never both. + const variant = effort === null ? (cfg.variant ?? null) : null; return { model: s.model, accuracy: s.accuracy, avgCostUsd: s.avgCostUsd ?? 0, + ...(variant !== null ? { variant } : {}), reasoningEffort: effort, }; }), diff --git a/services/auto-routing-benchmark/src/run-process-job.test.ts b/services/auto-routing-benchmark/src/run-process-job.test.ts index 5c2b2f9a86c..06c4d9d0f9a 100644 --- a/services/auto-routing-benchmark/src/run-process-job.test.ts +++ b/services/auto-routing-benchmark/src/run-process-job.test.ts @@ -48,10 +48,14 @@ import { } from './cli-runner'; import { countCaseResultsByLane, + getCaseResults, getExistingCaseResultIds, getRunWithModels, + getSummaries, listLaneFailures, + markRunCompleted, recordLaneFailure, + saveRoutingTable, upsertCaseResult, } from './db'; import { processJob } from './run'; @@ -435,3 +439,113 @@ describe('processJob — decider chunk chaining', () => { warn.mockRestore(); }); }); + +describe('processJob — saved canonical variant reaches the CLI and publish', () => { + it('passes a saved canonical variant to the CLI as variant', async () => { + mockRunSnapshot([{ model, variant: 'max', enqueued: true, reasoning_effort: null }]); + + await processJob(env, deciderMessage({ variant: 'max' })); + + expect(runDeciderCaseViaCli).toHaveBeenCalledWith( + env, + expect.objectContaining({ + model, + variant: 'max', + instanceName: `${runId}:${model}:max:0:0`, + }) + ); + expect(upsertCaseResult).toHaveBeenCalledWith( + env.BENCH_DB, + expect.objectContaining({ model, variant: 'max' }) + ); + }); + + // Publish-fidelity harness: a completed platform run whose snapshot row is + // variant-only publishes variant; an enum effort row keeps the effort shape. + function mockPublishHarness(variant: string, reasoningEffort: string | null) { + mockRunSnapshot([{ model, variant, enqueued: true, reasoning_effort: reasoningEffort }]); + vi.mocked(countCaseResultsByLane).mockResolvedValue([ + { model, variant, rep: 0, n: DECIDER_CASES.length }, + ]); + vi.mocked(getCaseResults).mockResolvedValue( + DECIDER_CASES.map(c => ({ + run_id: runId, + model, + variant, + case_id: c.id, + route_key: 'implementation/code_generation', + score: 1, + latency_ms: 10, + cost_usd: 0.001, + error: null, + fallback_reason: null, + retried: null, + exit_code: 0, + output_prefix: 'ok', + event_count: 1, + last_event_types: 'x', + rep: 0, + timed_out: 0, + })) + ); + vi.mocked(getExistingCaseResultIds).mockResolvedValue(new Set([benchCase.id])); + } + + it('publishes a routing table carrying variant for a non-enum snapshot key', async () => { + const { TAXONOMY_ROUTE_KEYS } = await import('@kilocode/auto-routing-contracts'); + mockPublishHarness('max', null); + vi.mocked(getSummaries).mockResolvedValue( + TAXONOMY_ROUTE_KEYS.map(routeKey => ({ + model, + variant: 'max', + routeKey, + accuracy: 0.9, + avgCostUsd: 0.001, + avgLatencyMs: 100, + p50LatencyMs: 90, + p95LatencyMs: 120, + cases: 5, + errors: 0, + timeouts: 0, + })) + ); + + await processJob(env, { ...deciderMessage({ variant: 'max' }), chunk: 9999 }); + + expect(markRunCompleted).toHaveBeenCalledWith(env.BENCH_DB, runId); + const table = vi.mocked(saveRoutingTable).mock.calls[0]?.[1]; + expect(table).toBeDefined(); + const firstRoute = Object.values(table.routes)[0]; + expect(firstRoute[0]).toMatchObject({ model, variant: 'max', reasoningEffort: null }); + }); + + it('publishes a routing table with the legacy effort shape for an enum key', async () => { + const { TAXONOMY_ROUTE_KEYS } = await import('@kilocode/auto-routing-contracts'); + mockPublishHarness('high', 'high'); + vi.mocked(getSummaries).mockResolvedValue( + TAXONOMY_ROUTE_KEYS.map(routeKey => ({ + model, + variant: 'high', + routeKey, + accuracy: 0.9, + avgCostUsd: 0.001, + avgLatencyMs: 100, + p50LatencyMs: 90, + p95LatencyMs: 120, + cases: 5, + errors: 0, + timeouts: 0, + })) + ); + + await processJob(env, { ...deciderMessage({ variant: 'high' }), chunk: 9999 }); + + const table = vi.mocked(saveRoutingTable).mock.calls[0]?.[1]; + expect(table).toBeDefined(); + const firstRoute = Object.values(table.routes)[0]; + expect(firstRoute[0]).toMatchObject({ model, reasoningEffort: 'high' }); + expect( + firstRoute[0] && 'variant' in firstRoute[0] ? firstRoute[0].variant : undefined + ).toBeUndefined(); + }); +}); diff --git a/services/auto-routing-benchmark/src/run.ts b/services/auto-routing-benchmark/src/run.ts index 04e3f30bac3..ebc66519290 100644 --- a/services/auto-routing-benchmark/src/run.ts +++ b/services/auto-routing-benchmark/src/run.ts @@ -1379,15 +1379,20 @@ async function finalizeRunIfComplete( try { // Built from the run's own model snapshot, not live config, so a mid-run // admin edit can't skew the published table. Platform tables still emit - // reasoningEffort (sourced from the run snapshot's effort key) — not - // variant — so the published artifact shape stays unchanged for rolling - // deploys of old auto-routing workers. - const deciderModels: BenchmarkDeciderModel[] = state.models.map(m => ({ - id: m.model, - reasoningEffort: parsePersistedReasoningEffort( - m.reasoning_effort ?? variantFromStorage(m.variant) - ), - })); + // reasoningEffort for enum efforts (sourced from the run snapshot's effort + // key) — not variant — so those published artifact shapes stay unchanged + // for rolling deploys of old auto-routing workers. A non-enum catalog + // variant can only be carried as `variant`; emitting effort would drop it. + const deciderModels: BenchmarkDeciderModel[] = state.models.map(m => { + const key = m.reasoning_effort ?? variantFromStorage(m.variant); + const effort = parsePersistedReasoningEffort(key); + // A catalog variant outside the legacy four-value effort enum (e.g. 'max') + // can only be carried as `variant`; emitting effort would drop it. + if (effort === null && key !== null) { + return { id: m.model, variant: key, reasoningEffort: null }; + } + return { id: m.model, reasoningEffort: effort }; + }); const table = buildRoutingTable({ runId, generatedAt, From 43859a27764fa9888ebec25b3ee398fe6ca59a7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20=C5=A0=C4=87eki=C4=87?= Date: Fri, 7 Aug 2026 20:02:54 +0200 Subject: [PATCH 3/5] feat(admin): add benchmark model pickers --- .../auto-routing/AutoRoutingAdminContent.tsx | 3 + .../auto-routing/BenchmarksSection.test.ts | 225 ++++++++++++- .../admin/auto-routing/BenchmarksSection.tsx | 315 +++++++++++++----- 3 files changed, 452 insertions(+), 91 deletions(-) diff --git a/apps/web/src/app/admin/auto-routing/AutoRoutingAdminContent.tsx b/apps/web/src/app/admin/auto-routing/AutoRoutingAdminContent.tsx index f6e262d43d4..e677cdd9ef4 100644 --- a/apps/web/src/app/admin/auto-routing/AutoRoutingAdminContent.tsx +++ b/apps/web/src/app/admin/auto-routing/AutoRoutingAdminContent.tsx @@ -452,6 +452,9 @@ export function AutoRoutingAdminContent() { void classifierModelQuery.refetch(); void analyticsQuery.refetch(); void openRouterModelsQuery.refetch(); + // Invalidate the shared selector catalog the benchmark pickers read + // from (query key prefix ['openrouter-models']). + void queryClient.invalidateQueries({ queryKey: ['openrouter-models'] }); }} disabled={isRefreshing} className="w-fit" diff --git a/apps/web/src/app/admin/auto-routing/BenchmarksSection.test.ts b/apps/web/src/app/admin/auto-routing/BenchmarksSection.test.ts index ad3a402e674..d1496cdf8b9 100644 --- a/apps/web/src/app/admin/auto-routing/BenchmarksSection.test.ts +++ b/apps/web/src/app/admin/auto-routing/BenchmarksSection.test.ts @@ -10,7 +10,11 @@ import { formatAccuracy, formatUsd, formStateToConfig, + modelOptionFromCatalog, + pinnedModelFor, RoutingTableView, + toBenchmarkModelOptions, + variantOptionsForModel, } from './BenchmarksSection'; describe('formatAccuracy', () => { @@ -76,6 +80,73 @@ describe('costPerAccuracy', () => { }); }); +describe('modelOptionFromCatalog', () => { + it('derives variant keys from catalog opencode metadata', () => { + const option = modelOptionFromCatalog({ + id: 'openai/gpt-5', + name: 'GPT-5', + opencode: { variants: { none: {}, low: {}, high: {}, ' ': {} } }, + }); + expect(option.id).toBe('openai/gpt-5'); + expect(option.name).toBe('GPT-5'); + expect(option.variants).toEqual(['none', 'low', 'high']); + }); + + it('omits variants when the catalog exposes none', () => { + const option = modelOptionFromCatalog({ + id: 'anthropic/claude-sonnet-4.5', + name: 'Claude Sonnet 4.5', + }); + expect(option.variants).toBeUndefined(); + }); +}); + +describe('toBenchmarkModelOptions', () => { + it('keeps eligible models and drops ineligible pool models', () => { + const options = toBenchmarkModelOptions([ + { id: 'anthropic/claude-sonnet-4.5', name: 'Claude Sonnet 4.5' }, + { id: 'kilo-auto/efficient', name: 'Efficient' }, + { id: 'kilo-internal/openai/custom', name: 'Custom' }, + { id: 'chutes-byok/m1', name: 'Chutes BYOK', hasUserByokAvailable: true }, + { id: 'openai/gpt-5', name: 'GPT-5', pricing: { prompt: '0' }, isFree: true }, + ]); + expect(options.map(option => option.id)).toEqual([ + 'anthropic/claude-sonnet-4.5', + 'openai/gpt-5', + ]); + }); +}); + +describe('pinnedModelFor', () => { + it('pins a saved id as a selectable option', () => { + expect(pinnedModelFor('anthropic/claude-sonnet-4.5')).toEqual({ + id: 'anthropic/claude-sonnet-4.5', + name: 'anthropic/claude-sonnet-4.5', + }); + }); +}); + +describe('variantOptionsForModel', () => { + it('offers the selected model catalog variant keys', () => { + const option = { id: 'openai/gpt-5', name: 'GPT-5', variants: ['none', 'low', 'high'] }; + expect(variantOptionsForModel(option, null)).toEqual(['none', 'low', 'high']); + }); + + it('appends a saved variant key omitted from the catalog', () => { + const option = { id: 'openai/gpt-5', name: 'GPT-5', variants: ['low', 'high'] }; + expect(variantOptionsForModel(option, 'max')).toEqual(['low', 'high', 'max']); + }); + + it('does not duplicate a saved variant key that is still in the catalog', () => { + const option = { id: 'openai/gpt-5', name: 'GPT-5', variants: ['low', 'high'] }; + expect(variantOptionsForModel(option, 'high')).toEqual(['low', 'high']); + }); + + it('hides when neither catalog nor saved variant key exists', () => { + expect(variantOptionsForModel({ id: 'model', name: 'Model' }, null)).toEqual([]); + }); +}); + describe('RoutingTableView', () => { it('renders candidates in the published serving rank order', () => { const html = renderToStaticMarkup( @@ -115,7 +186,7 @@ describe('RoutingTableView', () => { expect(html.indexOf('threshold-meeting')).toBeLessThan(html.indexOf('below-threshold-cheaper')); }); - it('renders reasoning effort next to the model name', () => { + it('renders canonical variant before effort/default', () => { const html = renderToStaticMarkup( React.createElement(RoutingTableView, { data: { @@ -134,6 +205,40 @@ describe('RoutingTableView', () => { accuracy: 0.8, avgCostUsd: 0.006, meetsThreshold: true, + variant: 'xhigh', + }, + ], + }, + }, + }, + }) + ); + + expect(html.indexOf('Model')).toBeLessThan(html.indexOf('Variant')); + expect(html.indexOf('Variant')).toBeLessThan(html.indexOf('Accuracy')); + expect(html.indexOf('openai/gpt-5')).toBeLessThan(html.indexOf('xhigh')); + expect(html.indexOf('xhigh')).toBeLessThan(html.indexOf('80.0%')); + }); + + it('renders legacy reasoning effort as the variant label when variant is absent', () => { + const html = renderToStaticMarkup( + React.createElement(RoutingTableView, { + data: { + publishedAt: '2026-06-17T00:00:00.000Z', + table: { + version: 'run-1', + generatedAt: '2026-06-17T00:00:00.000Z', + minAccuracy: 0.7, + switchCostFactor: 3, + bestAccuracySwitchThreshold: 0.05, + source: 'benchmark', + routes: { + 'implementation/code_generation': [ + { + model: 'anthropic/claude-sonnet-4.5', + accuracy: 0.8, + avgCostUsd: 0.006, + meetsThreshold: true, reasoningEffort: 'high', }, ], @@ -143,11 +248,39 @@ describe('RoutingTableView', () => { }) ); - expect(html.indexOf('Model')).toBeLessThan(html.indexOf('Reasoning effort')); - expect(html.indexOf('Reasoning effort')).toBeLessThan(html.indexOf('Accuracy')); - expect(html.indexOf('openai/gpt-5')).toBeLessThan(html.indexOf('high')); + expect(html.indexOf('anthropic/claude-sonnet-4.5')).toBeLessThan(html.indexOf('high')); expect(html.indexOf('high')).toBeLessThan(html.indexOf('80.0%')); }); + + it('renders default when neither variant nor effort exists', () => { + const html = renderToStaticMarkup( + React.createElement(RoutingTableView, { + data: { + publishedAt: '2026-06-17T00:00:00.000Z', + table: { + version: 'run-1', + generatedAt: '2026-06-17T00:00:00.000Z', + minAccuracy: 0.7, + switchCostFactor: 3, + bestAccuracySwitchThreshold: 0.05, + source: 'benchmark', + routes: { + 'implementation/code_generation': [ + { + model: 'openai/gpt-4o-mini', + accuracy: 0.8, + avgCostUsd: 0.006, + meetsThreshold: true, + }, + ], + }, + }, + }, + }) + ); + + expect(html.indexOf('openai/gpt-4o-mini')).toBeLessThan(html.indexOf('default')); + }); }); describe('configToFormState', () => { @@ -158,7 +291,7 @@ describe('configToFormState', () => { expect(state.classifierMaxP95LatencyMs).toBe('1000'); expect(state.autoDeciderMinCostUsd).toBe(15); expect(state.autoDeciderMaxCostUsd).toBe(25); - expect(state.classifierModels).toBe(''); + expect(state.classifierModels).toEqual([]); expect(state.deciderModels).toEqual([]); expect(state.autoDeciderModels).toEqual([]); expect(state.excludedAutoDeciderModels).toBe(''); @@ -201,7 +334,7 @@ describe('formStateToConfig round-trip', () => { expect(state.autoDeciderMinCostUsd).toBe(12); expect(state.autoDeciderMaxCostUsd).toBe(24); expect(state.benchmarkOrgId).toBe('org-123'); - expect(state.deciderModels).toEqual([{ id: 'manual-model', reasoningEffort: 'low' }]); + expect(state.deciderModels).toEqual([{ id: 'manual-model', variant: 'low' }]); expect(state.autoDeciderModels).toEqual(baseConfig.autoDeciderModels); expect(state.excludedAutoDeciderModels).toBe('excluded-auto-model'); @@ -212,14 +345,67 @@ describe('formStateToConfig round-trip', () => { expect(result.autoDeciderMinCostUsd).toBe(12); expect(result.autoDeciderMaxCostUsd).toBe(24); expect(result.benchmarkOrgId).toBe('org-123'); - expect(result.manualDeciderModels).toEqual([{ id: 'manual-model', reasoningEffort: 'low' }]); + expect(result.manualDeciderModels).toEqual([ + { id: 'manual-model', variant: 'low', reasoningEffort: null }, + ]); expect(result.excludedAutoDeciderModels).toEqual(['excluded-auto-model']); expect(result.deciderModels).toEqual([ - { id: 'manual-model', reasoningEffort: 'low' }, + { id: 'manual-model', variant: 'low', reasoningEffort: null }, + // Auto rows stay effort-only in the effective list. { id: 'auto-model', reasoningEffort: null }, ]); }); + it('round-trips classifierModels as a string array and drops blank rows', () => { + const state = configToFormState(baseConfig); + expect(state.classifierModels).toEqual(['model-a', 'model-b']); + const result = formStateToConfig( + { ...state, classifierModels: ['model-a', ' ', 'model-c'] }, + baseConfig + ); + expect(result.classifierModels).toEqual(['model-a', 'model-c']); + }); + + it('round-trips canonical variant rows and loads legacy effort rows as variant', () => { + const variantConfig: BenchmarkConfig = { + ...baseConfig, + manualDeciderModels: [ + { id: 'manual-v2', variant: 'high', reasoningEffort: null }, + { id: 'legacy', reasoningEffort: 'low' }, + ], + }; + const state = configToFormState(variantConfig); + expect(state.deciderModels).toEqual([ + { id: 'manual-v2', variant: 'high' }, + { id: 'legacy', variant: 'low' }, + ]); + + const result = formStateToConfig(state, variantConfig); + expect(result.manualDeciderModels).toEqual([ + { id: 'manual-v2', variant: 'high', reasoningEffort: null }, + { id: 'legacy', variant: 'low', reasoningEffort: null }, + ]); + // Manual rows save variant-only; the legacy effort field stays null. + expect(result.manualDeciderModels?.every(model => model.reasoningEffort === null)).toBe(true); + }); + + it('drops blank decider rows on save', () => { + const state = configToFormState(baseConfig); + const result = formStateToConfig( + { + ...state, + deciderModels: [ + { id: ' ', variant: null }, + { id: 'manual-model', variant: 'low' }, + ], + }, + baseConfig + ); + expect(result.manualDeciderModels).toEqual([ + { id: 'manual-model', variant: 'low', reasoningEffort: null }, + ]); + }); + it('converts empty-string classifierMaxP95LatencyMs form value to null in config', () => { const state = configToFormState(baseConfig); const stateWithEmpty = { ...state, classifierMaxP95LatencyMs: '' }; @@ -239,24 +425,35 @@ describe('formStateToConfig round-trip', () => { }); describe('effectiveDeciderModels', () => { - it('combines manual models with non-excluded auto models and lets manual override an auto duplicate', () => { + it('keeps manual rows variant-only and auto rows effort-only, drops excluded auto, and lets manual override an auto duplicate', () => { expect( effectiveDeciderModels({ manualDeciderModels: [ - { id: 'manual/model', reasoningEffort: null }, - { id: 'auto/duplicate', reasoningEffort: 'high' }, + { id: 'manual/model', variant: null }, + { id: 'auto/duplicate', variant: 'high' }, ], autoDeciderModels: [ { id: 'auto/duplicate', reasoningEffort: null, avgAttemptCostUsd: 20 }, { id: 'auto/included', reasoningEffort: 'low', avgAttemptCostUsd: 22 }, { id: 'auto/excluded', reasoningEffort: null, avgAttemptCostUsd: 23 }, + { + id: 'auto/variant-carrying', + variant: 'xhigh', + reasoningEffort: null, + avgAttemptCostUsd: 24, + }, ], excludedAutoDeciderModels: ['auto/excluded'], }) - ).toEqual([ - { id: 'manual/model', reasoningEffort: null }, - { id: 'auto/duplicate', reasoningEffort: 'high' }, + ).toStrictEqual([ + // Manual rows are canonical variant-only with the legacy effort null. + { id: 'manual/model', variant: null, reasoningEffort: null }, + // A manual row with the same id overrides the synced auto row. + { id: 'auto/duplicate', variant: 'high', reasoningEffort: null }, + // Auto rows stay effort-only and never emit the variant key, even when + // the synced row carries a variant. { id: 'auto/included', reasoningEffort: 'low' }, + { id: 'auto/variant-carrying', reasoningEffort: null }, ]); }); }); diff --git a/apps/web/src/app/admin/auto-routing/BenchmarksSection.tsx b/apps/web/src/app/admin/auto-routing/BenchmarksSection.tsx index 621a52e67b8..e58c73256a4 100644 --- a/apps/web/src/app/admin/auto-routing/BenchmarksSection.tsx +++ b/apps/web/src/app/admin/auto-routing/BenchmarksSection.tsx @@ -9,32 +9,29 @@ import { DEFAULT_BENCHMARK_ORG_ID, DEFAULT_BENCHMARK_USER_ID, StartBenchmarkRunResponseSchema, + type AutoBenchmarkDeciderModel, type BenchmarkConfig, + type BenchmarkDeciderModel, type BenchmarkKind, + type BenchmarkModelSummary, type BenchmarkRoutingTableResponse, type BenchmarkRun, - type BenchmarkModelSummary, type RankedCandidate, - type ReasoningEffort, - type AutoBenchmarkDeciderModel, } from '@kilocode/auto-routing-contracts'; -import React, { useCallback, useEffect, useRef, useState } from 'react'; +import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'; import { toast } from 'sonner'; import { ChevronDown, ChevronRight, Play, Plus, Save, Trash2 } from 'lucide-react'; +import { useModelSelectorList } from '@/app/api/openrouter/hooks'; +import { isEligiblePoolModel } from '@/components/auto-routing/AutoRoutingModeCard'; +import { ModelCombobox, type ModelOption } from '@/components/shared/ModelCombobox'; +import { VariantCombobox } from '@/components/shared/VariantCombobox'; import { Badge } from '@/components/ui/badge'; import { Button } from '@/components/ui/button'; import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'; import { Checkbox } from '@/components/ui/checkbox'; import { Input } from '@/components/ui/input'; import { Label } from '@/components/ui/label'; -import { - Select, - SelectContent, - SelectItem, - SelectTrigger, - SelectValue, -} from '@/components/ui/select'; import { Skeleton } from '@/components/ui/skeleton'; import { Table, @@ -44,7 +41,6 @@ import { TableHeader, TableRow, } from '@/components/ui/table'; -import { Textarea } from '@/components/ui/textarea'; import { parseAdminResponse } from './admin-fetch'; // --------------------------------------------------------------------------- @@ -75,6 +71,61 @@ export function formatCostPerAccuracy(candidate: Pick } | null; +}; + +export function modelOptionFromCatalog(model: CatalogModelShape): ModelOption { + const variantKeys = model.opencode?.variants + ? Object.keys(model.opencode.variants).filter(key => key.trim().length > 0) + : []; + return { + id: model.id, + name: model.name, + isFree: model.isFree, + mayTrainOnYourPrompts: model.mayTrainOnYourPrompts, + hasUserByokAvailable: model.hasUserByokAvailable, + variants: variantKeys.length > 0 ? variantKeys : undefined, + }; +} + +/** Eligible model options for the classifier and decider pickers. */ +export function toBenchmarkModelOptions(models: CatalogModelShape[]): ModelOption[] { + return models.filter(isEligiblePoolModel).map(modelOptionFromCatalog); +} + +/** Pins a saved model id that is absent from the eligible catalog so it stays selectable. */ +export function pinnedModelFor(id: string): ModelOption { + return { id, name: id }; +} + +/** + * Variant keys offered for a decider row: the selected model's catalog keys, + * plus a saved key the catalog no longer lists (kept so an existing row stays + * editable and round-trips). Hides only when no key exists. + */ +export function variantOptionsForModel( + modelOption: ModelOption | undefined, + savedVariant: string | null +): string[] { + const catalogKeys = modelOption?.variants ?? []; + if (savedVariant && !catalogKeys.includes(savedVariant)) { + return [...catalogKeys, savedVariant]; + } + return catalogKeys; +} + // --------------------------------------------------------------------------- // Fetch helpers // --------------------------------------------------------------------------- @@ -121,13 +172,13 @@ async function fetchBenchmarkRoutingTable() { type DeciderModelRow = { id: string; - reasoningEffort: ReasoningEffort | null; + variant: string | null; }; type AutoDeciderModelRow = AutoBenchmarkDeciderModel; export function configToFormState(config: BenchmarkConfig | null): { - classifierModels: string; + classifierModels: string[]; deciderModels: DeciderModelRow[]; autoDeciderModels: AutoDeciderModelRow[]; excludedAutoDeciderModels: string; @@ -147,7 +198,7 @@ export function configToFormState(config: BenchmarkConfig | null): { // No config saved yet: identity fields are overrides, so blank means the // worker uses its default benchmark user and org at run time. return { - classifierModels: '', + classifierModels: [], deciderModels: [], autoDeciderModels: [], excludedAutoDeciderModels: '', @@ -165,10 +216,12 @@ export function configToFormState(config: BenchmarkConfig | null): { }; } return { - classifierModels: config.classifierModels.join('\n'), + classifierModels: config.classifierModels, deciderModels: (config.manualDeciderModels ?? config.deciderModels).map(m => ({ id: m.id, - reasoningEffort: m.reasoningEffort ?? null, + // Legacy reasoningEffort-only rows load as the canonical form variant so + // a save emits variant-only manual rows. + variant: m.variant ?? m.reasoningEffort ?? null, })), autoDeciderModels: config.autoDeciderModels ?? [], excludedAutoDeciderModels: (config.excludedAutoDeciderModels ?? []).join('\n'), @@ -202,12 +255,15 @@ export function effectiveDeciderModels({ manualDeciderModels: DeciderModelRow[]; autoDeciderModels: AutoDeciderModelRow[]; excludedAutoDeciderModels: string[]; -}): DeciderModelRow[] { +}): BenchmarkDeciderModel[] { const manual = manualDeciderModels .filter(row => row.id.trim().length > 0) .map(row => ({ id: row.id.trim(), - reasoningEffort: row.reasoningEffort ?? null, + variant: row.variant ?? null, + // The contract type requires the legacy field; it stays null on + // variant-only rows (both non-null is malformed). + reasoningEffort: null, })); const manualIds = new Set(manual.map(model => model.id)); const excludedAuto = new Set(excludedAutoDeciderModels); @@ -218,6 +274,9 @@ export function effectiveDeciderModels({ .filter(model => !manualIds.has(model.id)) .map(model => ({ id: model.id, + // Auto rows stay effort-only: the benchmark worker writes and reads the + // legacy reasoningEffort for synced rows, so never emit the canonical + // variant key here. reasoningEffort: model.reasoningEffort ?? null, })), ]; @@ -227,11 +286,17 @@ export function formStateToConfig( state: ReturnType, base: BenchmarkConfig | null ): BenchmarkConfig { - const classifierModels = parseModelLines(state.classifierModels); + const classifierModels = state.classifierModels.map(id => id.trim()).filter(id => id.length > 0); const excludedAutoDeciderModels = parseModelLines(state.excludedAutoDeciderModels); const manualDeciderModels = state.deciderModels .filter(row => row.id.trim().length > 0) - .map(row => ({ id: row.id.trim(), reasoningEffort: row.reasoningEffort ?? null })); + .map(row => ({ + id: row.id.trim(), + variant: row.variant ?? null, + // Variant-only rows: the legacy effort field is always null so the two + // never collide (both non-null is malformed per the contract). + reasoningEffort: null, + })); const deciderModels = effectiveDeciderModels({ manualDeciderModels, autoDeciderModels: state.autoDeciderModels, @@ -270,9 +335,15 @@ export function formStateToConfig( function BenchmarkConfigEditor({ config, onSaved, + modelOptions, + modelsLoading, + modelsError, }: { config: BenchmarkConfig | null; onSaved: (next: { config: BenchmarkConfig | null }) => void; + modelOptions: ModelOption[]; + modelsLoading: boolean; + modelsError?: string; }) { const [form, setForm] = useState(() => configToFormState(config)); // Tracks unsaved local edits. A background config refetch (the runs list @@ -319,10 +390,37 @@ function BenchmarkConfigEditor({ }, }); + const handleAddClassifierRow = useCallback(() => { + updateForm(prev => ({ + ...prev, + classifierModels: [...prev.classifierModels, ''], + })); + }, [updateForm]); + + const handleRemoveClassifierRow = useCallback( + (index: number) => { + updateForm(prev => ({ + ...prev, + classifierModels: prev.classifierModels.filter((_, i) => i !== index), + })); + }, + [updateForm] + ); + + const handleClassifierModelChange = useCallback( + (index: number, value: string) => { + updateForm(prev => ({ + ...prev, + classifierModels: prev.classifierModels.map((id, i) => (i === index ? value : id)), + })); + }, + [updateForm] + ); + const handleAddDeciderRow = useCallback(() => { updateForm(prev => ({ ...prev, - deciderModels: [...prev.deciderModels, { id: '', reasoningEffort: null }], + deciderModels: [...prev.deciderModels, { id: '', variant: null }], })); }, [updateForm]); @@ -378,66 +476,34 @@ function BenchmarkConfigEditor({ {/* Classifier models */}
- -