From c8a7317f1986007963cd0504a4ff04a7f880db8a Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 16 Sep 2026 14:16:03 -0700 Subject: [PATCH 1/2] fix(metrics): price OpenRouter stealth previews as a real $0 `traces ask --model stealth/union-alpha` refused before the first call: "no pricing is configured for 'stealth/union-alpha'". The router bills these previews at $0, but the ledger only admits ids the table knows. A `stealth/*` family row with a zero price lets the budget ledger admit them while keeping unpriced ids refused. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 4 ++++ src/metrics.ts | 4 ++++ tests/metrics-pricing.test.ts | 7 +++++++ 3 files changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bdc79c410..6c37818b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval- ## Unreleased +### Fixed + +- Price OpenRouter stealth previews (`stealth/*`) as a real $0 so the DSPy RLM budget ledger admits them instead of refusing an unpriced model. + ## [0.182.0] — 2026-09-15 ### Breaking diff --git a/src/metrics.ts b/src/metrics.ts index e31a071d0..4258f4eb1 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -25,6 +25,10 @@ export const MODEL_PRICING: Record = { * none of which appear in the exact table above — without this they priced * to a silent $0, blanking every cost/Pareto axis downstream. */ const FAMILY_PRICING: Array<[RegExp, TokenPrice]> = [ + // OpenRouter stealth previews (`stealth/union-alpha`, …) bill $0 during + // their preview window. A real zero row lets the budget ledger admit them; + // an unpriced id is refused before the first call. + [/^stealth\//, { input: 0, output: 0 }], [/claude.*opus/, { input: 0.015, output: 0.075 }], [/claude.*haiku/, { input: 0.0008, output: 0.004 }], [/claude.*sonnet|claude-code|claude-sonnet/, { input: 0.003, output: 0.015 }], diff --git a/tests/metrics-pricing.test.ts b/tests/metrics-pricing.test.ts index ec0f221b7..feb4a5b9b 100644 --- a/tests/metrics-pricing.test.ts +++ b/tests/metrics-pricing.test.ts @@ -26,6 +26,13 @@ describe('estimateCost — real harness/router model ids price non-zero', () => }) } + it('prices OpenRouter stealth previews as a real $0, not unpriced', () => { + expect(isModelPriced('stealth/union-alpha')).toBe(true) + expect(resolveModelPricing('stealth/union-alpha')).toEqual({ input: 0, output: 0 }) + expect(estimateCost(1000, 1000, 'stealth/union-alpha')).toBe(0) + expect(isModelPriced('union-alpha')).toBe(false) + }) + it('still prices the exact legacy table entries', () => { expect(estimateCost(1000, 1000, 'gpt-4o')).toBeCloseTo(0.0125, 6) }) From f6ca07ed9ead84212101144483fc46ef5ed9c481 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 16 Sep 2026 14:26:35 -0700 Subject: [PATCH 2/2] chore(analyst): refresh the benchmark implementation digest for the pricing row `src/metrics.ts` is a bound source of the public analyst benchmark, so the stealth pricing row changes the pinned implementation digest. Co-Authored-By: Claude Opus 5 (1M context) --- src/analyst/benchmark-implementation.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/analyst/benchmark-implementation.ts b/src/analyst/benchmark-implementation.ts index b06de6964..0ad407ffe 100644 --- a/src/analyst/benchmark-implementation.ts +++ b/src/analyst/benchmark-implementation.ts @@ -139,7 +139,7 @@ export const ANALYST_BENCHMARK_IMPLEMENTATION_FILES = Object.freeze([ ]) export const ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 = - '53b4c9f453ed05e46968ed39a8e365f8593163b264c0b1a770f7781005d3aae7' + '956a7205aae4110a7b3dd927bef2c1098986e16cd89a17276791083219ade3fc' export function analystBenchmarkImplementationDigest() { return ANALYST_BENCHMARK_IMPLEMENTATION_SHA256