Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval-

## Unreleased

### Fixed

- Price OpenRouter stealth previews (`stealth/*`) as a real $0 so the DSPy RLM budget ledger admits them instead of refusing an unpriced model.

## [0.182.0] — 2026-09-15

### Breaking
Expand Down
2 changes: 1 addition & 1 deletion src/analyst/benchmark-implementation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ export const ANALYST_BENCHMARK_IMPLEMENTATION_FILES = Object.freeze([
])

export const ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 =
'53b4c9f453ed05e46968ed39a8e365f8593163b264c0b1a770f7781005d3aae7'
'956a7205aae4110a7b3dd927bef2c1098986e16cd89a17276791083219ade3fc'

export function analystBenchmarkImplementationDigest() {
return ANALYST_BENCHMARK_IMPLEMENTATION_SHA256
Expand Down
4 changes: 4 additions & 0 deletions src/metrics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,10 @@ export const MODEL_PRICING: Record<string, TokenPrice> = {
* none of which appear in the exact table above — without this they priced
* to a silent $0, blanking every cost/Pareto axis downstream. */
const FAMILY_PRICING: Array<[RegExp, TokenPrice]> = [
// OpenRouter stealth previews (`stealth/union-alpha`, …) bill $0 during
// their preview window. A real zero row lets the budget ledger admit them;
// an unpriced id is refused before the first call.
[/^stealth\//, { input: 0, output: 0 }],
[/claude.*opus/, { input: 0.015, output: 0.075 }],
[/claude.*haiku/, { input: 0.0008, output: 0.004 }],
[/claude.*sonnet|claude-code|claude-sonnet/, { input: 0.003, output: 0.015 }],
Expand Down
7 changes: 7 additions & 0 deletions tests/metrics-pricing.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,13 @@ describe('estimateCost — real harness/router model ids price non-zero', () =>
})
}

it('prices OpenRouter stealth previews as a real $0, not unpriced', () => {
expect(isModelPriced('stealth/union-alpha')).toBe(true)
expect(resolveModelPricing('stealth/union-alpha')).toEqual({ input: 0, output: 0 })
expect(estimateCost(1000, 1000, 'stealth/union-alpha')).toBe(0)
expect(isModelPriced('union-alpha')).toBe(false)
})

it('still prices the exact legacy table entries', () => {
expect(estimateCost(1000, 1000, 'gpt-4o')).toBeCloseTo(0.0125, 6)
})
Expand Down