From 5a8ac1279eb64102abb7c10bcb1c024948381a40 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Mon, 26 Oct 2026 05:24:54 -0700 Subject: [PATCH 1/4] feat(runtime): record the model that answered each native turn (TASK-028) An AgentRun turn kept only the model group it requested. Each turn now also records servedModel, the group LiteLLM names in x-litellm-model-group (the body's model when the header is absent), and fallbacks, the hop count from x-litellm-attempted-fallbacks when it is above zero. Native calls send disable_fallbacks (D81), and the 1.106.2 router honours it, so servedModel equals model today. The record is what shows it if something overrides the flag; the proxy lets a key's metadata set disable_fallbacks for every request on that key. The trace bound and the run-trace route carry both fields. The spend alarm prices a turn by servedModel, falling back to model for older turns. The run trace shows "Asked X, answered by Y" when they differ. Tested against a local HTTP server that answers with the proxy's headers, so the header names are read as they arrive. Written by Rivet, a Commonly agent: https://commonly.me/v2/pods/6addd4c17f8fc798a6f66431 Co-Authored-By: Claude Opus 5.5 Co-authored-by: rivet Agent-Model: claude/claude-opus-5-5/default --- .../__tests__/unit/routes/runTraces.test.js | 14 ++ .../nativeRuntimeService.servedModel.test.js | 135 ++++++++++++++++++ .../unit/services/spendAlarmService.test.js | 14 ++ backend/models/AgentRun.ts | 7 + backend/routes/runTraces.ts | 6 +- backend/services/agentRunLimits.ts | 3 +- backend/services/nativeRuntimeService.ts | 20 +++ backend/services/spendAlarmService.ts | 4 +- .../__tests__/zhNumberUnitSpacing.test.ts | 2 + frontend/src/i18n/locales/en.json | 1 + frontend/src/i18n/locales/zh-CN.json | 1 + frontend/src/v2/__tests__/V2RunTrace.test.tsx | 17 +++ frontend/src/v2/components/V2RunTrace.tsx | 7 +- 13 files changed, 225 insertions(+), 6 deletions(-) create mode 100644 backend/__tests__/unit/services/nativeRuntimeService.servedModel.test.js diff --git a/backend/__tests__/unit/routes/runTraces.test.js b/backend/__tests__/unit/routes/runTraces.test.js index 3769f4fc83..94983bcc09 100644 --- a/backend/__tests__/unit/routes/runTraces.test.js +++ b/backend/__tests__/unit/routes/runTraces.test.js @@ -65,6 +65,20 @@ test('returns durable turn + PG attribution after card retention without content expect(JSON.stringify(res.body)).not.toMatch(/PRIVATE_|private prompt|argsDigest/); expect(res.body.retention).toEqual({ cardDefaultDays: 30, traceExpiresAt: null }); }); +test('a turn keeps the model that answered beside the one requested, through save and read (TASK-028)', async () => { + run.turns = [ + { turnIndex: 0, model: 'openrouter/nvidia/nemotron-3-super-120b-a12b:free', servedModel: 'deepseek-v4-flash', fallbacks: 1, elapsedMs: 1 }, + { turnIndex: 1, model: 'deepseek-v4-flash', servedModel: 'deepseek-v4-flash', elapsedMs: 1 }, + { turnIndex: 2, model: 'model', elapsedMs: 1 }, + ]; + await run.save(); + const res = await get(); + expect(res.body.run.turns.map(({ model, servedModel, fallbacks }) => ({ model, servedModel, fallbacks }))).toEqual([ + { model: 'openrouter/nvidia/nemotron-3-super-120b-a12b:free', servedModel: 'deepseek-v4-flash', fallbacks: 1 }, + { model: 'deepseek-v4-flash', servedModel: 'deepseek-v4-flash' }, + { model: 'model' }, + ]); +}); test.each(['global admin', 'agent-dm observer'])( 'returns trace metadata without authorization identities to a non-member %s', async (viewerType) => { if (viewerType === 'global admin') { diff --git a/backend/__tests__/unit/services/nativeRuntimeService.servedModel.test.js b/backend/__tests__/unit/services/nativeRuntimeService.servedModel.test.js new file mode 100644 index 0000000000..e2aaac9bf1 --- /dev/null +++ b/backend/__tests__/unit/services/nativeRuntimeService.servedModel.test.js @@ -0,0 +1,135 @@ +/** + * TASK-028: a native turn records the model group that answered it beside the + * one it requested. LiteLLM names the answering group in x-litellm-model-group + * and counts fallback hops in x-litellm-attempted-fallbacks (v1.106.2); the + * body's `model` is the second source. + * + * Real HTTP: axios is not mocked. A local server stands in for the LiteLLM + * proxy and answers with the headers the proxy sends, so the header names and + * their parsing are exercised as they arrive over the wire. + */ +const http = require('http'); + +jest.mock('../../../models/Pod', () => ({ + findById: jest.fn(() => ({ + lean: jest.fn().mockResolvedValue({ name: 'Served pod' }), + })), +})); + +jest.mock('../../../models/AgentRun', () => ({ + create: jest.fn(), + countDocuments: jest.fn(), +})); + +jest.mock('../../../services/agentTypingService', () => ({ + emitAgentTypingStart: jest.fn(), + emitAgentTypingStop: jest.fn(), +})); + +jest.mock('../../../services/agentMessageService', () => ({ + postMessage: jest.fn().mockResolvedValue({ success: true }), +})); + +jest.mock('../../../services/messageClaimService', () => ({ + claim: jest.fn(), + release: jest.fn(), +})); + +const AgentRun = require('../../../models/AgentRun'); +const MessageClaimService = require('../../../services/messageClaimService'); +const { runAgent } = require('../../../services/nativeRuntimeService'); + +const NEMOTRON = 'openrouter/nvidia/nemotron-3-super-120b-a12b:free'; +const DEEPSEEK = 'deepseek-v4-flash'; + +let server; +let reply; +let requests; +let savedRun; +const previousEnv = { ...process.env }; + +beforeAll(async () => { + server = http.createServer((req, res) => { + let body = ''; + req.on('data', (chunk) => { body += chunk; }); + req.on('end', () => { + requests.push({ url: req.url, body: JSON.parse(body || '{}') }); + res.writeHead(200, { 'content-type': 'application/json', ...reply.headers }); + res.end(JSON.stringify({ + ...(reply.model ? { model: reply.model } : {}), + choices: [{ message: { role: 'assistant', content: 'NO_REPLY' } }], + usage: { prompt_tokens: 5, completion_tokens: 2, total_tokens: 7 }, + })); + }); + }); + await new Promise((resolve) => { server.listen(0, '127.0.0.1', resolve); }); + process.env.LITELLM_BASE_URL = `http://127.0.0.1:${server.address().port}`; + process.env.LITELLM_API_KEY = 'test-service-key'; +}); + +afterAll(async () => { + process.env = previousEnv; + await new Promise((resolve) => { server.close(resolve); }); +}); + +beforeEach(() => { + jest.clearAllMocks(); + MessageClaimService.claim.mockResolvedValue({ claimed: true, expiresAt: new Date() }); + MessageClaimService.release.mockResolvedValue({ released: true }); + AgentRun.create.mockImplementation(async (doc) => { + savedRun = { _id: 'run-1', ...doc, save: jest.fn().mockResolvedValue(undefined) }; + return savedRun; + }); +}); + +const installation = (model) => ({ + podId: 'pod-1', + agentName: 'served', + instanceId: 'default', + displayName: 'Served', + config: { + runtime: { runtimeType: 'native' }, systemPrompt: 'answer', model, tools: [], + }, +}); + +const mention = { + type: 'chat.mention', + eventId: 'evt-1', + payload: { messageId: 'msg-1', content: 'hi @served' }, +}; + +const turnOf = async (model, nextReply) => { + reply = nextReply; + requests = []; + const result = await runAgent(installation(model), mention); + expect(result.status).toBe('succeeded'); + expect(requests).toHaveLength(1); + const [turn] = savedRun.turns; + return { model: turn.model, servedModel: turn.servedModel, fallbacks: turn.fallbacks }; +}; + +test('a turn another group answered records both, and the hops', async () => { + const turn = await turnOf(NEMOTRON, { + // Through the proxy a fallback keeps the provider's own name in the body. + model: 'deepseek/deepseek-chat', + headers: { 'x-litellm-model-group': DEEPSEEK, 'x-litellm-attempted-fallbacks': '1' }, + }); + expect(turn).toEqual({ model: NEMOTRON, servedModel: DEEPSEEK, fallbacks: 1 }); + // The refusal still goes out: recording a fallback is not permission for one (D81). + expect(requests[0].body).toMatchObject({ model: NEMOTRON, disable_fallbacks: true }); +}); + +test('a normal turn records the two equal, with no hops', async () => { + const turn = await turnOf(DEEPSEEK, { + model: DEEPSEEK, + headers: { 'x-litellm-model-group': DEEPSEEK, 'x-litellm-attempted-fallbacks': '0' }, + }); + expect(turn).toEqual({ model: DEEPSEEK, servedModel: DEEPSEEK, fallbacks: undefined }); +}); + +test('without the group header the body model is used, and without either the request', async () => { + expect(await turnOf(NEMOTRON, { model: DEEPSEEK, headers: {} })) + .toEqual({ model: NEMOTRON, servedModel: DEEPSEEK, fallbacks: undefined }); + expect(await turnOf(DEEPSEEK, { headers: { 'x-litellm-attempted-fallbacks': 'not-a-number' } })) + .toEqual({ model: DEEPSEEK, servedModel: DEEPSEEK, fallbacks: undefined }); +}); diff --git a/backend/__tests__/unit/services/spendAlarmService.test.js b/backend/__tests__/unit/services/spendAlarmService.test.js index 9604f6d265..02be147b79 100644 --- a/backend/__tests__/unit/services/spendAlarmService.test.js +++ b/backend/__tests__/unit/services/spendAlarmService.test.js @@ -84,6 +84,20 @@ describe('userSpendForDay', () => { expect(rows.find((r) => r.userId === String(USER_A)).usd).toBeCloseTo(spend.UNPRICED_MODEL.input, 6); expect(rows.find((r) => r.userId === String(USER_B)).usd).toBeCloseTo(1.2, 6); }); + + test('a turn is priced by the model that answered it, and an older turn by the one it asked for', async () => { + await AgentRun.create([ + // Asked DeepSeek, answered by Sonnet: 1M out at Sonnet's $10, not DeepSeek's $1.20 (TASK-028). + run(USER_A, [{ model: 'deepseek-v4-flash', servedModel: 'claude-sonnet-5-5', fallbacks: 1, + promptTokens: 0, completionTokens: 1_000_000 }]), + // Recorded before servedModel existed: priced by model, as before. + run(USER_B, [{ model: 'claude-sonnet-5-5', promptTokens: 0, completionTokens: 1_000_000 }]), + ]); + + const rows = await spend.userSpendForDay(NOW); + expect(rows.find((r) => r.userId === String(USER_A)).usd).toBeCloseTo(10, 6); + expect(rows.find((r) => r.userId === String(USER_B)).usd).toBeCloseTo(10, 6); + }); }); describe('sandbox seconds (TASK-266)', () => { diff --git a/backend/models/AgentRun.ts b/backend/models/AgentRun.ts index 96ad06ea03..f6c98a779e 100644 --- a/backend/models/AgentRun.ts +++ b/backend/models/AgentRun.ts @@ -65,6 +65,11 @@ export interface IAgentRunTurn { llmResponseText?: string; elapsedMs: number; liteLLMCallId?: string; + // The model group that answered, when it can differ from `model`, the group + // requested (TASK-028). Absent on turns recorded before it was kept. + servedModel?: string; + // LiteLLM fallback hops taken before `servedModel` answered; absent means none. + fallbacks?: number; } export interface IAgentRun extends Document { @@ -132,6 +137,8 @@ const AgentRunTurnSchema = new Schema( llmResponseText: { type: String }, elapsedMs: { type: Number, default: 0 }, liteLLMCallId: { type: String }, + servedModel: { type: String }, + fallbacks: { type: Number }, }, { _id: false }, ); diff --git a/backend/routes/runTraces.ts b/backend/routes/runTraces.ts index b45e29efea..b5f9b8f374 100644 --- a/backend/routes/runTraces.ts +++ b/backend/routes/runTraces.ts @@ -37,7 +37,8 @@ router.get('/:podId/runs/:runId', traceReadLimit, auth, async (req, res) => { .select([ 'agentName instanceId status trigger startedAt completedAt totalTokens errorKind', 'traceTruncated omittedTurns omittedToolCalls', - 'turns.turnIndex turns.model turns.elapsedMs turns.promptTokens turns.completionTokens turns.totalTokens', + 'turns.turnIndex turns.model turns.servedModel turns.fallbacks turns.elapsedMs', + 'turns.promptTokens turns.completionTokens turns.totalTokens', 'turns.toolCalls.name turns.toolCalls.elapsedMs turns.toolCalls.callId turns.toolCalls.outcome', ].join(' ')) .lean(); @@ -59,7 +60,8 @@ router.get('/:podId/runs/:runId', traceReadLimit, auth, async (req, res) => { traceTruncated: Boolean(run.traceTruncated), omittedTurns: run.omittedTurns || 0, omittedToolCalls: run.omittedToolCalls || 0, turns: run.turns.map((turn) => ({ - turnIndex: turn.turnIndex, model: turn.model, elapsedMs: turn.elapsedMs, + turnIndex: turn.turnIndex, model: turn.model, servedModel: turn.servedModel, + fallbacks: turn.fallbacks, elapsedMs: turn.elapsedMs, promptTokens: turn.promptTokens, completionTokens: turn.completionTokens, totalTokens: turn.totalTokens, toolCalls: turn.toolCalls.map((call) => ({ name: call.name, elapsedMs: call.elapsedMs, callId: call.callId, diff --git a/backend/services/agentRunLimits.ts b/backend/services/agentRunLimits.ts index 29b061fd15..c1fda2da0a 100644 --- a/backend/services/agentRunLimits.ts +++ b/backend/services/agentRunLimits.ts @@ -45,7 +45,8 @@ export function boundAgentRun(run: { turnIndex: original.turnIndex, model: text(original.model) || 'unknown', promptTokens: original.promptTokens, completionTokens: original.completionTokens, totalTokens: original.totalTokens, elapsedMs: original.elapsedMs, - liteLLMCallId: text(original.liteLLMCallId), + liteLLMCallId: text(original.liteLLMCallId), servedModel: text(original.servedModel), + fallbacks: original.fallbacks, llmResponseText: text(original.llmResponseText, 4096), toolCalls: [], }; if (original.llmResponseText !== turn.llmResponseText) truncated = true; diff --git a/backend/services/nativeRuntimeService.ts b/backend/services/nativeRuntimeService.ts index 4ec7e856d8..4fe7c921f4 100644 --- a/backend/services/nativeRuntimeService.ts +++ b/backend/services/nativeRuntimeService.ts @@ -1395,6 +1395,7 @@ interface LiteLLMChoice { } interface LiteLLMResponse { + model?: string; choices?: LiteLLMChoice[]; usage?: { prompt_tokens?: number; @@ -1413,6 +1414,24 @@ function extractCallId(data: LiteLLMResponse, headers: Record): return headerId ? String(headerId) : undefined; } +// Which model group answered the call (TASK-028). LiteLLM's router names it in +// x-litellm-model-group on every response and counts the fallback hops it took +// in x-litellm-attempted-fallbacks (v1.106.2). The proxy restamps the body's +// `model` to the requested group unless a fallback happened, so the body comes +// second. Native calls send disable_fallbacks (D81), so the two differ only if +// something overrode it, such as a key whose metadata sets disable_fallbacks. +function servedModelOf( + data: LiteLLMResponse, + headers: Record, + requested: string, +): { servedModel: string; fallbacks?: number } { + const group = headers?.['x-litellm-model-group']; + const hops = Number(headers?.['x-litellm-attempted-fallbacks']); + const servedModel = (typeof group === 'string' && group) + || (typeof data?.model === 'string' && data.model) || requested; + return { servedModel, ...(Number.isInteger(hops) && hops > 0 ? { fallbacks: hops } : {}) }; +} + function mapTriggerType(raw: string): NativeRunTrigger['type'] { if (raw === 'chat.mention' || raw === 'thread.mention' || raw === 'mention') return 'mention'; if (raw === 'heartbeat') return 'heartbeat'; @@ -2142,6 +2161,7 @@ export async function runAgent( llmResponseText: (msg.content as string | null) || undefined, elapsedMs: 0, liteLLMCallId: extractCallId(llmResponse, responseHeaders), + ...servedModelOf(llmResponse, responseHeaders, model), }; const toolCalls = msg.tool_calls || []; diff --git a/backend/services/spendAlarmService.ts b/backend/services/spendAlarmService.ts index a7f58bd336..1bad45ff0c 100644 --- a/backend/services/spendAlarmService.ts +++ b/backend/services/spendAlarmService.ts @@ -95,7 +95,9 @@ export const userSpendForDay = async (now: Date = new Date()): Promise { + await act(async () => { await i18n.changeLanguage(language); }); + const nemotron = 'openrouter/nvidia/nemotron-3-super-120b-a12b:free'; + mockGet.mockResolvedValue({ run: { ...trace, turns: [ + { ...trace.turns[0], model: nemotron, servedModel: 'deepseek-v4-flash', fallbacks: 1 }, + { ...trace.turns[0], turnIndex: 1, model: 'deepseek-v4-flash', servedModel: 'deepseek-v4-flash' }, + { ...trace.turns[0], turnIndex: 2, model: 'older-turn' }, + ] } }); + const { container } = setup(); + await screen.findAllByText('vendor.read'); + const meta = Array.from(container.querySelectorAll('.v2-run-trace__meta'), (node) => node.textContent); + expect(meta[0]).toBe(`${i18n.t('runTrace.served', { requested: nemotron, served: 'deepseek-v4-flash' })} · 12 ms · 123 ${i18n.t('runTrace.tokens')}`); + expect(meta[0]).toContain(nemotron); + expect(meta[0]).toContain('deepseek-v4-flash'); + expect(meta[1]).toBe(`deepseek-v4-flash · 12 ms · 123 ${i18n.t('runTrace.tokens')}`); + expect(meta[2]).toBe(`older-turn · 12 ms · 123 ${i18n.t('runTrace.tokens')}`); +}); test('refusal can retry and never presents an empty successful trace', async () => { mockGet.mockRejectedValueOnce({ response: { status: 404 } }).mockResolvedValueOnce({ run: trace }); setup(); diff --git a/frontend/src/v2/components/V2RunTrace.tsx b/frontend/src/v2/components/V2RunTrace.tsx index 72182c1a3d..b72eb22bba 100644 --- a/frontend/src/v2/components/V2RunTrace.tsx +++ b/frontend/src/v2/components/V2RunTrace.tsx @@ -9,7 +9,8 @@ export interface RunTrace { trigger: string; startedAt: string; completedAt?: string; totalTokens: number; errorKind?: string; traceTruncated: boolean; omittedTurns: number; omittedToolCalls: number; turns: Array<{ - turnIndex: number; model: string; elapsedMs: number; totalTokens?: number; + turnIndex: number; model: string; servedModel?: string; fallbacks?: number; + elapsedMs: number; totalTokens?: number; toolCalls: Array<{ name: string; elapsedMs: number; callId?: string; outcome: string; trail?: { grantId?: string; credentialOwnerId?: string | null; outcome: string } | null; @@ -61,7 +62,9 @@ const V2RunTrace: React.FC = () => {
    {trace.turns.map((turn) =>

  1. {t('runTrace.turn', { number: turn.turnIndex + 1 })}

    -

    {turn.model} · {turn.elapsedMs} ms · {turn.totalTokens ?? '—'} {t('runTrace.tokens')}

    +

    {turn.servedModel && turn.servedModel !== turn.model + ? t('runTrace.served', { requested: turn.model, served: turn.servedModel }) + : turn.model} · {turn.elapsedMs} ms · {turn.totalTokens ?? '—'} {t('runTrace.tokens')}

    {!turn.toolCalls.length &&

    {t('runTrace.noTools')}

    } {turn.toolCalls.map((call, index) =>
    {call.name}{enumLabel('outcomes', call.outcome)} · {call.elapsedMs} ms From 46226306738050bd6bc73175b27840a9ce05e014 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Mon, 26 Oct 2026 05:33:03 -0700 Subject: [PATCH 2/4] docs(runtime): fallbacks says a fallback happened, not how many hops (TASK-028) On the real 1.106.2 router a two-hop chain returns x-litellm-attempted-fallbacks: 1, the outermost hop's depth. Written by Rivet, a Commonly agent: https://commonly.me/v2/pods/6addd4c17f8fc798a6f66431 Co-Authored-By: Claude Opus 5.5 Co-authored-by: rivet Agent-Model: claude/claude-opus-5-5/default --- backend/models/AgentRun.ts | 3 ++- backend/services/nativeRuntimeService.ts | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/backend/models/AgentRun.ts b/backend/models/AgentRun.ts index f6c98a779e..84f90eed17 100644 --- a/backend/models/AgentRun.ts +++ b/backend/models/AgentRun.ts @@ -68,7 +68,8 @@ export interface IAgentRunTurn { // The model group that answered, when it can differ from `model`, the group // requested (TASK-028). Absent on turns recorded before it was kept. servedModel?: string; - // LiteLLM fallback hops taken before `servedModel` answered; absent means none. + // LiteLLM's x-litellm-attempted-fallbacks; absent means none. The proxy reports + // the outermost hop's depth, so a two-hop chain records 1: a fallback happened. fallbacks?: number; } diff --git a/backend/services/nativeRuntimeService.ts b/backend/services/nativeRuntimeService.ts index 4fe7c921f4..fe5a0d27a5 100644 --- a/backend/services/nativeRuntimeService.ts +++ b/backend/services/nativeRuntimeService.ts @@ -1415,10 +1415,10 @@ function extractCallId(data: LiteLLMResponse, headers: Record): } // Which model group answered the call (TASK-028). LiteLLM's router names it in -// x-litellm-model-group on every response and counts the fallback hops it took -// in x-litellm-attempted-fallbacks (v1.106.2). The proxy restamps the body's -// `model` to the requested group unless a fallback happened, so the body comes -// second. Native calls send disable_fallbacks (D81), so the two differ only if +// x-litellm-model-group on every response, and x-litellm-attempted-fallbacks +// says a fallback happened (v1.106.2; it is the outermost hop's depth, so 1 on a +// two-hop chain). The proxy restamps the body's `model` to the requested group +// unless a fallback happened, so the body comes second. Native calls send disable_fallbacks (D81), so the two differ only if // something overrode it, such as a key whose metadata sets disable_fallbacks. function servedModelOf( data: LiteLLMResponse, From 08cbef8bcdad90c21f37fdf84b7c28040de1b84e Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Mon, 26 Oct 2026 05:33:13 -0700 Subject: [PATCH 3/4] style(runtime): rewrap the served-model comment (TASK-028) Written by Rivet, a Commonly agent: https://commonly.me/v2/pods/6addd4c17f8fc798a6f66431 Co-Authored-By: Claude Opus 5.5 Co-authored-by: rivet Agent-Model: claude/claude-opus-5-5/default --- backend/services/nativeRuntimeService.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/services/nativeRuntimeService.ts b/backend/services/nativeRuntimeService.ts index fe5a0d27a5..c89d425bcc 100644 --- a/backend/services/nativeRuntimeService.ts +++ b/backend/services/nativeRuntimeService.ts @@ -1418,8 +1418,9 @@ function extractCallId(data: LiteLLMResponse, headers: Record): // x-litellm-model-group on every response, and x-litellm-attempted-fallbacks // says a fallback happened (v1.106.2; it is the outermost hop's depth, so 1 on a // two-hop chain). The proxy restamps the body's `model` to the requested group -// unless a fallback happened, so the body comes second. Native calls send disable_fallbacks (D81), so the two differ only if -// something overrode it, such as a key whose metadata sets disable_fallbacks. +// unless a fallback happened, so the body comes second. Native calls send +// disable_fallbacks (D81), so the two differ only if something overrode it, +// such as a key whose metadata sets disable_fallbacks. function servedModelOf( data: LiteLLMResponse, headers: Record, From 72fcda9fad9ea44dfafcd51a53d80276aa4f624f Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Tue, 27 Oct 2026 00:02:34 -0700 Subject: [PATCH 4/4] fix(runtime): leave servedModel unset when the response names no model (TASK-028) When neither x-litellm-model-group nor the body named a model, the turn recorded the requested group as served. That was an inference, not a measurement. Leave the field unset instead. The spend grouping, the trace route and the run trace already fall back to the requested model when it is absent, so pricing and display don't change. Written by Rivet, a Commonly agent Co-Authored-By: Claude Opus 5.5 Co-authored-by: rivet Agent-Model: claude/claude-opus-5-5/default --- .../nativeRuntimeService.servedModel.test.js | 7 ++++--- backend/models/AgentRun.ts | 3 ++- backend/services/nativeRuntimeService.ts | 20 +++++++++++-------- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/backend/__tests__/unit/services/nativeRuntimeService.servedModel.test.js b/backend/__tests__/unit/services/nativeRuntimeService.servedModel.test.js index e2aaac9bf1..268d265a47 100644 --- a/backend/__tests__/unit/services/nativeRuntimeService.servedModel.test.js +++ b/backend/__tests__/unit/services/nativeRuntimeService.servedModel.test.js @@ -127,9 +127,10 @@ test('a normal turn records the two equal, with no hops', async () => { expect(turn).toEqual({ model: DEEPSEEK, servedModel: DEEPSEEK, fallbacks: undefined }); }); -test('without the group header the body model is used, and without either the request', async () => { +test('without the group header the body model is used, and without either nothing is recorded', async () => { expect(await turnOf(NEMOTRON, { model: DEEPSEEK, headers: {} })) .toEqual({ model: NEMOTRON, servedModel: DEEPSEEK, fallbacks: undefined }); - expect(await turnOf(DEEPSEEK, { headers: { 'x-litellm-attempted-fallbacks': 'not-a-number' } })) - .toEqual({ model: DEEPSEEK, servedModel: DEEPSEEK, fallbacks: undefined }); + // An unmeasured turn is not recorded as served by the requested model. + expect(await turnOf(NEMOTRON, { headers: { 'x-litellm-attempted-fallbacks': 'not-a-number' } })) + .toEqual({ model: NEMOTRON, servedModel: undefined, fallbacks: undefined }); }); diff --git a/backend/models/AgentRun.ts b/backend/models/AgentRun.ts index 84f90eed17..7ef6ea60dd 100644 --- a/backend/models/AgentRun.ts +++ b/backend/models/AgentRun.ts @@ -66,7 +66,8 @@ export interface IAgentRunTurn { elapsedMs: number; liteLLMCallId?: string; // The model group that answered, when it can differ from `model`, the group - // requested (TASK-028). Absent on turns recorded before it was kept. + // requested (TASK-028). Absent on turns recorded before it was kept, and on a + // turn whose response named no model. servedModel?: string; // LiteLLM's x-litellm-attempted-fallbacks; absent means none. The proxy reports // the outermost hop's depth, so a two-hop chain records 1: a fallback happened. diff --git a/backend/services/nativeRuntimeService.ts b/backend/services/nativeRuntimeService.ts index c89d425bcc..c95643fc76 100644 --- a/backend/services/nativeRuntimeService.ts +++ b/backend/services/nativeRuntimeService.ts @@ -1418,19 +1418,23 @@ function extractCallId(data: LiteLLMResponse, headers: Record): // x-litellm-model-group on every response, and x-litellm-attempted-fallbacks // says a fallback happened (v1.106.2; it is the outermost hop's depth, so 1 on a // two-hop chain). The proxy restamps the body's `model` to the requested group -// unless a fallback happened, so the body comes second. Native calls send -// disable_fallbacks (D81), so the two differ only if something overrode it, -// such as a key whose metadata sets disable_fallbacks. +// unless a fallback happened, so the body comes second. When neither names a +// model, nothing was measured and the field stays unset; every reader falls +// back to the requested model. Native calls send disable_fallbacks (D81), so +// the two differ only if something overrode it, such as a key whose metadata +// sets disable_fallbacks. function servedModelOf( data: LiteLLMResponse, headers: Record, - requested: string, -): { servedModel: string; fallbacks?: number } { +): { servedModel?: string; fallbacks?: number } { const group = headers?.['x-litellm-model-group']; const hops = Number(headers?.['x-litellm-attempted-fallbacks']); const servedModel = (typeof group === 'string' && group) - || (typeof data?.model === 'string' && data.model) || requested; - return { servedModel, ...(Number.isInteger(hops) && hops > 0 ? { fallbacks: hops } : {}) }; + || (typeof data?.model === 'string' && data.model) || undefined; + return { + ...(servedModel ? { servedModel } : {}), + ...(Number.isInteger(hops) && hops > 0 ? { fallbacks: hops } : {}), + }; } function mapTriggerType(raw: string): NativeRunTrigger['type'] { @@ -2162,7 +2166,7 @@ export async function runAgent( llmResponseText: (msg.content as string | null) || undefined, elapsedMs: 0, liteLLMCallId: extractCallId(llmResponse, responseHeaders), - ...servedModelOf(llmResponse, responseHeaders, model), + ...servedModelOf(llmResponse, responseHeaders), }; const toolCalls = msg.tool_calls || [];