diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 9925e111815..fdaa35096a5 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -76,6 +76,7 @@ "owner-registry-acl.test.ts": "config", "minimax-combo-image-input.test.ts": "providers", "web-search-forced-declaration.test.ts": "web-search", + "web-search-loop-iteration-usage.test.ts": "web-search", "claude-devin-output-order.test.ts": "claude-integration", "codex-config-preservation.test.ts": "codex-integration", "codex-credits.test.ts": "codex-integration", diff --git a/src/web-search/loop.ts b/src/web-search/loop.ts index 79d6c291f7c..e1fe62a8a24 100644 --- a/src/web-search/loop.ts +++ b/src/web-search/loop.ts @@ -7,6 +7,7 @@ import type { AttemptRecoveryKind } from "../usage/log"; import { isTruncatedStopReason } from "../responses/truncated-stop-reason"; import { bridgeToResponsesSSE } from "../bridge"; import { guardEmptyToolCatalogStream, type ClientToolCatalogRestriction } from "../lib/empty-tool-catalog"; +import { mergeUsage } from "../server/responses/empty-completion-guard"; import { runWebSearch, type SidecarOutcome, type SidecarOutcomeRecorder, type SidecarSettings } from "./executor"; import { runAnthropicWebSearch } from "./anthropic-executor"; import { runXaiWebSearch, type XaiSearchOptions } from "./xai-executor"; @@ -144,6 +145,18 @@ async function* replay(events: AdapterEvent[]): AsyncGenerator { for (const e of events) yield e; } +function terminalUsage(events: AdapterEvent[]): OcxUsage | undefined { + for (const e of events) if (e.type === "done" || e.type === "incomplete") return e.usage; + return undefined; +} + +/** Hidden iterations add counters only: their checkpoint and raw wire usage are not the final terminal's. */ +function countersOnly(usage: OcxUsage | undefined): OcxUsage | undefined { + if (!usage) return undefined; + const { contextTotalTokens: _context, rawUsage: _raw, ...counters } = usage; + return counters; +} + /** * Collect the thinking block that preceded a web_search call in this iteration's events, so the * replayed assistant turn can carry it. Anthropic extended thinking REQUIRES the assistant @@ -413,6 +426,12 @@ export async function runWithWebSearch(deps: WebSearchLoopDeps): Promise event.type))], })); + hiddenUsage = mergeUsage(hiddenUsage, terminalEvent.usage); if (!split.hasMalformedToolCall && !split.hasRealToolCall && emptyAnswerRetries === 0) { emptyAnswerRetries++; console.warn("[web-search-loop] empty forced answer — retrying once without tools"); @@ -972,6 +992,7 @@ export async function runWithWebSearch(deps: WebSearchLoopDeps): Promise deps.onUsage?.(mergeUsage(countersOnly(hiddenUsage), usage)) } : {}), ...(deps.onCompletedResponse ? { onCompletedResponse: deps.onCompletedResponse } : {}), }, ); diff --git a/structure/runtime.md b/structure/runtime.md index dc720c31f82..457286c6056 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -398,7 +398,7 @@ Automatic Codex pool selection and account status share the [plan exclusion cont ### Empty forced search answers -Both search loops keep `web_search` declared after the search budget is exhausted. Further calls receive a paired limit-reached result without another physical search; at most `maxSearches + 3` model iterations run before a terminal error. Ordinary caller tools and cancellation still end the loop. Empty-answer recovery alone removes all tools. `src/web-search/loop.ts` makes at most one extra answer attempt after a clean forced-answer terminal with no visible output or tool call, only while an iteration remains. The recovery has no tools and reuses gathered search results. Malformed calls fail before refusal/truncation passthrough, and well-formed recognized refusal/truncation terminals pass through unchanged, including empty or partial answers. The extra generation may incur provider usage. `src/web-search/run-turn-loop.ts` shares this recovery and checks the ceiling before dispatch so it cannot start an unused request; below the search cap, `emptyCompletionRetry` permits one identical empty-answer retry per request with current tools and results. Errors and invalid terminals never authorize another search. Batched results from one call share one 8,000-character budget in `src/web-search/format-result.ts`, planned per query before serialization (#6621): every query keeps its label and status, shortened answers and unlisted sources carry markers with their original counts outside the untrusted boundary, structured output stays one valid JSON document, and a batch whose fixed parts alone overflow is condensed to status lines with a per-status count of any query that still did not fit. +Both search loops keep `web_search` declared after the search budget is exhausted. Further calls receive a paired limit-reached result without another physical search; at most `maxSearches + 3` model iterations run before a terminal error. Ordinary caller tools and cancellation still end the loop. Empty-answer recovery alone removes all tools. `src/web-search/loop.ts` makes at most one extra answer attempt after a clean forced-answer terminal with no visible output or tool call, only while an iteration remains. The recovery has no tools and reuses gathered search results. Malformed calls fail before refusal/truncation passthrough, and well-formed recognized refusal/truncation terminals pass through unchanged, including empty or partial answers. The extra generation may incur provider usage. Every iteration re-sends the full context, so `src/web-search/loop.ts` adds the usage of each iteration whose terminal the client never sees (search-calling passes and the discarded empty answer) to what `onUsage` reports for the request log, including on an in-stream or translator-budget error; token and cache counts add, while `contextTotalTokens` and `rawUsage` come only from the final terminal, never an earlier iteration's. The client-facing terminal keeps the last iteration's usage, because Codex and Claude Code size the context window from it (`tests/web-search/web-search-loop-iteration-usage.test.ts`). `src/web-search/run-turn-loop.ts` shares this recovery and checks the ceiling before dispatch so it cannot start an unused request; below the search cap, `emptyCompletionRetry` permits one identical empty-answer retry per request with current tools and results. Errors and invalid terminals never authorize another search. Batched results from one call share one 8,000-character budget in `src/web-search/format-result.ts`, planned per query before serialization (#6621): every query keeps its label and status, shortened answers and unlisted sources carry markers with their original counts outside the untrusted boundary, structured output stays one valid JSON document, and a batch whose fixed parts alone overflow is condensed to status lines with a per-status count of any query that still did not fit. `src/web-search/loop.ts` uses `src/web-search/header-deadline.ts` to suspend the cumulative response-header deadline while each physical send awaits local pacing admission. Ordinary fetches, adapter executors, reset retries and pool-key rotations retain the remaining budget after admission; same-target 429 backoff still clears and restarts that budget. The default remains 200,000 ms. Client cancellation remains linked during queued admission and after final headers, and the existing pacing slot wrapper retains leases through body completion or cancellation. Streaming retry heartbeats and body inactivity deadlines keep their existing ownership. `tests/web-search/web-search-pacing-deadline.test.ts` covers queued success, real header stalls, queued cancellation, retry admission and cumulative rotation time. OpenAI sidecar 429 replays run only when their backoff fits the remaining sidecar deadline; otherwise the original 429 remains the routing-health outcome rather than becoming a timeout. Reset recovery and 429 replays share one three-send budget per search, so the two layers cannot multiply physical sends. Direct vision/search helpers and the alpha/search relay retain the resolver's live [stored-account credit policy](codex-account-controls.md#stored-account-authentication-policy) through physical sends and retries; a local hold prevents the send without becoming an upstream connection failure. ## Scoped provider quota for Combo selection diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index f5047535934..6805214c499 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -126,7 +126,8 @@ "cli-combo-partial-update.test.ts": "cli", "desktop-app-server-shim.test.ts": "clients", "desktop-app-server-shim-launcher.test.ts": "clients", "desktop-chatgpt-config.test.ts": "clients", "owner-registry-acl.test.ts": "config", "minimax-combo-image-input.test.ts": "providers", - "web-search-forced-declaration.test.ts": "web-search", "claude-devin-output-order.test.ts": "claude-integration", + "web-search-forced-declaration.test.ts": "web-search", "web-search-loop-iteration-usage.test.ts": "web-search", + "claude-devin-output-order.test.ts": "claude-integration", "codex-config-preservation.test.ts": "codex-integration", "codex-credits.test.ts": "codex-integration", "codex-credits-settings.test.ts": "codex-integration", "codex-credits-probes.test.ts": "codex-integration", "codex-credits-after-limit.test.ts": "codex-integration", diff --git a/tests/web-search/web-search-loop-iteration-usage.test.ts b/tests/web-search/web-search-loop-iteration-usage.test.ts new file mode 100644 index 00000000000..115257677c8 --- /dev/null +++ b/tests/web-search/web-search-loop-iteration-usage.test.ts @@ -0,0 +1,107 @@ +import { afterEach, describe, expect, spyOn, test } from "bun:test"; +import type { ProviderAdapter } from "../../src/adapters/base"; +import type { AdapterEvent, OcxParsedRequest, OcxUsage } from "../../src/types"; +import { runWithWebSearch } from "../../src/web-search/loop"; +import { buildWebSearchTool } from "../../src/web-search/synthetic-tool"; +import { createTestTranslatorBudget } from "../helpers/translator-budget"; + +// Every routed-model iteration re-sends the full context, so the usage ledger (onUsage) must get the +// sum of all of them. The client-facing terminal keeps the last iteration's usage: Codex and Claude +// Code size the context window from it, and a sum would trigger early auto-compaction. +const usage = (input: number, output: number, extra: Partial = {}): OcxUsage => + ({ inputTokens: input, outputTokens: output, totalTokens: input + output, ...extra }); +const search = (id: string, u: OcxUsage): AdapterEvent[] => [ + { type: "tool_call_start", id, name: "web_search" }, + { type: "tool_call_delta", arguments: JSON.stringify({ query: id }) }, + { type: "tool_call_end" }, { type: "done", usage: u }, +]; +const answer = (u: OcxUsage): AdapterEvent[] => [{ type: "text_delta", text: "answer" }, { type: "done", usage: u }]; + +const originalFetch = globalThis.fetch; +afterEach(() => { globalThis.fetch = originalFetch; }); + +async function drive(passes: AdapterEvent[][], refuseFrom = Infinity) { + spyOn(globalThis, "fetch").mockResolvedValue(Response.json({ results: [] })); + const reported: (OcxUsage | undefined)[] = []; + let events: AdapterEvent[] = []; + let iteration = 0; + const adapter: ProviderAdapter = { + name: "fixture", + buildRequest() { + events = passes[iteration++]!; + return { url: "https://routed.test/v1", method: "POST", headers: {}, body: "{}" }; + }, + fetchResponse: async () => iteration > refuseFrom ? new Response("refused", { status: 500 }) : new Response("wire"), + async *parseStream() { yield* events; }, + }; + const parsed: OcxParsedRequest = { + modelId: "fixture", stream: true, options: {}, context: { messages: [], tools: [buildWebSearchTool()] }, + }; + const response = await runWithWebSearch({ + parsed, adapter, backend: "exa", exaApiKey: "fixture", hostedTool: { type: "web_search" }, + settings: { model: "fixture", reasoning: "low", timeoutMs: 1000 }, maxSearches: 1, + incomingMeta: { headers: new Headers(), translatorBudget: createTestTranslatorBudget() }, + onUsage: u => reported.push(u), + }); + const text = await response.text(); + const terminal = text.split("\n").filter(line => line.startsWith("data: {")).map(line => JSON.parse(line.slice(6))) + .find(frame => frame.type === "response.completed" || frame.type === "response.failed"); + return { reported, text, clientUsage: terminal?.response?.usage }; +} + +describe("web-search loop iteration usage", () => { + test("search passes are added to the final answer's usage", async () => { + const { reported, text, clientUsage } = await drive([ + search("first", usage(1000, 50, { cachedInputTokens: 0, cacheCreationInputTokens: 900, contextTotalTokens: 1050 })), + answer(usage(1400, 80, { cachedInputTokens: 900, cacheReadInputTokens: 900, contextTotalTokens: 1480 })), + ]); + expect(text).toContain("response.completed"); + expect(reported).toHaveLength(1); + expect(reported[0]).toMatchObject({ + inputTokens: 2400, outputTokens: 130, totalTokens: 2530, + cachedInputTokens: 900, cacheReadInputTokens: 900, cacheCreationInputTokens: 900, + // An absolute context checkpoint is the last iteration's, never a sum. + contextTotalTokens: 1480, + }); + // Client wire derives input from the last checkpoint: 1480 - 80. + expect(clientUsage).toMatchObject({ input_tokens: 1400, output_tokens: 80, total_tokens: 1480 }); + }); + + test("the empty forced answer that was retried is counted too", async () => { + const { reported, text, clientUsage } = await drive([ + search("first", usage(1000, 50)), + [{ type: "done", usage: usage(1400, 5) }], + answer(usage(1500, 80)), + ]); + expect(text).toContain("response.completed"); + expect(reported).toEqual([expect.objectContaining({ inputTokens: 3900, outputTokens: 135, totalTokens: 4035 })]); + expect(clientUsage).toMatchObject({ input_tokens: 1500, output_tokens: 80, total_tokens: 1580 }); + }); + + test("an in-stream failure after a search still reports the iterations already sent", async () => { + const { reported, text, clientUsage } = await drive([search("first", usage(1000, 50)), answer(usage(1, 1))], 1); + expect(text).toContain("response.failed"); + expect(clientUsage).toBeFalsy(); + expect(reported).toEqual([expect.objectContaining({ inputTokens: 1000, outputTokens: 50 })]); + }); + + test("a translator-budget failure after a search still reports the iterations already sent", async () => { + const { reported, text } = await drive([ + search("first", usage(1000, 50)), + [{ type: "error", status: 502, code: "translation_buffer_limit", message: "over budget" }], + ]); + expect(text).toContain("translation_buffer_limit"); + expect(reported).toEqual([expect.objectContaining({ inputTokens: 1000, outputTokens: 50 })]); + }); + + test("an earlier iteration's checkpoint and raw usage never stand in for the final terminal's", async () => { + const { reported } = await drive([ + search("first", usage(1000, 50, { contextTotalTokens: 1050, rawUsage: { iteration: 1 } })), + answer(usage(1400, 80)), + ]); + expect(reported).toHaveLength(1); + expect(reported[0]).toMatchObject({ inputTokens: 2400, outputTokens: 130 }); + expect(reported[0]?.contextTotalTokens).toBeUndefined(); + expect(reported[0]?.rawUsage).toBeUndefined(); + }); +});