diff --git a/src/server/responses/context-overflow.ts b/src/server/responses/context-overflow.ts index 2cfc1f96175..e90b736e37f 100644 --- a/src/server/responses/context-overflow.ts +++ b/src/server/responses/context-overflow.ts @@ -17,21 +17,25 @@ export function lateAttributionLimitResponse(): Response { export const PROVIDER_INPUT_TOO_LARGE_MESSAGE = "The provider rejected this turn because its input exceeds the provider size or context limit. Reduce the current input or compact the conversation before retrying."; -/** Preserve non-streaming HTTP failure semantics without exposing an upstream body. */ -export function jsonContextOverflowResponse(): Response { +/** + * Preserve non-streaming HTTP failure semantics without exposing an upstream body. `message` is + * replaced only for the proxy's own wording, never with upstream text: when every combo target + * was refused by local input admission, no provider saw the turn and saying one did is false. + */ +export function jsonContextOverflowResponse(message = PROVIDER_INPUT_TOO_LARGE_MESSAGE): Response { return Response.json({ error: { - message: PROVIDER_INPUT_TOO_LARGE_MESSAGE, + message, type: "invalid_request_error", code: "context_length_exceeded", }, }, { status: 413, headers: { "Cache-Control": "no-store" } }); } -async function* contextOverflowEvents(): AsyncGenerator { +async function* contextOverflowEvents(message: string): AsyncGenerator { yield { type: "error", - message: PROVIDER_INPUT_TOO_LARGE_MESSAGE, + message, status: 413, errorType: "invalid_request_error", code: "context_length_exceeded", @@ -51,9 +55,10 @@ async function* contextOverflowEvents(): AsyncGenerator { export function streamingContextOverflowResponse( modelId: string, translatorBudget: TranslatorBudget, + message = PROVIDER_INPUT_TOO_LARGE_MESSAGE, ): Response { return new Response(bridgeToResponsesSSE( - contextOverflowEvents(), + contextOverflowEvents(message), modelId, undefined, undefined, diff --git a/src/server/responses/core-combo.ts b/src/server/responses/core-combo.ts index f4714f0c9b7..89a4daa4a4e 100644 --- a/src/server/responses/core-combo.ts +++ b/src/server/responses/core-combo.ts @@ -685,6 +685,9 @@ export async function executeComboResponses( // is gone, so carry the loop's own classification decision instead of re-deriving a // weaker one from the status alone (#4149). let lastFailureClassifiesOverflow = false; + // Local input admission refusals (#1524), while every failed target so far was one: such a + // target sent nothing, so an exhausted ladder must not be reported as a provider rejection. + let localAdmissionRefusals: string[] | undefined = []; // The first quota/rate-limit refusal of the ladder. A fallback that then refuses its own // credential or plan (401/403, or a 400 the ladder hopped past) says nothing about why the // request failed: the primary target ran out. Returning that fallback refusal told clients @@ -1052,6 +1055,12 @@ export async function executeComboResponses( Date.now() - started, failure.usage, ); + if (failure.upstreamCode === "input_admission_refused" && attempt.sendCount === 0) { + attempt.errorCode = failure.upstreamCode; + localAdmissionRefusals?.push(failure.upstreamMessage ?? ""); + } else { + localAdmissionRefusals = undefined; + } attemptRetained = true; lastFailure = failure.response; if (!quotaFailure && (lastFailure.status === 429 || lastFailure.status === 402)) { @@ -1232,9 +1241,14 @@ export async function executeComboResponses( lastFailure?.status === 413 && lastFailureClassifiesOverflow ) { + if (localAdmissionRefusals?.length) logCtx.errorCode = "input_admission_refused"; + const localMessage = localAdmissionRefusals?.length + ? `opencodex refused this turn locally; nothing was sent upstream. No target of combo ${comboId} ` + + `that was tried can fit its input: ${localAdmissionRefusals.join(" ")} Compact the conversation before retrying.` + : undefined; return (rawBody as { stream?: unknown } | null)?.stream === true - ? streamingContextOverflowResponse(requestedModel, options.translatorBudget) - : jsonContextOverflowResponse(); + ? streamingContextOverflowResponse(requestedModel, options.translatorBudget, localMessage) + : jsonContextOverflowResponse(localMessage); } return failure; } diff --git a/structure/transports/responses-failover.md b/structure/transports/responses-failover.md index f873d18f175..654a2480bb9 100644 --- a/structure/transports/responses-failover.md +++ b/structure/transports/responses-failover.md @@ -508,7 +508,7 @@ existing combo policy already treats as a safe hop. That ordering is the whole p total window cannot hold the turn plus the caller's allowance answers 200, emits a few hundred tokens and stops on `finish_reason: length`, which the Anthropic surface renders as an output-token error naming a limit the model never approached — and by then output has committed and no later -target may be tried. +target may be tried. When every failed target of an exhausted combo was refused this way with a send count of 0, `src/server/responses/core-combo.ts` keeps the `context_length_exceeded` overflow envelope that clients compact on, but replaces the provider wording from `context-overflow.ts` with a proxy-owned message saying the turn was refused locally, carrying each tried target's estimate and window (a target skipped for cooldown is not claimed to be too small). Those attempts, and the final row, keep the code `input_admission_refused` instead of `http_413`. One target that reached a provider keeps the provider wording for the whole combo. Scope is deliberately narrow. Direct and single-target requests keep the loose 2.5x pathological-input gate, because they have nowhere to hop. Compaction turns stay exempt. Unknown @@ -517,8 +517,8 @@ limits for custom providers. Canonical native slugs that the narrower pinned tab resolve their window from the generated in-tree bundle, which is what made the gate inert on the route where this was first observed; explicit provider and operator caps may only narrow it. -Regression coverage: `tests/server/input-admission.test.ts` and -`tests/helpers/combo-context-headroom-cases.ts`. +Regression coverage: `tests/server/input-admission.test.ts`, +`tests/helpers/combo-context-headroom-cases.ts` and `tests/responses/responses-context-overflow.test.ts`. Native steering retains fixed phase deadlines and reconciled replay output; see the [steering stability contract](../transports/streaming-health.md#steering-deadlines-and-replay-completeness). diff --git a/tests/responses/responses-context-overflow.test.ts b/tests/responses/responses-context-overflow.test.ts index ec81eb872ef..e7bf31c1278 100644 --- a/tests/responses/responses-context-overflow.test.ts +++ b/tests/responses/responses-context-overflow.test.ts @@ -4,11 +4,15 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { saveConfig } from "../../src/config"; import { startServer } from "../../src/server"; +import { handleResponses } from "../../src/server/responses/core"; +import type { RequestLogContext } from "../../src/server/request-log"; import { PROVIDER_INPUT_TOO_LARGE_MESSAGE } from "../../src/server/responses/context-overflow"; import { OLLAMA_LATE_ATTRIBUTION_LIMIT_MESSAGE } from "../../src/adapters/ollama-native"; import { isLateAttributionRefusal } from "../../src/server/responses/core-combo-failure"; import type { OcxConfig, OcxProviderConfig } from "../../src/types"; import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; +import { clearComboTargetCooldowns, coolComboTarget } from "../../src/combos"; let testDir = ""; let previousOcxHome: string | undefined; @@ -418,3 +422,78 @@ describe("combo handling of the Ollama late-attribution refusal", () => { expect(isLateAttributionRefusal(failure({ upstreamMessage: undefined }))).toBe(false); }); }); + +// Every target refused by the local admission gate means no provider saw the turn. The client +// must hear that from the proxy, with the estimate and each window, instead of the provider +// wording an operator reads as an upstream rejection. +describe("combo whose targets all fail local input admission", () => { + async function runCombo(stream: boolean, second: "local" | "upstream" | "cooled") { + let hits = 0; + const local = upstream413(() => { hits += 1; }); + const first = { ...provider("openai-chat", local), modelContextWindows: { "kimi-k3": 1_000 } }; + const next = config({ + first, + second: second === "upstream" + ? provider("openai-chat", freePromptCap413(() => { hits += 1; })) + : { ...provider("openai-chat", local), modelContextWindows: { "kimi-k3": 2_000 } }, + }); + next.combos = { fallback: { strategy: "failover", targets: [ + { provider: "first", model: "kimi-k3" }, { provider: "second", model: "kimi-k3" }, + ] } }; + clearComboTargetCooldowns(); + if (second === "cooled") coolComboTarget("fallback", { provider: "second", model: "kimi-k3" }, { retryAfter: "120" }); + const logCtx = { model: "", provider: "" } as RequestLogContext; + const release = acquireOwnedSpendHome(); + try { + const response = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "combo/fallback", stream, max_output_tokens: 500, input: "x".repeat(20_000) }), + }), next, logCtx); + const error = stream + ? (await responseFailed(response)).error as Record + : (expect(response.status).toBe(413), (await response.json() as { error: Record }).error); + return { error, hits, attempts: logCtx.attempts ?? [], logCtx }; + } finally { + release(); + clearComboTargetCooldowns(); + } + } + + test.each([true, false])("says the proxy refused it, with estimate and windows (stream=%s)", async stream => { + const { error, hits, attempts, logCtx } = await runCombo(stream, "local"); + expect(hits).toBe(0); + expect(error.code).toBe("context_length_exceeded"); + expect(error.type).toBe("invalid_request_error"); + const message = String(error.message); + expect(message).not.toBe(PROVIDER_INPUT_TOO_LARGE_MESSAGE); + expect(message).toContain("refused this turn locally"); + expect(message).toContain("No target of combo fallback that was tried"); + expect(message).toMatch(/~\d+ tokens/); + expect(message).toContain("(1000 tokens)"); + expect(message).toContain("(2000 tokens)"); + expect(attempts.map(attempt => [attempt.status, attempt.sendCount, attempt.errorCode])).toEqual([ + [413, 0, "input_admission_refused"], + [413, 0, "input_admission_refused"], + ]); + expect(logCtx.errorCode).toBe("input_admission_refused"); + }); + + test("a target skipped for cooldown is not claimed to be too small", async () => { + const { error, hits, attempts } = await runCombo(false, "cooled"); + expect(hits).toBe(0); + const message = String(error.message); + expect(message).toContain("No target of combo fallback that was tried can fit its input"); + expect(message).toContain("(1000 tokens)"); + expect(message).not.toContain("(2000 tokens)"); + expect(attempts.map(attempt => attempt.errorCode)).toEqual(["input_admission_refused"]); + }); + + test("a target that reached the provider keeps the provider wording", async () => { + const { error, hits, attempts, logCtx } = await runCombo(false, "upstream"); + expect(hits).toBe(1); + expect(logCtx.errorCode).toBeUndefined(); + expect(error.message).toBe(PROVIDER_INPUT_TOO_LARGE_MESSAGE); + expect(attempts.map(attempt => attempt.errorCode)).toEqual(["input_admission_refused", "http_413"]); + }); +});