diff --git a/docs-site/src/content/docs/guides/remote-hub.md b/docs-site/src/content/docs/guides/remote-hub.md index fd16df1c596..ab4e06cd75e 100644 --- a/docs-site/src/content/docs/guides/remote-hub.md +++ b/docs-site/src/content/docs/guides/remote-hub.md @@ -37,6 +37,11 @@ listener, not the management port. Local clients use the configured loopback com Advisor model used by managed native Anthropic Messages. Bare model IDs and the hub's configured `provider/model` names are accepted. A disallowed or unnamed Advisor destination returns HTTP 403 before provider dispatch; provider-only and unrestricted keys retain their existing scope. +- GitHub Copilot Auto rechecks the concrete model selected during negotiation and + renewal. A model outside the client's allowed scope ends the request with + `model_not_allowed_for_key` (HTTP 403 before streaming, or a terminal error in + an already-started stream). Credential rotation or a fallback target cannot + override that refusal. - A raw management admin token can perform ordinary administration, but it cannot mint a browser session or authorize consent-bearing actions such as starring the repository. Those actions require a server-issued `gui-session`, matching browser origin, and CSRF token. diff --git a/src/lib/translator-budget.ts b/src/lib/translator-budget.ts index f5a8d93c7a2..f6e184683dd 100644 --- a/src/lib/translator-budget.ts +++ b/src/lib/translator-budget.ts @@ -1,3 +1,5 @@ +import { isNonReplayableResponse, markResponseNonReplayable } from "./upstream-retry"; + export const TRANSLATOR_MAX_CALL_ARGUMENT_BYTES = 2 * 1024 * 1024; export const TRANSLATOR_MAX_TURN_BYTES = 32 * 1024 * 1024; export const TRANSLATOR_MAX_SSE_EVENT_BYTES = 32 * 1024 * 1024; @@ -410,5 +412,9 @@ export function finalizeTranslatorBudgetResponse(response: Response, budget: Tra }); if (signal?.aborted) finalize(); else signal?.addEventListener("abort", finalize, { once: true }); - return new Response(body, { status: response.status, statusText: response.statusText, headers: response.headers }); + const finalizedResponse = new Response(body, { status: response.status, statusText: response.statusText, headers: response.headers }); + // The rebuild above drops identity markers. A refusal that must not be replayed + // stays non-replayable through accounting finalization. + if (isNonReplayableResponse(response)) markResponseNonReplayable(finalizedResponse); + return finalizedResponse; } diff --git a/src/server/admission-model-scope.ts b/src/server/admission-model-scope.ts index 9390d2c3f34..54798ff98ce 100644 --- a/src/server/admission-model-scope.ts +++ b/src/server/admission-model-scope.ts @@ -1,5 +1,6 @@ import type { DataPlaneAdmission } from "./auth-cors"; import type { OcxApiKeyEntry, OcxConfig } from "../types"; +import { markResponseNonReplayable } from "../lib/upstream-retry"; /** * Per-admission-key model and provider scope. @@ -155,10 +156,12 @@ export function admissionModelDeniedBody(error: AdmissionModelDeniedError): { * indistinguishable from a typo in the model name. */ export function admissionModelDeniedResponse(error: AdmissionModelDeniedError): Response { - return new Response(JSON.stringify(admissionModelDeniedBody(error)), { + const response = new Response(JSON.stringify(admissionModelDeniedBody(error)), { status: 403, headers: { "Content-Type": "application/json" }, }); + markResponseNonReplayable(response); + return response; } /** diff --git a/src/server/responses/adapter-continuation.ts b/src/server/responses/adapter-continuation.ts index e0e5ae22de2..788932fef5a 100644 --- a/src/server/responses/adapter-continuation.ts +++ b/src/server/responses/adapter-continuation.ts @@ -30,6 +30,8 @@ import { isNonReplayableResponse, prepareSameTarget429Wait, } from "../../lib/upstream-retry"; +import { unwrapUpstreamRetryEvidenceError } from "../../codex/auth-context"; +import { AdmissionModelDeniedError, MODEL_NOT_ALLOWED_FOR_KEY } from "../admission-model-scope"; import { redactSecretString } from "../../lib/redact"; import { resolveWireProtocolOverride } from "../adapter-resolve"; import { bindRouteReasoningReplayScope } from "./core-replay"; @@ -146,6 +148,22 @@ export function createAdapterContinuations( sendBudgetExhausted, } = sendBudgetState; + const continuationFailureEvent = (error: unknown): Extract => { + if (options.abortSignal?.aborted || upstream.signal.aborted) { + return { type: "error", message: "client closed request during terminal continuation", status: 499 }; + } + const refusal = unwrapUpstreamRetryEvidenceError(error); + if (refusal instanceof AdmissionModelDeniedError) { + return { + type: "error", message: refusal.message, status: 403, + errorType: MODEL_NOT_ALLOWED_FOR_KEY, code: MODEL_NOT_ALLOWED_FOR_KEY, retryable: false, + }; + } + return { + type: "error", + message: `Provider continuation failed: ${redactSecretString(error instanceof Error ? error.message : String(error))}`, + }; + }; // One bounded internal continuation re-ask for clean end_turn turns that announced an edit // without emitting a tool call. Anthropic gets this by default; openai-chat providers opt in @@ -304,12 +322,7 @@ export function createAdapterContinuations( kiroRefusalPendingReplay = undefined; kiroReplaySetupFailed = true; } else { - if (options.abortSignal?.aborted || upstream.signal.aborted) { - yield { type: "error", message: "client closed request during terminal continuation", status: 499 }; - } else { - yield { type: "error", message: `Provider continuation failed: ${redactSecretString(error instanceof Error ? error.message : String(error))}` }; - } - return; + yield continuationFailureEvent(error); return; } } @@ -360,12 +373,7 @@ export function createAdapterContinuations( try { response = await fetchContinuation("rate-limit-429"); } catch (error) { - if (options.abortSignal?.aborted || upstream.signal.aborted) { - yield { type: "error", message: "client closed request during terminal continuation", status: 499 }; - } else { - yield { type: "error", message: `Provider continuation failed: ${redactSecretString(error instanceof Error ? error.message : String(error))}` }; - } - return; + yield continuationFailureEvent(error); return; } } @@ -381,9 +389,7 @@ export function createAdapterContinuations( route.provider = rotated; try { await transportState.resolveCopilotSelection(nextParsed); } catch (error) { - yield { type: "error", message: `Provider continuation failed: ${redactSecretString(error instanceof Error ? error.message : String(error))}`, - ...(options.abortSignal?.aborted || upstream.signal.aborted ? { status: 499 } : {}) }; - return; + yield continuationFailureEvent(error); return; } invalidateSameTargetRequest(); transportState.activeAdapter = resolveSelectionAdapter( @@ -630,13 +636,17 @@ export function createAdapterContinuations( nextContinuationRecoveryKind = "oauth-account-429"; continue; } - } catch { + } catch (error) { // Everything in this try runs before the replay: the send happens on the next // iteration, after `continue`. A throw here therefore leaves a reservation that // never dispatched, and holding it would refuse a later recovery in this same // request for a send that never left the process. hop.permit?.release(); - // fall through to emit continuation error below + if (unwrapUpstreamRetryEvidenceError(error) instanceof AdmissionModelDeniedError) { + yield continuationFailureEvent(error); + return; + } + // Other negotiation failures preserve the original provider refusal. } } } diff --git a/src/server/responses/adapter-dispatch.ts b/src/server/responses/adapter-dispatch.ts index fa0cfc62e4a..b87e6187bef 100644 --- a/src/server/responses/adapter-dispatch.ts +++ b/src/server/responses/adapter-dispatch.ts @@ -112,6 +112,7 @@ import { mapCodexAuthContextErrorToResponse } from "./codex-auth-error"; import { chargeWorkflowSends } from "../../lib/workflow-budget"; import { isAntigravityValidationRefusal } from "./antigravity-validation-refusal"; import { unboundPoolSpendRefusalResponse } from "../workflow-refusal"; +import { AdmissionModelDeniedError, admissionModelDeniedResponse } from "../admission-model-scope"; /** One responsibility of the Responses request pipeline; state owners are explicit. */ export async function prepareAdapterExchange( @@ -541,6 +542,10 @@ export async function prepareAdapterExchange( return storageRefusal; } const refusal = err instanceof UpstreamRetryEvidenceError ? err.cause : err; + if (refusal instanceof AdmissionModelDeniedError) { + releaseCodexAuthContextProbeLease(admissionState.authCtx); + return admissionModelDeniedResponse(refusal); + } const codexRefusal = mapCodexAuthContextErrorToResponse(unwrapUpstreamRetryEvidenceError(err), { now: Date.now(), accountSelector: route.codexAccountNamespace, }); @@ -810,6 +815,13 @@ export async function prepareAdapterExchange( releaseCodexAuthContextProbeLease(admissionState.authCtx); return { failed: storageRefusal }; } + const scopeRefusal = unwrapUpstreamRetryEvidenceError(err); + if (!options.abortSignal?.aborted && scopeRefusal instanceof AdmissionModelDeniedError) { + cleanupUpstreamAbort(); + upstream.abort(); + releaseCodexAuthContextProbeLease(admissionState.authCtx); + return { failed: admissionModelDeniedResponse(scopeRefusal) }; + } const codexRefusal = !options.abortSignal?.aborted && mapCodexAuthContextErrorToResponse(unwrapUpstreamRetryEvidenceError(err), { now: Date.now(), accountSelector: route.codexAccountNamespace, }); @@ -1539,7 +1551,17 @@ export async function prepareAdapterExchange( // The hop's permit is already settled by the dispatch boundary above; continuing // only skips the remaining arms, it does not abandon a reservation. if (isNonReplayableResponse(upstreamResponse)) continue recovery; - } catch { + } catch (err) { + // A local admission denial is terminal for the whole request: rotating to another + // account cannot re-scope the key, so the refusal answers in place of the 429. + const scopeRefusal = unwrapUpstreamRetryEvidenceError(err); + if (!options.abortSignal?.aborted && scopeRefusal instanceof AdmissionModelDeniedError) { + hop.permit?.release(); + cleanupUpstreamAbort(); + upstream.abort(); + releaseCodexAuthContextProbeLease(admissionState.authCtx); + return admissionModelDeniedResponse(scopeRefusal); + } // A throw before the send — snapshot fetch, credential application, adapter // resolution — must hand the reservation back. Without this the ladder charges the // request for a send it never made, and a later recovery in the same request is diff --git a/src/server/responses/core-combo-failure.ts b/src/server/responses/core-combo-failure.ts index fd5be61418d..90542a06cf7 100644 --- a/src/server/responses/core-combo-failure.ts +++ b/src/server/responses/core-combo-failure.ts @@ -16,6 +16,7 @@ import { resolveClientRetryAfter } from "../../lib/retry-after"; import { formatErrorResponse } from "../../bridge"; import { OLLAMA_LATE_ATTRIBUTION_LIMIT_MESSAGE } from "../../adapters/ollama-native"; import { isNonReplayableResponse, markResponseNonReplayable } from "../../lib/upstream-retry"; +import { MODEL_NOT_ALLOWED_FOR_KEY } from "../admission-model-scope"; import { usageFromResponsesPayload } from "../request-log"; import type { ResponsesTerminalStatus } from "../../bridge"; @@ -49,6 +50,9 @@ export async function consumeComboFailure( // display-safe body carries a recognized quota message. let quotaConfirmedByBody = false; const serverError = response.status >= 500 && response.status < 600; + // A local key-scope refusal is already the client's answer: rebuilding it as an upstream_error + // would hide the stable model_not_allowed_for_key type clients branch on. + let scopeRefusalBody: string | undefined; try { const body = await readBoundedResponseBody(response, { signal, reportUtf8Validity: serverError }); // A 5xx body counts as quota or classification evidence only when it decoded as valid @@ -71,6 +75,9 @@ export async function consumeComboFailure( upstreamMessage = normalized.message; upstreamType = normalized.type; } + if (nonReplayable && response.status === 403 && upstreamType === MODEL_NOT_ALLOWED_FOR_KEY) { + scopeRefusalBody = body.text; + } } } catch (error) { if (signal?.aborted) throw error; @@ -106,7 +113,9 @@ export async function consumeComboFailure( now, includeDefault: false, }); - const failureResponse = formatErrorResponse( + const failureResponse = scopeRefusalBody !== undefined + ? new Response(scopeRefusalBody, { status: 403, headers: { "Content-Type": "application/json" } }) + : formatErrorResponse( response.status, cyberFailure ? (upstreamType ?? CYBER_POLICY_ERROR_CODE) : "upstream_error", message, diff --git a/src/server/responses/core-lifetime.ts b/src/server/responses/core-lifetime.ts index 213f63d9a2a..2ecab488cd8 100644 --- a/src/server/responses/core-lifetime.ts +++ b/src/server/responses/core-lifetime.ts @@ -1,5 +1,6 @@ import type { TranslatorBudget } from "../../lib/translator-budget"; import { finalizeTranslatorBudgetResponse } from "../../lib/translator-budget"; +import { isNonReplayableResponse, markResponseNonReplayable } from "../../lib/upstream-retry"; import { isNativePassthroughSseResponse, markNativePassthroughSseResponse, @@ -67,6 +68,7 @@ export function finalizeAccountLease(response: Response, release: () => void): R async cancel(reason) { try { await reader.cancel(reason); } finally { finish(); } }, }); const wrapped = new Response(body, { status: response.status, statusText: response.statusText, headers: response.headers }); + if (isNonReplayableResponse(response)) markResponseNonReplayable(wrapped); if (isNativePassthroughSseResponse(response)) markNativePassthroughSseResponse(wrapped); if (isEagerRelaySseResponse(response)) markEagerRelaySseResponse(wrapped); if (isPreinspectedJsonResponse(response)) markPreinspectedJsonResponse(wrapped); diff --git a/src/server/responses/request-prepare.ts b/src/server/responses/request-prepare.ts index 76d3d000c9d..1c9f9bf6849 100644 --- a/src/server/responses/request-prepare.ts +++ b/src/server/responses/request-prepare.ts @@ -1452,6 +1452,10 @@ export async function prepareResponsesRequest( commitSkillsSnapshot?.(); return { inboundWire, + // Admission facts later route rewrites must not re-derive: the selector the caller sent and + // the key scope as it stood when this request was admitted. + inboundSelector, + admissionScope, translatorBudget, parsed, toolBridgeMaps, diff --git a/src/server/responses/request-transport.ts b/src/server/responses/request-transport.ts index 9e11b700647..1b6cb6f4c44 100644 --- a/src/server/responses/request-transport.ts +++ b/src/server/responses/request-transport.ts @@ -56,6 +56,11 @@ import { stampOAuthAccountLabel, usesApiKeyAccount } from "../../providers/label import { resolveProviderTransport } from "../../providers/xai-transport"; import { captureRouteStaticPolicy } from "../../router"; import { CopilotAutoHttpError, resolveCopilotAuto } from "../../providers/github-copilot-auto"; +import { + AdmissionModelDeniedError, + admissionModelDeniedResponse, + assertRouteAllowedByScope, +} from "../admission-model-scope"; import { renameRoutedIdentityInContext } from "../../adapters/identity"; import { resolveCopilotApiBaseUrl } from "../../oauth/github-copilot"; import { @@ -109,6 +114,8 @@ export async function prepareResponsesTransport( | "inboundWire" | "selectedForwardHeaders" | "translatorBudget" + | "inboundSelector" + | "admissionScope" >, ) { const { config, logCtx, options, req } = requestContext; @@ -297,6 +304,7 @@ export async function prepareResponsesTransport( const copilotRequestedModel = route.modelId; /** Project only bounded Copilot refusal metadata; upstream bodies and session tokens never reach the client. */ const copilotRefusalResponse = (error: unknown): Response | undefined => { + if (error instanceof AdmissionModelDeniedError) return admissionModelDeniedResponse(error); if (!(error instanceof CopilotAutoHttpError) || ![401, 403, 429].includes(error.status)) return; const response = formatErrorResponse(error.status, error.status === 429 ? "rate_limit_error" : "authentication_error", error.message); @@ -357,6 +365,13 @@ export async function prepareResponsesTransport( replayOAuthCredentialSnapshot = { accountId: refreshed.accountId, generation: refreshed.generation }; selected = await negotiate(); } + assertRouteAllowedByScope( + // The scope captured at admission: a key edited or removed while negotiation was in + // flight must not widen this request. The refusal names the caller's own selector. + requestState.admissionScope, + requestState.inboundSelector, + { providerName: route.providerName, modelId: selected.modelId }, + ); copilotSessionExpiresAt = selected.expiresAt ?? Infinity; route.provider = selected.provider; route.modelId = selected.modelId; @@ -725,6 +740,7 @@ export async function prepareResponsesTransport( let nextAdapter: ProviderAdapter; try { nextAdapter = await refreshDispatchAdapter(requestParsed); } catch (error) { + if (error instanceof AdmissionModelDeniedError) throw error; const refusal = copilotRefusalResponse(error); if (refusal) return refusal; throw error; diff --git a/src/server/responses/sidecar-execution.ts b/src/server/responses/sidecar-execution.ts index 7f277c4f715..355a3b4b2e7 100644 --- a/src/server/responses/sidecar-execution.ts +++ b/src/server/responses/sidecar-execution.ts @@ -30,6 +30,8 @@ import { } from "../../oauth/generic-account-failover"; import { classifyKiroRefusal } from "../../adapters/kiro-refusal"; import { isNonReplayableResponse } from "../../lib/upstream-retry"; +import { unwrapUpstreamRetryEvidenceError } from "../../codex/auth-context"; +import { AdmissionModelDeniedError, admissionModelDeniedResponse } from "../admission-model-scope"; import { noteKiroMonthlyRefusal, noteKiroServedSuccess } from "../../providers/kiro-usage"; import { persistKiroAccountState } from "../../providers/kiro-account-state-disk"; import { anthropicPoolCooldownResponse, readDisplaySafeErrorText } from "./core-errors"; @@ -198,6 +200,20 @@ export async function executeResponsesSidecars( // A single request may rotate once for a verification refusal. Keep this separate from the // broader OAuth failover count: a sidecar turn can also spend that count on rate-limit recovery. let antigravityValidationRotationAttempted = false; + // A replacement credential whose Copilot Auto negotiation leaves the key's scope ends the turn. + // The loops only see "no rotation" and fail with the original, replayable 429, so the refusal + // is held here and answers in place of that failure. + let sidecarScopeDenial: AdmissionModelDeniedError | undefined; + const noteSidecarScopeDenial = (error: unknown): void => { + const refusal = unwrapUpstreamRetryEvidenceError(error); + if (refusal instanceof AdmissionModelDeniedError) sidecarScopeDenial ??= refusal; + }; + const sidecarScopeRefusal = (response: Response): Response | undefined => { + if (!sidecarScopeDenial || response.ok) return undefined; + void response.body?.cancel().catch(() => {}); + releaseSearchProbeLease(); + return admissionModelDeniedResponse(sidecarScopeDenial); + }; /** Admit a replacement credential, or leave the loop's original refusal authoritative. */ const rotateSidecarProviderOn429 = async ( retryAfter: string | null, @@ -264,7 +280,8 @@ export async function executeResponsesSidecars( return null; } transportState.genericFailovers += 1; - } catch { + } catch (error) { + noteSidecarScopeDenial(error); hop.permit?.release(); return null; } @@ -284,7 +301,10 @@ export async function executeResponsesSidecars( } else if (rotated) { route.provider = rotated; try { await resolveCopilotSelection(retryParsed ?? parsed); } - catch { return null; } // Keep the sidecar's original refusal when replacement negotiation fails. + catch (error) { // Keep the sidecar's original refusal when replacement negotiation fails. + noteSidecarScopeDenial(error); + return null; + } } else if ( // A POSITIVE gate, not an early return. An early `return null` here made every later arm // unreachable: Anthropic never has a genericFailoverAccountId (isGenericFailoverProvider @@ -326,7 +346,8 @@ export async function executeResponsesSidecars( return null; } if (route.providerName === "kiro") transportState.genericFailovers += 1; - } catch { + } catch (error) { + noteSidecarScopeDenial(error); hop.permit?.release(); return null; } @@ -527,6 +548,8 @@ export async function executeResponsesSidecars( notifyResponseComplete(response); }, }); + const imgScopeRefusal = sidecarScopeRefusal(imgResponse); + if (imgScopeRefusal) return imgScopeRefusal; if (imgResponse.body) { const imgTurnAc = new AbortController(); imgTurnAc.signal.addEventListener("abort", cancelResponseCompletion, { once: true }); @@ -622,6 +645,8 @@ export async function executeResponsesSidecars( notifyResponseComplete(response); }, }); + const wsScopeRefusal = sidecarScopeRefusal(loopResponse); + if (wsScopeRefusal) return wsScopeRefusal; // A routed Anthropic model whose last pool account refused on spent quota fails the loop's // first iteration with a generic 429 before any search runs. Answer the usage limit instead, // as the main path does (#6991), so Codex stops rather than retrying the whole turn. diff --git a/structure/transports/responses.md b/structure/transports/responses.md index 723d5173da7..c88f5dce919 100644 --- a/structure/transports/responses.md +++ b/structure/transports/responses.md @@ -14,7 +14,7 @@ When a successful streamed native response has a missing or unrecognized non-JSO HTTP ingress applies the [caller header contract](../data-planes/inbound-compat.md#http-caller-conversation-identity) before admission and dispatch. Optional Codex memory selection enters `src/server/responses/request-prepare.ts` for HTTP and WebSocket frames, as specified in [memory phase routing](responses-failover.md#memory-phase-routing); an unset phase keeps its existing route and starts no background work. Responses request preparation stabilizes incoming `` under `skills.catalog_refresh`: `per_session` (default) reuses the first received catalog for a conversation; `per_turn` leaves the supplied catalog unchanged. Other instruction sections and user/tool content remain untouched. Requests without a reliable conversation identity bypass snapshots; shared prompt-cache cohorts are not conversation identities. Only a body with exactly one catalog block across its instructions and developer/system content takes part; two or more pass through unchanged. A known snapshot is substituted before parsing, but a new catalog is stored only when preparation reaches its success return, so a request rejected by parsing or admission pins nothing. Without a named principal, snapshots are shared by conversation id only on a server that requires no data-plane auth. Snapshots are process-local, expire after four idle hours, and use bounded LRU retention; oversized blocks bypass caching. The dashboard's `src/codex/prompt-layers.ts` and `src/codex/prompt-text-probe.ts` continue observing current files for previews and do not own session snapshots. `/v1/responses` is the main Codex-facing endpoint. The server parses Responses input, routes to a provider, lets the selected adapter speak the upstream protocol, then bridges adapter events back to -Responses-compatible streaming output. The routed provider name follows every adapter build, including retries and continuations; only `github-copilot` injects a configured `modelContextTiers` value as upstream `contextTier`, while unconfigured passthrough retains the caller field. For an opted-in key-auth provider, a hosted-search continuation stays bound to the API-key selection that served the first leg; the contract is the [hosted-search continuation binding](../providers-and-adapters.md#hosted-search-continuation-binding). +Responses-compatible streaming output. The routed provider name follows every adapter build, including retries and continuations; only `github-copilot` injects a configured `modelContextTiers` value as upstream `contextTier`, while unconfigured passthrough retains the caller field. For an opted-in key-auth provider, a hosted-search continuation stays bound to the API-key selection that served the first leg; the contract is the [hosted-search continuation binding](../providers-and-adapters.md#hosted-search-continuation-binding). GitHub Copilot Auto admission rechecks the negotiated provider/model against the key scope captured at request admission (a key edited or removed mid-request cannot widen it) before route mutation or dispatch, on initial and renewal negotiation; denial returns the standard `model_not_allowed_for_key` 403 naming the caller's inbound selector without disclosing the destination. The web-search and image sidecars answer a denial during their credential rotation with that 403 instead of the original 429, and a combo child's denial keeps the same envelope. Local model-scope refusals are non-replayable. A denial during Copilot dispatch renewal propagates as an admission exception to the request boundary, never as a provider fetch response; account recovery and combo/policy fallback cannot continue after that denial. A committed continuation, including OAuth account rotation after a 429, emits the same refusal type in-stream without dispatching another inference; ordinary negotiation failures still preserve the original provider refusal. The `openai-responses` adapter preserves caller `User-Agent` unless provider headers own it, in either auth mode. `forwardClientHeaders` additionally opts into `originator`, `x-client-request-id`, `x-codex-app-version`, or `user-agent`. diff --git a/tests/providers/github-copilot/github-copilot-auto-runtime.test.ts b/tests/providers/github-copilot/github-copilot-auto-runtime.test.ts index 824f0e31fcd..a891fb44fa0 100644 --- a/tests/providers/github-copilot/github-copilot-auto-runtime.test.ts +++ b/tests/providers/github-copilot/github-copilot-auto-runtime.test.ts @@ -13,6 +13,9 @@ import { filterCatalogVisibleModels } from "../../../src/codex/catalog/model-vis import { reconcileSuccessfulModelDiscoveries } from "../../../src/providers/new-model-policy"; import type { OcxConfig } from "../../../src/types"; import { createRequestExecutionBudget } from "../../../src/lib/request-execution-budget"; +import { isNonReplayableResponse } from "../../../src/lib/upstream-retry"; +import { clearComboSelectionState } from "../../../src/combos/resolve"; +import { clearComboTargetCooldowns } from "../../../src/combos/failover"; import { acquireOwnedSpendHome } from "../../helpers/owned-spend-home"; import { removeTreeWithRetry } from "../../helpers/remove-tree"; @@ -25,12 +28,16 @@ beforeEach(() => { process.env.OPENCODEX_HOME = home; clearCopilotAutoModelsForTests(); clearGenericFailoverHealth(); + clearComboSelectionState(); + clearComboTargetCooldowns(); release = acquireOwnedSpendHome(); }); afterEach(() => { release?.(); globalThis.fetch = previousFetch; clearGenericFailoverHealth(); + clearComboSelectionState(); + clearComboTargetCooldowns(); if (previousHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousHome; removeTreeWithRetry(home); @@ -41,6 +48,11 @@ function request(input: unknown, stream = true): Request { body: JSON.stringify({ model: "github-copilot/auto", input, stream, tools: [{ type: "function", name: "lookup", parameters: { type: "object", properties: { name: { type: "string" } } } }] }) }); } + +function scopedAdmission(config: OcxConfig, allowedModels: string[]) { + config.apiKeys = [{ id: "copilot-scope", name: "fixture", allowedModels }] as OcxConfig["apiKeys"]; + return { kind: "configured" as const, keyId: "copilot-scope", source: "bearer" as const }; +} /** Seed two synthetic OAuth accounts in the test-owned home, select A, and return store IDs for rotation assertions. */ async function accounts() { for (const account of ["a", "b"]) await saveCredential("github-copilot", { @@ -54,7 +66,7 @@ async function accounts() { return { a, b }; } /** Return an Auto provider config and send ledger; options drive refusal, expiry, rotation, and tool events through both inference wires. */ -function fixture(options: { nativeExpiry?: boolean; nativeFirst?: boolean; native429?: boolean; onNativeCancel?: () => void; chatRefresh?: boolean; closedNativeBody?: boolean; renewedChatRefusal?: boolean; rotate?: boolean; tool?: boolean; key?: boolean; refusal?: number; negotiationRefusal?: number; negotiationRefusalAt?: number; alwaysRefuse?: boolean; advanceAfterIntent?: () => void; onSession?: (count: number) => void; malformedSessionAt?: number; negotiationRetryAfter?: string } = {}) { +function fixture(options: { nativeExpiry?: boolean; nativeFirst?: boolean; native429?: boolean; onNativeCancel?: () => void; chatRefresh?: boolean; closedNativeBody?: boolean; renewedChatRefusal?: boolean; rotate?: boolean; rotateAt?: number; tool?: boolean; key?: boolean; refusal?: number; negotiationRefusal?: number; negotiationRefusalAt?: number; alwaysRefuse?: boolean; advanceAfterIntent?: () => void | Promise; onSession?: (count: number) => void | Promise; malformedSessionAt?: number; negotiationRetryAfter?: string; replyText?: string; onInference?: (count: number) => void | Promise } = {}) { const sent: Array<{ host: string; path: string; token: string | null; body: any; signal?: AbortSignal | null }> = []; let inferenceCount = 0; let sessionCount = 0; @@ -84,7 +96,7 @@ function fixture(options: { nativeExpiry?: boolean; nativeFirst?: boolean; nativ supported_endpoints: [options.nativeExpiry ? (expiredNative ? "/chat/completions" : "/responses") : options.native429 ? (account === "a" ? "/responses" : "/chat/completions") : options.chatRefresh ? (renewed ? "/responses" : "/chat/completions") : options.nativeFirst ? (headers.get("authorization")!.includes("-renewed") ? "/chat/completions" : "/responses") : account === "a" ? "/chat/completions" : "/responses"] }] }); if (destination.pathname === "/models/session") { sessionCount++; - options.onSession?.(sessionCount); + await options.onSession?.(sessionCount); if (sessionCount === options.malformedSessionAt) return Response.json({ session_token: "must-not-echo" }); if (options.negotiationRefusal && (sessionCount === (options.negotiationRefusalAt ?? 1) || (options.alwaysRefuse && sessionCount >= (options.negotiationRefusalAt ?? 1)))) return Response.json({ token: "must-not-echo" }, { status: options.negotiationRefusal, headers: { "retry-after": options.negotiationRetryAfter ?? "12" } }); @@ -92,7 +104,7 @@ function fixture(options: { nativeExpiry?: boolean; nativeFirst?: boolean; nativ } if (destination.pathname === "/models/session/intent") { intentCount++; - if (intentCount === 1) options.advanceAfterIntent?.(); + if (intentCount === 1) await options.advanceAfterIntent?.(); return Response.json({ candidate_models: [model] }); } expect(body.model).toBe(model); @@ -100,6 +112,7 @@ function fixture(options: { nativeExpiry?: boolean; nativeFirst?: boolean; nativ if (options.nativeExpiry && options.negotiationRefusal === 401) expect(headers.get("authorization")).toBe(`Bearer synthetic-access-${"a"}-renewed`); inferenceCount++; + await options.onInference?.(inferenceCount); if (options.nativeFirst && options.closedNativeBody && inferenceCount === 1) return Response.json({ error: { message: "native-inference-secret" } }, { status: 401 }); if (options.nativeFirst && inferenceCount === 1) return new Response(new ReadableStream({ @@ -107,7 +120,7 @@ function fixture(options: { nativeExpiry?: boolean; nativeFirst?: boolean; nativ cancel() { cancelledNativeBodies++; options.onNativeCancel?.(); }, }), { status: options.native429 ? 429 : 401, headers: { "content-type": "application/json", "retry-after": "1" } }); if (options.chatRefresh && inferenceCount === 1) return Response.json({ error: { message: "refresh" } }, { status: 401 }); - if (options.rotate && inferenceCount === 1) return Response.json({ error: { message: "limited" } }, { status: options.refusal ?? 429, headers: { "retry-after": "1" } }); + if (options.rotate && inferenceCount === (options.rotateAt ?? 1)) return Response.json({ error: { message: "limited" } }, { status: options.refusal ?? 429, headers: { "retry-after": "1" } }); if (destination.pathname === "/responses") return Response.json({ id: "resp-fixture", model, object: "response", status: "completed", output: [{ id: "msg-fixture", type: "message", role: "assistant", content: [{ type: "output_text", text: "ok" }] }], usage: { input_tokens: 2, output_tokens: 1 } }); @@ -117,7 +130,7 @@ function fixture(options: { nativeExpiry?: boolean; nativeFirst?: boolean; nativ choices: [{ message: { role: "assistant", content: "ok" }, finish_reason: "stop" }], usage: { prompt_tokens: 2, completion_tokens: 1 } }); const tool = options.tool && inferenceCount === (options.nativeFirst ? 2 : 1); - const delta = tool ? { tool_calls: [{ index: 0, id: "call_lookup", type: "function", function: { name: "lookup", arguments: '{"name":"file"}' } }] } : { content: "ok" }; + const delta = tool ? { tool_calls: [{ index: 0, id: "call_lookup", type: "function", function: { name: "lookup", arguments: '{"name":"file"}' } }] } : { content: options.replyText ?? "ok" }; return new Response(`data: ${JSON.stringify({ id: "chat-fixture", model, choices: [{ index: 0, delta, finish_reason: null }] })}\n\ndata: ${JSON.stringify({ id: "chat-fixture", model, choices: [{ index: 0, delta: {}, finish_reason: tool ? "tool_calls" : "stop" }] })}\n\ndata: [DONE]\n\n`, { headers: { "content-type": "text/event-stream" } }); }) as typeof fetch; const config: OcxConfig = { port: 0, defaultProvider: "github-copilot", providers: { "github-copilot": { @@ -427,6 +440,63 @@ describe("Copilot Auto through the Responses pipeline", () => { expect(await response.text()).not.toContain("must-not-echo"); expect(sent.filter(call => call.path === "/chat/completions")).toHaveLength(0); }); + test("Auto negotiation is denied when its selected model is outside the admission scope", async () => { + await accounts(); + const { config, sent } = fixture(); + const admission = scopedAdmission(config, ["github-copilot/auto"]); + const response = await handleResponses(request("hello"), config, { model: "", provider: "" }, { admission }); + + expect(response.status).toBe(403); + const body = await response.text(); + expect(body).toContain("model_not_allowed_for_key"); + expect(body).not.toContain("gpt-4o"); + expect(sent.filter(call => ["/chat/completions", "/responses"].includes(call.path))).toHaveLength(0); + }); + test("a negotiated denial names the caller's selector, not the resolved Auto model id", async () => { + await accounts(); + const { config } = fixture(); + const admission = scopedAdmission(config, ["github-copilot/auto"]); + const response = await handleResponses(request("hello", false), config, { model: "", provider: "" }, { admission }); + expect(response.status).toBe(403); + const { error } = await response.json() as { error: { type: string; model: string; message: string } }; + expect(error.type).toBe("model_not_allowed_for_key"); + expect(error.model).toBe("github-copilot/auto"); + expect(error.message).toContain("github-copilot/auto"); + }); + test("a key scope removed while negotiation is in flight does not widen the request", async () => { + await accounts(); + let config!: OcxConfig; + const f = fixture({ onSession: () => { config.apiKeys = []; } }); + config = f.config; + const admission = scopedAdmission(config, ["github-copilot/auto"]); + const response = await handleResponses(request("hello", false), config, { model: "", provider: "" }, { admission }); + expect(response.status).toBe(403); + expect(await response.text()).toContain("model_not_allowed_for_key"); + expect(f.sent.filter(call => ["/chat/completions", "/responses"].includes(call.path))).toHaveLength(0); + }); + test("Auto negotiation dispatches a model admitted by scope", async () => { + await accounts(); + const { config, sent } = fixture(); + const admission = scopedAdmission(config, ["github-copilot/auto", "gpt-4o"]); + const response = await handleResponses(request("hello"), config, { model: "", provider: "" }, { admission }); + + expect(response.status).toBe(200); + expect(sent.filter(call => call.path === "/chat/completions").map(call => call.body.model)).toEqual(["gpt-4o"]); + }); + test("dispatch-retried Auto selection outside admission scope returns 403", async () => { + const { b } = await accounts(); + const { config, sent } = fixture({ + advanceAfterIntent: async () => { await setActiveAccount("github-copilot", b); }, + }); + const admission = scopedAdmission(config, ["github-copilot/auto", "gpt-4o"]); + const response = await handleResponses(request("hello", false), config, { model: "", provider: "" }, { admission }); + + expect(response.status).toBe(403); + const body = await response.text(); + expect(body).toContain("model_not_allowed_for_key"); + expect(body).not.toContain("gpt-5.4"); + expect(sent.filter(call => ["/chat/completions", "/responses"].includes(call.path))).toHaveLength(0); + }); test.each([401, 429])("key pool %s rotation reacquires a new session and endpoint", async status => { const { config, sent } = fixture({ key: true, rotate: true, refusal: status }); await saveConfig(config); @@ -493,6 +563,23 @@ describe("Copilot Auto through the Responses pipeline", () => { expect(sent.filter(call => ["/chat/completions", "/responses"].includes(call.path))).toHaveLength(1); expect(sent.filter(call => call.path === "/models/session")).toHaveLength(2); }); + test("web-search key rotation into a model outside the key scope ends with the scope refusal", async () => { + const { config, sent } = fixture({ key: true, rotate: true }); + config.webSearchSidecar = { backend: "exa", exaApiKey: "synthetic-search-key" }; + saveConfig(config); + const admission = scopedAdmission(config, ["github-copilot/auto", "gpt-4o"]); + const req = new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "github-copilot/auto", input: "hello", stream: false, tools: [{ type: "web_search" }] }) }); + const response = await handleResponses(req, config, { model: "", provider: "" }, { admission }); + expect(response.status).toBe(403); + expect(isNonReplayableResponse(response)).toBe(true); + const text = await response.text(); + expect(JSON.parse(text).error.type).toBe("model_not_allowed_for_key"); + expect(text).not.toContain("gpt-5.4"); + // Account A's gpt-4o was sent once and refused; B's gpt-5.4 was negotiated but never sent. + expect(sent.filter(call => ["/chat/completions", "/responses"].includes(call.path)).map(call => call.body.model)) + .toEqual(["gpt-4o"]); + }); test("Auto-only catalog retains manual preferences and requires explicit picker selection", async () => { await accounts(); @@ -572,6 +659,105 @@ describe("Copilot Auto through the Responses pipeline", () => { ]); expect(sent.filter(call => call.path === "/models/session").map(call => call.host)).toEqual(["a.githubcopilot.com", "b.githubcopilot.com"]); }); + test.each([false, true])("expired native Auto scope denial is terminal before inference, stream=%s", async stream => { + await accounts(); + const actualNow = Date.now; + let offset = 0; + Date.now = () => actualNow() + offset; + try { + const { config, sent } = fixture({ + nativeExpiry: true, advanceAfterIntent: () => { offset = 61_000; }, + }); + const admission = scopedAdmission(config, ["github-copilot/auto", "gpt-4o"]); + const response = await handleResponses(request("hello", stream), config, { model: "", provider: "" }, { admission }); + expect(response.status).toBe(403); + expect(isNonReplayableResponse(response)).toBe(true); + const text = await response.text(); + expect(text).toContain("model_not_allowed_for_key"); + expect(text).not.toContain("gpt-5.4"); + expect(sent.filter(call => ["/chat/completions", "/responses"].includes(call.path))).toHaveLength(0); + } finally { Date.now = actualNow; } + }); + test.each([false, true])("Auto scope denial stops account recovery, key=%s", async key => { + await accounts(); + const { config, sent } = fixture({ key, rotate: true }); + if (key) await saveConfig(config); + const admission = scopedAdmission(config, ["github-copilot/auto", "gpt-4o"]); + const response = await handleResponses(request("hello", false), config, { model: "", provider: "" }, { admission }); + expect(response.status).toBe(403); + expect(isNonReplayableResponse(response)).toBe(true); + const text = await response.text(); + expect(text).toContain("model_not_allowed_for_key"); + expect(text).not.toContain("gpt-5.4"); + expect(sent.filter(call => ["/chat/completions", "/responses"].includes(call.path)).map(call => call.body.model)) + .toEqual(["gpt-4o"]); + }); + test("a local Auto scope refusal cannot hop to an otherwise allowed combo target", async () => { + await accounts(); + const { config } = fixture(); + let fallbackSends = 0; + config.providers.fallback = { + adapter: "openai-chat", authMode: "key", apiKey: "synthetic-fallback-key", + baseUrl: "https://fallback.invalid/v1", models: ["allowed"], + fetch: (async () => { + fallbackSends++; + return Response.json({ + id: "fallback-fixture", model: "allowed", + choices: [{ message: { role: "assistant", content: "unexpected fallback" }, finish_reason: "stop" }], + }); + }) as typeof fetch, + }; + config.combos = { + scopedAuto: { + strategy: "failover", + targets: [{ provider: "github-copilot", model: "auto" }, { provider: "fallback", model: "allowed" }], + }, + }; + const admission = scopedAdmission(config, ["github-copilot/auto", "fallback/allowed"]); + const req = new Request("http://localhost/v1/responses", { + method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "combo/scopedAuto", stream: false, input: "hello" }), + }); + const response = await handleResponses(req, config, { model: "", provider: "" }, { admission }); + expect(response.status).toBe(403); + expect(isNonReplayableResponse(response)).toBe(true); + // The combo boundary keeps the stable scope envelope instead of rewrapping it as upstream_error. + const { error } = await response.json() as { error: { type: string; model: string } }; + expect(error.type).toBe("model_not_allowed_for_key"); + expect(fallbackSends).toBe(0); + }); + test("terminal continuation emits the scope refusal without a replacement inference", async () => { + const { b } = await accounts(); + const { config, sent } = fixture({ + replyText: "I will modify the file now.", + onInference: async count => { if (count === 1) await setActiveAccount("github-copilot", b); }, + }); + config.providers["github-copilot"]!.terminalContinuationGuard = true; + const admission = scopedAdmission(config, ["github-copilot/auto", "gpt-4o"]); + const response = await handleResponses(request("Modify the file now."), config, { model: "", provider: "" }, { admission }); + const text = await response.text(); + expect(text).toContain("model_not_allowed_for_key"); + expect(text).not.toContain("gpt-5.4"); + expect(sent.filter(call => ["/chat/completions", "/responses"].includes(call.path))).toHaveLength(1); + }); + test("OAuth 429 during terminal continuation preserves the scope refusal", async () => { + await accounts(); + const { config, sent } = fixture({ rotate: true, rotateAt: 2, replyText: "I will modify the file now." }); + globalThis.fetch = config.providers["github-copilot"]!.fetch as typeof fetch; + config.providers["github-copilot"]!.terminalContinuationGuard = true; + const admission = scopedAdmission(config, ["github-copilot/auto", "gpt-4o"]); + const response = await handleResponses(request("Modify the file now."), config, { model: "", provider: "" }, { admission }); + const text = await response.text(); + const inference = sent.filter(call => ["/chat/completions", "/responses"].includes(call.path)); + expect(inference.map(call => [call.host, call.body.model])).toEqual([ + ["a.githubcopilot.com", "gpt-4o"], ["a.githubcopilot.com", "gpt-4o"], + ]); + const sessions = sent.filter(call => call.path === "/models/session").map(call => call.host); + expect(sessions[0]).toBe("a.githubcopilot.com"); + expect(sessions.at(-1)).toBe("b.githubcopilot.com"); + expect(text).toContain("model_not_allowed_for_key"); + expect(text).not.toContain("gpt-5.4"); + }); }); for (const bDiscovery of ["named", "outage"] as const) test(`leaving Auto on account rotation restores the configured Chat wire (${bDiscovery} discovery)`, async () => {