Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion docs-site/src/content/docs/guides/combos.md
Original file line number Diff line number Diff line change
Expand Up @@ -873,7 +873,10 @@ running opencodex instance that receives model requests.

Every target is currently ineligible: for example, its provider is disabled, it is cooling down,
it has already been attempted for this request, or an encrypted v2 task excludes it. Check target
provider state and recent upstream errors. For cooldowns, follow an observed `Retry-After` value first;
provider state and recent upstream errors. When no target was attempted, the error message lists
why each target was skipped, for example
`No available targets for combo: fast (openai/gpt-6: quota exhausted; anthropic/claude-opus-5-5: cooldown 42s)`.
For cooldowns, follow an observed `Retry-After` value first;
Codex reset headers also take precedence over `cooldownMs`.
If neither upstream signal is usable, the configured `cooldownMs` applies, or the upstream fallback applies
when it is unset (5 seconds for request-rate codes `1302`/`1305`, 10 minutes for a spent usage
Expand Down
1 change: 1 addition & 0 deletions src/combos/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ export {
export {
advanceComboAfterFailure,
clearComboSelectionState,
comboExclusionReasons,
NoAvailableComboTargetsError,
noteComboFailure,
noteComboSuccess,
Expand Down
46 changes: 39 additions & 7 deletions src/combos/resolve.ts
Original file line number Diff line number Diff line change
Expand Up @@ -55,17 +55,49 @@ export class UnknownComboError extends Error {
export class NoAvailableComboTargetsError extends Error {
readonly code = "combo_unavailable";

constructor(readonly comboId: string) {
super(`No available targets for combo: ${comboId}`);
constructor(readonly comboId: string, readonly reasons = "") {
super(`No available targets for combo: ${comboId}${reasons ? ` (${reasons})` : ""}`);
this.name = "NoAvailableComboTargetsError";
}
}

function providerExclusionReason(config: OcxConfig, target: OcxComboTarget, now: number): string | undefined {
const provider = Object.hasOwn(config.providers, target.provider) ? config.providers[target.provider] : undefined;
if (!provider) return "provider missing";
if (provider.disabled === true) return "provider disabled";
return cachedProviderQuotaIsExhausted(getCachedProviderRoutingQuota(target.provider, provider, now), now, target.model)
? "quota exhausted"
: undefined;
}

function targetProviderIsUsable(config: OcxConfig, target: OcxComboTarget, now: number): boolean {
if (!Object.hasOwn(config.providers, target.provider)) return false;
const provider = config.providers[target.provider];
if (!provider || provider.disabled === true) return false;
return !cachedProviderQuotaIsExhausted(getCachedProviderRoutingQuota(target.provider, provider, now), now, target.model);
return providerExclusionReason(config, target, now) === undefined;
}

/**
* Why each target of a combo cannot be picked right now, for a zero-attempt refusal. Only the
* configured provider/model and a coarse label are named, never account or request data.
*/
export function comboExclusionReasons(
config: OcxConfig,
comboId: string,
eligible?: (target: NormalizedComboTarget) => boolean,
now = Date.now(),
): string {
return (getCombo(config, comboId)?.targets ?? []).flatMap(target => {
let reason = providerExclusionReason(config, target, now);
const cooldown = reason ? undefined : earliestComboCooldown(comboId, [target], now);
if (cooldown) reason = `cooldown ${Math.max(1, Math.ceil((cooldown.expiry - now) / 1000))}s`;
if (!reason && eligible) {
try {
if (!eligible(target)) reason = "ineligible for this request";
} catch {
// An eligibility check can exhaust the translator budget; the target still was not picked.
reason = "ineligible for this request";
}
}
return reason ? [`${targetKey(target)}: ${reason}`] : [];
}).join("; ");
}

/** Capture usable providers' quota cooldowns; request eligibility is deferred to exhaustion. */
Expand Down Expand Up @@ -612,6 +644,6 @@ export function tryPickComboModel(config: OcxConfig, modelId: string, preview =
if (!comboId) return null;
if (!getCombo(config, comboId)) throw new UnknownComboError(comboId);
const picked = pickComboTarget(config, comboId, { preview });
if (!picked) throw new NoAvailableComboTargetsError(comboId);
if (!picked) throw new NoAvailableComboTargetsError(comboId, comboExclusionReasons(config, comboId));
return picked;
}
54 changes: 34 additions & 20 deletions src/server/responses/core-combo.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import {
comboDefaultEffort,
isComboTargetInCooldown,
snapshotComboQuotaCooldowns,
comboExclusionReasons,
noteComboSuccess,
comboFailureDecision,
advanceComboAfterFailure,
Expand Down Expand Up @@ -251,6 +252,30 @@ interface JevComboChoice {
candidate: JevCandidate;
}

/** A target's JEV effort choices, or undefined when an explicit allowlist has none still supported. */
function jevChoiceEfforts(
config: OcxConfig,
target: NonNullable<ReturnType<typeof getCombo>>["targets"][number],
): OcxComboDefaultEffort[] | undefined {
let ladder: string[] | undefined;
try {
const route = routeConcreteModel(config, targetKey(target));
ladder = supportedLadderFor({ provider: route.provider, modelId: route.modelId });
} catch {
// Preserve the existing routing-failure surface. Unknown capability becomes the explicit
// no-effort choice rather than broadening JEV's effort allowlist.
ladder = undefined;
}
const supportedEfforts = (ladder ?? []).filter(isCodexReasoningEffort) as OcxComboDefaultEffort[];
const configuredEfforts = target.reasoningEfforts;
const reasoningEfforts = configuredEfforts === undefined
? supportedEfforts
: configuredEfforts.filter(effort => supportedEfforts.includes(effort));
// An explicit allowlist is restrictive. If catalog capabilities drift until no configured
// effort remains supported, omit the target instead of silently broadening JEV's choices.
return configuredEfforts !== undefined && reasoningEfforts.length === 0 ? undefined : reasoningEfforts;
}

/** Enumerate the current ordinary Combo eligibility set without retaining attempted picks. */
function eligibleJevComboChoices(
config: OcxConfig,
Expand All @@ -269,23 +294,8 @@ function eligibleJevComboChoices(
excluded.add(key);
// The TypeSafe row owns a decision credential, not an inference transport.
if (pick.target.provider === JEV_PROVIDER_ID) continue;
let ladder: string[] | undefined;
try {
const route = routeConcreteModel(config, key);
ladder = supportedLadderFor({ provider: route.provider, modelId: route.modelId });
} catch {
// Preserve the existing routing-failure surface. Unknown capability becomes the explicit
// no-effort choice rather than broadening JEV's effort allowlist.
ladder = undefined;
}
const supportedEfforts = (ladder ?? []).filter(isCodexReasoningEffort) as OcxComboDefaultEffort[];
const configuredEfforts = pick.target.reasoningEfforts;
const reasoningEfforts = configuredEfforts === undefined
? supportedEfforts
: configuredEfforts.filter(effort => supportedEfforts.includes(effort));
// An explicit allowlist is restrictive. If catalog capabilities drift until no configured
// effort remains supported, omit the target instead of silently broadening JEV's choices.
if (configuredEfforts !== undefined && reasoningEfforts.length === 0) continue;
const reasoningEfforts = jevChoiceEfforts(config, pick.target);
if (!reasoningEfforts) continue;
choices.push({
pick: { ...pick, attempted: [key] },
candidate: {
Expand Down Expand Up @@ -549,7 +559,8 @@ export async function executeComboResponses(
);
return options.abortSignal?.aborted
? clientCancelledResponse()
: comboUnavailable(comboId);
: comboUnavailable(comboId, undefined,
comboExclusionReasons(config, comboId, target => meteredAllowed || !target.metered));
}
if (!(await recoverUnreadableEncryptedTask())) {
return options.abortSignal?.aborted
Expand All @@ -565,13 +576,16 @@ export async function executeComboResponses(
if (onlyReplayIncompatibleTargetsRemain()) return targetIncompatibleResponse();
return options.abortSignal?.aborted
? clientCancelledResponse()
: comboUnavailable(comboId);
: comboUnavailable(comboId, undefined, comboExclusionReasons(config, comboId, targetEligible));
}
let jevDecision: JevDecision | undefined;
if (combo.strategy === "jev") {
const choices = eligibleJevComboChoices(config, comboId, targetEligible, Date.now());
const first = choices[0];
if (!first) return comboUnavailable(comboId);
if (!first) {
return comboUnavailable(comboId, undefined, comboExclusionReasons(config, comboId, target =>
targetEligible(target) && target.provider !== JEV_PROVIDER_ID && jevChoiceEfforts(config, target) !== undefined));
}
const resolvedFailOpenEffort = resolveEffortAtOrBelow(
"medium",
first.candidate.reasoningEfforts,
Expand Down
6 changes: 3 additions & 3 deletions src/server/responses/core-errors.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import {
DecompressedBodyTooLargeError,
describeInboundBodyRefusal,
} from "../request-decompress";
import { comboCooldownRetryAfterSeconds } from "../../combos";
import { comboCooldownRetryAfterSeconds, NoAvailableComboTargetsError } from "../../combos";
import type { AgentTaskRecoveryFailureReason } from "./agent-task-recovery";

/**
Expand Down Expand Up @@ -115,8 +115,8 @@ export function comboUnavailableResponse(
}


export function comboUnavailable(comboId: string, now = Date.now()): Response {
return comboUnavailableResponse(`No available targets for combo: ${comboId}`, {
export function comboUnavailable(comboId: string, now = Date.now(), reasons = ""): Response {
return comboUnavailableResponse(new NoAvailableComboTargetsError(comboId, reasons).message, {
retryAfter: comboCooldownRetryAfterSeconds(comboId, now),
});
}
Expand Down
13 changes: 8 additions & 5 deletions src/server/responses/request-prepare.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import {
unreadableEncryptedAgentTaskResponse,
} from "./core-errors";
import { parseSyntheticRowId } from "../fast-row";
import { resolveComboId, comboIdFromRawBody, NoAvailableComboTargetsError, pickComboTarget } from "../../combos";
import { resolveComboId, comboIdFromRawBody, comboExclusionReasons, NoAvailableComboTargetsError, pickComboTarget } from "../../combos";
import { INTERCEPT_TARGET_UNAVAILABLE_CODE, interceptTargetUnavailableResponse, resolveChosenTarget, resolveShadowCallTarget } from "./shadow-target-availability";
import {
MEMORY_MODEL_TARGET_UNAVAILABLE_CODE,
Expand Down Expand Up @@ -645,7 +645,10 @@ export async function prepareResponsesRequest(
config, options.admission, config.combos?.[candidate.combo.comboId],
)) {
const pick = pickComboTarget(config, candidate.combo.comboId, { eligible: target => !target.metered });
if (!pick) throw new NoAvailableComboTargetsError(candidate.combo.comboId);
if (!pick) {
throw new NoAvailableComboTargetsError(candidate.combo.comboId,
comboExclusionReasons(config, candidate.combo.comboId, target => !target.metered));
}
const routed = routeConcreteModel(config, `${pick.target.provider}/${pick.target.model}`);
candidate = { ...routed, combo: pick, routeKind: "combo", routeReason: routed.routeReason === "blocked-model-redirect" ? "blocked-model-redirect" : "combo-pick" };
}
Expand Down Expand Up @@ -788,7 +791,7 @@ export async function prepareResponsesRequest(
if (policyRefusal) return policyRefusal;
if (err instanceof AdmissionModelDeniedError) return admissionModelDeniedResponse(err);
if (err instanceof NoAvailableComboTargetsError) {
return comboUnavailable(err.comboId);
return comboUnavailable(err.comboId, undefined, err.reasons);
}
if (err instanceof NoEligiblePolicyCandidateError) {
// Persist the evaluation trace (per-candidate exclusions + the
Expand Down Expand Up @@ -1006,7 +1009,7 @@ export async function prepareResponsesRequest(
if (policyRefusal) return policyRefusal;
if (err instanceof AdmissionModelDeniedError) return admissionModelDeniedResponse(err);
if (err instanceof NoAvailableComboTargetsError) {
return comboUnavailable(err.comboId);
return comboUnavailable(err.comboId, undefined, err.reasons);
}
if (err instanceof NoEligiblePolicyCandidateError) {
logCtx.routeDecision = err.trace;
Expand Down Expand Up @@ -1223,7 +1226,7 @@ export async function prepareResponsesRequest(
if (policyRefusal) return policyRefusal;
if (err instanceof AdmissionModelDeniedError) return admissionModelDeniedResponse(err);
if (err instanceof NoAvailableComboTargetsError) {
return comboUnavailable(err.comboId);
return comboUnavailable(err.comboId, undefined, err.reasons);
}
if (err instanceof NoEligiblePolicyCandidateError) {
logCtx.routeDecision = err.trace;
Expand Down
2 changes: 1 addition & 1 deletion structure/runtime.md
Original file line number Diff line number Diff line change
Expand Up @@ -504,7 +504,7 @@ This is also why the classifier cannot duplicate visible output. Native byte str

Regression coverage: `tests/responses/responses-forward-prompt-envelope.test.ts`, `tests/routing/router-combo-failover-classification.test.ts`, `tests/routing/routing-policy-fallback.test.ts`, `tests/helpers/combo-context-overflow-cases.ts`, and `tests/server/server-combo-failover-e2e.test.ts`.

`src/combos/failover.ts` uses a 10-minute fallback for a spent account usage window (codes `usage_limit_exceeded`, `usage_limit_reached`, `1308`, or `usage limit reached` / `usage limit has been reached` / `token-plan <window> quota has been exhausted` prose, including HTTP 502) and for provider-scoped credential or billing failure codes such as `invalid_api_key` and `insufficient_quota`. This duration does not change failure classification or cooldown scope; upstream retry/reset signals and configured durations retain precedence. It caps explicit upstream `Retry-After` target cooldowns at 24 hours while reset-derived, configured, and fallback cooldowns remain capped at 10 minutes. `src/combos/resolve.ts` omits a target cooldown for a 429 only when the failed child identifies one account in an Anthropic OAuth or canonical Codex pool and that account has a live pool cooldown. `src/server/responses/core-combo.ts` checks the pool's health record before passing that attribution; unknown accounts, ambiguous labels, API-key and generic OAuth providers, and other failures keep their ordinary target cooldown. The Anthropic OAuth pool's own all-accounts-cooled 429 is the one unattributed exception: `isAnthropicPoolLocalRefusal` in `core-combo.ts` requires the `anthropic` provider with `authMode: "oauth"`, status 429, no identified failed account, no eligible pool account and a live pool retry-after. The target then still cools, but for the local fallback instead of the restated per-account `Retry-After`, so a resumed or newly added account is not ignored until a 24-hour target cooldown expires. An upstream 429 on an API-key provider or on an identified pool account keeps its full `Retry-After`. Regression coverage: `tests/routing/always-on-429-failover.test.ts`. Cooldown entries in `src/combos/failover.ts` retain the failure status. The quota-only snapshot query records target/expiry pairs for unexpired 429/402 cooldowns using only cooldown map lookups. Its config-aware wrapper in `src/combos/resolve.ts` applies the picker's provider availability and cached quota checks before dispatch; request eligibility runs lazily only when exhaustion considers a 503, with throwing checks treated as ineligible. Responses exhaustion uses this evidence under the [failover contract](transports/responses-failover.md); ordinary cooldown selection and timing stay unchanged.
`src/combos/failover.ts` uses a 10-minute fallback for a spent account usage window (codes `usage_limit_exceeded`, `usage_limit_reached`, `1308`, or `usage limit reached` / `usage limit has been reached` / `token-plan <window> quota has been exhausted` prose, including HTTP 502) and for provider-scoped credential or billing failure codes such as `invalid_api_key` and `insufficient_quota`. This duration does not change failure classification or cooldown scope; upstream retry/reset signals and configured durations retain precedence. It caps explicit upstream `Retry-After` target cooldowns at 24 hours while reset-derived, configured, and fallback cooldowns remain capped at 10 minutes. `src/combos/resolve.ts` omits a target cooldown for a 429 only when the failed child identifies one account in an Anthropic OAuth or canonical Codex pool and that account has a live pool cooldown. `src/server/responses/core-combo.ts` checks the pool's health record before passing that attribution; unknown accounts, ambiguous labels, API-key and generic OAuth providers, and other failures keep their ordinary target cooldown. The Anthropic OAuth pool's own all-accounts-cooled 429 is the one unattributed exception: `isAnthropicPoolLocalRefusal` in `core-combo.ts` requires the `anthropic` provider with `authMode: "oauth"`, status 429, no identified failed account, no eligible pool account and a live pool retry-after. The target then still cools, but for the local fallback instead of the restated per-account `Retry-After`, so a resumed or newly added account is not ignored until a 24-hour target cooldown expires. An upstream 429 on an API-key provider or on an identified pool account keeps its full `Retry-After`. Regression coverage: `tests/routing/always-on-429-failover.test.ts`. Cooldown entries in `src/combos/failover.ts` retain the failure status. The quota-only snapshot query records target/expiry pairs for unexpired 429/402 cooldowns using only cooldown map lookups. Its config-aware wrapper in `src/combos/resolve.ts` applies the picker's provider availability and cached quota checks before dispatch; request eligibility runs lazily only when exhaustion considers a 503, with throwing checks treated as ineligible. Responses exhaustion uses this evidence under the [failover contract](transports/responses-failover.md); ordinary cooldown selection and timing stay unchanged. A zero-attempt `combo_unavailable` names why each target was skipped. `comboExclusionReasons` in `src/combos/resolve.ts` derives one coarse label per target from the picker's own checks, in picker order: `provider missing`, `provider disabled`, `quota exhausted`, `cooldown <n>s`, then `ineligible for this request` when the caller's eligibility predicate refuses it or throws. `NoAvailableComboTargetsError` and `comboUnavailable` append them to the message; the label carries only the configured provider/model, never account, credential or request data. Coverage: `tests/responses/responses-combo-exhausted-error.test.ts`.

## Combo default effort precedence

Expand Down
14 changes: 14 additions & 0 deletions tests/responses/jev-initial-effort-wire.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,20 @@ test("ordinary force invariant is not relaxed", () => {
.toThrow("force combo default effort requires a valid defaultEffort");
});

test("a JEV combo whose only allowlist is unsupported names that target in its 503", async () => {
const f = fixture("responses", "model", "high");
f.config.providers.a!.reasoningEfforts = [];
f.config.combos!.auto!.targets[0]!.reasoningEfforts = ["high"];
const response = await handleResponses(new Request("http://localhost/v1/responses", {
method: "POST", headers: { "content-type": "application/json" },
body: JSON.stringify({ model: "combo/auto", input: "Solve the fixture task.", stream: false }),
}), f.config, { model: "", provider: "" }, {});
expect(response.status).toBe(503);
expect(((await response.json()) as { error: { message: string } }).error.message)
.toBe("No available targets for combo: auto (a/m: ineligible for this request)");
expect(f.inference.bodies).toHaveLength(0);
});

for (const lane of ["native", "responses"] as const) {
test(`${lane}: unallowlisted service effort fails open without applying the rejected effort`, async () => {
const f = fixture(lane, "typesafe", "high");
Expand Down
Loading
Loading