From c7d97e274e3af8ca711098ae6bf9c20dae854638 Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Wed, 30 Sep 2026 04:05:35 +0000 Subject: [PATCH 01/15] fix(spend): keep OAuth logs out of pool identity --- src/server/messages-native.ts | 1 + src/server/request-log.ts | 2 ++ src/server/responses/request-spend.ts | 6 ++++-- structure/transports/responses-spend.md | 4 ++++ .../responses/responses-spend-ledger-wiring.test.ts | 12 ++++++++++++ 5 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/server/messages-native.ts b/src/server/messages-native.ts index b593af2d1b4..6e82bf2544e 100644 --- a/src/server/messages-native.ts +++ b/src/server/messages-native.ts @@ -290,6 +290,7 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions) logCtx.inboundProtocol = "messages"; logCtx.model = route.modelId; logCtx.provider = route.providerName; + logCtx.spendPoolId = route.providerName; logCtx.providerAdapter = route.provider.adapter; logCtx.requestedModel = requestedModel; if (route.routeReason === "model-alias" || route.modelId !== requestedModel) logCtx.requestedAlias = requestedModel; diff --git a/src/server/request-log.ts b/src/server/request-log.ts index f7986a07e0e..f984671f11f 100644 --- a/src/server/request-log.ts +++ b/src/server/request-log.ts @@ -199,6 +199,8 @@ export interface RequestLogContext { spendOutputCeilingTokens?: number; /** Pre-send input estimate reserved for spend only; unlike usageLogInputTokens it never enters usage. */ spendInputEstimateTokens?: number; + /** Canonical provider-pool spend identity; independent of mutable, account-specific log labels. */ + spendPoolId?: string; /** Settles this request's durable spend entries from `addFinalRequestLog`. */ spendTracker?: RequestSpendSettlement; attempts?: PersistedUsageAttempt[]; diff --git a/src/server/responses/request-spend.ts b/src/server/responses/request-spend.ts index 8485b3d79dd..f18d031c36b 100644 --- a/src/server/responses/request-spend.ts +++ b/src/server/responses/request-spend.ts @@ -43,7 +43,7 @@ export interface RequestSpendTracker extends RequestSendObserver, RequestSpendSe export function createRequestSpendTracker( logCtx: Pick< RequestLogContext, - "provider" | "accountLogLabel" | "usageLogInputTokens" | "spendOutputCeilingTokens" | "spendInputEstimateTokens" + "provider" | "accountLogLabel" | "usageLogInputTokens" | "spendOutputCeilingTokens" | "spendInputEstimateTokens" | "spendPoolId" > & Partial>, rootId: string | undefined, injected?: SpendReservationLedger, @@ -88,7 +88,9 @@ export function createRequestSpendTracker( // Already the privacy-safe label the request log uses, and the ledger aliases it // again on the way to disk. A raw credential never reaches either. ...(logCtx.accountLogLabel !== undefined ? { identityId: logCtx.accountLogLabel } : {}), - ...(logCtx.provider !== undefined ? { poolId: logCtx.provider } : {}), + ...((logCtx.spendPoolId ?? logCtx.provider) !== undefined + ? { poolId: logCtx.spendPoolId ?? logCtx.provider } + : {}), }, inputTokens: logCtx.spendInputEstimateTokens ?? logCtx.usageLogInputTokens ?? 0, outputCeilingTokens: logCtx.spendOutputCeilingTokens ?? 0, diff --git a/structure/transports/responses-spend.md b/structure/transports/responses-spend.md index 6bfc03a8938..735e8084e48 100644 --- a/structure/transports/responses-spend.md +++ b/structure/transports/responses-spend.md @@ -102,6 +102,10 @@ so one ledger entry per increment is one entry per send, and a dispatch path add forget to book. The previous attempt at this wiring shipped the whole reserve/dispatch/settle vocabulary with no caller at all (#4707), which is the failure mode this shape rules out. +Provider-pool accounting uses the canonical routed provider identity, not the mutable display label +that may identify an OAuth account in request logs. Native Messages therefore shares the same pool +ceiling as Responses even when its Anthropic log label includes an account ordinal. + A booking is confirmed dispatched only once a LATER send exists, because that later send proves the earlier one left. The newest booking stays open, so a reservation the budget hands back during this process's lifetime can still be released for free. diff --git a/tests/responses/responses-spend-ledger-wiring.test.ts b/tests/responses/responses-spend-ledger-wiring.test.ts index e1c973fc055..2aafa48cd02 100644 --- a/tests/responses/responses-spend-ledger-wiring.test.ts +++ b/tests/responses/responses-spend-ledger-wiring.test.ts @@ -88,6 +88,18 @@ describe("the request path books every physical send on the durable ledger", () expect(root?.unresolved).toBe(1000); }); + test("a canonical pool identity is independent of an account-specific display label", () => { + const ledger = createSpendReservationLedger({ journal: memoryJournal() }); + const tracker = createRequestSpendTracker(logContext({ + provider: "anthropic-p123abc", + spendPoolId: "anthropic", + }), undefined, ledger); + + expect(tracker.charge()).toBe(true); + expect(ledger.snapshot("pool", "anthropic")?.reserved).toBe(500); + expect(ledger.snapshot("pool", "anthropic-p123abc")).toBeUndefined(); + }); + test("a reservation the budget hands back releases its tokens instead of booking spend", () => { const ledger = createSpendReservationLedger({ journal: memoryJournal() }); const tracker = createRequestSpendTracker(logContext(), "root-c", ledger); From 44b3e14f8b46a9e49e0cbb2a77df5157b9e8e3c3 Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Thu, 1 Oct 2026 04:15:25 -0700 Subject: [PATCH 02/15] fix(spend): preserve explicit historical pool continuity Aggregate verified salted aliases without moving original balances; fail closed on unresolved pool history before dispatch. Preserve compatibility metadata in v1 checkpoints and document the supported rollback boundary. Add synthetic cross-version, retention, durability, and rootless preflight coverage. --- .../docs/reference/configuration/server.md | 56 ++ scripts/test-layout/layout.json | 1 + src/config/diagnostics.ts | 5 + src/lib/spend-pool-continuity.ts | 98 +++ src/lib/spend-reservation-ledger.ts | 109 +++- src/lib/workflow-budget.ts | 17 +- src/server/index.ts | 6 +- src/server/index/spend-ledger-lifecycle.ts | 6 +- src/server/responses/request-send-budget.ts | 10 +- src/server/responses/request-spend.ts | 9 +- src/server/workflow-refusal.ts | 12 + src/types/config.ts | 2 + structure/config.md | 2 +- structure/runtime.md | 2 +- structure/transports/responses-spend.md | 41 ++ tests/config/config-spend-ceilings.test.ts | 23 +- tests/fixtures/spend-ledger-f7a50dc3.ts.txt | 586 ++++++++++++++++++ tests/fixtures/test-layout-expected.json | 1 + tests/helpers/legacy-spend-ledger.ts | 16 + tests/lib/spend-pool-continuity.test.ts | 249 ++++++++ 20 files changed, 1216 insertions(+), 35 deletions(-) create mode 100644 src/lib/spend-pool-continuity.ts create mode 100644 tests/fixtures/spend-ledger-f7a50dc3.ts.txt create mode 100644 tests/helpers/legacy-spend-ledger.ts create mode 100644 tests/lib/spend-pool-continuity.test.ts diff --git a/docs-site/src/content/docs/reference/configuration/server.md b/docs-site/src/content/docs/reference/configuration/server.md index e97fab4cdca..ca72348a5fb 100644 --- a/docs-site/src/content/docs/reference/configuration/server.md +++ b/docs-site/src/content/docs/reference/configuration/server.md @@ -40,6 +40,7 @@ runs helper features around provider requests. | `codexProviderDisplayName?` | `string` | `"OpenCodex Proxy"` | Label Codex shows for the injected `opencodex` provider, written as its `name` field in `config.toml` and the reference profile. Presentation only: routing resolves through the provider id `opencodex`, so a rename never moves `model_provider = "opencodex"` or the `[model_providers.opencodex]` header and cannot orphan threads already tagged with that id. Codex refuses to load a provider with no name, so there is no way to omit the field — choose a neutral label instead. A blank, over-128-character, or control-character value is ignored and the default label is written. | | `resetCreditAutoRedeem?` | `{ enabled?: boolean; leadTimeMinutes?: number }` | off | Opt-in: redeem the main Codex account's soonest-expiring reset credit `leadTimeMinutes` (1–60, default 10) before it expires. Every attempt re-reads the upstream credit list first and skips when the credit is gone (for example, redeemed by hand); the `redeem_request_id` is journaled in `$OPENCODEX_HOME/reset-credit-auto-redeem.json` before the call so a crash replays the same idempotent request instead of spending a second credit. Servers sharing this configuration directory coordinate reservations and settlements so one process does not replace another's request record. Logs carry a hashed account key only. | | `syncResumeHistory?` | `boolean` | `true` | Reversible Codex App history compatibility. Original metadata is backed up and restored by `ocx stop` / `ocx restore`. | +| `spendPoolAliases?` | `Record` | unset | Explicit historical salted pool-alias to canonical provider mappings. Keep this key at the top level, outside `spend`. See [Historical pool continuity](#historical-pool-continuity). | | `shadowCallIntercept?` | `{ enabled?: boolean; model?: string; sourceModels?: string[] }` | off | Redirect recognized Codex helper/shadow calls to a chosen model while preserving the request's configured reasoning effort. The default source prefixes are `gpt-6-luna` and `gpt-5.6-luna`; older clients through 0.144.x used `gpt-5.4-mini`, which `sourceModels` can restore. | | `memoryModels?` | `{ extract?: { model: string; reasoningEffort?: string }; consolidation?: { model: string; reasoningEffort?: string } }` | off | Route Codex's two memory phases to a chosen model, with an optional reasoning effort per phase. See [Memory routing](#memory-routing). | | `webSearchSidecar?` | `OcxWebSearchSidecarConfig` | on when usable | Web-search sidecar options. | @@ -892,3 +893,58 @@ WebSocket control paths. See the canonical guide for [supported steering routes and settings](/guides/codex-integration/#steering-continuation-settings), [typed result and approval continuations](/guides/codex-integration/#rich-tool-results-and-explicit-approvals-after-response-completion), and [confirmation deadlines and retained context](/guides/codex-integration/#steering-confirmation-deadlines-and-retained-context). + + +## Historical pool continuity + +Provider-pool ceilings use the canonical routed provider, while request logs keep account-specific +display labels. Older journals may hold spend under those display labels. A journal stores salted +aliases, not the original provider/account names, so OpenCodex does not guess a mapping from a +current account roster, a label prefix, or a shortened account ID. + +If `spend.pool.maxTokens` is configured and positive historical pool balances remain unidentified, +inference admission returns local HTTP 429 with +`x-opencodex-local-refusal: workflow_pool_history_unresolved` before contacting a provider. +This can temporarily block otherwise valid requests, including requests without a workflow root. +After routing, preflight also refuses a canonical pool that is already exhausted by its combined +balances. This check does not turn post-reported passthrough sends into atomic reservations: +a crossing send, concurrent admissions or retries reported afterwards retain their existing limits. +Observe-only installs remain observe-only. Root and identity ceilings remain in force independently. + +To resolve it, an operator must verify which canonical provider each historical salted pool alias +belongs to, using their own retained evidence. Add a top-level `spendPoolAliases` object in +`config.json`: each key is the exact 32-character lowercase hexadecimal pool alias from that same +installation's journal; each value is its verified canonical provider ID. An old alias that already +represents the canonical provider still needs an explicit entry when it lacks identity metadata. +A positive alias that cannot be identified stays blocked. Do not infer a match from similar names, +account deletion, or a short-label collision, and do not share the journal or salt publicly. + +Keep `spendPoolAliases` outside `spend`: older versions reject unknown keys inside `spend` and can +disable the entire section. Invalid top-level mappings are rejected on configuration writes; +malformed `spendPoolAliases` hand edits retain existing ceilings and fail pool admission closed. Correct the mapping +and restart through the ordinary configuration workflow. Empty/removing mappings does not erase +links already recorded durably. A previously redirected alias cannot be reassigned to a different +group; verified canonical renames can join groups without splitting existing spend. + +Each original balance is counted once. Settled usage, in-flight reservations and unresolved usage +all count; unknown usage is never treated as a refund. Original reservation targets are retained. +New requests continue to record the canonical provider pool, and checkpoints retain salted identity +evidence without adding raw account/provider names. Unknown positive history and active or exhausted +groups are protected from cleanup; the existing dormant, under-limit retention rule still applies. +Identity evidence remains bounded; if its capacity is exhausted, pool admission stops rather than +forgetting it. No automatic tool reconstructs unverifiable history. + +### Safe rollback requirements + +A supported rollback must retain or backport both canonical request attribution and the +continuity-aware ledger reader/writer, with the same journal, salt and verified mappings. +Keep the latest journal: restoring a pre-upgrade copy would omit later spend. Do not remove records, +change the salt, raise ceilings, or disable enforcement to make a downgrade appear compatible. + +An unmodified older binary is **not a supported rollback**. It can read the v1 raw balances but +does not enforce the cross-alias provider total, may create a new account-label pool, and may discard +optional identity metadata when compacting. There is no automatic downgrade barrier. If such an old +writer has run, the new reader refuses unidentified positive history until the operator explicitly +verifies all remaining mappings again. Storage or journal-integrity denials are separate from an +alias problem and keep `workflow_spend_undurable`; unsafe-file/ownership failures may instead +propagate as storage errors, without admitting the request. diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 402eede4dc3..b5def6cb76c 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1632,6 +1632,7 @@ "spend-ledger-lifecycle.test.ts": "server", "spend-ledger-owner-startup.test.ts": "server", "spend-ledger-owner.test.ts": "lib", + "spend-pool-continuity.test.ts": "lib", "spend-reservation-ledger.test.ts": "lib", "sponsor-presets.test.ts": "providers", "sse-client-frame-bounds.test.ts": "responses", diff --git a/src/config/diagnostics.ts b/src/config/diagnostics.ts index dd559ed6097..bdbf180c909 100644 --- a/src/config/diagnostics.ts +++ b/src/config/diagnostics.ts @@ -1,3 +1,4 @@ +import { spendPoolAliasesError } from "../lib/spend-pool-continuity"; import { createHash } from "node:crypto"; import { lstatSync, readFileSync } from "node:fs"; import { join } from "node:path"; @@ -134,6 +135,9 @@ function validFileConfigDiagnostics(config: OcxConfig, rawParsed: unknown): Conf if (codexPoolWarning) warnings.push(codexPoolWarning); const spendWarning = malformedSpendWarning(rawParsed); if (spendWarning) warnings.push(spendWarning); + if (spendPoolAliasesError(rawConfigRecord(rawParsed)?.spendPoolAliases)) { + warnings.push("spendPoolAliases invalid: configured pool admission is refused until the mapping is corrected"); + } const plaintextWarning = malformedPlaintextV2AgentMessagesWarning(rawParsed); if (plaintextWarning) warnings.push(plaintextWarning); if (syncDisabledReason) { @@ -642,6 +646,7 @@ export function validateConfigCandidate(value: unknown): { ok: true; config: Ocx ?? agentTaskRecoveryError(value) ?? quotaResetNotifyError(value) ?? catalogAutoRefreshError(value) + ?? (spendPoolAliasesError(rawConfigRecord(value)?.spendPoolAliases) ? "schema_invalid: spendPoolAliases: invalid pool identity mapping" : null) ?? spendError(value) ?? codexPoolError(value) ?? googleAntigravityStaticCatalogVersionError(value) diff --git a/src/lib/spend-pool-continuity.ts b/src/lib/spend-pool-continuity.ts new file mode 100644 index 00000000000..991f8fe4541 --- /dev/null +++ b/src/lib/spend-pool-continuity.ts @@ -0,0 +1,98 @@ +/** Explicit, salted pool identity evidence. Never infer ownership from a label prefix. */ +export interface PoolBinding { readonly alias: string; readonly canonical: string } +export interface PoolContinuityRecord { + readonly v: 1; + readonly kind: "pool-continuity"; + readonly at: number; + readonly bindings: readonly PoolBinding[]; +} + +const isAlias = (value: unknown): value is string => + typeof value === "string" && /^[0-9a-f]{32}$/.test(value); + +/** Kept outside `spend`: older strict spend schemas must keep all configured ceilings. */ +export function spendPoolAliasesError(value: unknown): string | undefined { + if (value === undefined) return undefined; + if (!value || typeof value !== "object" || Array.isArray(value)) return "spendPoolAliases must be an object"; + if (Object.keys(value).length > 4_096) return "spendPoolAliases has too many entries"; + for (const [alias, provider] of Object.entries(value)) { + if (!isAlias(alias) || typeof provider !== "string" || !provider.trim() || provider.length > 256) { + return "spendPoolAliases requires exact 32-character salted pool aliases and nonempty provider IDs"; + } + } + return undefined; +} + +export function parsePoolContinuityRecord(value: unknown): PoolContinuityRecord | undefined { + if (!value || typeof value !== "object") return undefined; + const record = value as Record; + if (record.v !== 1 || record.kind !== "pool-continuity" + || typeof record.at !== "number" || !Number.isFinite(record.at) || record.at < 0 + || !Array.isArray(record.bindings) || record.bindings.length > 16_384) return undefined; + const bindings: PoolBinding[] = []; + const seen = new Set(); + for (const entry of record.bindings) { + if (!entry || typeof entry !== "object") return undefined; + const { alias, canonical } = entry as Partial; + if (!isAlias(alias) || !isAlias(canonical) || seen.has(alias)) return undefined; + seen.add(alias); + bindings.push({ alias, canonical }); + } + return { v: 1, kind: "pool-continuity", at: record.at, bindings }; +} + +/** A directed union: links may merge proven groups, never split or erase accounted spend. */ +export function createPoolContinuity() { + let bindings = new Map(); + const resolve = (alias: string, map = bindings): string => { + const seen = new Set(); + while (map.has(alias) && map.get(alias) !== alias) { + if (seen.has(alias)) throw new Error("cyclic pool identity evidence"); + seen.add(alias); + alias = map.get(alias)!; + } + return alias; + }; + const link = (map: Map, alias: string, canonical: string): boolean => { + const existing = map.get(alias); + const target = resolve(canonical, map); + if (existing !== undefined && existing !== alias && resolve(alias, map) !== target) return false; + // An explicit rename can join a formerly canonical alias to its successor. The old + // name still resolves to this same group, so neither a reload nor an old caller splits it. + if (target !== alias || existing === undefined) map.set(alias, target); + return true; + }; + return { + resolve: (alias: string): string => resolve(alias), + known: (alias: string): boolean => bindings.has(alias), + size: (): number => bindings.size, + record: (at: number): PoolContinuityRecord => ({ + v: 1, kind: "pool-continuity", at, + bindings: [...bindings].map(([alias, canonical]) => ({ alias, canonical })), + }), + restore(record: PoolContinuityRecord): boolean { + const next = new Map(bindings); + try { + for (const { alias, canonical } of record.bindings) if (!link(next, alias, canonical)) return false; + for (const alias of next.keys()) resolve(alias, next); + } catch { return false; } + bindings = next; + return true; + }, + prepare(config: unknown, requested: string | undefined, historical: ReadonlySet, + hash: (provider: string) => string, at: number, capacity: number): PoolContinuityRecord | false | undefined { + if (spendPoolAliasesError(config)) return false; + const next = new Map(bindings); + // Sort for deterministic multi-hop mappings regardless of JSON property order. Link + // conflicts fail closed; repeat entries and already-joined destinations are idempotent. + for (const [alias, provider] of Object.entries(config ?? {}).sort(([a], [b]) => a.localeCompare(b))) { + if (!link(next, alias, hash(provider as string))) return false; + } + if (requested !== undefined && !next.has(requested) && !historical.has(requested)) next.set(requested, requested); + if (next.size > Math.min(16_384, capacity)) return false; + if (next.size === bindings.size && [...next].every(([key, value]) => bindings.get(key) === value)) return undefined; + return { v: 1, kind: "pool-continuity", at, + bindings: [...next].map(([alias, canonical]) => ({ alias, canonical })) }; + }, + }; +} diff --git a/src/lib/spend-reservation-ledger.ts b/src/lib/spend-reservation-ledger.ts index 1a7ff2b4c64..04a5a313339 100644 --- a/src/lib/spend-reservation-ledger.ts +++ b/src/lib/spend-reservation-ledger.ts @@ -62,6 +62,7 @@ import { getConfigDir } from "../config/paths"; // Type-only, so it is erased before this module has a runtime import graph at all. The // config SHAPE is what this file needs; the config loader is what the note above keeps out. import type { OcxSpendConfig, OcxSpendScopeConfig } from "../types/config"; +import { createPoolContinuity, parsePoolContinuityRecord, type PoolContinuityRecord } from "./spend-pool-continuity"; import { assertNotRealHomeUnderTest } from "./test-home-guard"; // Windows chmod does not remove inherited ACEs; this is the repository's icacls path. import { hardenSecretPath } from "./windows-secret-acl"; @@ -93,6 +94,8 @@ export interface SpendScopeLimit { } export interface SpendReservationPolicy { + /** Exact historical salted pool alias -> canonical provider. Malformed input fails closed. */ + readonly poolAliases?: unknown; readonly root: SpendScopeLimit; readonly identity: SpendScopeLimit; readonly pool: SpendScopeLimit; @@ -181,6 +184,7 @@ export interface SpendReservationRequest { * to buy an unlimited number of physical sends while the scope totals never moved. */ export type SpendDenial = + | { readonly reason: "pool-history-unresolved" } | { readonly reason: "spend-limit-exceeded"; readonly scope: SpendScope; @@ -258,6 +262,7 @@ type JournalRecord = v: 1; kind: "checkpoint"; at: number; + poolContinuity?: PoolContinuityRecord; scopes: { scope: SpendScope; alias: string; settled: number; unresolved: number; seenAt: number }[]; sends: { send: string; status: ReservationStatus; targets: ScopeRef[]; tokens: number; at: number; resolvedAt: number }[]; }; @@ -350,7 +355,9 @@ export function parseSpendJournalRecord(line: string): JournalRecord | undefined if (!isCountable(e.tokens) || !isCountable(e.at) || !isCountable(e.resolvedAt)) return undefined; sends.push({ send: e.send, status: e.status, targets, tokens: e.tokens, at: e.at, resolvedAt: e.resolvedAt }); } - return { v: 1, kind: "checkpoint", at, scopes, sends }; + const poolContinuity = record.poolContinuity === undefined ? undefined : parsePoolContinuityRecord(record.poolContinuity); + if (record.poolContinuity !== undefined && !poolContinuity) return undefined; + return { v: 1, kind: "checkpoint", at, scopes, sends, ...(poolContinuity ? { poolContinuity } : {}) }; } default: return undefined; @@ -574,6 +581,8 @@ export interface ScopeSpendSnapshot { export interface SpendReservationLedger { reserve(request: SpendReservationRequest): SpendReservationDecision; + /** Pre-dispatch guard, including transports which report their sends after dispatch. */ + checkPoolContinuity(): SpendDenial | undefined; /** * The send left for upstream. Until this is called the reservation may be abandoned for * free; after it, a missing usage frame becomes unresolved spend. Returns false when the @@ -667,6 +676,7 @@ export function createSpendReservationLedger(options: { const compactAfterRecords = (): number => policy.compactAfterRecords ?? DEFAULT_COMPACT_AFTER_RECORDS; const scopes = new Map(); const reservations = new Map(); + const poolContinuity = createPoolContinuity(); let persistFailures = 0; let corruptRecords = 0; let recordsOnDisk = 0; @@ -690,6 +700,26 @@ export function createSpendReservationLedger(options: { return state; }; + const historicalPools = (): Set => new Set([...scopes].filter(([key, state]) => + key.startsWith("pool\0") && state.settled + state.reserved + state.unresolved > 0, + ).map(([key]) => key.slice(5))); + const unknownPoolHistory = (): boolean => [...historicalPools()].some(alias => !poolContinuity.known(alias)); + const poolState = (alias: string): ScopeState | undefined => { + const group = poolContinuity.resolve(alias); + let total: ScopeState | undefined; + for (const [key, state] of scopes) { + if (!key.startsWith("pool\0") || poolContinuity.resolve(key.slice(5)) !== group) continue; + total ??= { settled: 0, reserved: 0, unresolved: 0, lastSeenAt: 0 }; + total.settled += state.settled; + total.reserved += state.reserved; + total.unresolved += state.unresolved; + total.lastSeenAt = Math.max(total.lastSeenAt, state.lastSeenAt); + } + return total; + }; + const stateFor = (scope: SpendScope, alias: string): ScopeState | undefined => + scope === "pool" ? poolState(alias) : scopes.get(scopeKey(scope, alias)); + const limitFor = (scope: SpendScope): number | undefined => policy[scope].maxTokens; const isExhausted = (scope: SpendScope, state: ScopeState): boolean => { @@ -777,6 +807,7 @@ export function createSpendReservationLedger(options: { }; const applyCheckpoint = (record: Extract): void => { + if (record.poolContinuity && !poolContinuity.restore(record.poolContinuity)) corruptRecords += 1; scopes.clear(); reservations.clear(); for (const entry of record.scopes) { @@ -814,12 +845,13 @@ export function createSpendReservationLedger(options: { const line = lines[index] as string; const record = parseSpendJournalRecord(line); if (!record) { - // A rejected FINAL line is a torn tail write -- the process died between the write - // and its newline -- and is dropped quietly, because that record never completed and - // therefore never authorised anything. A rejected line ANYWHERE ELSE is different: - // the records after it did complete, so skipping it silently undercounts a scope and - // hands back budget. It is counted, and a configured limit refuses on it below. - if (index < lines.length - 1) corruptRecords += 1; + // A malformed final JSON line can be an incomplete tail write. Rejected records + // elsewhere cannot be skipped: later complete writes may have authorized spend. + // Only malformed JSON can be a torn tail. A complete unknown/invalid final + // record is evidence of an unsupported format, not permission to forget accounting. + let completeJson = false; + try { JSON.parse(line); completeJson = true; } catch { /* torn tail */ } + if (completeJson || index < lines.length - 1) corruptRecords += 1; continue; } switch (record.kind) { @@ -870,9 +902,12 @@ export function createSpendReservationLedger(options: { for (const [key, state] of scopes) { const separator = key.indexOf("\0"); const scope = key.slice(0, separator) as SpendScope; - if (state.reserved > 0) continue; - if (isExhausted(scope, state)) continue; - if (!force && state.lastSeenAt >= cutoff) continue; + const effective = scope === "pool" ? poolState(key.slice(separator + 1))! : state; + if (effective.reserved > 0) continue; + if (scope === "pool" && state.settled + state.unresolved > 0 + && !poolContinuity.known(key.slice(separator + 1))) continue; + if (isExhausted(scope, effective)) continue; + if (!force && effective.lastSeenAt >= cutoff) continue; candidates.push({ key, scope, alias: key.slice(separator + 1), seenAt: state.lastSeenAt }); } if (force) { @@ -915,10 +950,7 @@ export function createSpendReservationLedger(options: { * Bounded maps are not enough on their own: the file behind them is what replay reads, and * an uncompacted file grows forever on unique root and send ids. */ - const compact = (at: number): void => { - const rewrite = journal?.rewrite; - if (!journal || !rewrite || recordsOnDisk < compactAfterRecords()) return; - const checkpoint: JournalRecord = { + const checkpointRecord = (at: number, evidence = poolContinuity.record(at)): Extract => ({ v: 1, kind: "checkpoint", at, @@ -940,9 +972,15 @@ export function createSpendReservationLedger(options: { at: reservation.at, resolvedAt: reservation.resolvedAt, })), - }; + ...(evidence.bindings.length > 0 ? { poolContinuity: evidence } : {}), + }); + + const compact = (at: number): void => { + const rewrite = journal?.rewrite; + // A checkpoint must never erase evidence of replay corruption. + if (!journal || !rewrite || corruptRecords > 0 || recordsOnDisk < compactAfterRecords()) return; try { - rewrite.call(journal, [JSON.stringify(checkpoint)]); + rewrite.call(journal, [JSON.stringify(checkpointRecord(at))]); recordsOnDisk = 1; } catch (error) { if (error instanceof SpendLedgerOwnerError) throw error; @@ -952,6 +990,19 @@ export function createSpendReservationLedger(options: { } }; + const preparePoolContinuity = (at: number, requested?: string): SpendDenial | undefined => { + const evidence = poolContinuity.prepare(policy.poolAliases, requested, historicalPools(), + provider => aliasFor("pool", provider), at, maxTrackedScopes()); + if (evidence === false) return { reason: "pool-history-unresolved" }; + if (evidence) { + // A v1 checkpoint atomically carries the unchanged balances AND salted identity + // evidence. Older parsers can still read its balances; no unknown-record fence. + if (!append(checkpointRecord(at, evidence))) return { reason: "reserve-not-durable", sendId: "" }; + if (!poolContinuity.restore(evidence)) return { reason: "pool-history-unresolved" }; + } + return unknownPoolHistory() ? { reason: "pool-history-unresolved" } : undefined; + }; + /** The denial when tracking cannot fit this request, or undefined when it can. */ const makeRoom = (refs: readonly ScopeRef[], at: number): SpendDenial | undefined => { evictSends(at, false); @@ -977,6 +1028,13 @@ export function createSpendReservationLedger(options: { get degraded() { assertOwnedAccounting?.(); return persistFailures > 0 || corruptRecords > 0; }, get policy() { assertOwnedAccounting?.(); return policy; }, + checkPoolContinuity(): SpendDenial | undefined { + assertOwnedAccounting?.(); + if (policy.pool.maxTokens === undefined) return undefined; + if (corruptRecords > 0) return { reason: "journal-corrupt", corruptRecords }; + return preparePoolContinuity(now()); + }, + reserve(request: SpendReservationRequest): SpendReservationDecision { assertOwnedAccounting?.(); const tokens = sanitizeTokens(request.inputTokens) + sanitizeTokens(request.outputCeilingTokens); @@ -996,6 +1054,11 @@ export function createSpendReservationLedger(options: { if (enforced && corruptRecords > 0) { return { reserved: false, denial: { reason: "journal-corrupt", corruptRecords } }; } + const poolRef = refs.find(ref => ref.scope === "pool"); + const continuityDenial = preparePoolContinuity(at, poolRef?.alias); + if (continuityDenial && policy.pool.maxTokens !== undefined && request.alreadySent !== true) { + return { reserved: false, denial: continuityDenial }; + } const capacity = makeRoom(refs, at); if (capacity) return { reserved: false, denial: capacity }; @@ -1007,7 +1070,7 @@ export function createSpendReservationLedger(options: { // it is to let the total go OVER the ceiling so the next request can be refused. const limit = request.alreadySent === true ? undefined : limitFor(ref.scope); if (limit === undefined) continue; - const state = scopes.get(scopeKey(ref.scope, ref.alias)); + const state = stateFor(ref.scope, ref.alias); const projected = (state ? state.settled + state.reserved + state.unresolved : 0) + tokens; if (projected > limit) { const scopeId = ref.scope === "root" @@ -1088,7 +1151,7 @@ export function createSpendReservationLedger(options: { // Reading accounting from a handle whose ownership has ended is as wrong as writing it: // the figures describe a journal this process no longer owns. assertOwnedAccounting?.(); - const state = scopes.get(scopeKey(scope, aliasFor(scope, scopeId))); + const state = stateFor(scope, aliasFor(scope, scopeId)); if (!state) return undefined; return { settled: state.settled, @@ -1100,7 +1163,7 @@ export function createSpendReservationLedger(options: { exhausted(scope: SpendScope, scopeId: string): boolean { assertOwnedAccounting?.(); - const state = scopes.get(scopeKey(scope, aliasFor(scope, scopeId))); + const state = stateFor(scope, aliasFor(scope, scopeId)); return state !== undefined && isExhausted(scope, state); }, @@ -1120,6 +1183,11 @@ export function createSpendReservationLedger(options: { }; } +/** Does no I/O for installations without a pool ceiling. */ +export function sharedPoolContinuityDenial(): SpendDenial | undefined { + return sharedPolicy.pool.maxTokens === undefined ? undefined : sharedSpendLedger().checkPoolContinuity(); +} + let sharedLedger: SpendReservationLedger | undefined; /** * The operator policy in effect. Held beside the ledger rather than inside it because the @@ -1155,9 +1223,10 @@ const spendScopeLimitFromConfig = (scope: OcxSpendScopeConfig | undefined): Spen * ceiling would start refusing real traffic on the first upgrade that ran this code, against * a number nobody chose. There is deliberately no default figure here at all. */ -export function spendPolicyFromConfig(spend: OcxSpendConfig | undefined): SpendReservationPolicy { +export function spendPolicyFromConfig(spend: OcxSpendConfig | undefined, poolAliases?: unknown): SpendReservationPolicy { const retentionDays = spend?.retentionDays; return { + ...(poolAliases !== undefined ? { poolAliases } : {}), root: spendScopeLimitFromConfig(spend?.root), identity: spendScopeLimitFromConfig(spend?.identity), pool: spendScopeLimitFromConfig(spend?.pool), diff --git a/src/lib/workflow-budget.ts b/src/lib/workflow-budget.ts index ae12fee0753..41b72265b36 100644 --- a/src/lib/workflow-budget.ts +++ b/src/lib/workflow-budget.ts @@ -196,7 +196,8 @@ export type WorkflowDenial = /** This send id was already reserved once; a repeat buys no second dispatch. */ | "workflow-send-replayed" /** The reservation could not be made durable, and a configured ceiling requires it. */ - | "workflow-spend-undurable"; + | "workflow-spend-undurable" + | "workflow-pool-history-unresolved"; /** * The sentence an operator reads, plus the machine-readable name of the ceiling that fired. @@ -265,6 +266,8 @@ export function workflowDenialSummary( message: "This proxy refused the request locally: this send was already reserved once, and" + " a repeat buys no second dispatch.", }; + case "workflow-pool-history-unresolved": + return { code: "workflow_pool_history_unresolved", message: "Historical provider-pool spend needs verified alias mappings before dispatch. No provider was contacted." }; case "workflow-spend-undurable": return { code: "workflow_spend_undurable", @@ -582,6 +585,7 @@ export function admitWorkflowTurn( // an operator reading a 429 needs to know which of the three happened. const reason: WorkflowDenial = denial.reason === "duplicate-send-id" ? "workflow-send-replayed" + : denial.reason === "pool-history-unresolved" ? "workflow-pool-history-unresolved" : denial.reason === "reserve-not-durable" || denial.reason === "journal-corrupt" ? "workflow-spend-undurable" : denial.reason === "tracking-capacity-exhausted" @@ -718,7 +722,7 @@ function spentRootCeiling( } /** - * The token ceiling a root has already spent, or undefined when it has room or has none. + * A root or routed pool ceiling already spent, or undefined when neither is exhausted. * * The count-side twin of {@link workflowSendCeilingReached}, and the responses path calls both * at the same seam for the same reason: a refusal decided before dispatch can be reported as @@ -731,10 +735,15 @@ function spentRootCeiling( export function workflowSpendCeilingReached( rootId: string | undefined, spendLedger?: SpendReservationLedger, + poolId?: string, ): WorkflowSpendDenialDetail | undefined { - if (!rootId) return undefined; + if (!rootId && !poolId) return undefined; const ledger = spendLedger ?? (spendCeilingsConfigured() ? sharedSpendLedger() : undefined); - return ledger ? spentRootCeiling(rootId, ledger) : undefined; + if (!ledger) return undefined; + const root = rootId ? spentRootCeiling(rootId, ledger) : undefined; + if (root) return root; + const limit = ledger.policy.pool.maxTokens; + return poolId && limit !== undefined && ledger.exhausted("pool", poolId) ? { scope: "pool", limit } : undefined; } export interface WorkflowBudgetSnapshot { diff --git a/src/server/index.ts b/src/server/index.ts index bc827af06fe..78bb9fc8498 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -302,7 +302,7 @@ function startServerWithSpendLedgerOwner(port: number | undefined, deps: StartSe configureAppOwnedMemoryBudget(resolveAppOwnedMemoryBudgetBytes(config.appOwnedMemoryBudgetMb)); enforceAppOwnedMemoryBudget(); // Observe-only mode still journals physical sends, so every server owns before configuring. - spendLedgerLifecycle.configure(config.spend); + spendLedgerLifecycle.configure(config.spend, config.spendPoolAliases); // After ownership: a second server on the same home is refused above, so the process running // this line is the only one appending to usage.jsonl and the only one that may compact it. setUsageLedgerRetention(config.usageLedgerMaxBytes); @@ -507,7 +507,9 @@ function startServerWithSpendLedgerOwner(port: number | undefined, deps: StartSe const lease = tryAdmitTurn(sessionLaneIdFromRequest(req.headers)); if (!lease) return serverBusyResponse(req, "active turns", policy); // Root, lane, and the refusal that follows from them, all live in ./workflow-refusal. - const workflow = admitHttpWorkflowTurn(req.headers); + let workflow: ReturnType; + try { workflow = admitHttpWorkflowTurn(req.headers); } + catch (error) { lease.release(); throw error; } if (workflow && !workflow.admitted) { lease.release(); // withCors, because without Access-Control-Allow-Origin the exposed refusal header is diff --git a/src/server/index/spend-ledger-lifecycle.ts b/src/server/index/spend-ledger-lifecycle.ts index ddcc70a6a3b..a87f469cdbb 100644 --- a/src/server/index/spend-ledger-lifecycle.ts +++ b/src/server/index/spend-ledger-lifecycle.ts @@ -25,7 +25,7 @@ export function waitForFailedStartRollback(error: unknown): Promise { } export interface SpendLedgerServerLifecycle { - configure(spend: OcxSpendConfig | undefined): void; + configure(spend: OcxSpendConfig | undefined, poolAliases?: unknown): void; track }>(server: T): T; release(): void; releaseAfterFailedStart(): Promise; @@ -45,8 +45,8 @@ export function acquireSpendLedgerServerLifecycle(configDir: string): SpendLedge owner.release(); }; return { - configure(spend): void { - configureSharedSpendLedger(spendPolicyFromConfig(spend)); + configure(spend, poolAliases): void { + configureSharedSpendLedger(spendPolicyFromConfig(spend, poolAliases)); }, track }>(server: T): T { // Capture the raw stop before startServer replaces the public method with full teardown. diff --git a/src/server/responses/request-send-budget.ts b/src/server/responses/request-send-budget.ts index b8bc18e4fd9..bad16cf6426 100644 --- a/src/server/responses/request-send-budget.ts +++ b/src/server/responses/request-send-budget.ts @@ -5,7 +5,7 @@ import { workflowSendCeilingReached, workflowSpendCeilingReached, } from "../../lib/workflow-budget"; -import { workflowRefusalResponse } from "../workflow-refusal"; +import { poolContinuityRefusalReason, workflowRefusalResponse } from "../workflow-refusal"; import type { AttemptRecoveryKind, AttemptRecoveryWithheld } from "../../usage/log"; import { noteAttemptRecoveryWithheld, noteAttemptSend } from "../request-log"; import { TRANSIENT_RETRY_MAX_ATTEMPTS } from "../../lib/upstream-retry"; @@ -78,7 +78,13 @@ export function createResponsesSendBudget( // returns -- a refusal an operator cannot tell from an ordinary budget exhaustion, on a // ceiling they configured themselves. Asked before dispatch, it names the scope and the // number instead. Returns undefined and touches no ledger when no ceiling is configured. - const spentCeiling = workflowSpendCeilingReached(workflowRootId); + // Passthrough reports sends after they leave. Historical identity uncertainty must + // refuse here as well as in reserve(), including requests with no workflow root. + const continuityRefusal = poolContinuityRefusalReason(); + if (continuityRefusal) { + return workflowRefusalResponse(continuityRefusal, logCtx, undefined, workflowRootId); + } + const spentCeiling = workflowSpendCeilingReached(workflowRootId, undefined, logCtx.spendPoolId ?? logCtx.provider); if (spentCeiling) { return workflowRefusalResponse( "workflow-spend-exhausted", diff --git a/src/server/responses/request-spend.ts b/src/server/responses/request-spend.ts index f18d031c36b..8e86aac475d 100644 --- a/src/server/responses/request-spend.ts +++ b/src/server/responses/request-spend.ts @@ -106,7 +106,14 @@ export function createRequestSpendTracker( // configured -- so an install that configured nothing is still never refused here. // Capacity and a duplicate send id stay permissive: they say the ledger cannot account // for this send, which is a degradation to report, not an outage to cause. - if (denial.reason === "reserve-not-durable" || denial.reason === "journal-corrupt") { + if (denial.reason === "reserve-not-durable" || denial.reason === "journal-corrupt" + || denial.reason === "pool-history-unresolved") { + if (denial.reason === "pool-history-unresolved" && !alreadySent) { + const summary = workflowDenialSummary("workflow-pool-history-unresolved"); + markLocalRequestLogRefusal(logCtx, summary.code); + logCtx.errorCode = summary.code; + recordWorkflowRefusalEvent(rootId, "workflow-pool-history-unresolved", Date.now()); + } return alreadySent; } if (denial.reason !== "spend-limit-exceeded") return true; diff --git a/src/server/workflow-refusal.ts b/src/server/workflow-refusal.ts index 38bfd559584..e20c8e178d1 100644 --- a/src/server/workflow-refusal.ts +++ b/src/server/workflow-refusal.ts @@ -1,3 +1,4 @@ +import { sharedPoolContinuityDenial } from "../lib/spend-reservation-ledger"; /** * The one place that knows how this proxy refuses a turn on its own workflow budget. * @@ -104,8 +105,19 @@ export function workflowRefusalResponse( * children. A request that names a parent is treated as that fan-out; a top-level request is * the conversation and may use the reserved slots. */ +/** Keep storage/integrity failures distinct from unresolved identity evidence. */ +export function poolContinuityRefusalReason(): WorkflowDenial | undefined { + const denial = sharedPoolContinuityDenial(); + if (!denial) return undefined; + return denial.reason === "pool-history-unresolved" ? "workflow-pool-history-unresolved" : "workflow-spend-undurable"; +} + export function admitHttpWorkflowTurn(headers: Headers): WorkflowDecision | undefined { const rootId = headers.get("x-codex-parent-thread-id")?.trim() || undefined; + const continuityRefusal = poolContinuityRefusalReason(); + if (continuityRefusal) { + return { admitted: false, reason: continuityRefusal, rootId: rootId ?? "" }; + } const threadId = headers.get("thread-id")?.trim() || undefined; const lane: WorkflowLane = rootId !== undefined && threadId !== undefined && threadId !== rootId ? "worker" diff --git a/src/types/config.ts b/src/types/config.ts index 1c0f7c5c83c..b60d8e9f606 100644 --- a/src/types/config.ts +++ b/src/types/config.ts @@ -1033,6 +1033,8 @@ export interface OcxConfig { * upgrade against a number nobody chose. */ spend?: OcxSpendConfig; + /** Exact historical salted pool aliases mapped to canonical providers; retained verbatim for fail-closed validation. */ + spendPoolAliases?: Record; /** Opt-in per-account activation of newly reset Codex quota windows. */ codexQuotaAutoRefresh?: Record.proxy`, `providers..noProxy` | An absent `proxy` inherits global egress; `"direct"` or `null` forces direct egress; HTTP(S) and SOCKS5(H) URLs select a provider-owned proxy. `noProxy` uses NO_PROXY syntax and sends a matching destination direct across either a provider-owned or inherited global proxy. `src/lib/provider-egress.ts` owns parsing and request-local resolution. | diff --git a/structure/runtime.md b/structure/runtime.md index 85e0a73cb16..cb56f5ce573 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -216,7 +216,7 @@ and build a ledger by replaying that directory's own journal. A second process o directory is refused even for observe-only spend configuration, while a separate directory is independent. Ordinary stop releases the final reference after listener teardown, and every thrown startup path releases its reference. SQLite and the OS release a crashed owner; no PID, timestamp, -TTL or lock-file deletion participates in recovery. +TTL or lock-file deletion participates in recovery. Startup passes top-level pool-alias evidence with the spend policy. HTTP admission checks [historical continuity](transports/responses-spend.md#historical-pool-continuity-and-rollback) even without a root; failed accounting reads release the active-turn lease. An explicit Codex integration OFF skips startup cache invalidation before the user-scoped catalog serialization lock is resolved. Explicit `sync` and `sync-cache` retain their catalog-only override. diff --git a/structure/transports/responses-spend.md b/structure/transports/responses-spend.md index d401f8a35bc..74e82f1b33c 100644 --- a/structure/transports/responses-spend.md +++ b/structure/transports/responses-spend.md @@ -111,6 +111,47 @@ includes an account ordinal; root and account identity scopes remain independent Before reserving each combo hop, `core-combo.ts` updates the parent tracker's pool to the resolved target provider; child account labels and the logical `combo` label do not create separate pools. +### Historical pool continuity and rollback + +`src/lib/spend-pool-continuity.ts` accepts only explicit operator mappings from exact salted +historical pool aliases to canonical provider IDs, configured in top-level `spendPoolAliases`. +Current account rosters, label prefixes, short IDs, and renamed/deleted providers are not evidence +for automatically assigning old balances. Old canonical-looking aliases also need explicit +mapping when their positive history predates identity metadata. Zero-balance history needs none. + +The ledger retains original scope balances and reservation targets. The canonical view adds each +member once, keeping settled, reserved and unresolved buckets separate. Explicit links can join +previously canonical groups for a verified rename; an already redirected alias cannot be assigned +to a different group. A removed config entry never removes a journaled link. Group activity, +last-seen time and exhaustion govern retention; unidentified positive balances cannot be evicted. +Identity evidence is bounded and retained even when dormant under-limit scopes are evicted. + +Before a mapping authorizes admission, a v1 checkpoint durably carries both unchanged accounting +and optional salted `poolContinuity` metadata. No raw provider/account names are added to the +journal. New reservations still use the routed canonical pool ID. Replays and compaction preserve +the links; complete invalid metadata fails closed, including at the final line, and corruption is +never compacted away. Unparseable torn final JSON keeps the existing conservative replay rule. + +With a configured pool ceiling, any remaining unidentified positive pool history or invalid/ +conflicting mapping refuses admission. HTTP workflow admission and the Responses pre-dispatch +seam check this even without a root ID, before passthrough transports that report sends afterwards. +After routing, that seam also refuses an already-exhausted canonical pool, including mapped historical totals. +It is a snapshot check, not a new atomic reservation for report-only transports: crossing sends, +concurrent preflight admissions and retries reported afterwards retain their existing limitations. +`workflow_pool_history_unresolved` identifies the local 429 without disclosing aliases; storage +or replay failures keep `workflow_spend_undurable`. Already-sent reports still book actual spend. +Observe-only mode has no new token refusal, and root/identity accounting remains independent. + +Supported rollback retains/backports **both** canonical route attribution and the continuity-aware +reader/writer, using the same journal, salt and verified bindings. Restoring an older journal or +salt loses newer spend and is not a supported rollback. Unmodified older binaries are unsupported: +they can read v1 counters but do not enforce the canonical aggregate, can introduce fresh account +label pools, and can discard optional identity metadata during compaction. There is no automatic +downgrade barrier and no unknown-record fence. If that unsupported write has happened, the current +reader requires explicit mappings for the remaining unidentified balances rather than assuming +zero. `tests/lib/spend-pool-continuity.test.ts` exercises this using the frozen pre-change reader, +as well as exact aggregation, active/unresolved sends, retention, failures and rootless preflight. + A booking is confirmed dispatched only once a LATER send exists, because that later send proves the earlier one left. The newest booking stays open, so a reservation the budget hands back during this process's lifetime can still be released for free. diff --git a/tests/config/config-spend-ceilings.test.ts b/tests/config/config-spend-ceilings.test.ts index e80cc9f4154..87eab546feb 100644 --- a/tests/config/config-spend-ceilings.test.ts +++ b/tests/config/config-spend-ceilings.test.ts @@ -16,7 +16,7 @@ import { validateConfigCandidate, } from "../../src/config"; import { configDiagnosticsFromRaw } from "../../src/config/diagnostics"; -import { spendCeilingsConfigured, spendPolicyFromConfig } from "../../src/lib/spend-reservation-ledger"; +import { createSpendReservationLedger, spendCeilingsConfigured, spendPolicyFromConfig } from "../../src/lib/spend-reservation-ledger"; import { removeTreeWithRetry } from "../helpers/remove-tree"; let home = ""; @@ -125,3 +125,24 @@ test("a ceiling on any one scope is enough to turn enforcement on", () => { expect(spendCeilingsConfigured(spendPolicyFromConfig({ pool: { maxTokens: 1 } }))).toBe(true); expect(spendCeilingsConfigured(spendPolicyFromConfig({ retentionDays: 30 }))).toBe(false); }); + + +test("top-level pool aliases preserve all ceilings and malformed hand edits fail closed", () => { + const alias = "a".repeat(32); + const spend = { root: { maxTokens: 200 }, identity: { maxTokens: 150 }, pool: { maxTokens: 100 } }; + const valid = { ...candidate(spend), spendPoolAliases: { [alias]: "fixture-provider" } }; + expect(validateConfigCandidate(valid).ok).toBe(true); + for (const aliases of [null, [], { wrong: "fixture-provider" }, { [alias]: 1 }]) { + const raw = { ...valid, spendPoolAliases: aliases }; + expect(validateConfigCandidate(raw).ok).toBe(false); + writeFileSync(getConfigPath(), JSON.stringify(raw), "utf8"); + const loaded = loadConfig(); + expect(loaded.spend).toEqual(spend); + const resolved = spendPolicyFromConfig(loaded.spend, loaded.spendPoolAliases); + expect(resolved.pool.maxTokens).toBe(100); + expect(createSpendReservationLedger({ policy: resolved }).checkPoolContinuity()?.reason).toBe("pool-history-unresolved"); + } + // The old schema's passthrough top level retains this key without routing it through + // the strict spend object. A nested spelling remains rejected rather than blessed. + expect(validateConfigCandidate(candidate({ ...spend, poolAliases: { [alias]: "fixture-provider" } })).ok).toBe(false); +}); diff --git a/tests/fixtures/spend-ledger-f7a50dc3.ts.txt b/tests/fixtures/spend-ledger-f7a50dc3.ts.txt new file mode 100644 index 00000000000..48c5dae3d18 --- /dev/null +++ b/tests/fixtures/spend-ledger-f7a50dc3.ts.txt @@ -0,0 +1,586 @@ +// Unchanged parser and ledger function excerpts from lidge-jun/opencodex +// f7a50dc35b493d29f7a5c6bf2a459e5441549580:src/lib/spend-reservation-ledger.ts +// Frozen deliberately: tests execute the old implementation, never emulate it. +const isCountable = (value: unknown): value is number => + typeof value === "number" && Number.isFinite(value) && value >= 0; + +const isAlias = (value: unknown): value is string => + typeof value === "string" && value.length > 0 && value.length <= 256; + +const isScopeName = (value: unknown): value is SpendScope => + value === "root" || value === "identity" || value === "pool"; + +const isStatus = (value: unknown): value is ReservationStatus => + value === "open" || value === "dispatched" || value === "settled" + || value === "lost" || value === "abandoned"; + +const parseTargets = (value: unknown): ScopeRef[] | undefined => { + if (!Array.isArray(value) || value.length > 3) return undefined; + const targets: ScopeRef[] = []; + for (const entry of value) { + if (typeof entry !== "object" || entry === null) return undefined; + const { scope, alias } = entry as { scope?: unknown; alias?: unknown }; + if (!isScopeName(scope) || !isAlias(alias)) return undefined; + targets.push({ scope, alias }); + } + return targets; +}; + +/** + * Validate one journal line into a record, or reject it. + * + * Exported because this is the boundary where a hostile or damaged file meets the accounting: + * `JSON.parse(line) as JournalRecord` type-asserts a lie, and a bare `null` line or a + * `{"v":1,"kind":"reserve"}` with no fields crashed the rebuild rather than being rejected. + * Every field is checked, including that numbers are finite and non-negative. + */ +export function parseSpendJournalRecord(line: string): JournalRecord | undefined { + let raw: unknown; + try { + raw = JSON.parse(line); + } catch { + return undefined; + } + if (typeof raw !== "object" || raw === null || Array.isArray(raw)) return undefined; + const record = raw as Record; + if (record.v !== 1) return undefined; + if (!isCountable(record.at)) return undefined; + const at = record.at; + switch (record.kind) { + case "reserve": { + const targets = parseTargets(record.targets); + if (!isAlias(record.send) || targets === undefined || !isCountable(record.tokens)) return undefined; + return { v: 1, kind: "reserve", send: record.send, targets, tokens: record.tokens, at }; + } + case "settle": + if (!isAlias(record.send) || !isCountable(record.tokens)) return undefined; + return { v: 1, kind: "settle", send: record.send, tokens: record.tokens, at }; + case "dispatch": + if (!isAlias(record.send)) return undefined; + return { v: 1, kind: "dispatch", send: record.send, at }; + case "lost": + if (!isAlias(record.send)) return undefined; + return { v: 1, kind: "lost", send: record.send, at }; + case "abandon": + if (!isAlias(record.send)) return undefined; + return { v: 1, kind: "abandon", send: record.send, at }; + case "forget": + if (!isAlias(record.send)) return undefined; + return { v: 1, kind: "forget", send: record.send, at }; + case "drop": + if (!isScopeName(record.scope) || !isAlias(record.alias)) return undefined; + return { v: 1, kind: "drop", scope: record.scope, alias: record.alias, at }; + case "checkpoint": { + if (!Array.isArray(record.scopes) || !Array.isArray(record.sends)) return undefined; + const scopes: { scope: SpendScope; alias: string; settled: number; unresolved: number; seenAt: number }[] = []; + for (const entry of record.scopes) { + if (typeof entry !== "object" || entry === null) return undefined; + const e = entry as Record; + if (!isScopeName(e.scope) || !isAlias(e.alias)) return undefined; + if (!isCountable(e.settled) || !isCountable(e.unresolved) || !isCountable(e.seenAt)) return undefined; + scopes.push({ scope: e.scope, alias: e.alias, settled: e.settled, unresolved: e.unresolved, seenAt: e.seenAt }); + } + const sends: { send: string; status: ReservationStatus; targets: ScopeRef[]; tokens: number; at: number; resolvedAt: number }[] = []; + for (const entry of record.sends) { + if (typeof entry !== "object" || entry === null) return undefined; + const e = entry as Record; + const targets = parseTargets(e.targets); + if (!isAlias(e.send) || !isStatus(e.status) || targets === undefined) return undefined; + if (!isCountable(e.tokens) || !isCountable(e.at) || !isCountable(e.resolvedAt)) return undefined; + sends.push({ send: e.send, status: e.status, targets, tokens: e.tokens, at: e.at, resolvedAt: e.resolvedAt }); + } + return { v: 1, kind: "checkpoint", at, scopes, sends }; + } + default: + return undefined; + } +} + +const scopeKey = (scope: SpendScope, alias: string): string => scope + "\0" + alias; + +const sanitizeTokens = (value: number): number => + Number.isFinite(value) ? Math.max(0, Math.trunc(value)) : 0; + +export function createSpendReservationLedger(options: { + readonly journal?: SpendJournal; + readonly policy?: SpendReservationPolicy; + readonly now?: () => number; + /** + * Per-install alias salt. Production passes the file-backed value from + * `loadOrCreateSpendLedgerSalt`; an empty default is for in-memory journals, which have + * no file anyone could correlate. + */ + readonly salt?: string; + /** + * Shared production ledgers prove their exact ownership before reading or changing + * accounting. Identity, not just the directory name: a handle kept across a release and a + * reacquire describes a journal another writer may have changed in between. + */ + readonly assertOwnedAccounting?: () => void; +} = {}): SpendReservationLedger { + // Mutable because the ceilings are operator configuration, and configuration is reloadable. + // The three bounds below are read through functions for the same reason: a value captured + // at construction would answer for the policy this ledger was BUILT with, and an operator + // who raised a bound would keep the old one until the process restarted. + let policy = options.policy ?? DEFAULT_SPEND_RESERVATION_POLICY; + const journal = options.journal; + const now = options.now ?? (() => Date.now()); + const salt = options.salt ?? ""; + const assertOwnedAccounting = options.assertOwnedAccounting; + const maxTrackedScopes = (): number => policy.maxTrackedScopes ?? DEFAULT_MAX_TRACKED_SCOPES; + const maxTrackedSends = (): number => policy.maxTrackedSends ?? DEFAULT_MAX_TRACKED_SENDS; + const compactAfterRecords = (): number => policy.compactAfterRecords ?? DEFAULT_COMPACT_AFTER_RECORDS; + const scopes = new Map(); + const reservations = new Map(); + let persistFailures = 0; + let corruptRecords = 0; + let recordsOnDisk = 0; + + /** + * Salted alias for one identifier. The raw value -- a client-supplied root header, a + * credential id, a pool name -- never leaves this function, so nothing identifying is + * written to disk or held in a map key. + */ + const aliasFor = (kind: SpendScope | "send", id: string): string => + createHash("sha256").update(salt).update("\u0000").update(kind).update("\u0000").update(id) + .digest("hex").slice(0, 32); + + const scopeState = (scope: SpendScope, alias: string): ScopeState => { + const key = scopeKey(scope, alias); + let state = scopes.get(key); + if (!state) { + state = { settled: 0, reserved: 0, unresolved: 0, lastSeenAt: 0 }; + scopes.set(key, state); + } + return state; + }; + + const limitFor = (scope: SpendScope): number | undefined => policy[scope].maxTokens; + + const isExhausted = (scope: SpendScope, state: ScopeState): boolean => { + const limit = limitFor(scope); + return limit !== undefined && state.settled + state.reserved + state.unresolved >= limit; + }; + + /** The scopes a request touches, as aliases. Creates no state: a refusal must leave none. */ + const refsFor = (targets: SpendScopes): ScopeRef[] => { + const refs: ScopeRef[] = []; + if (targets.rootId !== undefined) refs.push({ scope: "root", alias: aliasFor("root", targets.rootId) }); + if (targets.identityId !== undefined) refs.push({ scope: "identity", alias: aliasFor("identity", targets.identityId) }); + if (targets.poolId !== undefined) refs.push({ scope: "pool", alias: aliasFor("pool", targets.poolId) }); + return refs; + }; + + /** + * Returns whether the record reached storage. With no journal there is nothing to fail, + * and the caller's durability question is vacuously satisfied. + */ + const append = (record: JournalRecord): boolean => { + if (!journal) return true; + try { + journal.append(JSON.stringify(record)); + recordsOnDisk += 1; + return true; + } catch (error) { + if (error instanceof SpendLedgerOwnerError) throw error; + // In-memory state still bounds this process; the counter is how a caller learns the + // restart guarantee degraded instead of discovering it after the fact. + persistFailures += 1; + return false; + } + }; + + const applyReserve = (send: string, targets: readonly ScopeRef[], tokens: number, at: number): void => { + if (reservations.has(send)) return; + reservations.set(send, { targets, tokens, status: "open", at, resolvedAt: at }); + for (const ref of targets) { + const state = scopeState(ref.scope, ref.alias); + state.reserved += tokens; + state.lastSeenAt = Math.max(state.lastSeenAt, at); + } + }; + + const isLive = (status: ReservationStatus): boolean => status === "open" || status === "dispatched"; + + /** + * Resolve a live reservation. `settled` books the real figure, `lost` keeps the whole + * reservation as unresolved spend because it may have been billed, and `abandoned` + * releases it because no byte ever left this process. + */ + const applyResolve = (send: string, outcome: "settled" | "lost" | "abandoned", tokens: number, at: number): void => { + const reservation = reservations.get(send); + if (!reservation || !isLive(reservation.status)) return; + reservation.status = outcome; + reservation.resolvedAt = at; + for (const ref of reservation.targets) { + const state = scopeState(ref.scope, ref.alias); + state.reserved = Math.max(0, state.reserved - reservation.tokens); + if (outcome === "lost") state.unresolved += reservation.tokens; + else if (outcome === "settled") state.settled += tokens; + state.lastSeenAt = Math.max(state.lastSeenAt, at); + } + }; + + const applyDispatch = (send: string, at: number): void => { + const reservation = reservations.get(send); + if (!reservation || reservation.status !== "open") return; + reservation.status = "dispatched"; + reservation.resolvedAt = at; + }; + + /** Tombstone replay: the entry is gone, so a later reuse of the id books a fresh charge. */ + const applyForget = (send: string): void => { + const reservation = reservations.get(send); + if (!reservation || isLive(reservation.status)) return; + reservations.delete(send); + }; + + const applyDrop = (scope: SpendScope, alias: string): void => { + const state = scopes.get(scopeKey(scope, alias)); + if (!state || state.reserved > 0) return; + scopes.delete(scopeKey(scope, alias)); + }; + + const applyCheckpoint = (record: Extract): void => { + scopes.clear(); + reservations.clear(); + for (const entry of record.scopes) { + scopes.set(scopeKey(entry.scope, entry.alias), { + settled: entry.settled, + reserved: 0, + unresolved: entry.unresolved, + lastSeenAt: entry.seenAt, + }); + } + for (const entry of record.sends) { + // `reserved` is rebuilt from the live entries rather than trusted from the snapshot, + // so the two can never disagree about the same tokens. + if (isLive(entry.status)) { + applyReserve(entry.send, entry.targets, entry.tokens, entry.at); + if (entry.status === "dispatched") applyDispatch(entry.send, entry.resolvedAt); + continue; + } + reservations.set(entry.send, { + targets: entry.targets, + tokens: entry.tokens, + status: entry.status, + at: entry.at, + resolvedAt: entry.resolvedAt, + }); + } + }; + + // Rebuild from the journal before serving: an exhausted scope must still be exhausted + // after a restart, which is the whole reason this store exists. + if (journal) { + const lines = journal.read(); + recordsOnDisk = lines.length; + for (let index = 0; index < lines.length; index += 1) { + const line = lines[index] as string; + const record = parseSpendJournalRecord(line); + if (!record) { + // A rejected FINAL line is a torn tail write -- the process died between the write + // and its newline -- and is dropped quietly, because that record never completed and + // therefore never authorised anything. A rejected line ANYWHERE ELSE is different: + // the records after it did complete, so skipping it silently undercounts a scope and + // hands back budget. It is counted, and a configured limit refuses on it below. + if (index < lines.length - 1) corruptRecords += 1; + continue; + } + switch (record.kind) { + case "reserve": applyReserve(record.send, record.targets, sanitizeTokens(record.tokens), record.at); break; + case "dispatch": applyDispatch(record.send, record.at); break; + case "settle": applyResolve(record.send, "settled", sanitizeTokens(record.tokens), record.at); break; + case "lost": applyResolve(record.send, "lost", 0, record.at); break; + case "abandon": applyResolve(record.send, "abandoned", 0, record.at); break; + case "forget": applyForget(record.send); break; + case "drop": applyDrop(record.scope, record.alias); break; + case "checkpoint": applyCheckpoint(record); break; + } + } + // A reservation that survived replay has no owner left. The process that made it is gone, + // so nothing in this one can ever settle it, and leaving it live means the send stays + // pending forever against a scope that can never resolve it. Deleting the entry is not the + // alternative either: that would hand the same send id a second reservation. + // + // Both live states resolve to UNRESOLVED, including an undispatched one. The tempting + // distinction -- open never reached the wire, so give its tokens back -- assumes the + // journal is complete up to the crash, and the torn-tail handling above says it is not: a + // send can dispatch and die before its dispatch record lands. Abandoning that reservation + // returns tokens for a send that may have been billed, and worse, it RESETS a ceiling that + // had already fired. An exhausted scope staying exhausted across a restart is the whole + // reason this store is on disk. + const reconciledAt = now(); + for (const [send, reservation] of reservations) { + if (!isLive(reservation.status)) continue; + applyResolve(send, "lost", 0, reconciledAt); + append({ v: 1, kind: "lost", send, at: reconciledAt }); + } + } + + /** + * Bounded cleanup. It runs before every admission, so nothing depends on a caller + * remembering `prune()` -- the first draft exported one and no production path called it. + * Every removal writes a tombstone: without one, replay rebuilds precisely what cleanup + * removed and the file keeps growing while the maps look bounded. + * + * `force` is the at-capacity pass. It ignores the retention window but never the safety + * rule: an ACTIVE or EXHAUSTED scope is not a candidate at any pressure, because dropping + * one hands it a fresh allowance under the same id. When that leaves nothing to remove, + * the caller refuses admission rather than making room by forgetting a spent scope. + */ + const evictScopes = (at: number, force: boolean): number => { + const cutoff = at - policy.retentionMs; + const candidates: { key: string; scope: SpendScope; alias: string; seenAt: number }[] = []; + for (const [key, state] of scopes) { + const separator = key.indexOf("\0"); + const scope = key.slice(0, separator) as SpendScope; + if (state.reserved > 0) continue; + if (isExhausted(scope, state)) continue; + if (!force && state.lastSeenAt >= cutoff) continue; + candidates.push({ key, scope, alias: key.slice(separator + 1), seenAt: state.lastSeenAt }); + } + if (force) { + candidates.sort((a, b) => a.seenAt - b.seenAt); + candidates.length = Math.min(candidates.length, 1); + } + for (const candidate of candidates) { + scopes.delete(candidate.key); + append({ v: 1, kind: "drop", scope: candidate.scope, alias: candidate.alias, at }); + } + return candidates.length; + }; + + /** + * Forget resolved send ids. A forgotten id is forgotten COMPLETELY: reusing it later books + * a fresh reservation against every scope, which is conservative. The state this must never + * produce is the middle one -- an id the ledger recognises but charges nothing for. + */ + const evictSends = (at: number, force: boolean): number => { + const cutoff = at - policy.retentionMs; + const candidates: { send: string; resolvedAt: number }[] = []; + for (const [send, reservation] of reservations) { + if (isLive(reservation.status)) continue; + if (!force && reservation.resolvedAt >= cutoff) continue; + candidates.push({ send, resolvedAt: reservation.resolvedAt }); + } + if (force) { + candidates.sort((a, b) => a.resolvedAt - b.resolvedAt); + candidates.length = Math.min(candidates.length, 1); + } + for (const candidate of candidates) { + reservations.delete(candidate.send); + append({ v: 1, kind: "forget", send: candidate.send, at }); + } + return candidates.length; + }; + + /** + * Replace the journal with a single checkpoint once it has grown past its record budget. + * Bounded maps are not enough on their own: the file behind them is what replay reads, and + * an uncompacted file grows forever on unique root and send ids. + */ + const compact = (at: number): void => { + const rewrite = journal?.rewrite; + if (!journal || !rewrite || recordsOnDisk < compactAfterRecords()) return; + const checkpoint: JournalRecord = { + v: 1, + kind: "checkpoint", + at, + scopes: [...scopes].map(([key, state]) => { + const separator = key.indexOf("\0"); + return { + scope: key.slice(0, separator) as SpendScope, + alias: key.slice(separator + 1), + settled: state.settled, + unresolved: state.unresolved, + seenAt: state.lastSeenAt, + }; + }), + sends: [...reservations].map(([send, reservation]) => ({ + send, + status: reservation.status, + targets: [...reservation.targets], + tokens: reservation.tokens, + at: reservation.at, + resolvedAt: reservation.resolvedAt, + })), + }; + try { + rewrite.call(journal, [JSON.stringify(checkpoint)]); + recordsOnDisk = 1; + } catch (error) { + if (error instanceof SpendLedgerOwnerError) throw error; + // Compaction is maintenance, not accounting: a failed rewrite leaves the previous + // journal intact and every figure in it still replayable. + persistFailures += 1; + } + }; + + /** The denial when tracking cannot fit this request, or undefined when it can. */ + const makeRoom = (refs: readonly ScopeRef[], at: number): SpendDenial | undefined => { + evictSends(at, false); + evictScopes(at, false); + while (reservations.size >= maxTrackedSends()) { + if (evictSends(at, true) === 0) return { reason: "tracking-capacity-exhausted" }; + } + let fresh = 0; + for (const ref of refs) if (!scopes.has(scopeKey(ref.scope, ref.alias))) fresh += 1; + while (scopes.size + fresh > maxTrackedScopes()) { + if (evictScopes(at, true) === 0) { + return { reason: "tracking-capacity-exhausted", scope: refs[0]?.scope }; + } + } + return undefined; + }; + + return { + // Every figure this ledger reports describes a journal it must still own. Reporting one + // after ownership ended is the same error as writing then, with a quieter symptom. + get persistFailures() { assertOwnedAccounting?.(); return persistFailures; }, + get corruptRecords() { assertOwnedAccounting?.(); return corruptRecords; }, + get degraded() { assertOwnedAccounting?.(); return persistFailures > 0 || corruptRecords > 0; }, + get policy() { assertOwnedAccounting?.(); return policy; }, + + reserve(request: SpendReservationRequest): SpendReservationDecision { + assertOwnedAccounting?.(); + const tokens = sanitizeTokens(request.inputTokens) + sanitizeTokens(request.outputCeilingTokens); + const at = request.at ?? now(); + const send = aliasFor("send", request.sendId); + const refs = refsFor(request.scopes); + const enforced = refs.some((ref) => limitFor(ref.scope) !== undefined); + + // A send id this ledger already knows is REFUSED. Returning success while booking + // nothing -- the old behaviour -- let one id authorise an unlimited number of physical + // sends with the scope totals never moving. + if (reservations.has(send)) { + return { reserved: false, denial: { reason: "duplicate-send-id", sendId: request.sendId } }; + } + // Replay could not prove these totals are complete, so a configured ceiling cannot be + // enforced on them. Observe-only accounting continues and reports the degradation. + if (enforced && corruptRecords > 0) { + return { reserved: false, denial: { reason: "journal-corrupt", corruptRecords } }; + } + const capacity = makeRoom(refs, at); + if (capacity) return { reserved: false, denial: capacity }; + + // Check every scope before mutating any: a refusal must not leave a partial + // reservation booked on the scopes that would have passed. Reading state without + // creating it matters here -- a denied request must not leave a tracked scope behind. + for (const ref of refs) { + // A recorded send has no limit to fail: it already happened, and the point of booking + // it is to let the total go OVER the ceiling so the next request can be refused. + const limit = request.alreadySent === true ? undefined : limitFor(ref.scope); + if (limit === undefined) continue; + const state = scopes.get(scopeKey(ref.scope, ref.alias)); + const projected = (state ? state.settled + state.reserved + state.unresolved : 0) + tokens; + if (projected > limit) { + const scopeId = ref.scope === "root" + ? request.scopes.rootId + : ref.scope === "identity" ? request.scopes.identityId : request.scopes.poolId; + return { + reserved: false, + denial: { reason: "spend-limit-exceeded", scope: ref.scope, scopeId: scopeId ?? "", limit, projected }, + }; + } + } + + // Durability BEFORE admission. The record goes to disk first, and under a configured + // limit a failed write refuses the request rather than admitting one that a restart + // would forget -- which is exactly the disk-full and permission case durability is for. + const durable = append({ v: 1, kind: "reserve", send, targets: refs, tokens, at }); + if (!durable && enforced && request.alreadySent !== true) { + return { reserved: false, denial: { reason: "reserve-not-durable", sendId: request.sendId } }; + } + applyReserve(send, refs, tokens, at); + compact(at); + return { reserved: true, sendId: request.sendId, tokens, durable }; + }, + + markDispatched(sendId: string): boolean { + assertOwnedAccounting?.(); + const send = aliasFor("send", sendId); + const reservation = reservations.get(send); + if (!reservation || reservation.status !== "open") return false; + const at = now(); + applyDispatch(send, at); + append({ v: 1, kind: "dispatch", send, at }); + return true; + }, + + abandon(sendId: string): boolean { + assertOwnedAccounting?.(); + const send = aliasFor("send", sendId); + const reservation = reservations.get(send); + // Only an UNDISPATCHED reservation may be released for free. Once bytes have left for + // upstream the tokens may already be billed, so the caller owes settle or markLost. + if (!reservation || reservation.status !== "open") return false; + const at = now(); + applyResolve(send, "abandoned", 0, at); + append({ v: 1, kind: "abandon", send, at }); + return true; + }, + + settle(sendId: string, usage: SpendUsage): boolean { + assertOwnedAccounting?.(); + const send = aliasFor("send", sendId); + const reservation = reservations.get(send); + if (!reservation || !isLive(reservation.status)) return false; + const tokens = sanitizeTokens(usage.inputTokens) + sanitizeTokens(usage.outputTokens); + const at = now(); + applyResolve(send, "settled", tokens, at); + append({ v: 1, kind: "settle", send, tokens, at }); + return true; + }, + + markLost(sendId: string): boolean { + assertOwnedAccounting?.(); + const send = aliasFor("send", sendId); + const reservation = reservations.get(send); + if (!reservation || !isLive(reservation.status)) return false; + const at = now(); + applyResolve(send, "lost", 0, at); + append({ v: 1, kind: "lost", send, at }); + return true; + }, + + knows(sendId: string): boolean { + assertOwnedAccounting?.(); + return reservations.has(aliasFor("send", sendId)); + }, + + snapshot(scope: SpendScope, scopeId: string): ScopeSpendSnapshot | undefined { + // Reading accounting from a handle whose ownership has ended is as wrong as writing it: + // the figures describe a journal this process no longer owns. + assertOwnedAccounting?.(); + const state = scopes.get(scopeKey(scope, aliasFor(scope, scopeId))); + if (!state) return undefined; + return { + settled: state.settled, + reserved: state.reserved, + unresolved: state.unresolved, + exhausted: isExhausted(scope, state), + }; + }, + + exhausted(scope: SpendScope, scopeId: string): boolean { + assertOwnedAccounting?.(); + const state = scopes.get(scopeKey(scope, aliasFor(scope, scopeId))); + return state !== undefined && isExhausted(scope, state); + }, + + prune(at: number = now()): void { + assertOwnedAccounting?.(); + // Removal requires BOTH inactive and not exhausted inside the window. An + // exhausted-but-idle scope that was dropped would be recreated fresh under the + // same id -- the exact laundering the ceiling exists to stop. + evictSends(at, false); + evictScopes(at, false); + }, + + reconfigure(next: SpendReservationPolicy): void { + assertOwnedAccounting?.(); + policy = next; + }, + }; +} diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 4303c879bac..22cd290dade 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -1646,6 +1646,7 @@ "spend-ledger-lifecycle.test.ts": "server", "spend-ledger-owner-startup.test.ts": "server", "spend-ledger-owner.test.ts": "lib", + "spend-pool-continuity.test.ts": "lib", "spend-reservation-ledger.test.ts": "lib", "sponsor-presets.test.ts": "providers", "sse-client-frame-bounds.test.ts": "responses", diff --git a/tests/helpers/legacy-spend-ledger.ts b/tests/helpers/legacy-spend-ledger.ts new file mode 100644 index 00000000000..2de285d36ed --- /dev/null +++ b/tests/helpers/legacy-spend-ledger.ts @@ -0,0 +1,16 @@ +import { readFileSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { fixturePath } from "./repo-root"; +import { DEFAULT_SPEND_RESERVATION_POLICY, type SpendJournal, type SpendReservationLedger, type SpendReservationPolicy } from "../../src/lib/spend-reservation-ledger"; + +/** Execute immutable pre-continuity production code, with only its ordinary imports supplied. */ +const source = readFileSync(fixturePath("spend-ledger-f7a50dc3.ts.txt"), "utf8"); +const code = new Bun.Transpiler({ loader: "ts", target: "bun" }) + .transformSync(source.replaceAll("export function", "function")); +export const createLegacySpendLedger = new Function( + "createHash", "DEFAULT_SPEND_RESERVATION_POLICY", "DEFAULT_MAX_TRACKED_SCOPES", + "DEFAULT_MAX_TRACKED_SENDS", "DEFAULT_COMPACT_AFTER_RECORDS", "SpendLedgerOwnerError", + `${code}\nreturn createSpendReservationLedger;`, +)(createHash, DEFAULT_SPEND_RESERVATION_POLICY, 4_096, 16_384, 8_192, class SpendLedgerOwnerError extends Error {}) as + (options: { journal: SpendJournal; salt?: string; policy: SpendReservationPolicy; now: () => number }) => + Pick; diff --git a/tests/lib/spend-pool-continuity.test.ts b/tests/lib/spend-pool-continuity.test.ts new file mode 100644 index 00000000000..5f47660df84 --- /dev/null +++ b/tests/lib/spend-pool-continuity.test.ts @@ -0,0 +1,249 @@ +import { createLegacySpendLedger } from "../helpers/legacy-spend-ledger"; +import { describe, expect, test } from "bun:test"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createHash } from "node:crypto"; +import { + createSpendReservationLedger, configureSharedSpendLedger, DEFAULT_SPEND_RESERVATION_POLICY, parseSpendJournalRecord, + type SpendJournal, type SpendReservationPolicy, +} from "../../src/lib/spend-reservation-ledger"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { admitHttpWorkflowTurn, workflowDecisionRefusalResponse } from "../../src/server/workflow-refusal"; +import { createResponsesSendBudget } from "../../src/server/responses/request-send-budget"; +import { createRequestSpendTracker } from "../../src/server/responses/request-spend"; + +const salt = "5".repeat(64); +const alias = (kind: string, id: string) => createHash("sha256").update(salt).update("\0").update(kind).update("\0").update(id).digest("hex").slice(0, 32); +const pool = (id: string) => alias("pool", id); +const policy = (poolAliases?: unknown, overrides: Partial = {}): SpendReservationPolicy => ({ + ...DEFAULT_SPEND_RESERVATION_POLICY, pool: { maxTokens: 100 }, poolAliases, ...overrides, +}); +const journal = (records: unknown[] = []): SpendJournal & { lines: string[] } => { + const lines = records.map(record => JSON.stringify(record)); + return { lines, read: () => [...lines], append: line => { lines.push(line); }, + rewrite: next => { lines.splice(0, lines.length, ...next); } }; +}; +const checkpoint = (entries: Array<[string, number, number]>) => ({ + v: 1, kind: "checkpoint", at: 1, + scopes: entries.map(([id, settled, unresolved]) => ({ scope: "pool", alias: pool(id), settled, unresolved, seenAt: 1 })), sends: [], +}); +const reserve = (ledger: ReturnType, sendId: string, poolId = "provider", tokens = 1, alreadySent = false) => + ledger.reserve({ sendId, scopes: { poolId }, inputTokens: tokens, outputCeilingTokens: 0, alreadySent }); + +describe("historical pool identity continuity", () => { + test("unmapped historical debt fails closed across labels, providers, pruning and restart", () => { + const disk = journal([checkpoint([["provider-old-label", 100, 0]])]); + for (let restart = 0; restart < 2; restart += 1) { + const ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(), now: () => 100_000 }); + ledger.prune(); + for (const id of ["provider", "provider-old-label", "unrelated-provider"]) { + expect(reserve(ledger, `${restart}-${id}`, id)).toMatchObject({ reserved: false, denial: { reason: "pool-history-unresolved" } }); + } + expect(ledger.snapshot("pool", "provider-old-label")?.settled).toBe(100); + } + }); + + test("explicit aliases aggregate each original balance once, including canonical history", () => { + const disk = journal([checkpoint([["label-a", 40, 0], ["label-b", 0, 30], ["provider", 20, 0]])]); + const aliases = { [pool("label-a")]: "provider", [pool("label-b")]: "provider", [pool("provider")]: "provider" }; + let ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(aliases, { compactAfterRecords: 1 }), now: () => 2 }); + expect(ledger.checkPoolContinuity()).toBeUndefined(); + expect(ledger.snapshot("pool", "provider")).toEqual({ settled: 60, unresolved: 30, reserved: 0, exhausted: false }); + expect(reserve(ledger, "new", "provider", 10).reserved).toBe(true); + expect(ledger.settle("new", { inputTokens: 10, outputTokens: 0 })).toBe(true); + expect(ledger.settle("new", { inputTokens: 10, outputTokens: 0 })).toBe(false); + // Clearing config never removes durable evidence. Repeated replay/compaction never adds + // a migrated copy of a balance or charges a canonical self-alias twice. + for (let restart = 0; restart < 3; restart += 1) { + ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(undefined, { compactAfterRecords: 1 }), now: () => 3 }); + expect(ledger.checkPoolContinuity()).toBeUndefined(); + expect(ledger.snapshot("pool", "provider")).toEqual({ settled: 70, unresolved: 30, reserved: 0, exhausted: true }); + expect(reserve(ledger, `denied-${restart}`)).toMatchObject({ reserved: false, denial: { reason: "spend-limit-exceeded", projected: 101 } }); + } + expect(disk.lines.join("\n")).not.toContain("label-a"); + expect(disk.lines.join("\n")).not.toContain("provider"); + }); + + test("old open/dispatched sends become unresolved exactly once; duplicate send IDs remain refused", () => { + const old = ["a", "b"].flatMap(id => [ + { v: 1, kind: "reserve", send: alias("send", id), targets: [{ scope: "pool", alias: pool(`label-${id}`) }], tokens: 30, at: 1 }, + ...(id === "b" ? [{ v: 1, kind: "dispatch", send: alias("send", id), at: 1 }] : []), + ]); + const disk = journal(old); + const aliases = { [pool("label-a")]: "provider", [pool("label-b")]: "provider" }; + for (let count = 0; count < 2; count += 1) { + const ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(aliases), now: () => 2 }); + expect(ledger.checkPoolContinuity()).toBeUndefined(); + expect(ledger.snapshot("pool", "provider")?.unresolved).toBe(60); + expect(reserve(ledger, "a")).toMatchObject({ reserved: false, denial: { reason: "duplicate-send-id" } }); + } + }); + + test("a live original reservation settles/refunds once after explicit linking", () => { + const ledger = createSpendReservationLedger({ salt, policy: policy(), now: () => 2 }); + expect(reserve(ledger, "pending", "label", 40).reserved).toBe(true); + expect(reserve(ledger, "refund", "label", 10).reserved).toBe(true); + ledger.reconfigure(policy({ [pool("label")]: "provider" })); + expect(ledger.checkPoolContinuity()).toBeUndefined(); + expect(ledger.abandon("refund")).toBe(true); + expect(ledger.settle("pending", { inputTokens: 30, outputTokens: 0 })).toBe(true); + expect(ledger.snapshot("pool", "provider")).toEqual({ settled: 30, reserved: 0, unresolved: 0, exhausted: false }); + }); + + test("zero/abandoned-only historical scopes do not create debt", () => { + const ledger = createSpendReservationLedger({ journal: journal([checkpoint([["empty-old", 0, 0]])]), salt, policy: policy(), now: () => 2 }); + expect(reserve(ledger, "new").reserved).toBe(true); + }); + + test("unknown history and aggregate exhaustion survive retention and capacity pressure", () => { + const disk = journal([checkpoint([["label-a", 60, 0], ["label-b", 40, 0]])]); + const ledger = createSpendReservationLedger({ journal: disk, salt, + policy: policy({ [pool("label-a")]: "provider", [pool("label-b")]: "provider" }, { retentionMs: 1, maxTrackedScopes: 2 }), now: () => 100 }); + expect(ledger.checkPoolContinuity()).toBeUndefined(); + ledger.prune(); + expect(ledger.snapshot("pool", "provider")?.settled).toBe(100); + expect(reserve(ledger, "new").reserved).toBe(false); + expect(ledger.snapshot("pool", "provider")?.settled).toBe(100); + }); + + test("under-limit historical components remain while their canonical group is active", () => { + const disk = journal([checkpoint([["label", 40, 0]])]); + let now = 2; + const ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy({ [pool("label")]: "provider" }, { retentionMs: 5 }), now: () => now }); + expect(reserve(ledger, "pending", "provider", 10).reserved).toBe(true); + now = 100; + ledger.prune(); + expect(ledger.snapshot("pool", "provider")).toMatchObject({ settled: 40, reserved: 10 }); + }); + + test("observe-only records already-sent requests while ambiguity still blocks new dispatches", () => { + const disk = journal([checkpoint([["unknown-label", 40, 0]])]); + const ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(), now: () => 2 }); + const tracker = createRequestSpendTracker({ provider: "provider-display", spendPoolId: "provider", usageLogInputTokens: 10 }, undefined, ledger); + expect(tracker.charge()).toBe(false); + expect(tracker.charge({ alreadySent: true })).toBe(true); + tracker.settle({ inputTokens: 8, outputTokens: 0 }); + expect(ledger.snapshot("pool", "provider")?.settled).toBe(8); + expect(ledger.snapshot("pool", "unknown-label")?.settled).toBe(40); + expect(reserve(ledger, "new").reserved).toBe(false); + }); + + test("malformed or conflicting maps fail closed without clearing ceilings or saved links", () => { + const disk = journal([checkpoint([["label", 40, 0]])]); + const ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy({ [pool("label")]: "provider" }), now: () => 2 }); + expect(ledger.checkPoolContinuity()).toBeUndefined(); + for (const aliases of [null, [], { label: "provider" }, { [pool("label")]: "different-provider" }]) { + ledger.reconfigure(policy(aliases)); + expect(ledger.policy.pool.maxTokens).toBe(100); + expect(ledger.checkPoolContinuity()?.reason).toBe("pool-history-unresolved"); + expect(reserve(ledger, "new").reserved).toBe(false); + expect(ledger.snapshot("pool", "provider")?.settled).toBe(40); + } + ledger.reconfigure(policy()); + expect(ledger.checkPoolContinuity()).toBeUndefined(); + }); + + test("evidence write failure cannot publish a mapping or erase historical balances", () => { + const disk = journal([checkpoint([["label", 40, 0]])]); + disk.append = () => { throw new Error("synthetic write failure"); }; + const ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy({ [pool("label")]: "provider" }), now: () => 2 }); + expect(reserve(ledger, "new")).toMatchObject({ reserved: false, denial: { reason: "reserve-not-durable" } }); + expect(ledger.snapshot("pool", "provider")).toBeUndefined(); + expect(ledger.snapshot("pool", "label")?.settled).toBe(40); + expect(disk.lines).toHaveLength(1); + }); + + test("complete invalid metadata at the final line fails closed and survives attempted compaction", () => { + const invalid = { ...checkpoint([["label", 40, 0]]), poolContinuity: { v: 1, kind: "pool-continuity", at: 1, bindings: [{ alias: "bad", canonical: pool("provider") }] } }; + const disk = journal([invalid]); + let ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(), now: () => 2 }); + expect(ledger.corruptRecords).toBe(1); + expect(ledger.checkPoolContinuity()?.reason).toBe("journal-corrupt"); + ledger.reconfigure(policy(undefined, { pool: {}, compactAfterRecords: 1 })); + reserve(ledger, "observed", "provider", 1, true); + ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(), now: () => 3 }); + expect(ledger.corruptRecords).toBe(1); + }); + + test("a v1 checkpoint remains parseable when compatibility metadata is omitted by an old reader", () => { + const disk = journal(); + const ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(undefined, { compactAfterRecords: 1 }), now: () => 2 }); + reserve(ledger, "new", "provider", 30); + ledger.settle("new", { inputTokens: 25, outputTokens: 0 }); + expect(disk.lines.every(line => parseSpendJournalRecord(line) !== undefined)).toBe(true); + const oldView = JSON.parse(disk.lines[0]!); + delete oldView.poolContinuity; + // Unmodified old readers retain raw v1 counters, but cannot prove canonical continuity. + expect(parseSpendJournalRecord(JSON.stringify(oldView))).toBeDefined(); + const restart = createSpendReservationLedger({ journal: disk, salt, policy: policy(), now: () => 3 }); + expect(restart.checkPoolContinuity()).toBeUndefined(); + expect(restart.snapshot("pool", "provider")?.settled).toBe(25); + }); +}); + + +test("rootless HTTP and passthrough preflight refuse before any synthetic fetch", () => { + const previous = process.env.OPENCODEX_HOME; + const home = mkdtempSync(join(tmpdir(), "ocx-pool-history-")); + process.env.OPENCODEX_HOME = home; + const release = acquireOwnedSpendHome(); + try { + writeFileSync(join(home, "spend-ledger.salt"), salt + "\n", { mode: 0o600 }); + writeFileSync(join(home, "spend-ledger.jsonl"), JSON.stringify(checkpoint([["old-label", 100, 0]])) + "\n", { mode: 0o600 }); + configureSharedSpendLedger(policy()); + let syntheticFetches = 0; + const decision = admitHttpWorkflowTurn(new Headers()); + expect(decision).toMatchObject({ admitted: false, reason: "workflow-pool-history-unresolved" }); + if (!decision || decision.admitted) syntheticFetches += 1; + else { + const response = workflowDecisionRefusalResponse(decision); + expect(response.status).toBe(429); + expect(response.headers.get("x-opencodex-local-refusal")).toBe("workflow_pool_history_unresolved"); + } + const budget = createResponsesSendBudget({ req: new Request("https://fixture.example.test/v1/responses"), options: {}, logCtx: { model: "fixture", provider: "provider" } }); + expect(budget).toBeInstanceOf(Response); + if (!(budget instanceof Response)) syntheticFetches += 1; + expect(syntheticFetches).toBe(0); + configureSharedSpendLedger(policy({ [pool("old-label")]: "provider" })); + expect(admitHttpWorkflowTurn(new Headers())).toBeUndefined(); + const mappedBudget = createResponsesSendBudget({ req: new Request("https://fixture.example.test/v1/responses"), options: {}, logCtx: { model: "fixture", provider: "provider-display", spendPoolId: "provider" } }); + expect(mappedBudget).toBeInstanceOf(Response); + if (mappedBudget instanceof Response) { + expect(mappedBudget.status).toBe(429); + expect(mappedBudget.headers.get("x-opencodex-local-refusal")).toBe("workflow_spend_exhausted"); + } else syntheticFetches += 1; + expect(syntheticFetches).toBe(0); + const otherPool = createResponsesSendBudget({ req: new Request("https://fixture.example.test/v1/responses"), options: {}, logCtx: { model: "fixture", provider: "provider", spendPoolId: "unspent-provider" } }); + expect(otherPool).not.toBeInstanceOf(Response); + } finally { + release(); + if (previous === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previous; + removeTreeWithRetry(home); + } +}); + +test("actual old-reader compaction preserves raw spend; compatible rollback resolves every alias exactly once", () => { + const disk = journal([checkpoint([["old-label", 40, 0]])]); + const modern = createSpendReservationLedger({ journal: disk, salt, + policy: policy({ [pool("old-label")]: "provider" }, { compactAfterRecords: 1 }), now: () => 2 }); + expect(reserve(modern, "modern", "provider", 8).reserved).toBe(true); + modern.settle("modern", { inputTokens: 8, outputTokens: 0 }); + expect(modern.snapshot("pool", "provider")?.settled).toBe(48); + const old = createLegacySpendLedger({ journal: disk, salt, + policy: { ...DEFAULT_SPEND_RESERVATION_POLICY, compactAfterRecords: 1 }, now: () => 3 }); + expect(old.corruptRecords).toBe(0); + // Unsupported old binaries can still write a newly account-qualified label and strip + // compatibility metadata. Do not claim an automatic downgrade barrier or its enforcement. + expect(old.reserve({ sendId: "old-again", scopes: { poolId: "new-old-label" }, inputTokens: 12, outputCeilingTokens: 0 }).reserved).toBe(true); + old.settle("old-again", { inputTokens: 12, outputTokens: 0 }); + const returned = createSpendReservationLedger({ journal: disk, salt, + policy: policy({ [pool("old-label")]: "provider" }), now: () => 4 }); + expect(returned.checkPoolContinuity()?.reason).toBe("pool-history-unresolved"); + expect(returned.snapshot("pool", "new-old-label")?.settled).toBe(12); + returned.reconfigure(policy({ [pool("old-label")]: "provider", [pool("provider")]: "provider", [pool("new-old-label")]: "provider" })); + expect(returned.checkPoolContinuity()).toBeUndefined(); + expect(returned.snapshot("pool", "provider")?.settled).toBe(60); +}); From 22027eb28bf0f6c545bfa06f69693080ea2e9f07 Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Thu, 1 Oct 2026 05:50:31 -0700 Subject: [PATCH 03/15] fix(spend): honor verified pool merges and prepaid dispatches Validate historical identity proposals atomically, retain exact reservation ownership through child preflight, and record rooted continuity refusals once. Add synthetic regressions for graph ordering, permit lifecycle and scope, exact-limit recovery/combo dispatch, and refusal event accounting. --- .../docs/reference/configuration/server.md | 7 +- src/lib/request-execution-budget.ts | 110 +++++---- src/lib/spend-pool-continuity.ts | 36 ++- src/lib/spend-reservation-ledger.ts | 20 +- src/lib/workflow-budget.ts | 10 +- src/server/responses/core-combo.ts | 1 + src/server/responses/core-options.ts | 2 + src/server/responses/request-send-budget.ts | 6 +- src/server/responses/request-spend.ts | 6 +- src/server/workflow-refusal.ts | 1 + structure/runtime.md | 2 +- structure/transports/responses-spend.md | 7 +- tests/lib/spend-pool-continuity.test.ts | 231 +++++++++++++++++- 13 files changed, 358 insertions(+), 81 deletions(-) diff --git a/docs-site/src/content/docs/reference/configuration/server.md b/docs-site/src/content/docs/reference/configuration/server.md index ca72348a5fb..429123ac244 100644 --- a/docs-site/src/content/docs/reference/configuration/server.md +++ b/docs-site/src/content/docs/reference/configuration/server.md @@ -907,7 +907,9 @@ inference admission returns local HTTP 429 with `x-opencodex-local-refusal: workflow_pool_history_unresolved` before contacting a provider. This can temporarily block otherwise valid requests, including requests without a workflow root. After routing, preflight also refuses a canonical pool that is already exhausted by its combined -balances. This check does not turn post-reported passthrough sends into atomic reservations: +balances. A recovery or combo send already admitted by a reservation does not count that same +reservation against itself a second time; all other reservations remain counted. +This check does not turn post-reported passthrough sends into atomic reservations: a crossing send, concurrent admissions or retries reported afterwards retain their existing limits. Observe-only installs remain observe-only. Root and identity ceilings remain in force independently. @@ -924,7 +926,8 @@ disable the entire section. Invalid top-level mappings are rejected on configura malformed `spendPoolAliases` hand edits retain existing ceilings and fail pool admission closed. Correct the mapping and restart through the ordinary configuration workflow. Empty/removing mappings does not erase links already recorded durably. A previously redirected alias cannot be reassigned to a different -group; verified canonical renames can join groups without splitting existing spend. +group; verified canonical renames can join groups without splitting existing spend. The complete +mapping is checked together, so a valid group merge does not depend on alias-key order. Each original balance is counted once. Settled usage, in-flight reservations and unresolved usage all count; unknown usage is never treated as a refund. Original reservation targets are retained. diff --git a/src/lib/request-execution-budget.ts b/src/lib/request-execution-budget.ts index cd74c808598..2bb250d30d6 100644 --- a/src/lib/request-execution-budget.ts +++ b/src/lib/request-execution-budget.ts @@ -14,6 +14,7 @@ * sends plus one alternate -- by funding the alternate from a reserve that a validated * sanitized repair can spend instead, but never both. */ +import type { SpendReservationProof } from "./spend-reservation-ledger"; import type { TransientSendBudget } from "./upstream-retry"; export type SendClass = @@ -137,7 +138,7 @@ export interface RequestSendObserver { * would cross a ceiling never joins the total, the total stays just under, and the ceiling * never fires for any later request either. */ - charge(options?: { alreadySent?: boolean }): boolean; + charge(options?: { alreadySent?: boolean; onReserved?: (proof: SpendReservationProof) => void }): boolean; /** Give back a booking whose send never happened. */ refund(): void; } @@ -215,7 +216,7 @@ let logicalRequestSeq = 0; */ interface SharedSendLedger { spent: number; - pendingExternalSends: number; + pendingExternalSends: Set; /** * Spend one of this logical request's replacements for an ambiguous failure. Beside `spent` * for the same reason `pendingExternalSends` is: a derived scope that shared one without the @@ -231,6 +232,16 @@ interface SharedSendLedger { } const sharedSendLedgers = new WeakMap(); +const dispatchSpendProofs = new WeakMap(); + +/** One preflight for the exact prepaid dispatch, never another budget or a replayed permit. */ +export function claimDispatchSpendProof(budget: RequestExecutionBudget, permit?: SingleUseDispatchPermit): SpendReservationProof | undefined { + const proof = permit && dispatchSpendProofs.get(permit); + return proof && proof.owner === sharedSendLedgers.get(budget) ? proof.claim() : undefined; +} /** * One logical request's replacement grant: how many it has spent, and the ceiling it is held @@ -285,8 +296,10 @@ function createRequestExecutionBudgetWithLedger( counter.spent = Math.max(0, next); return; } - const settled = Math.min(delta, counter.pendingExternalSends); - counter.pendingExternalSends -= settled; + const settled = Math.min(delta, counter.pendingExternalSends.size); + for (let index = 0; index < settled; index += 1) { + counter.pendingExternalSends.delete(counter.pendingExternalSends.values().next().value!); + } const charged = delta - settled; counter.spent += charged; // These sends have already left. The ledger records them even past a ceiling it would @@ -339,7 +352,10 @@ function createRequestExecutionBudgetWithLedger( // Consulted last, because it is the only bound here that WRITES. A ledger entry booked // for a dispatch a cheaper check above would have refused is spend this request never // makes, and it would hold those tokens against the scope until retention expired. - if (observer && !observer.charge()) return { allowed: false, reason: "spend-exhausted" }; + let spendProof: SpendReservationProof | undefined; + if (observer && !observer.charge({ onReserved: proof => { spendProof = proof; } })) { + return { allowed: false, reason: "spend-exhausted" }; + } // THE RESERVATION IS THE CHARGE. Deciding here and charging in `use()` left a window in // which two legs read the same remainder, both received a permit, and both dispatched: @@ -347,51 +363,54 @@ function createRequestExecutionBudgetWithLedger( // this budget exists to stop. Everything is booked now; `release()` is the way back. const previousTargetKey = lastTargetKey; counter.spent += 1; - if (intent.countedExternally === true) counter.pendingExternalSends += 1; + const receipt = {}; + if (intent.countedExternally === true) counter.pendingExternalSends.add(receipt); if (drawsReserve) reserveSpent = true; if (isAlternateTarget) alternateTargetSends += 1; if (changesTarget) targetTransitions += 1; lastTargetKey = intent.targetKey; let settled: "open" | "used" | "released" = "open"; - return { - allowed: true, - permit: { - sendClass: intent.sendClass, - use(): boolean { - if (settled !== "open") return false; - settled = "used"; - return true; - }, - assumeCharge(): boolean { - if (settled !== "open") return false; - settled = "used"; - // The booking this reservation made for an external reporter is now owned by the - // caller. Leaving it pending is not harmless: the next `used` report of this request - // would settle against it and one real send would go uncharged. - if (intent.countedExternally === true && counter.pendingExternalSends > 0) { - counter.pendingExternalSends -= 1; - } - return true; - }, - release(): void { - if (settled !== "open") return; - settled = "released"; - // An externally counted reservation the reporter already settled paid for a send - // that physically happened. Refunding it would hand the request a free send back. - if (intent.countedExternally === true) { - if (counter.pendingExternalSends === 0) return; - counter.pendingExternalSends -= 1; - } - counter.spent -= 1; - observer?.refund(); - if (drawsReserve) reserveSpent = false; - if (isAlternateTarget) alternateTargetSends -= 1; - if (changesTarget) targetTransitions -= 1; - lastTargetKey = previousTargetKey; - }, + const permit: SingleUseDispatchPermit = { + sendClass: intent.sendClass, + use(): boolean { + if (settled !== "open") return false; + settled = "used"; + return true; + }, + assumeCharge(): boolean { + if (settled !== "open") return false; + settled = "used"; + // The booking this reservation made for an external reporter is now owned by the + // caller. Leaving it pending is not harmless: the next `used` report of this request + // would settle against it and one real send would go uncharged. + if (intent.countedExternally === true) counter.pendingExternalSends.delete(receipt); + return true; + }, + release(): void { + if (settled !== "open") return; + settled = "released"; + // An externally counted reservation the reporter already settled paid for a send + // that physically happened. Refunding it would hand the request a free send back. + if (intent.countedExternally === true) { + if (!counter.pendingExternalSends.delete(receipt)) return; + } + counter.spent -= 1; + observer?.refund(); + if (drawsReserve) reserveSpent = false; + if (isAlternateTarget) alternateTargetSends -= 1; + if (changesTarget) targetTransitions -= 1; + lastTargetKey = previousTargetKey; }, }; + let preflightClaimed = false; + dispatchSpendProofs.set(permit, { owner: counter, claim: () => { + if (preflightClaimed || settled === "released" + || (intent.countedExternally === true ? !counter.pendingExternalSends.has(receipt) : settled !== "open")) return undefined; + preflightClaimed = true; + return spendProof; + } }); + return { allowed: true, permit }; }, }; sharedSendLedgers.set(budget, counter); @@ -406,7 +425,7 @@ export function createRequestExecutionBudget( const grant = createAmbiguousResendGrant(); return createRequestExecutionBudgetWithLedger(policy, logicalRequestId, { spent: 0, - pendingExternalSends: 0, + pendingExternalSends: new Set(), claimAmbiguousResend: grant.claimAmbiguousResend, get ambiguousResendSpent(): boolean { return grant.ambiguousResendSpent; }, ...(observer ? { observer } : {}), @@ -450,7 +469,7 @@ const bridgedGrantClaims = new WeakMap(); let bridged = bridgedGrantClaims.get(parent); if (!bridged) { bridged = { claimed: false }; @@ -460,8 +479,7 @@ function ledgerFor(parent: RequestExecutionBudget): SharedSendLedger { return { get spent(): number { return parent.used; }, set spent(next: number) { parent.used = next; }, - get pendingExternalSends(): number { return pendingExternalSends; }, - set pendingExternalSends(next: number) { pendingExternalSends = next; }, + pendingExternalSends, // Asked of the parent rather than counted here. A local counter is a SECOND grant: two // scopes derived from one bridged parent, or one scope beside the parent it was derived // from, each replaced an unknown-state send once. Pending bookings and the durable-spend diff --git a/src/lib/spend-pool-continuity.ts b/src/lib/spend-pool-continuity.ts index 991f8fe4541..89b8243afe1 100644 --- a/src/lib/spend-pool-continuity.ts +++ b/src/lib/spend-pool-continuity.ts @@ -53,14 +53,19 @@ export function createPoolContinuity() { } return alias; }; - const link = (map: Map, alias: string, canonical: string): boolean => { - const existing = map.get(alias); - const target = resolve(canonical, map); - if (existing !== undefined && existing !== alias && resolve(alias, map) !== target) return false; - // An explicit rename can join a formerly canonical alias to its successor. The old - // name still resolves to this same group, so neither a reload nor an old caller splits it. - if (target !== alias || existing === undefined) map.set(alias, target); - return true; + const merge = (proposed: Iterable): Map | undefined => { + const next = new Map(bindings); + for (const [alias, canonical] of proposed) next.set(alias, canonical); + try { + // Validate the whole proposed graph, not an intermediate sorted prefix. A verified + // A -> B -> C rename may repeat A -> C without splitting the existing A/B group. + for (const alias of next.keys()) resolve(alias, next); + for (const [alias, canonical] of bindings) { + if (resolve(alias, next) !== resolve(canonical, next)) return undefined; + } + for (const alias of next.keys()) next.set(alias, resolve(alias, next)); + } catch { return undefined; } + return next; }; return { resolve: (alias: string): string => resolve(alias), @@ -71,23 +76,16 @@ export function createPoolContinuity() { bindings: [...bindings].map(([alias, canonical]) => ({ alias, canonical })), }), restore(record: PoolContinuityRecord): boolean { - const next = new Map(bindings); - try { - for (const { alias, canonical } of record.bindings) if (!link(next, alias, canonical)) return false; - for (const alias of next.keys()) resolve(alias, next); - } catch { return false; } + const next = merge(record.bindings.map(({ alias, canonical }) => [alias, canonical] as const)); + if (!next) return false; bindings = next; return true; }, prepare(config: unknown, requested: string | undefined, historical: ReadonlySet, hash: (provider: string) => string, at: number, capacity: number): PoolContinuityRecord | false | undefined { if (spendPoolAliasesError(config)) return false; - const next = new Map(bindings); - // Sort for deterministic multi-hop mappings regardless of JSON property order. Link - // conflicts fail closed; repeat entries and already-joined destinations are idempotent. - for (const [alias, provider] of Object.entries(config ?? {}).sort(([a], [b]) => a.localeCompare(b))) { - if (!link(next, alias, hash(provider as string))) return false; - } + const next = merge(Object.entries(config ?? {}).map(([alias, provider]) => [alias, hash(provider as string)] as const)); + if (!next) return false; if (requested !== undefined && !next.has(requested) && !historical.has(requested)) next.set(requested, requested); if (next.size > Math.min(16_384, capacity)) return false; if (next.size === bindings.size && [...next].every(([key, value]) => bindings.get(key) === value)) return undefined; diff --git a/src/lib/spend-reservation-ledger.ts b/src/lib/spend-reservation-ledger.ts index 04a5a313339..1e774d50f72 100644 --- a/src/lib/spend-reservation-ledger.ts +++ b/src/lib/spend-reservation-ledger.ts @@ -579,6 +579,12 @@ export interface ScopeSpendSnapshot { readonly exhausted: boolean; } +/** In-memory proof of one booked send; never journaled or exposed in request logs. */ +export interface SpendReservationProof { + readonly ledger: SpendReservationLedger; + readonly sendId: string; +} + export interface SpendReservationLedger { reserve(request: SpendReservationRequest): SpendReservationDecision; /** Pre-dispatch guard, including transports which report their sends after dispatch. */ @@ -607,7 +613,8 @@ export interface SpendReservationLedger { */ markLost(sendId: string): boolean; snapshot(scope: SpendScope, scopeId: string): ScopeSpendSnapshot | undefined; - exhausted(scope: SpendScope, scopeId: string): boolean; + /** Exclude only a proven current dispatch's still-open reservation in this scope. */ + exhausted(scope: SpendScope, scopeId: string, excludingSendId?: string): boolean; /** * Drop dormant scopes per the retention rule in SpendReservationPolicy. Cleanup also runs * automatically on every reservation, so nothing depends on a caller remembering this. @@ -1161,10 +1168,15 @@ export function createSpendReservationLedger(options: { }; }, - exhausted(scope: SpendScope, scopeId: string): boolean { + exhausted(scope: SpendScope, scopeId: string, excludingSendId?: string): boolean { assertOwnedAccounting?.(); - const state = stateFor(scope, aliasFor(scope, scopeId)); - return state !== undefined && isExhausted(scope, state); + const alias = aliasFor(scope, scopeId); + const state = stateFor(scope, alias); + if (!state) return false; + const own = excludingSendId === undefined ? undefined : reservations.get(aliasFor("send", excludingSendId)); + const matches = own?.status === "open" && own.targets.some(target => target.scope === scope + && (scope === "pool" ? poolContinuity.resolve(target.alias) === poolContinuity.resolve(alias) : target.alias === alias)); + return isExhausted(scope, matches ? { ...state, reserved: Math.max(0, state.reserved - own.tokens) } : state); }, prune(at: number = now()): void { diff --git a/src/lib/workflow-budget.ts b/src/lib/workflow-budget.ts index 41b72265b36..0c2e518045d 100644 --- a/src/lib/workflow-budget.ts +++ b/src/lib/workflow-budget.ts @@ -23,6 +23,7 @@ import { sharedSpendLedger, spendCeilingsConfigured, type SpendReservationLedger, + type SpendReservationProof, type SpendScope, type SpendUsage, } from "./spend-reservation-ledger"; @@ -715,10 +716,11 @@ export function workflowSendCeilingReached( function spentRootCeiling( rootId: string, ledger: SpendReservationLedger, + excludingSendId?: string, ): WorkflowSpendDenialDetail | undefined { const limit = ledger.policy.root.maxTokens; if (limit === undefined) return undefined; - return ledger.exhausted("root", rootId) ? { scope: "root", limit } : undefined; + return ledger.exhausted("root", rootId, excludingSendId) ? { scope: "root", limit } : undefined; } /** @@ -736,14 +738,16 @@ export function workflowSpendCeilingReached( rootId: string | undefined, spendLedger?: SpendReservationLedger, poolId?: string, + reservation?: SpendReservationProof, ): WorkflowSpendDenialDetail | undefined { if (!rootId && !poolId) return undefined; const ledger = spendLedger ?? (spendCeilingsConfigured() ? sharedSpendLedger() : undefined); if (!ledger) return undefined; - const root = rootId ? spentRootCeiling(rootId, ledger) : undefined; + const excludingSendId = reservation?.ledger === ledger ? reservation.sendId : undefined; + const root = rootId ? spentRootCeiling(rootId, ledger, excludingSendId) : undefined; if (root) return root; const limit = ledger.policy.pool.maxTokens; - return poolId && limit !== undefined && ledger.exhausted("pool", poolId) ? { scope: "pool", limit } : undefined; + return poolId && limit !== undefined && ledger.exhausted("pool", poolId, excludingSendId) ? { scope: "pool", limit } : undefined; } export interface WorkflowBudgetSnapshot { diff --git a/src/server/responses/core-combo.ts b/src/server/responses/core-combo.ts index a9669bbe327..d1f9466853f 100644 --- a/src/server/responses/core-combo.ts +++ b/src/server/responses/core-combo.ts @@ -791,6 +791,7 @@ export async function executeComboResponses( // parent arrived with. sendBudget: targetSendBudget, comboAttempt: true, + comboDispatchPermit: hopDecision?.allowed ? hopDecision.permit : undefined, comboReplaySnapshot, deferCodexResetDerivedCooldown, // Attempt-relative TTFT is recorded HERE (not via childLog.firstOutputMs — a later diff --git a/src/server/responses/core-options.ts b/src/server/responses/core-options.ts index aea7f850800..0fa5ba9e0fb 100644 --- a/src/server/responses/core-options.ts +++ b/src/server/responses/core-options.ts @@ -143,6 +143,8 @@ export interface HandleResponsesOptions { callerDirectAuth?: CallerDirectAuth | null; /** Internal recursion guard; callers outside this module must not set it. */ comboAttempt?: boolean; + /** Exact externally booked combo hop, used only for its child's spend preflight. */ + comboDispatchPermit?: SingleUseDispatchPermit; /** Internal handoff: this combo was selected by shadow-call interception. */ shadowCallIntercepted?: boolean; /** Internal handoff: the memory phase this turn belongs to, so combo children keep its routing. */ diff --git a/src/server/responses/request-send-budget.ts b/src/server/responses/request-send-budget.ts index bad16cf6426..33f1cf954b2 100644 --- a/src/server/responses/request-send-budget.ts +++ b/src/server/responses/request-send-budget.ts @@ -1,5 +1,5 @@ import type { ResponsesRequestContext } from "./core-options"; -import { createRequestExecutionBudget, isRequestExecutionBudget } from "../../lib/request-execution-budget"; +import { claimDispatchSpendProof, createRequestExecutionBudget, isRequestExecutionBudget } from "../../lib/request-execution-budget"; import { chargeWorkflowSends, workflowSendCeilingReached, @@ -84,7 +84,9 @@ export function createResponsesSendBudget( if (continuityRefusal) { return workflowRefusalResponse(continuityRefusal, logCtx, undefined, workflowRootId); } - const spentCeiling = workflowSpendCeilingReached(workflowRootId, undefined, logCtx.spendPoolId ?? logCtx.provider); + const prepaid = isRequestExecutionBudget(sendBudget) + ? claimDispatchSpendProof(sendBudget, options.compactionRecoveryPermit ?? options.comboDispatchPermit) : undefined; + const spentCeiling = workflowSpendCeilingReached(workflowRootId, undefined, logCtx.spendPoolId ?? logCtx.provider, prepaid); if (spentCeiling) { return workflowRefusalResponse( "workflow-spend-exhausted", diff --git a/src/server/responses/request-spend.ts b/src/server/responses/request-spend.ts index 8e86aac475d..9ddab48c3e5 100644 --- a/src/server/responses/request-spend.ts +++ b/src/server/responses/request-spend.ts @@ -73,7 +73,7 @@ export function createRequestSpendTracker( for (let index = 0; index < live.length - 1; index += 1) ledger().markDispatched(live[index] as string); }; return { - charge(options?: { alreadySent?: boolean }): boolean { + charge(options?: Parameters[0]): boolean { // A send that has already left is RECORDED, never refused: the tokens are spent, and a // booking the ledger drops is a booking the ceiling can never see. This is the reporting // transports' path -- the passthrough ladder reports through `onSendsConsumed` after the @@ -81,7 +81,8 @@ export function createRequestSpendTracker( // short of its limit forever and refuse nothing. const alreadySent = options?.alreadySent === true; const sendId = randomUUID(); - const decision = ledger().reserve({ + const bookedLedger = ledger(); + const decision = bookedLedger.reserve({ sendId, scopes: { ...(rootId !== undefined ? { rootId } : {}), @@ -130,6 +131,7 @@ export function createRequestSpendTracker( recordWorkflowRefusalEvent(rootId, "workflow-spend-exhausted", Date.now(), detail); return false; } + if (!alreadySent) options?.onReserved?.({ ledger: bookedLedger, sendId }); live.push(sendId); confirmOlderSends(); // It has already left, so the reservation cannot be handed back for free: from here only diff --git a/src/server/workflow-refusal.ts b/src/server/workflow-refusal.ts index e20c8e178d1..a1457fed130 100644 --- a/src/server/workflow-refusal.ts +++ b/src/server/workflow-refusal.ts @@ -116,6 +116,7 @@ export function admitHttpWorkflowTurn(headers: Headers): WorkflowDecision | unde const rootId = headers.get("x-codex-parent-thread-id")?.trim() || undefined; const continuityRefusal = poolContinuityRefusalReason(); if (continuityRefusal) { + recordWorkflowRefusalEvent(rootId, continuityRefusal); return { admitted: false, reason: continuityRefusal, rootId: rootId ?? "" }; } const threadId = headers.get("thread-id")?.trim() || undefined; diff --git a/structure/runtime.md b/structure/runtime.md index cb56f5ce573..82983125301 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -216,7 +216,7 @@ and build a ledger by replaying that directory's own journal. A second process o directory is refused even for observe-only spend configuration, while a separate directory is independent. Ordinary stop releases the final reference after listener teardown, and every thrown startup path releases its reference. SQLite and the OS release a crashed owner; no PID, timestamp, -TTL or lock-file deletion participates in recovery. Startup passes top-level pool-alias evidence with the spend policy. HTTP admission checks [historical continuity](transports/responses-spend.md#historical-pool-continuity-and-rollback) even without a root; failed accounting reads release the active-turn lease. +TTL or lock-file deletion participates in recovery. Startup passes top-level pool-alias evidence with the spend policy. HTTP admission checks [historical continuity](transports/responses-spend.md#historical-pool-continuity-and-rollback) even without a root; failed accounting reads release the active-turn lease. Rooted continuity denials record exactly one workflow refusal event before response formatting. An explicit Codex integration OFF skips startup cache invalidation before the user-scoped catalog serialization lock is resolved. Explicit `sync` and `sync-cache` retain their catalog-only override. diff --git a/structure/transports/responses-spend.md b/structure/transports/responses-spend.md index 74e82f1b33c..a7e43c36ccb 100644 --- a/structure/transports/responses-spend.md +++ b/structure/transports/responses-spend.md @@ -122,7 +122,9 @@ mapping when their positive history predates identity metadata. Zero-balance his The ledger retains original scope balances and reservation targets. The canonical view adds each member once, keeping settled, reserved and unresolved buckets separate. Explicit links can join previously canonical groups for a verified rename; an already redirected alias cannot be assigned -to a different group. A removed config entry never removes a journaled link. Group activity, +to a different group. The complete proposed graph is validated atomically, so a verified merge +that repeats existing member aliases is independent of salted-key order. A removed config entry +never removes a journaled link. Group activity, last-seen time and exhaustion govern retention; unidentified positive balances cannot be evicted. Identity evidence is bounded and retained even when dormant under-limit scopes are evicted. @@ -136,6 +138,9 @@ With a configured pool ceiling, any remaining unidentified positive pool history conflicting mapping refuses admission. HTTP workflow admission and the Responses pre-dispatch seam check this even without a root ID, before passthrough transports that report sends afterwards. After routing, that seam also refuses an already-exhausted canonical pool, including mapped historical totals. +A prepaid child excludes only its own open reservation, proven by its exact permit, shared send +ledger and still-pending receipt. Proof is single-use; unrelated reservations and other pool groups +remain counted. HTTP continuity refusals record one rooted workflow event; rootless ones record none. It is a snapshot check, not a new atomic reservation for report-only transports: crossing sends, concurrent preflight admissions and retries reported afterwards retain their existing limitations. `workflow_pool_history_unresolved` identifies the local 429 without disclosing aliases; storage diff --git a/tests/lib/spend-pool-continuity.test.ts b/tests/lib/spend-pool-continuity.test.ts index 5f47660df84..dffaf3ea575 100644 --- a/tests/lib/spend-pool-continuity.test.ts +++ b/tests/lib/spend-pool-continuity.test.ts @@ -1,3 +1,7 @@ +import { executeComboResponses } from "../../src/server/responses/core-combo"; +import { clearComboSelectionState, clearComboTargetCooldowns } from "../../src/combos"; +import { createTranslatorBudget } from "../../src/lib/translator-budget"; +import type { OcxConfig } from "../../src/types"; import { createLegacySpendLedger } from "../helpers/legacy-spend-ledger"; import { describe, expect, test } from "bun:test"; import { mkdtempSync, writeFileSync } from "node:fs"; @@ -6,12 +10,15 @@ import { join } from "node:path"; import { createHash } from "node:crypto"; import { createSpendReservationLedger, configureSharedSpendLedger, DEFAULT_SPEND_RESERVATION_POLICY, parseSpendJournalRecord, - type SpendJournal, type SpendReservationPolicy, + sharedSpendLedger, type SpendJournal, type SpendReservationPolicy, } from "../../src/lib/spend-reservation-ledger"; import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; import { admitHttpWorkflowTurn, workflowDecisionRefusalResponse } from "../../src/server/workflow-refusal"; import { createResponsesSendBudget } from "../../src/server/responses/request-send-budget"; +import { claimDispatchSpendProof, createRequestExecutionBudget, deriveRequestExecutionBudget } from "../../src/lib/request-execution-budget"; +import { createPoolContinuity } from "../../src/lib/spend-pool-continuity"; +import { listWorkflowBudgetEvents, resetWorkflowBudgetsForTest, workflowSpendCeilingReached } from "../../src/lib/workflow-budget"; import { createRequestSpendTracker } from "../../src/server/responses/request-spend"; const salt = "5".repeat(64); @@ -33,6 +40,30 @@ const reserve = (ledger: ReturnType, sendId ledger.reserve({ sendId, scopes: { poolId }, inputTokens: tokens, outputCeilingTokens: 0, alreadySent }); describe("historical pool identity continuity", () => { + test("verified group merges are atomic and independent of salted key order", () => { + const low = "1".repeat(32), high = "2".repeat(32), target = "3".repeat(32); + for (const [a, b] of [[low, high], [high, low]]) { + const continuity = createPoolContinuity(); + expect(continuity.restore({ v: 1, kind: "pool-continuity", at: 1, + bindings: [{ alias: a!, canonical: b! }] })).toBe(true); + const record = continuity.prepare({ [a!]: target, [b!]: target }, undefined, new Set(), id => id, 2, 100); + expect(record).not.toBe(false); + if (!record) throw new Error("merge was refused"); + expect(continuity.resolve(a!)).toBe(b!); // prepare does not publish evidence + expect(continuity.restore(record)).toBe(true); + expect(continuity.resolve(a!)).toBe(target); + expect(continuity.resolve(b!)).toBe(target); + const before = continuity.record(3); + expect(continuity.prepare({ [a!]: "4".repeat(32) }, undefined, new Set(), id => id, 3, 100)).toBe(false); + expect(continuity.record(3)).toEqual(before); + expect(continuity.restore({ v: 1, kind: "pool-continuity", at: 3, + bindings: [{ alias: a!, canonical: "4".repeat(32) }] })).toBe(false); + expect(continuity.record(3)).toEqual(before); + expect(continuity.prepare({ [target]: a!, [a!]: target }, undefined, new Set(), id => id, 3, 100)).toBe(false); + expect(continuity.record(3)).toEqual(before); + } + }); + test("unmapped historical debt fails closed across labels, providers, pruning and restart", () => { const disk = journal([checkpoint([["provider-old-label", 100, 0]])]); for (let restart = 0; restart < 2; restart += 1) { @@ -193,6 +224,12 @@ test("rootless HTTP and passthrough preflight refuse before any synthetic fetch" writeFileSync(join(home, "spend-ledger.salt"), salt + "\n", { mode: 0o600 }); writeFileSync(join(home, "spend-ledger.jsonl"), JSON.stringify(checkpoint([["old-label", 100, 0]])) + "\n", { mode: 0o600 }); configureSharedSpendLedger(policy()); + resetWorkflowBudgetsForTest(); + const rooted = admitHttpWorkflowTurn(new Headers({ "x-codex-parent-thread-id": "synthetic-root" })); + expect(rooted).toMatchObject({ admitted: false, reason: "workflow-pool-history-unresolved" }); + expect(listWorkflowBudgetEvents()).toMatchObject([{ rootId: "synthetic-root", reason: "workflow-pool-history-unresolved" }]); + if (rooted && !rooted.admitted) workflowDecisionRefusalResponse(rooted); + expect(listWorkflowBudgetEvents()).toHaveLength(1); let syntheticFetches = 0; const decision = admitHttpWorkflowTurn(new Headers()); expect(decision).toMatchObject({ admitted: false, reason: "workflow-pool-history-unresolved" }); @@ -206,6 +243,7 @@ test("rootless HTTP and passthrough preflight refuse before any synthetic fetch" expect(budget).toBeInstanceOf(Response); if (!(budget instanceof Response)) syntheticFetches += 1; expect(syntheticFetches).toBe(0); + expect(listWorkflowBudgetEvents()).toHaveLength(1); // rootless refusals add no event configureSharedSpendLedger(policy({ [pool("old-label")]: "provider" })); expect(admitHttpWorkflowTurn(new Headers())).toBeUndefined(); const mappedBudget = createResponsesSendBudget({ req: new Request("https://fixture.example.test/v1/responses"), options: {}, logCtx: { model: "fixture", provider: "provider-display", spendPoolId: "provider" } }); @@ -218,6 +256,7 @@ test("rootless HTTP and passthrough preflight refuse before any synthetic fetch" const otherPool = createResponsesSendBudget({ req: new Request("https://fixture.example.test/v1/responses"), options: {}, logCtx: { model: "fixture", provider: "provider", spendPoolId: "unspent-provider" } }); expect(otherPool).not.toBeInstanceOf(Response); } finally { + resetWorkflowBudgetsForTest(); release(); if (previous === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previous; @@ -247,3 +286,193 @@ test("actual old-reader compaction preserves raw spend; compatible rollback reso expect(returned.checkPoolContinuity()).toBeUndefined(); expect(returned.snapshot("pool", "provider")?.settled).toBe(60); }); + +for (const kind of ["compaction", "combo"] as const) { + for (const rootId of [undefined, "reservation-root"]) { + test(`${kind} child spends its own exact-limit reservation (${rootId ?? "rootless"})`, () => { + const previous = process.env.OPENCODEX_HOME; + const home = mkdtempSync(join(tmpdir(), "ocx-prepaid-spend-")); + process.env.OPENCODEX_HOME = home; + const release = acquireOwnedSpendHome(); + try { + configureSharedSpendLedger(policy(undefined, { root: { maxTokens: 100 } })); + const logCtx = { model: "fixture", provider: "provider-display", spendPoolId: "provider", usageLogInputTokens: 100 }; + const tracker = createRequestSpendTracker(logCtx, rootId); + const sendBudget = createRequestExecutionBudget(undefined, undefined, tracker); + const reservation = sendBudget.reserveDispatch({ sendClass: "initial", targetKey: "provider/fixture", countedExternally: true }); + expect(reservation.allowed).toBe(true); + if (!reservation.allowed) throw new Error("synthetic reservation refused"); + if (kind === "combo") expect(reservation.permit.use()).toBe(true); + expect(sharedSpendLedger().snapshot("pool", "provider")?.reserved).toBe(100); + const req = new Request("https://fixture.example.test/v1/responses", { + headers: rootId ? { "x-codex-parent-thread-id": rootId } : {}, + }); + const options = { sendBudget, ...(kind === "compaction" + ? { compactionRecoveryPermit: reservation.permit } : { comboDispatchPermit: reservation.permit }) }; + const child = createResponsesSendBudget({ req, options, logCtx }); + expect(child).not.toBeInstanceOf(Response); + if (child instanceof Response) throw new Error("own reservation refused"); + expect(createResponsesSendBudget({ req, options, logCtx })).toBeInstanceOf(Response); // proof is single-use + if (kind === "compaction") { + const dispatch = child.adapterDispatchBudget!.reserveDispatch({ sendClass: "initial", targetKey: "provider/fixture" }); + expect(dispatch.allowed).toBe(true); + if (dispatch.allowed) expect(dispatch.permit.use()).toBe(true); + } else child.noteTransientSends(1); // synthetic report-only combo dispatch, no fetch + expect(sendBudget.used).toBe(1); + tracker.settle({ inputTokens: 100, outputTokens: 0 }); + expect(sharedSpendLedger().snapshot("pool", "provider")).toMatchObject({ settled: 100, reserved: 0 }); + expect(createResponsesSendBudget({ req, options: {}, logCtx })).toBeInstanceOf(Response); + } finally { + release(); + if (previous === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previous; + removeTreeWithRetry(home); + } + }); + } +} + +function prepaidFixture(tokens = 100, ceiling = 100) { + const ledger = createSpendReservationLedger({ salt, policy: policy(undefined, { root: { maxTokens: ceiling }, pool: { maxTokens: ceiling } }) }); + const tracker = createRequestSpendTracker({ provider: "provider", usageLogInputTokens: tokens }, "root", ledger); + const budget = createRequestExecutionBudget(undefined, undefined, tracker); + const reservePermit = () => { + const decision = budget.reserveDispatch({ sendClass: "initial", targetKey: "provider/fixture", countedExternally: true }); + if (!decision.allowed) throw new Error("synthetic permit refused"); + return decision.permit; + }; + return { ledger, tracker, budget, reservePermit }; +} + +test("prepaid proof belongs to one shared budget and one still-pending dispatch", () => { + for (const end of ["release", "report", "assume"] as const) { + const { budget, reservePermit } = prepaidFixture(); + const permit = reservePermit(); + if (end === "release") permit.release(); + if (end === "report") budget.used += 1; + if (end === "assume") expect(permit.assumeCharge()).toBe(true); + expect(claimDispatchSpendProof(budget, permit)).toBeUndefined(); + } + const { ledger, budget, reservePermit } = prepaidFixture(); + const permit = reservePermit(); + expect(permit.use()).toBe(true); // combo use leaves its external receipt pending + expect(claimDispatchSpendProof(createRequestExecutionBudget(), permit)).toBeUndefined(); + const childBudget = deriveRequestExecutionBudget(budget, budget.policy); + const proof = claimDispatchSpendProof(childBudget, permit); + expect(proof).toBeDefined(); + expect(workflowSpendCeilingReached("root", ledger, "provider", proof)).toBeUndefined(); + expect(claimDispatchSpendProof(budget, permit)).toBeUndefined(); + expect(workflowSpendCeilingReached("root", ledger, "provider")).toMatchObject({ scope: "root" }); +}); + +test("receipt identity survives out-of-order handoff and reports cannot authorize another permit", () => { + const first = prepaidFixture(10, 100); + const a = first.reservePermit(), b = first.reservePermit(); + expect(b.assumeCharge()).toBe(true); + expect(claimDispatchSpendProof(first.budget, b)).toBeUndefined(); + expect(claimDispatchSpendProof(first.budget, a)).toBeDefined(); + first.budget.used += 1; + expect(first.budget.used).toBe(2); // report consumed A; B was already assumed + + const second = prepaidFixture(10, 100); + const reported = second.reservePermit(), pending = second.reservePermit(); + second.budget.used += 1; + expect(claimDispatchSpendProof(second.budget, reported)).toBeUndefined(); + reported.release(); + expect(second.budget.used).toBe(2); // cannot refund a different pending receipt + expect(claimDispatchSpendProof(second.budget, pending)).toBeDefined(); + pending.release(); + expect(second.budget.used).toBe(1); + expect(second.ledger.snapshot("pool", "provider")).toMatchObject({ reserved: 10 }); +}); + +test("preflight retains unrelated reservations, debt and mismatched scope or ledger", () => { + const { ledger, budget, reservePermit } = prepaidFixture(100, 200); + const permit = reservePermit(); + expect(reserve(ledger, "unrelated", "provider", 100).reserved).toBe(true); + ledger.reconfigure(policy(undefined, { root: { maxTokens: 200 } })); + const proof = claimDispatchSpendProof(budget, permit)!; + expect(workflowSpendCeilingReached(undefined, ledger, "provider", proof)).toMatchObject({ scope: "pool" }); + const foreign = prepaidFixture(); + foreign.reservePermit(); + expect(workflowSpendCeilingReached(undefined, foreign.ledger, "provider", proof)).toMatchObject({ scope: "pool" }); + expect(ledger.markDispatched(proof.sendId)).toBe(true); + expect(ledger.exhausted("pool", "provider", proof.sendId)).toBe(true); + ledger.settle(proof.sendId, { inputTokens: 100, outputTokens: 0 }); + expect(ledger.exhausted("pool", "provider", proof.sendId)).toBe(true); + + const zero = prepaidFixture(0, 100); + expect(reserve(zero.ledger, "full", "provider", 100).reserved).toBe(true); + const zeroProof = claimDispatchSpendProof(zero.budget, zero.reservePermit()); + expect(workflowSpendCeilingReached(undefined, zero.ledger, "provider", zeroProof)).toMatchObject({ scope: "pool" }); +}); + +test("prepaid exclusion follows only its canonical pool and retains settled/unresolved history", () => { + const disk = journal([checkpoint([["historical", 30, 10]])]); + const ledger = createSpendReservationLedger({ salt, journal: disk, policy: policy({ [pool("historical")]: "provider" }), now: () => 2 }); + const tracker = createRequestSpendTracker({ provider: "provider", usageLogInputTokens: 60 }, undefined, ledger); + const budget = createRequestExecutionBudget(undefined, undefined, tracker); + const decision = budget.reserveDispatch({ sendClass: "initial", targetKey: "provider", countedExternally: true }); + if (!decision.allowed) throw new Error("synthetic permit refused"); + const proof = claimDispatchSpendProof(budget, decision.permit)!; + expect(ledger.snapshot("pool", "provider")).toMatchObject({ settled: 30, unresolved: 10, reserved: 60 }); + expect(workflowSpendCeilingReached(undefined, ledger, "provider", proof)).toBeUndefined(); + ledger.reconfigure(policy({ [pool("historical")]: "renamed", [pool("provider")]: "renamed" })); + expect(ledger.checkPoolContinuity()).toBeUndefined(); + expect(workflowSpendCeilingReached(undefined, ledger, "renamed", proof)).toBeUndefined(); + expect(reserve(ledger, "other", "unrelated", 100).reserved).toBe(true); + expect(workflowSpendCeilingReached(undefined, ledger, "unrelated", proof)).toMatchObject({ scope: "pool" }); + ledger.reconfigure(policy(undefined, { pool: { maxTokens: 40 } })); + expect(workflowSpendCeilingReached(undefined, ledger, "renamed", proof)).toMatchObject({ scope: "pool" }); +}); + + +test("actual combo dispatch forwards only its own prepaid permit into the child preflight", async () => { + const previous = process.env.OPENCODEX_HOME; + const home = mkdtempSync(join(tmpdir(), "ocx-combo-prepaid-")); + process.env.OPENCODEX_HOME = home; + const release = acquireOwnedSpendHome(); + const originalFetch = globalThis.fetch; + globalThis.fetch = (() => { throw new Error("unexpected network in synthetic combo"); }) as typeof fetch; + const translatorBudget = createTranslatorBudget(); + clearComboSelectionState(); + clearComboTargetCooldowns(); + try { + configureSharedSpendLedger(policy()); + const config: OcxConfig = { port: 0, defaultProvider: "provider", providers: { + provider: { adapter: "openai-chat", apiKey: "fixture-only", baseUrl: "https://provider.example.test/v1" }, + }, combos: { prepaid: { strategy: "failover", targets: [{ provider: "provider", model: "fixture" }] } } }; + const logCtx = { model: "", provider: "", usageLogInputTokens: 100 }; + const tracker = createRequestSpendTracker(logCtx, undefined); + const sendBudget = createRequestExecutionBudget(undefined, undefined, tracker); + const body = { model: "combo/prepaid", input: [] }; + let sends = 0; + const response = await executeComboResponses(new Request("https://fixture.example.test/v1/responses", { + method: "POST", body: JSON.stringify(body), headers: { "content-type": "application/json" }, + }), body, "prepaid", config, logCtx, { sendBudget, translatorBudget }, { + handleResponses: async (req, _config, childLog, options) => { + const child = createResponsesSendBudget({ req, logCtx: childLog, options: options! }); + expect(child).not.toBeInstanceOf(Response); + if (child instanceof Response) return child; + sends += 1; + child.noteTransientSends(1); + return Response.json({ id: "synthetic-response", output: [] }); + }, + handleComboResponses: async () => { throw new Error("unexpected nested combo"); }, + }); + expect(response.status).toBe(200); + expect(sends).toBe(1); + expect(sendBudget.used).toBe(1); + tracker.settle({ inputTokens: 100, outputTokens: 0 }); + expect(sharedSpendLedger().snapshot("pool", "provider")).toMatchObject({ settled: 100, reserved: 0 }); + } finally { + globalThis.fetch = originalFetch; + translatorBudget.dispose(); + clearComboSelectionState(); + clearComboTargetCooldowns(); + release(); + if (previous === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previous; + removeTreeWithRetry(home); + } +}); From ade5bcf0a064709de05e03186c06e5952283b969 Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Thu, 1 Oct 2026 22:13:52 -0700 Subject: [PATCH 04/15] fix(spend): attribute external reports to exact dispatch receipts --- src/lib/request-execution-budget.ts | 68 +++++++----- src/server/chat-native.ts | 6 +- src/server/responses/adapter-continuation.ts | 18 +-- src/server/responses/adapter-dispatch.ts | 8 +- src/server/responses/core-combo-native.ts | 5 +- src/server/responses/core-combo.ts | 1 + src/server/responses/passthrough-dispatch.ts | 12 +- src/server/responses/request-send-budget.ts | 14 ++- src/server/responses/request-spend.ts | 34 ++++-- structure/transports/responses-spend.md | 17 ++- tests/lib/execution-budget-permits.test.ts | 29 ++++- tests/lib/spend-pool-continuity.test.ts | 105 +++++++++++++++++- .../lib/transient-budget-scope-source.test.ts | 20 +++- tests/responses/chat-native-combo.test.ts | 2 + .../responses/responses-core-modules.test.ts | 2 +- .../responses-spend-ledger-wiring.test.ts | 4 +- tests/server/inference-send-budget.test.ts | 2 +- 17 files changed, 268 insertions(+), 79 deletions(-) diff --git a/src/lib/request-execution-budget.ts b/src/lib/request-execution-budget.ts index fcc85734f6f..b2da4cf8b80 100644 --- a/src/lib/request-execution-budget.ts +++ b/src/lib/request-execution-budget.ts @@ -146,9 +146,11 @@ export interface RequestSendObserver { * would cross a ceiling never joins the total, the total stays just under, and the ceiling * never fires for any later request either. */ - charge(options?: { alreadySent?: boolean; onReserved?: (proof: SpendReservationProof) => void }): boolean; + charge(options?: { alreadySent?: boolean; deferDispatch?: boolean; onReserved?: (proof: SpendReservationProof) => void }): boolean; + /** Confirm the exact reservation once its dispatch is known. */ + dispatch?(proof: SpendReservationProof): void; /** Give back a booking whose send never happened. */ - refund(): void; + refund(proof?: SpendReservationProof | null): void; } /** @@ -243,6 +245,7 @@ const sharedSendLedgers = new WeakMap( const dispatchSpendProofs = new WeakMap(); /** One preflight for the exact prepaid dispatch, never another budget or a replayed permit. */ @@ -251,6 +254,21 @@ export function claimDispatchSpendProof(budget: RequestExecutionBudget, permit?: return proof && proof.owner === sharedSendLedgers.get(budget) ? proof.claim() : undefined; } +/** Reports actual sends against only the named permit on this shared ledger. */ +export function reportDispatchSends( + budget: TransientSendBudget, + sends: number, + permit?: SingleUseDispatchPermit, +): void { + const count = Number.isFinite(sends) ? Math.max(0, Math.trunc(sends)) : 0; + if (count === 0) return; + const receipt = permit && dispatchSpendProofs.get(permit); + const prepaid = receipt && receipt.owner === sharedSendLedgers.get(budget as RequestExecutionBudget) + && receipt.report() ? 1 : 0; + // Unnamed, foreign, released or already-reported permits cannot consume another receipt. + budget.used += count - prepaid; +} + /** * One logical request's replacement grant: how many it has spent, and the ceiling it is held * to. @@ -292,24 +310,15 @@ function createRequestExecutionBudgetWithLedger( let alternateTargetSends = 0; let targetTransitions = 0; let lastTargetKey: string | undefined; + const targetReservations: Array<{ targetKey: string }> = []; const budget: RequestExecutionBudget = { get used(): number { return counter.spent; }, set used(next: number) { - // The retry helpers report their real send count by assigning through this field. A - // reservation taken with `countedExternally` has already booked one of those sends, so - // the report settles the pending booking first and only the surplus is charged. - const delta = next - counter.spent; - if (delta <= 0) { - counter.spent = Math.max(0, next); - return; - } - const settled = Math.min(delta, counter.pendingExternalSends.size); - for (let index = 0; index < settled; index += 1) { - counter.pendingExternalSends.delete(counter.pendingExternalSends.values().next().value!); - } - const charged = delta - settled; - counter.spent += charged; + // A numeric report has no receipt identity. Only reportDispatchSends may settle a + // prepaid permit; guessing by reservation order can refund another leg's actual send. + const charged = next - counter.spent; + counter.spent = Math.max(0, next); // These sends have already left. The ledger records them even past a ceiling it would // have refused, because refusing after the fact only hides spend that was really // incurred -- the refusal has to happen at the reservation below, or not at all. @@ -363,7 +372,7 @@ function createRequestExecutionBudgetWithLedger( // for a dispatch a cheaper check above would have refused is spend this request never // makes, and it would hold those tokens against the scope until retention expired. let spendProof: SpendReservationProof | undefined; - if (observer && !observer.charge({ onReserved: proof => { spendProof = proof; } })) { + if (observer && !observer.charge({ deferDispatch: true, onReserved: proof => { spendProof = proof; } })) { return { allowed: false, reason: "spend-exhausted" }; } @@ -371,9 +380,9 @@ function createRequestExecutionBudgetWithLedger( // which two legs read the same remainder, both received a permit, and both dispatched: // one remaining send admitted two physical sends, which is the per-request multiplication // this budget exists to stop. Everything is booked now; `release()` is the way back. - const previousTargetKey = lastTargetKey; counter.spent += 1; - const receipt = {}; + const receipt = { targetKey: intent.targetKey }; + targetReservations.push(receipt); if (intent.countedExternally === true) counter.pendingExternalSends.add(receipt); if (drawsReserve) reserveSpent = true; if (chargesAlternateTarget) alternateTargetSends += 1; @@ -386,15 +395,16 @@ function createRequestExecutionBudgetWithLedger( use(): boolean { if (settled !== "open") return false; settled = "used"; + if (intent.countedExternally !== true && spendProof) observer?.dispatch?.(spendProof); return true; }, assumeCharge(): boolean { - if (settled !== "open") return false; + if (settled !== "open" || (intent.countedExternally === true && !counter.pendingExternalSends.has(receipt))) return false; settled = "used"; // The booking this reservation made for an external reporter is now owned by the - // caller. Leaving it pending is not harmless: the next `used` report of this request - // would settle against it and one real send would go uncharged. + // caller. Close only its own receipt so a later reporter cannot spend it again. if (intent.countedExternally === true) counter.pendingExternalSends.delete(receipt); + if (spendProof) observer?.dispatch?.(spendProof); return true; }, release(): void { @@ -406,15 +416,23 @@ function createRequestExecutionBudgetWithLedger( if (!counter.pendingExternalSends.delete(receipt)) return; } counter.spent -= 1; - observer?.refund(); + observer?.refund(spendProof ?? null); if (drawsReserve) reserveSpent = false; if (chargesAlternateTarget) alternateTargetSends -= 1; if (chargesTransition) targetTransitions -= 1; - lastTargetKey = previousTargetKey; + targetReservations.splice(targetReservations.indexOf(receipt), 1); + lastTargetKey = targetReservations.at(-1)?.targetKey; }, }; let preflightClaimed = false; - dispatchSpendProofs.set(permit, { owner: counter, claim: () => { + dispatchSpendProofs.set(permit, { owner: counter, report: () => { + if (!counter.pendingExternalSends.has(receipt)) return false; + if (spendProof) observer?.dispatch?.(spendProof); + counter.pendingExternalSends.delete(receipt); + // Reset-only helpers report just BEFORE calling the dispatch thunk. Its one use() + // remains available, but release/proof cannot refund or reuse this reported receipt. + return true; + }, claim: () => { if (preflightClaimed || settled === "released" || (intent.countedExternally === true ? !counter.pendingExternalSends.has(receipt) : settled !== "open")) return undefined; preflightClaimed = true; diff --git a/src/server/chat-native.ts b/src/server/chat-native.ts index 3e50b5e02c0..2e394d23350 100644 --- a/src/server/chat-native.ts +++ b/src/server/chat-native.ts @@ -17,7 +17,7 @@ import { isCyberPolicyMessage, SEND_BUDGET_EXHAUSTED_CODE, } from "../lib/errors"; -import type { RequestExecutionBudget } from "../lib/request-execution-budget"; +import { reportDispatchSends, type RequestExecutionBudget, type SingleUseDispatchPermit } from "../lib/request-execution-budget"; import type { AdmissionLease } from "../lib/admission"; import { readBoundedResponseBody } from "../lib/bounded-body"; import { redactSecretString } from "../lib/redact"; @@ -198,6 +198,7 @@ export interface NativeChatExecution extends HandleNativeChatOptions { * the parent row, replace the one the final log settles. */ sendBudget?: RequestExecutionBudget; + comboDispatchPermit?: SingleUseDispatchPermit; /** Replaces the request-relative first-output mark; a combo child records its own. */ onFirstOutput?: () => void; /** The lease a streamed body holds; defaults to `logIds.turnAdmissionLease`. */ @@ -254,6 +255,7 @@ export function createNativeChatComboSource(input: { translatorBudget: input.translatorBudget, finishLog: child.finishLog, sendBudget: child.sendBudget, + comboDispatchPermit: child.comboDispatchPermit, onFirstOutput: child.onFirstOutput, ...(child.turnAdmissionLease ? { turnAdmissionLease: child.turnAdmissionLease } : {}), }, child.attemptHandle), @@ -457,7 +459,7 @@ export async function runNativeChatAttempt( throw new SendBudgetExhaustedError(safeHostLabel(request.url)); } physicalSends += 1; - sendBudget.used += 1; + reportDispatchSends(sendBudget, 1, execution.comboDispatchPermit); } else if (!spendTracker?.charge()) throw new NativeChatSpendRefusal(); noteProviderAttemptSend(logCtx, route.providerName, activeProvider, logCtx.usageLogInputTokens, transportRecovery ?? recovery); // A reselected provider transport is still a physical send: the connection policy diff --git a/src/server/responses/adapter-continuation.ts b/src/server/responses/adapter-continuation.ts index 2d3aa39e5e7..a9fcab36385 100644 --- a/src/server/responses/adapter-continuation.ts +++ b/src/server/responses/adapter-continuation.ts @@ -102,7 +102,7 @@ export function createAdapterContinuations( | "noteAdapterPhysicalSend" | "noteAdapterRecoveryWithheld" | "remainingTransientSendBudget" - | "noteTransientSends" + | "transientSendReporter" | "reserveCredentialHop" | "pendingHopPermit" | "sendBudgetExhausted" @@ -135,7 +135,7 @@ export function createAdapterContinuations( noteAdapterPhysicalSend, noteAdapterRecoveryWithheld, remainingTransientSendBudget, - noteTransientSends, + transientSendReporter, reserveCredentialHop, sendBudgetExhausted, } = sendBudgetState; @@ -264,7 +264,7 @@ export function createAdapterContinuations( ...(continuationTransientPolicy ? { attempts: remainingTransientSendBudget(continuationTransientPolicy.attempts), - onSendsConsumed: noteTransientSends, + onSendsConsumed: transientSendReporter(), } : {}), }, @@ -513,9 +513,9 @@ export function createAdapterContinuations( recordAttemptCredentialSource(logCtx.activeAttempt, route.providerName, route.provider, transportState.activeAdapter.name); // The replay goes out on the next iteration. An adapter that owns its ladder // reserves for that send itself, so hand this reservation down rather than let it - // take a second one for the same replay. A helper-routed replay needs no handoff: - // its reporter settles the booking made above. - if (adapterOwnsDispatch) sendBudgetState.pendingHopPermit = hop.permit; + // take a second one for the same replay. A helper-routed replay carries the + // same permit so its reporter settles only this booking. + sendBudgetState.pendingHopPermit = hop.permit; nextContinuationRecoveryKind = "oauth-account-429"; kiroRefusalPendingReplay = response; continue; @@ -604,9 +604,9 @@ export function createAdapterContinuations( recordAttemptCredentialSource(logCtx.activeAttempt, route.providerName, route.provider, transportState.activeAdapter.name); // The replay goes out on the next iteration. An adapter that owns its ladder // reserves for that send itself, so hand this reservation down rather than let it - // take a second one for the same replay. A helper-routed replay needs no handoff: - // its reporter settles the booking made above. - if (adapterOwnsDispatch) sendBudgetState.pendingHopPermit = hop.permit; + // take a second one for the same replay. A helper-routed replay carries the + // same permit so its reporter settles only this booking. + sendBudgetState.pendingHopPermit = hop.permit; nextContinuationRecoveryKind = "oauth-account-429"; continue; } diff --git a/src/server/responses/adapter-dispatch.ts b/src/server/responses/adapter-dispatch.ts index 3cdede083cd..f228eff3886 100644 --- a/src/server/responses/adapter-dispatch.ts +++ b/src/server/responses/adapter-dispatch.ts @@ -147,7 +147,7 @@ export async function prepareAdapterExchange( | "noteAdapterPhysicalSend" | "noteAdapterRecoveryWithheld" | "remainingTransientSendBudget" - | "noteTransientSends" + | "transientSendReporter" | "recoverySendAllowance" | "recoveryClassFor" | "sendBudgetExhausted" @@ -182,7 +182,7 @@ export async function prepareAdapterExchange( noteAdapterPhysicalSend, noteAdapterRecoveryWithheld, remainingTransientSendBudget, - noteTransientSends, + transientSendReporter, recoverySendAllowance, recoveryClassFor, sendBudgetExhausted, @@ -374,7 +374,7 @@ export async function prepareAdapterExchange( // remaining allowance; treating the booking as unavailable blocks a cap of two. attempts: Math.min(transientPolicy?.attempts ?? 1, remainingTransientSendBudget(transientPolicy?.attempts ?? 1) + (compactPrepaid ? 1 : 0)), - onSendsConsumed: noteTransientSends, + onSendsConsumed: transientSendReporter(compactPrepaid), } : {}), }, @@ -581,7 +581,7 @@ export async function prepareAdapterExchange( ...(refetchAllowance ? { attempts: refetchAllowance.attempts, - onSendsConsumed: noteTransientSends, + onSendsConsumed: transientSendReporter(refetchAllowance.permit), } : {}), }, diff --git a/src/server/responses/core-combo-native.ts b/src/server/responses/core-combo-native.ts index c70af618f51..3a039f741db 100644 --- a/src/server/responses/core-combo-native.ts +++ b/src/server/responses/core-combo-native.ts @@ -18,7 +18,7 @@ import type { OcxConfig } from "../../types"; import type { RouteResult } from "../../router"; import type { DataPlaneAdmission } from "../auth-cors"; import type { AdmissionLease } from "../../lib/admission"; -import type { RequestExecutionBudget } from "../../lib/request-execution-budget"; +import type { RequestExecutionBudget, SingleUseDispatchPermit } from "../../lib/request-execution-budget"; import type { TransientSendBudget } from "../../lib/upstream-retry"; import type { ResponsesTerminalStatus } from "../../bridge"; import type { NativeChatFinishLog } from "../chat-native"; @@ -61,6 +61,7 @@ export interface NativeComboChildRun { childLog: RequestLogContext; attemptHandle: InferenceAttempt; sendBudget: RequestExecutionBudget; + comboDispatchPermit?: SingleUseDispatchPermit; turnAdmissionLease?: AdmissionLease; onFirstOutput: () => void; /** Reports to the combo's child callbacks; it never writes the parent's final row itself. */ @@ -266,6 +267,7 @@ export interface ComboChildCallbacks { export async function dispatchNativeComboChild(input: { source: ComboProtocolSource; plan: NativeComboChildPlan; + comboDispatchPermit?: SingleUseDispatchPermit; logCtx: RequestLogContext; childLog: RequestLogContext; attempt: PersistedUsageAttempt; @@ -311,6 +313,7 @@ export async function dispatchNativeComboChild(input: { childLog, attemptHandle, sendBudget: plan.sendBudget, + comboDispatchPermit: input.comboDispatchPermit, ...(input.turnAdmissionLease ? { turnAdmissionLease: input.turnAdmissionLease } : {}), onFirstOutput: () => { outputSeen = true; diff --git a/src/server/responses/core-combo.ts b/src/server/responses/core-combo.ts index d730349e086..7a9accdd518 100644 --- a/src/server/responses/core-combo.ts +++ b/src/server/responses/core-combo.ts @@ -816,6 +816,7 @@ export async function executeComboResponses( response = nativeChild ? await dispatchNativeComboChild({ source: options.protocolSource!, plan: nativeChild, + comboDispatchPermit: hopDecision?.allowed ? hopDecision.permit : undefined, logCtx, childLog, attempt, diff --git a/src/server/responses/passthrough-dispatch.ts b/src/server/responses/passthrough-dispatch.ts index ec47d4e6e77..bdda62fdfb1 100644 --- a/src/server/responses/passthrough-dispatch.ts +++ b/src/server/responses/passthrough-dispatch.ts @@ -215,6 +215,7 @@ export async function preparePassthroughExchange( ResponsesSendBudget, | "remainingTransientSendBudget" | "noteTransientSends" + | "transientSendReporter" | "recoverySendAllowance" | "recoveryClassFor" | "sendBudgetExhausted" @@ -250,6 +251,7 @@ export async function preparePassthroughExchange( const { remainingTransientSendBudget, noteTransientSends, + transientSendReporter, recoverySendAllowance, recoveryClassFor, sendBudgetExhausted, @@ -992,7 +994,7 @@ export async function preparePassthroughExchange( .then(adoptObservedResponse); }, { abortSignal: upstream.signal, label: safeHostLabel(request.url), - attempts: remainingTransientSendBudget(transientSendAttempts()), onSendsConsumed: noteTransientSends, + attempts: remainingTransientSendBudget(transientSendAttempts()), onSendsConsumed: transientSendReporter(), claimAmbiguousResend: claimPreHeaderResend, }, ); @@ -1095,7 +1097,7 @@ export async function preparePassthroughExchange( .then(adoptObservedResponse); }, { abortSignal: upstream.signal, label: safeHostLabel(request.url), attempts: allowance.attempts, - onSendsConsumed: noteTransientSends, claimAmbiguousResend: claimPreHeaderResend }, + onSendsConsumed: transientSendReporter(allowance.permit), claimAmbiguousResend: claimPreHeaderResend }, ); } catch (err) { return { failed: transportFailureResponse(err) }; @@ -1224,7 +1226,7 @@ export async function preparePassthroughExchange( }, { abortSignal: upstream.signal, label: safeHostLabel(request.url), attempts: remainingTransientSendBudget(transientSendAttempts()), - onSendsConsumed: noteTransientSends, claimAmbiguousResend: claimPreHeaderResend }, + onSendsConsumed: transientSendReporter(), claimAmbiguousResend: claimPreHeaderResend }, ); } catch (err) { return transportFailureResponse(err); @@ -1352,7 +1354,7 @@ export async function preparePassthroughExchange( .then(adoptObservedResponse); }, { abortSignal: upstream.signal, label: safeHostLabel(request.url), attempts: remainingTransientSendBudget(transientSendAttempts()), - onSendsConsumed: noteTransientSends, claimAmbiguousResend: claimPreHeaderResend }, + onSendsConsumed: transientSendReporter(), claimAmbiguousResend: claimPreHeaderResend }, ); } catch (err) { return transportFailureResponse(err); @@ -1487,7 +1489,7 @@ export async function preparePassthroughExchange( .then(adoptObservedResponse); }, { abortSignal: upstream.signal, label: safeHostLabel(request.url), attempts: remainingTransientSendBudget(transientSendAttempts()), - onSendsConsumed: noteTransientSends, claimAmbiguousResend: claimPreHeaderResend }, + onSendsConsumed: transientSendReporter(), claimAmbiguousResend: claimPreHeaderResend }, ); } catch (err) { return transportFailureResponse(err); diff --git a/src/server/responses/request-send-budget.ts b/src/server/responses/request-send-budget.ts index 33f1cf954b2..5b4d3e26eaa 100644 --- a/src/server/responses/request-send-budget.ts +++ b/src/server/responses/request-send-budget.ts @@ -1,5 +1,5 @@ import type { ResponsesRequestContext } from "./core-options"; -import { claimDispatchSpendProof, createRequestExecutionBudget, isRequestExecutionBudget } from "../../lib/request-execution-budget"; +import { claimDispatchSpendProof, createRequestExecutionBudget, isRequestExecutionBudget, reportDispatchSends } from "../../lib/request-execution-budget"; import { chargeWorkflowSends, workflowSendCeilingReached, @@ -57,12 +57,18 @@ export function createResponsesSendBudget( // sends once per child seven hundred times, so every send charged to the request is charged // to the root as well (#4546). const workflowRootId = req.headers.get("x-codex-parent-thread-id")?.trim() || undefined; - const noteTransientSends = (used: number): void => { + const inheritedPermit = options.compactionRecoveryPermit ?? options.comboDispatchPermit; + let pendingHopPermit: SingleUseDispatchPermit | undefined = options.compactionRecoveryPermit; + const recordTransientSends = (used: number, permit?: SingleUseDispatchPermit): void => { const charged = Math.max(0, used); - sendBudget.used += charged; + reportDispatchSends(sendBudget, charged, permit); options.onCompactionRecoverySendsReported?.(charged); chargeWorkflowSends(workflowRootId, charged); }; + const noteTransientSends = (used: number): void => recordTransientSends(used, pendingHopPermit ?? inheritedPermit); + // Capture at helper creation, before another leg can replace the pending handoff. + const transientSendReporter = (permit = pendingHopPermit ?? inheritedPermit) => + (used: number): void => recordTransientSends(used, permit); // Refused before any dispatch, and deliberately not by evicting the root's ledger entry: // dropping the record to make room would hand the fan-out a fresh allowance, which is the // laundering this ceiling exists to stop. The client is told the task needs a new grant @@ -165,7 +171,6 @@ export function createResponsesSendBudget( * refused and the request would answer with a synthetic 502 in place of the real 429 the hop * was recovering from. */ - let pendingHopPermit: SingleUseDispatchPermit | undefined = options.compactionRecoveryPermit; /** * The budget an adapter's OWN dispatch ladder reserves against. * @@ -276,6 +281,7 @@ export function createResponsesSendBudget( return { workflowRootId, noteTransientSends, + transientSendReporter, remainingTransientSendBudget, /** * Physical sends this logical request has already made. diff --git a/src/server/responses/request-spend.ts b/src/server/responses/request-spend.ts index 9ddab48c3e5..ae6d896a6fb 100644 --- a/src/server/responses/request-spend.ts +++ b/src/server/responses/request-spend.ts @@ -54,23 +54,25 @@ export function createRequestSpendTracker( // one. It also means the home in effect at dispatch is the one that gets written. let ledgerRef: SpendReservationLedger | undefined = injected; const ledger = (): SpendReservationLedger => (ledgerRef ??= sharedSpendLedger()); - // Every send this request still owes the ledger an answer for, oldest first. + // Outstanding entries; exact dispatch reports move their reservation to the end. const live: string[] = []; + const pendingDispatch = new Set(); let refusals = 0; let resolved = false; let terminalProcessed = false; /** * Confirm the sends this request has already moved past. * - * A booking is only marked dispatched once a LATER send exists, because that later send - * proves the earlier one left. The newest booking stays open until it is settled, so a - * reservation the budget hands back -- a rotation that found no alternate, a rebuild - * abandoned before the wire -- can still be released for free while this process is alive. + * Legacy direct charges infer dispatch from a later send. Exact budget reservations wait + * for their own dispatch/report instead: reserving B does not prove that A left, and A + * must remain refundable if B reports first. The newest direct charge stays open. * A crash resolves every surviving reservation as unresolved spend regardless of this mark, * because a journal that lost its tail cannot prove a send never left. */ const confirmOlderSends = (): void => { - for (let index = 0; index < live.length - 1; index += 1) ledger().markDispatched(live[index] as string); + for (const sendId of live.slice(0, -1)) { + if (!pendingDispatch.has(sendId)) ledger().markDispatched(sendId); + } }; return { charge(options?: Parameters[0]): boolean { @@ -133,15 +135,29 @@ export function createRequestSpendTracker( } if (!alreadySent) options?.onReserved?.({ ledger: bookedLedger, sendId }); live.push(sendId); - confirmOlderSends(); + if (options?.deferDispatch) pendingDispatch.add(sendId); + else confirmOlderSends(); // It has already left, so the reservation cannot be handed back for free: from here only // a settlement or unresolved spend is honest about it. if (alreadySent) ledger().markDispatched(sendId); return true; }, - refund(): void { - const sendId = live.pop(); + dispatch(proof): void { + if (proof.ledger !== ledgerRef || !pendingDispatch.has(proof.sendId)) return; + ledger().markDispatched(proof.sendId); + pendingDispatch.delete(proof.sendId); + // Terminal usage follows dispatch/report order, not reservation order. + const index = live.indexOf(proof.sendId); + if (index >= 0) live.push(...live.splice(index, 1)); + }, + refund(proof): void { + // null is an exact budget reservation that obtained no durable booking. + if (proof === null || (proof && proof.ledger !== ledgerRef)) return; + const index = proof ? live.indexOf(proof.sendId) : live.length - 1; + if (index < 0) return; + const [sendId] = live.splice(index, 1); if (sendId === undefined) return; + pendingDispatch.delete(sendId); // Undispatched, so this returns the tokens. If the send was already confirmed by a later // one, `abandon` refuses and unresolved is the only honest outcome left. if (!ledger().abandon(sendId)) ledger().markLost(sendId); diff --git a/structure/transports/responses-spend.md b/structure/transports/responses-spend.md index 1e4e06bfa90..53afd9bb089 100644 --- a/structure/transports/responses-spend.md +++ b/structure/transports/responses-spend.md @@ -38,7 +38,13 @@ refunds only charges it made; reporting or assuming that receipt prevents a late The hop pays for a replay that some *other* layer dispatches, so which layer settles the reservation follows the dispatcher, not the ladder. A helper-routed replay reports the same physical send back through `onSendsConsumed`; that is what `countedExternally: true` names, and the -reporter's first send settles the pending booking instead of adding a second charge. An adapter +reporter's first send settles only its named pending permit instead of adding a second charge. +`transientSendReporter` captures that permit before entering a helper; a later handoff cannot +replace it. `reportDispatchSends` verifies shared-ledger ownership and consumes one receipt only. +Native Chat combo children carry the same exact permit to their physical-send boundary. +Numeric `used` updates and unnamed, foreign, released or already-reported permits charge actual +sends without consuming another reservation. Extra retries remain full charges. Reports may arrive +out of reservation order; no FIFO ordering is required. An adapter that owns its transport — Kiro's reset ladder, Cursor's transport ladder, or Devin's bounded pre-output stated-reset replay — reserves once per physical send instead, so no reporter ever arrives. Those ladders are handed @@ -166,9 +172,12 @@ reader requires explicit mappings for the remaining unidentified balances rather zero. `tests/lib/spend-pool-continuity.test.ts` exercises this using the frozen pre-change reader, as well as exact aggregation, active/unresolved sends, retention, failures and rootless preflight. -A booking is confirmed dispatched only once a LATER send exists, because that later send proves -the earlier one left. The newest booking stays open, so a reservation the budget hands back -during this process's lifetime can still be released for free. +Budget reservations retain their exact durable proof until their own dispatch/report confirms +them. A later reservation does not confirm an earlier pending send. Refunds remove the exact +send and original pool, and releasing an older permit preserves the latest surviving target. +Legacy direct charges still infer dispatch from a later charge and leave their newest booking +open. Report order updates terminal attribution; unrelated pending reservations remain independent. +`tests/lib/spend-pool-continuity.test.ts` covers reversed child reports and exact-pool cancellation. Settlement follows what the request learned. The terminal usage belongs to the last send that left, so that one settles with the real figure; every earlier send failed without reporting usage diff --git a/tests/lib/execution-budget-permits.test.ts b/tests/lib/execution-budget-permits.test.ts index 0084f8798f0..a5973d1c231 100644 --- a/tests/lib/execution-budget-permits.test.ts +++ b/tests/lib/execution-budget-permits.test.ts @@ -3,6 +3,7 @@ import { CODEX_TEXT_GUARDED_BUDGET_POLICY, createRequestExecutionBudget, deriveRequestExecutionBudget, + reportDispatchSends, type RequestExecutionBudgetPolicy, } from "../../src/lib/request-execution-budget"; @@ -102,7 +103,7 @@ describe("atomic dispatch permits", () => { expect(leg.permit.use()).toBe(true); // `onSendsConsumed` reporting one physical send settles the pending booking instead of // charging a second time. Charging both is how a four-send cap became a two-send cap. - budget.used += 1; + reportDispatchSends(budget, 1, leg.permit); expect(budget.used).toBe(1); // Sends the helper made beyond the reserved one are still charged in full. @@ -118,7 +119,7 @@ describe("atomic dispatch permits", () => { countedExternally: true, }); if (!leg.allowed) throw new Error("unreachable"); - budget.used += 1; + reportDispatchSends(budget, 1, leg.permit); expect(budget.used).toBe(1); // The send physically happened. A refund here would hand the request a free one back. leg.permit.release(); @@ -293,7 +294,8 @@ describe("a credential hop is settled by whichever layer dispatches its replay", expect(budget.used).toBe(1); // The helper names the same physical send the hop already booked. - budget.used += 1; + if (!hop.allowed) throw new Error("unreachable"); + reportDispatchSends(budget, 1, hop.permit); expect(budget.used).toBe(1); // A genuinely second send is charged in full. budget.used += 1; @@ -373,7 +375,8 @@ describe("derived policy scopes", () => { const target = deriveRequestExecutionBudget(scope, wide); // The reporter names the send that the booking above already paid for. - target.used += 1; + if (!hop.allowed) throw new Error("unreachable"); + reportDispatchSends(target, 1, hop.permit); expect(parent.used).toBe(1); // Anything beyond it is a genuinely new send. target.used += 2; @@ -455,8 +458,10 @@ describe("derived scopes and the durable spend observer", () => { const spy = recordingObserver(); const parent = createRequestExecutionBudget(wide, "lr-once", spy.observer); const scope = deriveRequestExecutionBudget(parent, wide); - expect(scope.reserveDispatch({ sendClass: "initial", targetKey: "a/m", countedExternally: true }).allowed).toBe(true); - deriveRequestExecutionBudget(scope, wide).used += 1; + const hop = scope.reserveDispatch({ sendClass: "initial", targetKey: "a/m", countedExternally: true }); + expect(hop.allowed).toBe(true); + if (!hop.allowed) throw new Error("unreachable"); + reportDispatchSends(deriveRequestExecutionBudget(scope, wide), 1, hop.permit); expect(spy.events).toEqual(["charge"]); expect(parent.used).toBe(1); }); @@ -559,3 +564,15 @@ test("a validated rebase remains admissible after alternate-target spend and ref expect(budget.alternateTargetSends).toBe(1); expect(budget.targetTransitions).toBe(1); }); + +test("a reported external receipt cannot be taken over by an adapter", () => { + const budget = createRequestExecutionBudget(); + const decision = budget.reserveDispatch({ sendClass: "initial", targetKey: "same", countedExternally: true }); + if (!decision.allowed) throw new Error("synthetic reservation refused"); + reportDispatchSends(budget, 1, decision.permit); + expect(decision.permit.assumeCharge()).toBe(false); + expect(decision.permit.use()).toBe(true); // reset helper reports before its dispatch thunk + expect(decision.permit.use()).toBe(false); + decision.permit.release(); + expect(budget.used).toBe(1); +}); diff --git a/tests/lib/spend-pool-continuity.test.ts b/tests/lib/spend-pool-continuity.test.ts index 3ee0ba59051..34923a6445c 100644 --- a/tests/lib/spend-pool-continuity.test.ts +++ b/tests/lib/spend-pool-continuity.test.ts @@ -16,7 +16,7 @@ import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; import { admitHttpWorkflowTurn, workflowDecisionRefusalResponse } from "../../src/server/workflow-refusal"; import { createResponsesSendBudget } from "../../src/server/responses/request-send-budget"; -import { claimDispatchSpendProof, createRequestExecutionBudget, deriveRequestExecutionBudget } from "../../src/lib/request-execution-budget"; +import { claimDispatchSpendProof, createRequestExecutionBudget, deriveRequestExecutionBudget, reportDispatchSends } from "../../src/lib/request-execution-budget"; import { createPoolContinuity } from "../../src/lib/spend-pool-continuity"; import { listWorkflowBudgetEvents, resetWorkflowBudgetsForTest, workflowSpendCeilingReached } from "../../src/lib/workflow-budget"; import { createRequestSpendTracker } from "../../src/server/responses/request-spend"; @@ -377,7 +377,7 @@ for (const end of ["release", "report", "assume"] as const) { expect(workflowSpendCeilingReached(undefined, ledger, "provider", proof)).toBeUndefined(); expect(claimDispatchSpendProof(budget, permit)).toBeUndefined(); } - if (end === "report") budget.used += 1; + if (end === "report") reportDispatchSends(budget, 1, permit); if (end === "assume") expect(permit.assumeCharge()).toBe(true); permit.release(); permit.release(); @@ -398,7 +398,7 @@ test("prepaid proof belongs to one shared budget and one still-pending dispatch" const { budget, reservePermit } = prepaidFixture(); const permit = reservePermit(); if (end === "release") permit.release(); - if (end === "report") budget.used += 1; + if (end === "report") reportDispatchSends(budget, 1, permit); if (end === "assume") expect(permit.assumeCharge()).toBe(true); expect(claimDispatchSpendProof(budget, permit)).toBeUndefined(); } @@ -420,12 +420,12 @@ test("receipt identity survives out-of-order handoff and reports cannot authoriz expect(b.assumeCharge()).toBe(true); expect(claimDispatchSpendProof(first.budget, b)).toBeUndefined(); expect(claimDispatchSpendProof(first.budget, a)).toBeDefined(); - first.budget.used += 1; + reportDispatchSends(first.budget, 1, a); expect(first.budget.used).toBe(2); // report consumed A; B was already assumed const second = prepaidFixture(10, 100); const reported = second.reservePermit(), pending = second.reservePermit(); - second.budget.used += 1; + reportDispatchSends(second.budget, 1, reported); expect(claimDispatchSpendProof(second.budget, reported)).toBeUndefined(); reported.release(); expect(second.budget.used).toBe(2); // cannot refund a different pending receipt @@ -525,3 +525,98 @@ test("actual combo dispatch forwards only its own prepaid permit into the child removeTreeWithRetry(home); } }); + +test("a later child report preserves the earlier receipt and refunds its exact pool", () => { + const ledger = createSpendReservationLedger({ salt }); + const logCtx = { provider: "earlier", usageLogInputTokens: 10 }; + const tracker = createRequestSpendTracker(logCtx, "root", ledger); + const budget = createRequestExecutionBudget(undefined, undefined, tracker); + const first = budget.reserveDispatch({ sendClass: "initial", targetKey: "same", countedExternally: true }); + logCtx.provider = "later"; + logCtx.usageLogInputTokens = 30; + const second = budget.reserveDispatch({ sendClass: "initial", targetKey: "same", countedExternally: true }); + if (!first.allowed || !second.allowed) throw new Error("synthetic reservation refused"); + const child = createResponsesSendBudget({ + req: new Request("http://localhost/v1/responses"), logCtx: {}, + options: { sendBudget: deriveRequestExecutionBudget(budget, budget.policy), comboDispatchPermit: second.permit }, + }); + if (child instanceof Response) throw new Error("synthetic child refused"); + child.noteTransientSends(1); + expect(claimDispatchSpendProof(budget, first.permit)).toBeDefined(); + expect(claimDispatchSpendProof(budget, second.permit)).toBeUndefined(); + second.permit.release(); + expect(budget.used).toBe(2); + first.permit.release(); + expect(budget.used).toBe(1); + expect(ledger.snapshot("pool", "earlier")).toMatchObject({ reserved: 0, unresolved: 0 }); + expect(ledger.snapshot("pool", "later")).toMatchObject({ reserved: 30, unresolved: 0 }); + tracker.settle({ inputTokens: 25, outputTokens: 0 }); + expect(ledger.snapshot("pool", "later")).toMatchObject({ settled: 25, reserved: 0, unresolved: 0 }); +}); + +test("captured reporters retain receipt ownership across handoffs, cancellation and retries", () => { + for (const finish of ["release", "report", "assume"] as const) { + const { ledger, tracker, budget, reservePermit } = prepaidFixture(10, 100); + const a = reservePermit(); + const owner = createResponsesSendBudget({ + req: new Request("http://localhost/v1/responses"), logCtx: {}, options: { sendBudget: budget }, + }); + if (owner instanceof Response) throw new Error("synthetic owner refused"); + owner.pendingHopPermit = a; + const reportA = owner.transientSendReporter(); + const b = reservePermit(); + owner.pendingHopPermit = b; + const reportB = owner.transientSendReporter(); + reportB(0); // A cancelled/no-send helper cannot settle a receipt. + reportB(1); + b.release(); + expect(budget.used).toBe(2); + expect(claimDispatchSpendProof(budget, b)).toBeUndefined(); + expect(claimDispatchSpendProof(budget, a)).toBeDefined(); + if (finish === "report") reportA(1); + if (finish === "assume") expect(a.assumeCharge()).toBe(true); + a.release(); + a.release(); + expect(budget.used).toBe(finish === "release" ? 1 : 2); + // The same reporter's next count is a real retry, never another prepaid receipt. + reportB(1); + expect(budget.used).toBe(finish === "release" ? 2 : 3); + tracker.settle(undefined); + expect(ledger.snapshot("pool", "provider")).toMatchObject({ + reserved: 0, settled: 0, unresolved: finish === "release" ? 20 : 30, + }); + } +}); + +test("unnamed and foreign reports cannot consume a pending receipt", () => { + const { ledger, budget, reservePermit } = prepaidFixture(10, 100); + const a = reservePermit(), b = reservePermit(); + const foreign = prepaidFixture(10, 100).reservePermit(); + budget.used += 1; + reportDispatchSends(deriveRequestExecutionBudget(budget, budget.policy), 1, foreign); + expect(budget.used).toBe(4); + expect(claimDispatchSpendProof(budget, a)).toBeDefined(); + expect(claimDispatchSpendProof(budget, b)).toBeDefined(); + a.release(); + b.release(); + expect(budget.used).toBe(2); + expect(ledger.snapshot("pool", "provider")).toMatchObject({ reserved: 20, unresolved: 0 }); + reportDispatchSends(budget, 1, a); // A late physical report is counted; no proof is revived. + expect(budget.used).toBe(3); + expect(claimDispatchSpendProof(budget, a)).toBeUndefined(); +}); + +test("releasing an older receipt preserves the later target and its recovery charges", () => { + const budget = createRequestExecutionBudget(); + const a = budget.reserveDispatch({ sendClass: "initial", targetKey: "a", countedExternally: true }); + const b = budget.reserveDispatch({ sendClass: "account-failover", targetKey: "b", countedExternally: true }); + if (!a.allowed || !b.allowed) throw new Error("synthetic reservation refused"); + reportDispatchSends(budget, 1, b.permit); + a.permit.release(); + expect(budget.used).toBe(1); + expect(budget.lastTargetKey).toBe("b"); + expect(budget.alternateTargetSends).toBe(1); + expect(budget.targetTransitions).toBe(1); + b.permit.release(); + expect(budget.used).toBe(1); +}); diff --git a/tests/lib/transient-budget-scope-source.test.ts b/tests/lib/transient-budget-scope-source.test.ts index ecf41ca3613..9f314bbc5d8 100644 --- a/tests/lib/transient-budget-scope-source.test.ts +++ b/tests/lib/transient-budget-scope-source.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from "bun:test"; import { createRequestExecutionBudget, deriveRequestExecutionBudget, + reportDispatchSends, type RequestExecutionBudget, type RequestExecutionBudgetPolicy, type RequestSendObserver, @@ -91,7 +92,7 @@ describe("transient send accounting stays request-scoped", () => { return new Response("ok"); }, { attempts: 1, - onSendsConsumed: (count) => { budget.used += count; }, + onSendsConsumed: (count) => { reportDispatchSends(budget, count, reserved.permit); }, }); expect(response.status).toBe(200); @@ -102,6 +103,23 @@ describe("transient send accounting stays request-scoped", () => { expect(reserved.permit.use()).toBe(false); }); + test("a reset helper can report its exact receipt before the single-use dispatch thunk", async () => { + const budget = createRequestExecutionBudget(THREE_SEND_POLICY); + const earlier = budget.reserveDispatch({ sendClass: "initial", targetKey: "same", countedExternally: true }); + const later = budget.reserveDispatch({ sendClass: "initial", targetKey: "same", countedExternally: true }); + if (!earlier.allowed || !later.allowed) throw new Error("synthetic reservation refused"); + let physicalSends = 0; + await fetchWithResetRetry(async () => { + expect(later.permit.use()).toBe(true); + physicalSends += 1; + return new Response("ok"); + }, { attempts: 1, onSendsConsumed: count => reportDispatchSends(budget, count, later.permit) }); + later.permit.release(); + earlier.permit.release(); + expect(budget.used).toBe(physicalSends); + expect(physicalSends).toBe(1); + }); + test("the transient wrapper reports a rejected physical send exactly once", async () => { const observer = recordingObserver(); const budget = createRequestExecutionBudget(THREE_SEND_POLICY, "lr-rejected", observer); diff --git a/tests/responses/chat-native-combo.test.ts b/tests/responses/chat-native-combo.test.ts index 55b9d15a679..c527364b37e 100644 --- a/tests/responses/chat-native-combo.test.ts +++ b/tests/responses/chat-native-combo.test.ts @@ -168,6 +168,7 @@ describe("native Chat candidates in a combo", () => { expect(b.bodies).toHaveLength(0); expect(rows).toHaveLength(1); expect(rows[0]!.status).toBe(200); + expect(rows[0]!.spend).toMatchObject({ sends: 1, unresolved: 0 }); expect(rows[0]!.provider).toBe("combo"); expect(rows[0]!.protocolTrace).toMatchObject({ inbound: "chat", mode: "native", requestPath: ["chat", "chat"], @@ -201,6 +202,7 @@ describe("native Chat candidates in a combo", () => { expect(b.bodies[0]).not.toHaveProperty("messages"); expect(rows).toHaveLength(1); expect(rows[0]!.attempts?.map(attempt => attempt.status)).toEqual([503, 200]); + expect(rows[0]!.spend).toMatchObject({ sends: 4, unresolved: 0 }); expect(rows[0]!.protocolTrace).toMatchObject({ inbound: "chat", requestPath: ["chat", "responses"], diff --git a/tests/responses/responses-core-modules.test.ts b/tests/responses/responses-core-modules.test.ts index 6d92a3ab551..426889d924e 100644 --- a/tests/responses/responses-core-modules.test.ts +++ b/tests/responses/responses-core-modules.test.ts @@ -161,7 +161,7 @@ describe("Responses request-owned send budget after extraction", () => { expect(owner.pendingHopPermit).toBeUndefined(); expect(hop.permit.use()).toBe(true); expect(hop.permit.use()).toBe(false); - owner.noteTransientSends(1); + owner.transientSendReporter(allowance.permit)(1); expect(holder.used).toBe(4); expect(owner.remainingTransientSendBudget(3)).toBe(0); } finally { dispose(); } diff --git a/tests/responses/responses-spend-ledger-wiring.test.ts b/tests/responses/responses-spend-ledger-wiring.test.ts index 4fa5fc1d6ee..152e69f0f6e 100644 --- a/tests/responses/responses-spend-ledger-wiring.test.ts +++ b/tests/responses/responses-spend-ledger-wiring.test.ts @@ -7,7 +7,7 @@ import { DEFAULT_SPEND_RESERVATION_POLICY, type SpendJournal, } from "../../src/lib/spend-reservation-ledger"; -import { createRequestExecutionBudget } from "../../src/lib/request-execution-budget"; +import { createRequestExecutionBudget, reportDispatchSends } from "../../src/lib/request-execution-budget"; import { createRequestSpendTracker } from "../../src/server/responses/request-spend"; import { SpendLedgerOwnerError, type SpendLedgerOwnerErrorCode } from "../../src/lib/spend-ledger-owner"; import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; @@ -189,7 +189,7 @@ describe("the request path books every physical send on the durable ledger", () }), body, "spend", config, logCtx, { sendBudget, translatorBudget }, { handleResponses: async (_req, _config, childLog, options) => { children += 1; - options!.sendBudget!.used += 1; // Report the send already reserved by the combo. + reportDispatchSends(options!.sendBudget!, 1, options!.comboDispatchPermit); childLog.provider += `-account-${children}`; return children === 1 ? Response.json({ error: { message: "fixture outage" } }, { status: 503 }) diff --git a/tests/server/inference-send-budget.test.ts b/tests/server/inference-send-budget.test.ts index 0e3ca1597a9..4870996cf16 100644 --- a/tests/server/inference-send-budget.test.ts +++ b/tests/server/inference-send-budget.test.ts @@ -64,7 +64,7 @@ describe("createInferenceSendBudget", () => { if (sends === 0) hop.permit?.use(); sends++; return new Response("", { status: sends === 3 ? 200 : 503 }); - }, { attempts: allowance.attempts, onSendsConsumed: owner.noteTransientSends }); + }, { attempts: allowance.attempts, onSendsConsumed: owner.transientSendReporter(allowance.permit) }); expect(response.status).toBe(200); expect(sends).toBe(3); expect(budget.used).toBe(12); From 056941866fdc815708313a6d2744bf1e4e221c49 Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Thu, 1 Oct 2026 22:32:02 -0700 Subject: [PATCH 05/15] fix(spend): report prepaid reset-only combo sends --- src/server/responses/adapter-dispatch.ts | 3 +++ src/server/responses/core-options.ts | 4 ++-- structure/transports/responses-spend.md | 2 ++ .../lib/ambiguous-resend-composition.test.ts | 22 ++++++++++--------- .../responses-compaction-recovery.test.ts | 8 ++++++- 5 files changed, 26 insertions(+), 13 deletions(-) diff --git a/src/server/responses/adapter-dispatch.ts b/src/server/responses/adapter-dispatch.ts index f228eff3886..500039924b0 100644 --- a/src/server/responses/adapter-dispatch.ts +++ b/src/server/responses/adapter-dispatch.ts @@ -365,6 +365,9 @@ export async function prepareAdapterExchange( { abortSignal: upstream.signal, label: safeHostLabel(builtInitialRequest.url), + // Even the reset-only combo leg must settle its named prepaid receipt. + ...(options.comboDispatchPermit + ? { onSendsConsumed: transientSendReporter(options.comboDispatchPermit) } : {}), ...(transientPolicy || compactPrepaid // Draws the remainder, not the raw policy. A combo child inherits the parent's // holder but used to take a fresh full allowance on its own first send, so the diff --git a/src/server/responses/core-options.ts b/src/server/responses/core-options.ts index 445b004ef40..68e7a9eb062 100644 --- a/src/server/responses/core-options.ts +++ b/src/server/responses/core-options.ts @@ -68,7 +68,7 @@ export interface HandleResponsesOptions { compactionRecoveryKind?: "compaction-v1" | "compaction-v2"; onCompactionRecoveryRoute?: (route: RouteResult) => void; onCompactionRecoveryAdapterEvent?: (event: AdapterEvent) => void; - /** Physical-send reports already delivered to the shared used setter, including booking settlement. */ + /** Physical-send reports already delivered to the shared ledger, including exact booking settlement. */ onCompactionRecoverySendsReported?: (count: number) => void; /** Private holder for the Kiro serving-account lease. */ accountLoad?: { lease: AccountLease | null; cancelled: boolean }; @@ -153,7 +153,7 @@ export interface HandleResponsesOptions { callerDirectAuth?: CallerDirectAuth | null; /** Internal recursion guard; callers outside this module must not set it. */ comboAttempt?: boolean; - /** Exact externally booked combo hop, used only for its child's spend preflight. */ + /** Exact externally booked combo hop, used for its child's spend preflight and send reports. */ comboDispatchPermit?: SingleUseDispatchPermit; /** Internal handoff: this combo was selected by shadow-call interception. */ shadowCallIntercepted?: boolean; diff --git a/structure/transports/responses-spend.md b/structure/transports/responses-spend.md index 53afd9bb089..339349a5844 100644 --- a/structure/transports/responses-spend.md +++ b/structure/transports/responses-spend.md @@ -42,6 +42,8 @@ reporter's first send settles only its named pending permit instead of adding a `transientSendReporter` captures that permit before entering a helper; a later handoff cannot replace it. `reportDispatchSends` verifies shared-ledger ownership and consumes one receipt only. Native Chat combo children carry the same exact permit to their physical-send boundary. +Reset-only generic combo helpers report their named prepaid receipt too, without changing +the selected retry cap; compaction reconciliation therefore does not count that source again. Numeric `used` updates and unnamed, foreign, released or already-reported permits charge actual sends without consuming another reservation. Extra retries remain full charges. Reports may arrive out of reservation order; no FIFO ordering is required. An adapter diff --git a/tests/lib/ambiguous-resend-composition.test.ts b/tests/lib/ambiguous-resend-composition.test.ts index f82bf06bcd2..093e4a61932 100644 --- a/tests/lib/ambiguous-resend-composition.test.ts +++ b/tests/lib/ambiguous-resend-composition.test.ts @@ -5,6 +5,8 @@ import { CODEX_TEXT_GUARDED_BUDGET_POLICY, createRequestExecutionBudget, deriveRequestExecutionBudget, + reportDispatchSends, + type SingleUseDispatchPermit, type RequestExecutionBudget, type SendClass, } from "../../src/lib/request-execution-budget"; @@ -96,23 +98,23 @@ function oneLogicalRequest() { * What src/server/responses/request-send-budget.ts hands a recovery leg: the base allowance * while it lasts, then the single final-recovery reserve, and nothing after that. */ - const recoveryAttempts = (sendClass: SendClass, targetKey: string): number => { + const recoveryAllowance = (sendClass: SendClass, targetKey: string): { attempts: number; permit?: SingleUseDispatchPermit } => { const base = budget.remainingBaseSends(TRANSIENT_RETRY_MAX_ATTEMPTS); - if (base > 0) return base; + if (base > 0) return { attempts: base }; const decision = budget.reserveDispatch({ sendClass, targetKey, countedExternally: true }); - return decision.allowed ? 1 : 0; + return decision.allowed ? { attempts: 1, permit: decision.permit } : { attempts: 0 }; }; return { budget, get physicalSends(): number { return counts.physical; }, get ambiguousSends(): number { return counts.ambiguous; }, - recoveryAttempts, + recoveryAllowance, /** A leg that fails before any response head, through the real reset ladder. */ - preHeader: (outcomes: Array, row: ProviderRow, attempts?: number): Promise => + preHeader: (outcomes: Array, row: ProviderRow, allowance?: { attempts: number; permit?: SingleUseDispatchPermit }): Promise => fetchWithResetRetry(dispatcher(outcomes), { - attempts: attempts ?? budget.remainingBaseSends(TRANSIENT_RETRY_MAX_ATTEMPTS), - onSendsConsumed: sends => { budget.used += sends; }, + attempts: allowance?.attempts ?? budget.remainingBaseSends(TRANSIENT_RETRY_MAX_ATTEMPTS), + onSendsConsumed: sends => reportDispatchSends(budget, sends, allowance?.permit), claimAmbiguousResend: () => authorize("pre-header", row).allowed, }), /** A stream that died after the head while carrying only control events. */ @@ -191,9 +193,9 @@ describe("one resend budget across composed recovery legs", () => { // the account it moves to is a row the operator tuned HIGHER. A ceiling read from the // asking leg let that row buy a second duplicate inference on the way out; the ceiling // is the smallest any leg presented, so it buys nothing. - const attempts = request.recoveryAttempts("account-failover", "other-account"); - expect(attempts).toBe(1); - const lastSend = await request.preHeader([reset(), new Response("duplicate")], MORE_PERMISSIVE, attempts); + const allowance = request.recoveryAllowance("account-failover", "other-account"); + expect(allowance.attempts).toBe(1); + const lastSend = await request.preHeader([reset(), new Response("duplicate")], MORE_PERMISSIVE, allowance); expect(isNonReplayableResponse(lastSend)).toBe(true); expect(request.ambiguousSends).toBe(GRANT); diff --git a/tests/responses/responses-compaction-recovery.test.ts b/tests/responses/responses-compaction-recovery.test.ts index 076909a06d3..0fc68284d06 100644 --- a/tests/responses/responses-compaction-recovery.test.ts +++ b/tests/responses/responses-compaction-recovery.test.ts @@ -322,11 +322,17 @@ describe("routed compaction emergency integration", () => { const budget = createRequestExecutionBudget({ maxTotalModelSends: 2, baseSendAllowance: 2, finalRecoveryAllowance: 0, maxAlternateTargetSends: 1, maxTargetTransitions: 1 }); const reservation = budget.reserveDispatch({ sendClass: "initial", targetKey: "source/swe-2", countedExternally: true }); expect(reservation.allowed).toBe(true); - const response = await handleResponses(request(), config, { model: "", provider: "" }, { sendBudget: budget }); + if (!reservation.allowed) throw new Error("synthetic source reservation refused"); + const response = await handleResponses(request(), config, { model: "", provider: "" }, { + sendBudget: budget, comboDispatchPermit: reservation.permit, + }); expect((await response.json()).status).toBe("completed"); expect(sourceRequests).toBe(1); expect(calls.map(call => call.model)).toEqual(["rescue"]); expect(budget.used).toBe(2); + expect(reservation.permit.assumeCharge()).toBe(false); + reservation.permit.release(); + expect(budget.used).toBe(2); }); test("runTurn emergency consumes its prepaid permit once rather than taking another send", async () => { From 023c3d8c4c1fc7a9fabda8df8a1b2fe05e80384b Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Fri, 2 Oct 2026 00:00:59 -0700 Subject: [PATCH 06/15] fix(spend): aggregate pool eviction once and sync locale guidance --- .../docs/ja/reference/configuration/server.md | 57 +++++++++++++++++ .../docs/ko/reference/configuration/server.md | 53 ++++++++++++++++ .../docs/ru/reference/configuration/server.md | 63 +++++++++++++++++++ .../zh-cn/reference/configuration/server.md | 47 ++++++++++++++ src/lib/spend-reservation-ledger.ts | 15 ++++- structure/transports/responses-spend.md | 3 + tests/lib/spend-pool-continuity.test.ts | 44 +++++++++++++ 7 files changed, 281 insertions(+), 1 deletion(-) diff --git a/docs-site/src/content/docs/ja/reference/configuration/server.md b/docs-site/src/content/docs/ja/reference/configuration/server.md index c11d7af21c0..0c062f1befb 100644 --- a/docs-site/src/content/docs/ja/reference/configuration/server.md +++ b/docs-site/src/content/docs/ja/reference/configuration/server.md @@ -187,3 +187,60 @@ Anthropic OAuth サイドカーは、opencodex の既存のクロード コー メイン Codex アカウント行の `quotaRefresh` はクォータ取得の診断情報であり、残量やモデルへのアクセス権を示すものではありません。キャッシュ利用時や取得を行わない場合は省略されることがあります。取得には操作中のシェルではなく、実行中のプロキシサービスの環境が使われます。`proxy` 未設定では既存の環境を維持し、`"auto"` は起動時の Windows または macOS の静的 HTTP/HTTPS 設定を読みます。macOS では継承したプロキシがある場合、読み取りを行いません。macOS では有効な `*.` を `.` に変換します。`*.local` は `foo.local` と基底名 `local` を直接接続にしますが、`xlocal` は対象外です。`169.254/16`、`169.254.0.0/16`、`fe80::/10` は診断を出して省略し、リンクローカル IP アドレスはプロキシを使います。IP アドレスと `*` は受け入れますが、その他の CIDR、glob、単純ホスト名の例外では環境を変更せず検出を中止します。PAC/WPAD、SOCKS のみの設定、実行中の変更は自動反映されません。TUN での成功だけでは HTTP プロキシ経路の正常性は確認できません。[コマンドと状態の説明（英語）](/reference/configuration/server/#codex-quota-network-diagnostics)を参照してください。 `dropCodexSafetyBuffering`: プロバイダーの安全性の適用と拒否応答は変更しません。native `codex.response.metadata.headers` WebSocket メタデータと `/responses/compact` は対象外です。 + +## 過去のプール使用量の継承 + +プロバイダープールの上限にはルーティング先の正規プロバイダーを使い、リクエストログにはアカウント別の +表示ラベルを残します。古いジャーナルでは、その表示ラベルに使用量が計上されている場合があります。 +ジャーナルに保存されるのはソルト付きのエイリアスであり、元のプロバイダー名やアカウント名ではありません。 +そのため、OpenCodex は現在のアカウント一覧、ラベルの接頭辞、短縮されたアカウント ID から対応関係を推測しません。 + +`spend.pool.maxTokens` が設定されていて、正の使用量が残る過去のプールを識別できない場合、推論リクエストは +プロバイダーへ接続する前にローカルの HTTP 429 と +`x-opencodex-local-refusal: workflow_pool_history_unresolved` で拒否されます。 +これにより、ワークフロールートを持たないリクエストを含め、本来は有効なリクエストが一時的にブロックされることがあります。 +ルーティング後の事前確認でも、合算した使用量がすでに上限に達している正規プールを拒否します。 +予約によってすでに許可されたリカバリー送信やコンボ送信では、その予約を自分自身に対して再度計上しません。 +それ以外の予約はすべて計上されます。この確認により、送信後に使用量を報告するパススルー送信が +アトミックな予約に変わるわけではありません。上限をまたぐ送信、同時に許可されるリクエスト、 +事後報告される再試行には、従来の制約が残ります。監視のみの構成は引き続き監視のみです。 +ルートと ID の上限もそれぞれ独立して適用されます。 + +解決するには、運用者自身が保持している証拠を使い、過去の各ソルト付きプールエイリアスが +どの正規プロバイダーに属するかを確認する必要があります。`config.json` のトップレベルに +`spendPoolAliases` オブジェクトを追加してください。各キーには、その同じインストールのジャーナルにある +32 文字の小文字 16 進数のプールエイリアスを正確に指定し、各値には確認済みの正規プロバイダー ID を指定します。 +古いエイリアスがすでに正規プロバイダーを表していても、ID メタデータがなければ明示的なエントリが必要です。 +正の使用量を持ち、識別できないエイリアスはブロックされたままです。似た名前、アカウントの削除、 +短縮ラベルの衝突から対応関係を推測しないでください。ジャーナルやソルトを公開しないでください。 + +`spendPoolAliases` は `spend` の外に置いてください。古いバージョンは `spend` 内の未知のキーを拒否し、 +セクション全体を無効にする場合があります。無効なトップレベルのマッピングは設定の書き込み時に拒否されます。 +手動編集で `spendPoolAliases` が不正になった場合、既存の上限を維持したままプールへのリクエスト受け入れを拒否します。 +通常の設定手順でマッピングを修正し、再起動してください。マッピングを空にしたり削除したりしても、 +すでに永続的に記録された対応関係は消えません。一度リダイレクトされたエイリアスを別のグループへ再割り当てすることはできません。 +確認済みの正規プロバイダー名の変更では、既存の使用量を分割せずにグループを統合できます。 +マッピング全体をまとめて検証するため、有効なグループ統合はエイリアスキーの順序に依存しません。 + +元の使用量はそれぞれ一度だけ計上されます。確定済みの使用量、処理中の予約、未解決の使用量はすべて計上し、 +不明な使用量を払い戻しとして扱うことはありません。元の予約先は保持されます。 +新しいリクエストでは引き続き正規プロバイダーのプールを記録し、チェックポイントには生のアカウント名や +プロバイダー名を追加せず、ソルト付きの ID 識別情報を保持します。識別できない正の使用履歴と、稼働中または +上限に達したグループはクリーンアップから保護されます。休眠中で上限未満の項目には従来の保持ルールが引き続き適用されます。 +ID 識別情報の保存容量には上限があり、容量を使い切った場合は情報を忘れるのではなくプールへの受け入れを停止します。 +検証できない履歴を復元する自動ツールはありません。 + +### 安全なロールバックの要件 + +サポートされるロールバックでは、リクエストを正規プロバイダーに帰属させる処理と、プール間の継承に対応した +台帳の読み書き処理の両方を維持するかバックポートし、同じジャーナル、ソルト、確認済みマッピングを使う必要があります。 +最新のジャーナルを保持してください。アップグレード前のコピーを復元すると、それ以降の使用量が欠落します。 +ダウングレードを互換性があるように見せるために、記録の削除、ソルトの変更、上限の引き上げ、制限の無効化をしないでください。 + +変更を加えていない古いバイナリへの切り戻しは、**サポート対象のロールバックではありません**。 +古いバイナリは v1 の生の計上値を読めても、エイリアスをまたぐプロバイダー合計に上限を適用せず、 +新たなアカウントラベルのプールを作成したり、圧縮時に任意の ID メタデータを破棄したりする場合があります。 +ダウングレードを自動的に防ぐ仕組みはありません。そのような古い書き込み処理が実行された場合、新しい読み取り処理は、 +運用者が残っているすべてのマッピングを明示的に再確認するまで、識別できない正の使用履歴を拒否します。 +ストレージやジャーナルの整合性による拒否はエイリアスの問題とは別で、`workflow_spend_undurable` のままです。 +安全でないファイルや所有権の問題は、代わりにストレージエラーとして伝播する場合がありますが、リクエストは許可されません。 diff --git a/docs-site/src/content/docs/ko/reference/configuration/server.md b/docs-site/src/content/docs/ko/reference/configuration/server.md index ee24fd4f16e..d0980a0d407 100644 --- a/docs-site/src/content/docs/ko/reference/configuration/server.md +++ b/docs-site/src/content/docs/ko/reference/configuration/server.md @@ -244,3 +244,56 @@ Anthropic OAuth 사이드카는 opencodex의 기존 Claude Code OAuth fingerprin ## Codex 할당량 네트워크 진단 메인 Codex 계정 행의 `quotaRefresh`는 할당량 조회 결과를 분류하는 진단값입니다. 남은 할당량이나 모델 접근 권한을 뜻하지 않으며, 캐시를 쓰거나 조회하지 않았다면 생략될 수 있습니다. 요청은 명령을 입력한 터미널이 아니라 실행 중인 프록시 서비스의 환경을 따릅니다. `proxy`를 지정하지 않으면 기존 환경을 유지하고, `"auto"`는 시작 시 Windows 또는 macOS의 정적 HTTP/HTTPS 설정을 읽습니다. macOS에서는 상속된 프록시가 있으면 읽지 않습니다. macOS에서는 유효한 `*.`을 `.`으로 바꿉니다. `*.local`은 `foo.local`과 최상위 이름 `local`을 직접 연결하지만 `xlocal`은 제외합니다. `169.254/16`, `169.254.0.0/16`, `fe80::/10`은 진단 메시지와 함께 생략하므로 링크 로컬 IP 주소는 프록시를 사용합니다. IP 주소와 `*`는 허용하지만 다른 CIDR, glob, 단순 호스트명 예외는 환경 변경 전에 탐색을 거부합니다. PAC/WPAD, SOCKS 전용 설정과 실행 중 변경은 자동으로 반영하지 않습니다. TUN에서 성공했다고 HTTP 프록시 경로도 정상이라는 뜻은 아닙니다. 명령과 상태값은 [네트워크 진단(영문)](/reference/configuration/server/#codex-quota-network-diagnostics)에서 확인하세요. + +## 과거 풀 사용량의 연속성 + +제공자 풀 상한은 라우팅된 정규 제공자를 기준으로 적용하고, 요청 로그에는 계정별 표시 레이블을 유지합니다. +이전 저널에는 이 표시 레이블에 사용량이 기록되어 있을 수 있습니다. 저널은 원래 제공자·계정 이름이 아닌 +솔트가 적용된 별칭을 저장하므로, OpenCodex는 현재 계정 목록, 레이블 접두사, 축약된 계정 ID로 매핑을 추측하지 않습니다. + +`spend.pool.maxTokens`가 설정되어 있고 양수의 사용량이 남은 과거 풀을 식별하지 못하면, 추론 요청은 +제공자에 접속하기 전에 로컬 HTTP 429와 +`x-opencodex-local-refusal: workflow_pool_history_unresolved`로 거부됩니다. +워크플로 루트가 없는 요청을 포함해 원래 유효한 요청도 일시적으로 차단될 수 있습니다. +라우팅 후 사전 검사에서도 합산 사용량이 이미 한도에 도달한 정규 풀을 거부합니다. +예약을 통해 이미 허용된 복구 또는 콤보 전송에서는 그 예약을 자신에게 다시 계산하지 않습니다. +다른 예약은 모두 계산합니다. 이 검사가 사용량을 사후 보고하는 패스스루 전송을 원자적 예약으로 바꾸지는 않습니다. +한도를 넘기는 전송, 동시 요청 허용, 사후 보고되는 재시도에는 기존 제약이 그대로 남습니다. +관찰 전용 구성은 계속 관찰 전용으로 동작합니다. 루트와 ID 상한도 각각 독립적으로 적용됩니다. + +해결하려면 운영자가 보관한 근거를 사용해 과거의 각 솔트 적용 풀 별칭이 어느 정규 제공자에 속하는지 확인해야 합니다. +`config.json`의 최상위에 `spendPoolAliases` 객체를 추가하세요. 각 키는 동일 설치의 저널에 있는 +32자리 소문자 16진수 풀 별칭과 정확히 일치해야 하고, 각 값은 확인된 정규 제공자 ID여야 합니다. +이전 별칭이 이미 정규 제공자를 나타내더라도 ID 메타데이터가 없으면 명시적 항목이 필요합니다. +식별할 수 없는 양수 사용량의 별칭은 계속 차단됩니다. 비슷한 이름, 계정 삭제, 축약 레이블 충돌로 +매핑을 추측하지 말고, 저널이나 솔트를 공개하지 마세요. + +`spendPoolAliases`는 `spend` 밖에 두세요. 이전 버전은 `spend` 안의 알 수 없는 키를 거부하며 +섹션 전체를 비활성화할 수 있습니다. 잘못된 최상위 매핑은 설정 쓰기 시 거부됩니다. +수동 편집으로 `spendPoolAliases`가 잘못된 형식이 되면 기존 상한을 유지한 채 풀 요청 허용을 차단합니다. +일반적인 설정 절차로 매핑을 수정하고 재시작하세요. 매핑을 비우거나 삭제해도 이미 영구 기록된 연결은 지워지지 않습니다. +이전에 다른 대상으로 연결한 별칭은 다른 그룹에 재할당할 수 없습니다. 확인된 정규 제공자 이름 변경은 +기존 사용량을 나누지 않고 그룹을 합칠 수 있습니다. 전체 매핑을 함께 검증하므로 유효한 그룹 병합은 별칭 키 순서에 의존하지 않습니다. + +각 원래 사용량은 한 번만 계산합니다. 확정된 사용량, 진행 중 예약, 미해결 사용량을 모두 계산하며, +알 수 없는 사용량을 환급으로 취급하지 않습니다. 원래 예약 대상은 유지됩니다. +새 요청은 계속 정규 제공자 풀을 기록하고, 체크포인트는 원래 계정·제공자 이름을 추가하지 않은 채 +솔트가 적용된 ID 식별 근거를 보존합니다. 알 수 없는 양수 사용 이력과 활성 상태 또는 한도에 도달한 그룹은 +정리 대상에서 보호됩니다. 휴면 상태이고 한도 미만인 항목에는 기존 보존 규칙이 계속 적용됩니다. +ID 식별 근거의 저장 용량에는 한도가 있으며, 용량을 소진하면 근거를 잊는 대신 풀 요청 허용을 중단합니다. +검증할 수 없는 이력을 복원하는 자동 도구는 없습니다. + +### 안전한 롤백 요건 + +지원되는 롤백은 요청을 정규 제공자에 귀속시키는 처리와 연속성을 인식하는 원장 읽기·쓰기 처리를 모두 유지하거나 +백포트해야 하며, 동일한 저널, 솔트, 검증된 매핑을 사용해야 합니다. +최신 저널을 유지하세요. 업그레이드 전 사본을 복원하면 이후 사용량이 누락됩니다. +다운그레이드가 호환되는 것처럼 보이게 하려고 기록을 삭제하거나, 솔트를 바꾸거나, 상한을 높이거나, 제한 적용을 끄지 마세요. + +수정하지 않은 이전 바이너리로 되돌리는 것은 **지원되는 롤백이 아닙니다**. +이전 바이너리는 v1 원시 사용량을 읽을 수 있지만 별칭 간 제공자 합계에 상한을 적용하지 않으며, +새 계정 레이블 풀을 만들거나 압축할 때 선택적 ID 메타데이터를 버릴 수 있습니다. +다운그레이드를 자동으로 막는 장치는 없습니다. 이런 이전 쓰기 처리가 실행되었다면, 새 읽기 처리는 +운영자가 남아 있는 모든 매핑을 다시 명시적으로 확인할 때까지 식별되지 않은 양수 사용 이력을 거부합니다. +스토리지 또는 저널 무결성에 따른 거부는 별칭 문제와 별개이며 `workflow_spend_undurable`을 유지합니다. +안전하지 않은 파일이나 소유권 문제는 대신 스토리지 오류로 전파될 수 있으며, 이 경우에도 요청을 허용하지 않습니다. diff --git a/docs-site/src/content/docs/ru/reference/configuration/server.md b/docs-site/src/content/docs/ru/reference/configuration/server.md index 73af6c0fbe8..7ec344d4f92 100644 --- a/docs-site/src/content/docs/ru/reference/configuration/server.md +++ b/docs-site/src/content/docs/ru/reference/configuration/server.md @@ -235,3 +235,66 @@ opencodex. Перед использованием прогоните soak-test Поле `quotaRefresh` в строке основного аккаунта Codex описывает получение квоты, а не её остаток или право доступа к модели. Оно может отсутствовать при чтении кэша или если запрос не выполнялся. Используется окружение работающего прокси-сервиса, а не текущего терминала. Если `proxy` не задан, существующее окружение сохраняется; `"auto"` при запуске читает статические настройки HTTP/HTTPS Windows или macOS. На macOS унаследованный прокси отменяет это чтение. На macOS допустимый шаблон `*.` преобразуется в `.`: для `*.local` прямое соединение получают `foo.local` и само имя `local`, но не `xlocal`. Точные диапазоны `169.254/16`, `169.254.0.0/16` и `fe80::/10` пропускаются с диагностикой: link-local IP-адреса используют прокси. IP-адреса и `*` принимаются; прочие CIDR, glob-шаблоны и исключения простых имён отменяют обнаружение без изменения окружения. PAC/WPAD, настройки только SOCKS и изменения во время работы автоматически не учитываются. Успех через TUN сам по себе не подтверждает исправность пути HTTP-прокси. См. [команды и состояния на английском](/reference/configuration/server/#codex-quota-network-diagnostics). `dropCodexSafetyBuffering`: не меняет проверки безопасности провайдера или отказы. Native WebSocket `codex.response.metadata.headers` и `/responses/compact` не входят в область фильтра. + +## Непрерывный учёт истории пулов + +Лимиты пулов провайдеров применяются к каноническому провайдеру, выбранному маршрутизацией, а журналы +запросов сохраняют отображаемые метки отдельных аккаунтов. В старых журналах расход мог учитываться +под этими метками. Журнал хранит псевдонимы с солью, а не исходные имена провайдеров и аккаунтов, +поэтому OpenCodex не выводит соответствия из текущего списка аккаунтов, префикса метки или сокращённого ID аккаунта. + +Если задан `spend.pool.maxTokens` и остаётся положительный расход в неопознанных исторических пулах, +допуск запроса на инференс завершается локальным HTTP 429 с +`x-opencodex-local-refusal: workflow_pool_history_unresolved` до обращения к провайдеру. +Это может временно блокировать корректные запросы, в том числе запросы без корня workflow. +После маршрутизации предварительная проверка также отклоняет канонический пул, суммарный расход которого +уже исчерпал лимит. Отправка при восстановлении или через combo, уже допущенная по резервированию, +не учитывает то же резервирование повторно против самой себя; все остальные резервирования учитываются. +Эта проверка не превращает passthrough-отправки с последующим отчётом об использовании в атомарные резервирования: +отправка, пересекающая лимит, одновременные допуски и повторы с последующим отчётом сохраняют существующие ограничения. +Установки в режиме только наблюдения остаются в этом режиме. Лимиты корня и идентичности продолжают действовать независимо. + +Для устранения блокировки оператор должен по собственным сохранённым подтверждениям установить, +какому каноническому провайдеру принадлежит каждый исторический псевдоним пула с солью. +Добавьте объект `spendPoolAliases` на верхнем уровне `config.json`: каждый ключ должен точно совпадать +с 32-символьным шестнадцатеричным псевдонимом пула в нижнем регистре из журнала той же установки, +а каждое значение должно быть подтверждённым каноническим ID провайдера. Даже старый псевдоним, +уже обозначающий канонического провайдера, требует явной записи, если у него нет метаданных идентичности. +Неопознанный псевдоним с положительным расходом остаётся заблокированным. Не выводите соответствие +из похожих имён, удаления аккаунта или совпадения коротких меток и не публикуйте журнал или соль. + +Размещайте `spendPoolAliases` вне `spend`: старые версии отклоняют неизвестные ключи внутри `spend` +и могут отключить весь раздел. Некорректные соответствия верхнего уровня отклоняются при записи конфигурации; +если ручная правка нарушила формат `spendPoolAliases`, существующие лимиты сохраняются, а допуск к пулам блокируется. +Исправьте соответствия и перезапустите сервис обычным способом изменения конфигурации. +Очистка или удаление соответствий не стирает уже надёжно записанные связи. Ранее перенаправленный псевдоним +нельзя назначить другой группе; подтверждённые переименования канонического провайдера могут объединять +группы без разделения существующего расхода. Все соответствия проверяются вместе, поэтому корректное +объединение групп не зависит от порядка ключей псевдонимов. + +Каждая исходная величина расхода учитывается ровно один раз. Учитываются подтверждённое использование, +резервирования в процессе выполнения и неразрешённое использование; неизвестное использование никогда +не считается возвратом. Исходные цели резервирования сохраняются. Новые запросы продолжают записываться +в пул канонического провайдера, а контрольные точки сохраняют сведения об идентичности с солью, +не добавляя исходные имена аккаунтов и провайдеров. Неопознанная история с положительным расходом, +активные группы и группы с исчерпанным лимитом защищены от очистки; существующее правило хранения +неактивных записей ниже лимита продолжает действовать. Объём сведений об идентичности ограничен: +при его исчерпании допуск к пулам прекращается, а сведения не забываются. +Автоматического инструмента для восстановления неподтверждаемой истории нет. + +### Требования к безопасному откату + +Поддерживаемый откат должен сохранять или переносить в старую версию как отнесение запросов к каноническому +провайдеру, так и чтение и запись реестра с учётом непрерывности истории, используя тот же журнал, +соль и подтверждённые соответствия. Сохраните актуальный журнал: восстановление копии до обновления +исключит более поздний расход. Не удаляйте записи, не меняйте соль, не повышайте лимиты и не отключайте +их применение, чтобы создать видимость совместимости при понижении версии. + +Неизменённый старый бинарный файл **не является поддерживаемым вариантом отката**. Он может читать +исходные величины расхода v1, но не применяет лимит к сумме по всем псевдонимам провайдера, +может создать новый пул с меткой аккаунта и отбросить необязательные метаданные идентичности при уплотнении. +Автоматического запрета понижения версии нет. Если такая старая версия выполняла запись, новая версия +при чтении отклоняет неопознанную историю с положительным расходом, пока оператор явно не подтвердит +все оставшиеся соответствия заново. Отказы из-за хранилища или целостности журнала не связаны с проблемой +псевдонимов и сохраняют `workflow_spend_undurable`; проблемы небезопасных файлов или прав владения +могут вместо этого передаваться как ошибки хранилища, при этом запрос не допускается. diff --git a/docs-site/src/content/docs/zh-cn/reference/configuration/server.md b/docs-site/src/content/docs/zh-cn/reference/configuration/server.md index dc96d8ff4b9..fed411d91fe 100644 --- a/docs-site/src/content/docs/zh-cn/reference/configuration/server.md +++ b/docs-site/src/content/docs/zh-cn/reference/configuration/server.md @@ -201,3 +201,50 @@ Anthropic OAuth 侧车会复用 opencodex 现有的 Claude Code OAuth 指纹。 主 Codex 账户行中的 `quotaRefresh` 描述额度查询结果，并不代表剩余额度或模型访问权限。读取缓存或未执行查询时，该字段可能省略。查询使用正在运行的代理服务的环境，而不是当前终端的环境。未设置 `proxy` 时保留现有环境；`"auto"` 在启动时读取 Windows 或 macOS 静态 HTTP/HTTPS 设置；macOS 上若有继承代理则跳过读取。macOS 将有效的 `*.` 转为 `.`：`*.local` 使 `foo.local` 和裸域名 `local` 直连，但不匹配 `xlocal`。精确的 `169.254/16`、`169.254.0.0/16`、`fe80::/10` 网段会跳过并给出诊断，因此链路本地 IP 地址使用代理。IP 地址和 `*` 仍可用；其他 CIDR、通配形式和简单主机名例外会在修改环境前拒绝自动发现。不自动处理 PAC/WPAD、仅 SOCKS 的设置或运行中的更改。TUN 测试成功并不能单独证明 HTTP 代理路径正常。命令和状态说明见[英文网络诊断章节](/reference/configuration/server/#codex-quota-network-diagnostics)。 `dropCodexSafetyBuffering`: 不会改变供应商安全策略或拒绝响应。原生 WebSocket `codex.response.metadata.headers` 和 `/responses/compact` 不在过滤范围内。 + +## 历史池用量的连续性 + +提供方池上限按路由选定的规范提供方计算，而请求日志保留各账户的显示标签。 +旧日志可能将用量记在这些显示标签下。日志保存的是加盐别名，而不是原始提供方或账户名称， +因此 OpenCodex 不会根据当前账户列表、标签前缀或缩短的账户 ID 推测映射。 + +如果配置了 `spend.pool.maxTokens`，并且仍有已记账用量大于零的历史池无法识别， +推理请求会在联系提供方之前被本地 HTTP 429 拒绝，并返回 +`x-opencodex-local-refusal: workflow_pool_history_unresolved`。 +这可能暂时阻止原本有效的请求，包括没有工作流根节点的请求。 +路由后的预检也会拒绝合计用量已耗尽额度的规范池。 +已通过预留获准的恢复或组合发送，不会再次将同一预留计入自身；其他所有预留仍会计入。 +这项检查不会将事后报告用量的透传发送变成原子预留：跨越上限的发送、并发准入或事后报告的重试， +仍受现有机制的限制。仅观察模式的安装仍保持仅观察模式。根节点和身份上限继续独立生效。 + +要解除阻止，操作员必须使用自己保留的证据，确认每个历史加盐池别名所属的规范提供方。 +在 `config.json` 顶层添加 `spendPoolAliases` 对象：每个键必须与同一安装实例日志中的 +32 位小写十六进制池别名完全一致，每个值则是已经核实的规范提供方 ID。 +旧别名即使已代表规范提供方，只要缺少身份元数据，仍然需要显式条目。 +用量大于零且无法识别的别名会继续被阻止。不要根据名称相似、账户删除或短标签冲突推测映射， +也不要公开日志或盐值。 + +请将 `spendPoolAliases` 放在 `spend` 之外：旧版本会拒绝 `spend` 内的未知键，并可能禁用整个部分。 +无效的顶层映射会在写入配置时被拒绝；手动编辑导致 `spendPoolAliases` 格式错误时， +现有上限会被保留，池准入会关闭并拒绝请求。请按正常配置流程修正映射并重启。 +清空或删除映射不会抹去已经持久记录的关联。先前已重定向的别名不能重新分配给其他组； +经核实的规范提供方重命名可以合并组，而不会拆分已有用量。 +系统会一起检查完整映射，因此有效的分组合并不依赖别名键的顺序。 + +每份原始用量只计算一次。已结算用量、进行中的预留和未解决的用量都会计入；未知用量绝不会被视为退款。 +原始预留目标会被保留。新请求继续记录规范提供方池，检查点保留加盐身份依据， +而不加入原始账户或提供方名称。未知且用量大于零的历史记录、活跃组以及额度耗尽的组都受保护，不会被清理； +现有的休眠且未达上限记录保留规则仍然适用。身份依据的存储容量仍有上限；容量耗尽时， +系统会停止池准入，而不会遗忘这些依据。没有自动工具可以重建无法核实的历史记录。 + +### 安全回滚要求 + +受支持的回滚必须保留或回移两项功能：按规范提供方归属请求，以及识别历史连续性的账本读写， +并使用相同的日志、盐值和已核实的映射。请保留最新日志：恢复升级前的副本会遗漏之后的用量。 +不要通过删除记录、更改盐值、提高上限或关闭限制，来让降级看起来兼容。 + +未经修改的旧二进制文件**不是受支持的回滚方式**。它可以读取 v1 原始用量，但不会对提供方的跨别名合计用量实施限制， +可能创建新的账户标签池，也可能在压缩时丢弃可选身份元数据。没有自动阻止降级的机制。 +如果这样的旧写入程序已经运行，新读取程序会拒绝无法识别且用量大于零的历史记录， +直到操作员再次显式核实所有剩余映射。存储或日志完整性导致的拒绝与别名问题分开，仍使用 +`workflow_spend_undurable`；不安全文件或所有权问题可能改为作为存储错误向上传递，同样不会允许请求通过。 diff --git a/src/lib/spend-reservation-ledger.ts b/src/lib/spend-reservation-ledger.ts index e5fe7189379..ac100b30491 100644 --- a/src/lib/spend-reservation-ledger.ts +++ b/src/lib/spend-reservation-ledger.ts @@ -939,11 +939,24 @@ export function createSpendReservationLedger(options: { */ const evictScopes = (at: number, force: boolean): number => { const cutoff = at - policy.retentionMs; + // Candidate selection uses one snapshot of each pool group for the whole pass. + // Re-scanning all scopes per pool member repeats work on every reservation. + const pools = new Map(); + for (const [key, state] of scopes) { + if (!key.startsWith("pool\0")) continue; + const group = poolContinuity.resolve(key.slice(5)); + let total = pools.get(group); + if (!total) pools.set(group, total = { settled: 0, reserved: 0, unresolved: 0, lastSeenAt: 0 }); + total.settled += state.settled; + total.reserved += state.reserved; + total.unresolved += state.unresolved; + total.lastSeenAt = Math.max(total.lastSeenAt, state.lastSeenAt); + } const candidates: { key: string; scope: SpendScope; alias: string; seenAt: number }[] = []; for (const [key, state] of scopes) { const separator = key.indexOf("\0"); const scope = key.slice(0, separator) as SpendScope; - const effective = scope === "pool" ? poolState(key.slice(separator + 1))! : state; + const effective = scope === "pool" ? pools.get(poolContinuity.resolve(key.slice(separator + 1)))! : state; if (effective.reserved > 0) continue; if (scope === "pool" && state.settled + state.unresolved > 0 && !poolContinuity.known(key.slice(separator + 1))) continue; diff --git a/structure/transports/responses-spend.md b/structure/transports/responses-spend.md index 339349a5844..2c6eb430de8 100644 --- a/structure/transports/responses-spend.md +++ b/structure/transports/responses-spend.md @@ -144,6 +144,9 @@ that repeats existing member aliases is independent of salted-key order. A remov never removes a journaled link. Group activity, last-seen time and exhaustion govern retention; unidentified positive balances cannot be evicted. Identity evidence is bounded and retained even when dormant under-limit scopes are evicted. +Each eviction pass aggregates pool groups once before choosing candidates. Retention uses the +group's newest activity; capacity pressure still removes only the oldest eligible individual +scope per pass, and every removal journals its original scope alias. Before a mapping authorizes admission, a v1 checkpoint durably carries both unchanged accounting and optional salted `poolContinuity` metadata. No raw provider/account names are added to the diff --git a/tests/lib/spend-pool-continuity.test.ts b/tests/lib/spend-pool-continuity.test.ts index 34923a6445c..298fcbeb7b2 100644 --- a/tests/lib/spend-pool-continuity.test.ts +++ b/tests/lib/spend-pool-continuity.test.ts @@ -149,6 +149,50 @@ describe("historical pool identity continuity", () => { expect(ledger.snapshot("pool", "provider")).toMatchObject({ settled: 40, reserved: 10 }); }); + test("retention uses the newest pool member and journals every dormant member removal", () => { + const record = checkpoint([["older", 10, 0], ["newer", 0, 20]]); + record.scopes[1]!.seenAt = 95; + const disk = journal([record]); + const aliases = { [pool("older")]: "provider", [pool("newer")]: "provider" }; + const config = policy(aliases, { retentionMs: 5 }); + const ledger = createSpendReservationLedger({ journal: disk, salt, policy: config, now: () => 100 }); + expect(ledger.checkPoolContinuity()).toBeUndefined(); + ledger.prune(100); // The newest member is exactly at the retention cutoff. + expect(ledger.snapshot("pool", "provider")).toMatchObject({ settled: 10, unresolved: 20 }); + expect(disk.lines.map(line => JSON.parse(line)).filter(record => record.kind === "drop")).toEqual([]); + ledger.prune(101); + expect(ledger.snapshot("pool", "provider")).toBeUndefined(); + expect(disk.lines.map(line => JSON.parse(line)).filter(record => record.kind === "drop")).toEqual([ + { v: 1, kind: "drop", scope: "pool", alias: pool("older"), at: 101 }, + { v: 1, kind: "drop", scope: "pool", alias: pool("newer"), at: 101 }, + ]); + const restarted = createSpendReservationLedger({ journal: disk, salt, policy: config, now: () => 101 }); + expect(restarted.snapshot("pool", "provider")).toBeUndefined(); + expect(restarted.checkPoolContinuity()).toBeUndefined(); + }); + + test("capacity eviction chooses the oldest individual member and drops only one scope", () => { + const record = checkpoint([["oldest", 10, 0], ["recent", 20, 0], ["other", 0, 0]]); + record.scopes[1]!.seenAt = 40; + record.scopes[2]!.seenAt = 2; + const disk = journal([record]); + const config = policy({ [pool("oldest")]: "provider", [pool("recent")]: "provider" }, + { retentionMs: 100, maxTrackedScopes: 3 }); + const ledger = createSpendReservationLedger({ journal: disk, salt, policy: config, now: () => 50 }); + expect(ledger.checkPoolContinuity()).toBeUndefined(); + expect(ledger.reserve({ sendId: "new-root", scopes: { rootId: "new-root" }, inputTokens: 1, outputCeilingTokens: 0 }).reserved).toBe(true); + expect(disk.lines.map(line => JSON.parse(line)).filter(record => record.kind === "drop")).toEqual([ + { v: 1, kind: "drop", scope: "pool", alias: pool("oldest"), at: 50 }, + ]); + expect(ledger.snapshot("pool", "provider")?.settled).toBe(20); + expect(ledger.snapshot("pool", "other")?.settled).toBe(0); + expect(ledger.snapshot("root", "new-root")?.reserved).toBe(1); + const restarted = createSpendReservationLedger({ journal: disk, salt, policy: config, now: () => 50 }); + expect(restarted.snapshot("pool", "provider")?.settled).toBe(20); + expect(restarted.snapshot("pool", "other")?.settled).toBe(0); + expect(restarted.snapshot("root", "new-root")?.unresolved).toBe(1); + }); + test("observe-only records already-sent requests while ambiguity still blocks new dispatches", () => { const disk = journal([checkpoint([["unknown-label", 40, 0]])]); const ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(), now: () => 2 }); From 9892f8ad57c506ac16ebcf5d2edfd65cc5942c68 Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Sun, 4 Oct 2026 01:36:34 -0700 Subject: [PATCH 07/15] fix(spend): isolate legacy pool history and explain refusals --- .../docs/ja/reference/configuration/server.md | 18 +- .../docs/ko/reference/configuration/server.md | 18 +- .../docs/reference/configuration/server.md | 31 ++- .../docs/ru/reference/configuration/server.md | 22 +- .../zh-cn/reference/configuration/server.md | 18 +- src/lib/spend-reservation-ledger.ts | 99 ++++++-- src/lib/workflow-budget.ts | 32 ++- src/server/request-log.ts | 3 + src/server/responses/adapter-dispatch.ts | 7 +- src/server/responses/passthrough-dispatch.ts | 4 +- src/server/responses/request-send-budget.ts | 5 +- src/server/responses/request-spend.ts | 11 +- src/server/responses/run-turn-execution.ts | 3 +- src/server/workflow-refusal.ts | 17 ++ structure/transports/responses-spend.md | 46 ++-- tests/lib/spend-ceiling-enforcement.test.ts | 3 +- tests/lib/spend-pool-continuity.test.ts | 233 +++++++++++++++--- .../responses-send-budget-errors.test.ts | 9 + 18 files changed, 447 insertions(+), 132 deletions(-) diff --git a/docs-site/src/content/docs/ja/reference/configuration/server.md b/docs-site/src/content/docs/ja/reference/configuration/server.md index a345d9c50bf..dab41bbda05 100644 --- a/docs-site/src/content/docs/ja/reference/configuration/server.md +++ b/docs-site/src/content/docs/ja/reference/configuration/server.md @@ -207,11 +207,12 @@ The dashboard warns about old or unknown CLI versions, unavailable targets, and ジャーナルに保存されるのはソルト付きのエイリアスであり、元のプロバイダー名やアカウント名ではありません。 そのため、OpenCodex は現在のアカウント一覧、ラベルの接頭辞、短縮されたアカウント ID から対応関係を推測しません。 -`spend.pool.maxTokens` が設定されていて、正の使用量が残る過去のプールを識別できない場合、推論リクエストは -プロバイダーへ接続する前にローカルの HTTP 429 と -`x-opencodex-local-refusal: workflow_pool_history_unresolved` で拒否されます。 -これにより、ワークフロールートを持たないリクエストを含め、本来は有効なリクエストが一時的にブロックされることがあります。 -ルーティング後の事前確認でも、合算した使用量がすでに上限に達している正規プールを拒否します。 +`spend.pool.maxTokens` が設定され、正の過去残高の所有先が不明な場合、元のスコープに残したまま、 +各残高を候補となるすべてのプロバイダープールに一度ずつ加算し、そのプロバイダーの確認済みグループと合算します。 +この合計と予約分が上限に収まる場合だけリクエストを許可します。そのため、未使用のプールも一時的に制限されることがあります。 +ローカル拒否では、エイリアスやジャーナル内容を公開せず、この保守的な加算を説明します。 +ルートが判明する前は、不明な履歴だけを理由に HTTP admission を拒否しません。 +ルーティング後の事前確認では、保守的に合算した使用量がすでに上限に達している正規プールを拒否します。 予約によってすでに許可されたリカバリー送信やコンボ送信では、その予約を自分自身に対して再度計上しません。 それ以外の予約はすべて計上されます。この確認により、送信後に使用量を報告するパススルー送信が アトミックな予約に変わるわけではありません。上限をまたぐ送信、同時に許可されるリクエスト、 @@ -223,8 +224,9 @@ The dashboard warns about old or unknown CLI versions, unavailable targets, and `spendPoolAliases` オブジェクトを追加してください。各キーには、その同じインストールのジャーナルにある 32 文字の小文字 16 進数のプールエイリアスを正確に指定し、各値には確認済みの正規プロバイダー ID を指定します。 古いエイリアスがすでに正規プロバイダーを表していても、ID メタデータがなければ明示的なエントリが必要です。 -正の使用量を持ち、識別できないエイリアスはブロックされたままです。似た名前、アカウントの削除、 -短縮ラベルの衝突から対応関係を推測しないでください。ジャーナルやソルトを公開しないでください。 +正の使用量を持ち、識別できないエイリアスは、すべての候補プールに保守的に加算されます。 +似た名前、アカウントの削除、現在のプロバイダー名やハッシュ、短縮ラベルの衝突から対応関係を推測しないでください。 +ジャーナルやソルトを公開しないでください。 `spendPoolAliases` は `spend` の外に置いてください。古いバージョンは `spend` 内の未知のキーを拒否し、 セクション全体を無効にする場合があります。無効なトップレベルのマッピングは設定の書き込み時に拒否されます。 @@ -253,7 +255,7 @@ ID 識別情報の保存容量には上限があり、容量を使い切った 古いバイナリは v1 の生の計上値を読めても、エイリアスをまたぐプロバイダー合計に上限を適用せず、 新たなアカウントラベルのプールを作成したり、圧縮時に任意の ID メタデータを破棄したりする場合があります。 ダウングレードを自動的に防ぐ仕組みはありません。そのような古い書き込み処理が実行された場合、新しい読み取り処理は、 -運用者が残っているすべてのマッピングを明示的に再確認するまで、識別できない正の使用履歴を拒否します。 +運用者が残っているマッピングを確認するまで、識別できない正の使用履歴を各候補プールに保守的に加算します。 ストレージやジャーナルの整合性による拒否はエイリアスの問題とは別で、`workflow_spend_undurable` のままです。 安全でないファイルや所有権の問題は、代わりにストレージエラーとして伝播する場合がありますが、リクエストは許可されません。 diff --git a/docs-site/src/content/docs/ko/reference/configuration/server.md b/docs-site/src/content/docs/ko/reference/configuration/server.md index 63d2f3be220..32dc569716b 100644 --- a/docs-site/src/content/docs/ko/reference/configuration/server.md +++ b/docs-site/src/content/docs/ko/reference/configuration/server.md @@ -263,11 +263,12 @@ The dashboard warns about old or unknown CLI versions, unavailable targets, and 이전 저널에는 이 표시 레이블에 사용량이 기록되어 있을 수 있습니다. 저널은 원래 제공자·계정 이름이 아닌 솔트가 적용된 별칭을 저장하므로, OpenCodex는 현재 계정 목록, 레이블 접두사, 축약된 계정 ID로 매핑을 추측하지 않습니다. -`spend.pool.maxTokens`가 설정되어 있고 양수의 사용량이 남은 과거 풀을 식별하지 못하면, 추론 요청은 -제공자에 접속하기 전에 로컬 HTTP 429와 -`x-opencodex-local-refusal: workflow_pool_history_unresolved`로 거부됩니다. -워크플로 루트가 없는 요청을 포함해 원래 유효한 요청도 일시적으로 차단될 수 있습니다. -라우팅 후 사전 검사에서도 합산 사용량이 이미 한도에 도달한 정규 풀을 거부합니다. +`spend.pool.maxTokens`가 설정되어 있고 양수인 과거 잔액의 소유자를 알 수 없으면, +각 잔액은 원래 스코프에 보존되며 후보 제공자 풀마다 한 번씩 더해져 확인된 그룹과 합산됩니다. +합계와 예약량이 한도에 들어오는 경우에만 요청을 허용합니다. 이 방식은 사용하지 않는 풀도 일시적으로 제한할 수 있습니다. +로컬 거부 메시지는 별칭이나 저널 내용을 공개하지 않고 이러한 보수적 계산을 설명합니다. +라우트를 알기 전에는 불명확한 이력만으로 HTTP admission을 거부하지 않습니다. +라우팅 후 사전 검사에서도 보수적으로 합산한 사용량이 이미 한도에 도달한 정규 풀을 거부합니다. 예약을 통해 이미 허용된 복구 또는 콤보 전송에서는 그 예약을 자신에게 다시 계산하지 않습니다. 다른 예약은 모두 계산합니다. 이 검사가 사용량을 사후 보고하는 패스스루 전송을 원자적 예약으로 바꾸지는 않습니다. 한도를 넘기는 전송, 동시 요청 허용, 사후 보고되는 재시도에는 기존 제약이 그대로 남습니다. @@ -277,8 +278,9 @@ The dashboard warns about old or unknown CLI versions, unavailable targets, and `config.json`의 최상위에 `spendPoolAliases` 객체를 추가하세요. 각 키는 동일 설치의 저널에 있는 32자리 소문자 16진수 풀 별칭과 정확히 일치해야 하고, 각 값은 확인된 정규 제공자 ID여야 합니다. 이전 별칭이 이미 정규 제공자를 나타내더라도 ID 메타데이터가 없으면 명시적 항목이 필요합니다. -식별할 수 없는 양수 사용량의 별칭은 계속 차단됩니다. 비슷한 이름, 계정 삭제, 축약 레이블 충돌로 -매핑을 추측하지 말고, 저널이나 솔트를 공개하지 마세요. +식별할 수 없는 양수 사용량의 별칭은 모든 후보 풀에 보수적으로 계산됩니다. +비슷한 이름, 계정 삭제, 현재 제공자 이름이나 해시, 축약 레이블 충돌로 매핑을 추측하지 말고, +저널이나 솔트를 공개하지 마세요. `spendPoolAliases`는 `spend` 밖에 두세요. 이전 버전은 `spend` 안의 알 수 없는 키를 거부하며 섹션 전체를 비활성화할 수 있습니다. 잘못된 최상위 매핑은 설정 쓰기 시 거부됩니다. @@ -306,7 +308,7 @@ ID 식별 근거의 저장 용량에는 한도가 있으며, 용량을 소진하 이전 바이너리는 v1 원시 사용량을 읽을 수 있지만 별칭 간 제공자 합계에 상한을 적용하지 않으며, 새 계정 레이블 풀을 만들거나 압축할 때 선택적 ID 메타데이터를 버릴 수 있습니다. 다운그레이드를 자동으로 막는 장치는 없습니다. 이런 이전 쓰기 처리가 실행되었다면, 새 읽기 처리는 -운영자가 남아 있는 모든 매핑을 다시 명시적으로 확인할 때까지 식별되지 않은 양수 사용 이력을 거부합니다. +운영자가 남은 매핑을 확인할 때까지 식별되지 않은 양수 사용 이력을 모든 후보 제공자 풀에 보수적으로 계산합니다. 스토리지 또는 저널 무결성에 따른 거부는 별칭 문제와 별개이며 `workflow_spend_undurable`을 유지합니다. 안전하지 않은 파일이나 소유권 문제는 대신 스토리지 오류로 전파될 수 있으며, 이 경우에도 요청을 허용하지 않습니다. diff --git a/docs-site/src/content/docs/reference/configuration/server.md b/docs-site/src/content/docs/reference/configuration/server.md index bcecb76abfb..c9d03f06235 100644 --- a/docs-site/src/content/docs/reference/configuration/server.md +++ b/docs-site/src/content/docs/reference/configuration/server.md @@ -949,23 +949,27 @@ aliases, not the original provider/account names, so OpenCodex does not guess a current account roster, a label prefix, or a shortened account ID. If `spend.pool.maxTokens` is configured and positive historical pool balances remain unidentified, -inference admission returns local HTTP 429 with -`x-opencodex-local-refusal: workflow_pool_history_unresolved` before contacting a provider. -This can temporarily block otherwise valid requests, including requests without a workflow root. -After routing, preflight also refuses a canonical pool that is already exhausted by its combined -balances. A recovery or combo send already admitted by a reservation does not count that same -reservation against itself a second time; all other reservations remain counted. +the ledger keeps each balance in its original scope and conservatively counts it once against every +candidate provider pool, in addition to that provider's verified group. A request is admitted only +when that total plus its reservation fits the ceiling. This may restrict an otherwise unused pool +until the operator verifies mappings; the request-local refusal explains this overrestriction +without exposing aliases or journal contents. Unidentified history alone does not block admission +before a route is known. After routing, preflight also refuses a canonical pool whose conservative +total is already exhausted. A recovery or combo send already admitted by a reservation does not +count that same reservation against itself a second time; all other reservations remain counted. This check does not turn post-reported passthrough sends into atomic reservations: a crossing send, concurrent admissions or retries reported afterwards retain their existing limits. Observe-only installs remain observe-only. Root and identity ceilings remain in force independently. To resolve it, an operator must verify which canonical provider each historical salted pool alias -belongs to, using their own retained evidence. Add a top-level `spendPoolAliases` object in +belongs to, using their own retained evidence. Verified mappings keep an old balance from being +charged to every candidate pool. Add a top-level `spendPoolAliases` object in `config.json`: each key is the exact 32-character lowercase hexadecimal pool alias from that same installation's journal; each value is its verified canonical provider ID. An old alias that already represents the canonical provider still needs an explicit entry when it lacks identity metadata. -A positive alias that cannot be identified stays blocked. Do not infer a match from similar names, -account deletion, or a short-label collision, and do not share the journal or salt publicly. +A positive alias that cannot be identified remains conservatively charged to every candidate pool. +Do not infer a match from similar names, account deletion, a current provider name/hash, or a +short-label collision, and do not share the journal or salt publicly. Keep `spendPoolAliases` outside `spend`: older versions reject unknown keys inside `spend` and can disable the entire section. Invalid top-level mappings are rejected on configuration writes; @@ -993,7 +997,8 @@ change the salt, raise ceilings, or disable enforcement to make a downgrade appe An unmodified older binary is **not a supported rollback**. It can read the v1 raw balances but does not enforce the cross-alias provider total, may create a new account-label pool, and may discard optional identity metadata when compacting. There is no automatic downgrade barrier. If such an old -writer has run, the new reader refuses unidentified positive history until the operator explicitly -verifies all remaining mappings again. Storage or journal-integrity denials are separate from an -alias problem and keep `workflow_spend_undurable`; unsafe-file/ownership failures may instead -propagate as storage errors, without admitting the request. +writer has run, the new reader conservatively counts unidentified positive history against every +candidate provider pool until the operator verifies the remaining mappings. Storage or +journal-integrity denials are separate from an alias problem and keep `workflow_spend_undurable`; +unsafe-file/ownership failures may instead propagate as storage errors, without admitting the +request. diff --git a/docs-site/src/content/docs/ru/reference/configuration/server.md b/docs-site/src/content/docs/ru/reference/configuration/server.md index 3d8ec782188..b65b39c977f 100644 --- a/docs-site/src/content/docs/ru/reference/configuration/server.md +++ b/docs-site/src/content/docs/ru/reference/configuration/server.md @@ -255,12 +255,13 @@ The dashboard warns about old or unknown CLI versions, unavailable targets, and под этими метками. Журнал хранит псевдонимы с солью, а не исходные имена провайдеров и аккаунтов, поэтому OpenCodex не выводит соответствия из текущего списка аккаунтов, префикса метки или сокращённого ID аккаунта. -Если задан `spend.pool.maxTokens` и остаётся положительный расход в неопознанных исторических пулах, -допуск запроса на инференс завершается локальным HTTP 429 с -`x-opencodex-local-refusal: workflow_pool_history_unresolved` до обращения к провайдеру. -Это может временно блокировать корректные запросы, в том числе запросы без корня workflow. -После маршрутизации предварительная проверка также отклоняет канонический пул, суммарный расход которого -уже исчерпал лимит. Отправка при восстановлении или через combo, уже допущенная по резервированию, +Если задан `spend.pool.maxTokens`, положительные исторические балансы с неизвестным владельцем +сохраняются в исходных областях и один раз учитываются для каждого возможного пула провайдера, +дополнительно к подтверждённой группе этого провайдера. Запрос допускается, только если сумма и резерв +укладываются в лимит. Это может временно ограничить даже неиспользуемые пулы. Локальный отказ объясняет +такой консервативный учёт, не раскрывая псевдонимы или содержимое журнала. Неизвестная история сама по себе +не блокирует HTTP-допуск до определения маршрута. После маршрутизации предварительная проверка отклоняет +канонический пул, консервативная сумма которого уже исчерпала лимит. Отправка при восстановлении или через combo, уже допущенная по резервированию, не учитывает то же резервирование повторно против самой себя; все остальные резервирования учитываются. Эта проверка не превращает passthrough-отправки с последующим отчётом об использовании в атомарные резервирования: отправка, пересекающая лимит, одновременные допуски и повторы с последующим отчётом сохраняют существующие ограничения. @@ -272,8 +273,9 @@ The dashboard warns about old or unknown CLI versions, unavailable targets, and с 32-символьным шестнадцатеричным псевдонимом пула в нижнем регистре из журнала той же установки, а каждое значение должно быть подтверждённым каноническим ID провайдера. Даже старый псевдоним, уже обозначающий канонического провайдера, требует явной записи, если у него нет метаданных идентичности. -Неопознанный псевдоним с положительным расходом остаётся заблокированным. Не выводите соответствие -из похожих имён, удаления аккаунта или совпадения коротких меток и не публикуйте журнал или соль. +Неопознанный псевдоним с положительным расходом консервативно учитывается для каждого возможного пула. +Не выводите соответствие из похожих имён, удаления аккаунта, текущего имени или хэша провайдера, +либо совпадения коротких меток; не публикуйте журнал или соль. Размещайте `spendPoolAliases` вне `spend`: старые версии отклоняют неизвестные ключи внутри `spend` и могут отключить весь раздел. Некорректные соответствия верхнего уровня отклоняются при записи конфигурации; @@ -306,8 +308,8 @@ The dashboard warns about old or unknown CLI versions, unavailable targets, and исходные величины расхода v1, но не применяет лимит к сумме по всем псевдонимам провайдера, может создать новый пул с меткой аккаунта и отбросить необязательные метаданные идентичности при уплотнении. Автоматического запрета понижения версии нет. Если такая старая версия выполняла запись, новая версия -при чтении отклоняет неопознанную историю с положительным расходом, пока оператор явно не подтвердит -все оставшиеся соответствия заново. Отказы из-за хранилища или целостности журнала не связаны с проблемой +при чтении консервативно учитывает неопознанную историю с положительным расходом для каждого возможного +пула провайдера, пока оператор не подтвердит оставшиеся соответствия. Отказы из-за хранилища или целостности журнала не связаны с проблемой псевдонимов и сохраняют `workflow_spend_undurable`; проблемы небезопасных файлов или прав владения могут вместо этого передаваться как ошибки хранилища, при этом запрос не допускается. diff --git a/docs-site/src/content/docs/zh-cn/reference/configuration/server.md b/docs-site/src/content/docs/zh-cn/reference/configuration/server.md index 21966a05af8..190a8fe6580 100644 --- a/docs-site/src/content/docs/zh-cn/reference/configuration/server.md +++ b/docs-site/src/content/docs/zh-cn/reference/configuration/server.md @@ -220,11 +220,11 @@ The dashboard warns about old or unknown CLI versions, unavailable targets, and 旧日志可能将用量记在这些显示标签下。日志保存的是加盐别名，而不是原始提供方或账户名称， 因此 OpenCodex 不会根据当前账户列表、标签前缀或缩短的账户 ID 推测映射。 -如果配置了 `spend.pool.maxTokens`，并且仍有已记账用量大于零的历史池无法识别， -推理请求会在联系提供方之前被本地 HTTP 429 拒绝，并返回 -`x-opencodex-local-refusal: workflow_pool_history_unresolved`。 -这可能暂时阻止原本有效的请求，包括没有工作流根节点的请求。 -路由后的预检也会拒绝合计用量已耗尽额度的规范池。 +如果配置了 `spend.pool.maxTokens`，无法确认所有者的正历史余额会保留在原始范围内， +并且会对每个候选提供方池各计入一次，再与该提供方已验证的组相加。 +只有合计用量加上本次预留不超过额度时才会准入。这可能暂时限制原本未使用的池。 +本地拒绝会说明这种保守计算，不会公开别名或日志内容。路由确定前，仅有未识别历史不会阻止 HTTP 准入。 +路由后的预检也会拒绝保守合计已耗尽额度的规范池。 已通过预留获准的恢复或组合发送，不会再次将同一预留计入自身；其他所有预留仍会计入。 这项检查不会将事后报告用量的透传发送变成原子预留：跨越上限的发送、并发准入或事后报告的重试， 仍受现有机制的限制。仅观察模式的安装仍保持仅观察模式。根节点和身份上限继续独立生效。 @@ -233,8 +233,8 @@ The dashboard warns about old or unknown CLI versions, unavailable targets, and 在 `config.json` 顶层添加 `spendPoolAliases` 对象：每个键必须与同一安装实例日志中的 32 位小写十六进制池别名完全一致，每个值则是已经核实的规范提供方 ID。 旧别名即使已代表规范提供方，只要缺少身份元数据，仍然需要显式条目。 -用量大于零且无法识别的别名会继续被阻止。不要根据名称相似、账户删除或短标签冲突推测映射， -也不要公开日志或盐值。 +用量大于零且无法识别的别名会保守地计入每个候选池。不要根据名称相似、账户删除、 +当前提供方名称或哈希、短标签冲突推测映射，也不要公开日志或盐值。 请将 `spendPoolAliases` 放在 `spend` 之外：旧版本会拒绝 `spend` 内的未知键，并可能禁用整个部分。 无效的顶层映射会在写入配置时被拒绝；手动编辑导致 `spendPoolAliases` 格式错误时， @@ -257,8 +257,8 @@ The dashboard warns about old or unknown CLI versions, unavailable targets, and 未经修改的旧二进制文件**不是受支持的回滚方式**。它可以读取 v1 原始用量，但不会对提供方的跨别名合计用量实施限制， 可能创建新的账户标签池，也可能在压缩时丢弃可选身份元数据。没有自动阻止降级的机制。 -如果这样的旧写入程序已经运行，新读取程序会拒绝无法识别且用量大于零的历史记录， -直到操作员再次显式核实所有剩余映射。存储或日志完整性导致的拒绝与别名问题分开，仍使用 +如果这样的旧写入程序已经运行，新读取程序会将无法识别且用量大于零的历史记录保守地计入 +每个候选提供方池，直到操作员核实剩余映射。存储或日志完整性导致的拒绝与别名问题分开，仍使用 `workflow_spend_undurable`；不安全文件或所有权问题可能改为作为存储错误向上传递，同样不会允许请求通过。 ## 令牌预留与额度 diff --git a/src/lib/spend-reservation-ledger.ts b/src/lib/spend-reservation-ledger.ts index ac100b30491..e513d7bf2df 100644 --- a/src/lib/spend-reservation-ledger.ts +++ b/src/lib/spend-reservation-ledger.ts @@ -191,6 +191,8 @@ export type SpendDenial = readonly scopeId: string; readonly limit: number; readonly projected: number; + /** Pool checks include all unbound positive history until an operator verifies its owner. */ + readonly includesUnboundPoolHistory?: boolean; } /** This send id is already known -- open, settled, lost or abandoned. */ | { readonly reason: "duplicate-send-id"; readonly sendId: string } @@ -623,6 +625,8 @@ export interface SpendReservationLedger { reserve(request: SpendReservationRequest): SpendReservationDecision; /** Pre-dispatch guard, including transports which report their sends after dispatch. */ checkPoolContinuity(): SpendDenial | undefined; + /** Whether any positive pool balance is still unbound and therefore charged to each candidate pool. */ + hasUnboundPositivePoolHistory(): boolean; /** * The send left for upstream. Until this is called the reservation may be abandoned for * free; after it, a missing usage frame becomes unresolved spend. Returns false when the @@ -727,10 +731,24 @@ export function createSpendReservationLedger(options: { * credential id, a pool name -- never leaves this function, so nothing identifying is * written to disk or held in a map key. */ - const aliasFor = (kind: SpendScope | "send", id: string): string => + const aliasFor = (kind: SpendScope | "send" | "pool-current", id: string): string => createHash("sha256").update(salt).update("\u0000").update(kind).update("\u0000").update(id) .digest("hex").slice(0, 32); + // An old unbound label may hash exactly like today's provider ID. Only in that ambiguous + // case, put new routed spend in a separate hash domain so it belongs to this candidate + // without treating the old balance as verified ownership. Existing proven groups keep + // their alias, and explicit mappings of old aliases target the same current alias. + const poolAliasFor = (provider: string): string => { + const legacyAlias = aliasFor("pool", provider); + const legacy = scopes.get(scopeKey("pool", legacyAlias)); + const hasPositiveHistory = legacy !== undefined + && legacy.settled + legacy.reserved + legacy.unresolved > 0; + const ambiguous = hasPositiveHistory + && (!poolContinuity.known(legacyAlias) || poolContinuity.resolve(legacyAlias) !== legacyAlias); + return ambiguous ? aliasFor("pool-current", provider) : legacyAlias; + }; + const scopeState = (scope: SpendScope, alias: string): ScopeState => { const key = scopeKey(scope, alias); let state = scopes.get(key); @@ -744,22 +762,34 @@ export function createSpendReservationLedger(options: { const historicalPools = (): Set => new Set([...scopes].filter(([key, state]) => key.startsWith("pool\0") && state.settled + state.reserved + state.unresolved > 0, ).map(([key]) => key.slice(5))); - const unknownPoolHistory = (): boolean => [...historicalPools()].some(alias => !poolContinuity.known(alias)); - const poolState = (alias: string): ScopeState | undefined => { + const hasUnboundPositivePoolHistory = (): boolean => [...historicalPools()].some(alias => !poolContinuity.known(alias)); + const poolState = (alias: string): { totals: ScopeState; includesUnboundHistory: boolean } | undefined => { const group = poolContinuity.resolve(alias); let total: ScopeState | undefined; + let includesUnboundHistory = false; for (const [key, state] of scopes) { - if (!key.startsWith("pool\0") || poolContinuity.resolve(key.slice(5)) !== group) continue; + if (!key.startsWith("pool\0")) continue; + const member = key.slice(5); + const unbound = !poolContinuity.known(member); + if (unbound) { + // Ownership cannot be inferred because an old alias hashes to a current provider ID. + // Keep that original balance out of the proven group, then charge it once through the + // conservative overlay below for every candidate provider. + if (state.settled + state.reserved + state.unresolved === 0) continue; + includesUnboundHistory = true; + } else if (poolContinuity.resolve(member) !== group) { + continue; + } total ??= { settled: 0, reserved: 0, unresolved: 0, lastSeenAt: 0 }; total.settled += state.settled; total.reserved += state.reserved; total.unresolved += state.unresolved; total.lastSeenAt = Math.max(total.lastSeenAt, state.lastSeenAt); } - return total; + return total ? { totals: total, includesUnboundHistory } : undefined; }; const stateFor = (scope: SpendScope, alias: string): ScopeState | undefined => - scope === "pool" ? poolState(alias) : scopes.get(scopeKey(scope, alias)); + scope === "pool" ? poolState(alias)?.totals : scopes.get(scopeKey(scope, alias)); const limitFor = (scope: SpendScope): number | undefined => policy[scope].maxTokens; @@ -773,7 +803,7 @@ export function createSpendReservationLedger(options: { const refs: ScopeRef[] = []; if (targets.rootId !== undefined) refs.push({ scope: "root", alias: aliasFor("root", targets.rootId) }); if (targets.identityId !== undefined) refs.push({ scope: "identity", alias: aliasFor("identity", targets.identityId) }); - if (targets.poolId !== undefined) refs.push({ scope: "pool", alias: aliasFor("pool", targets.poolId) }); + if (targets.poolId !== undefined) refs.push({ scope: "pool", alias: poolAliasFor(targets.poolId) }); return refs; }; @@ -939,12 +969,23 @@ export function createSpendReservationLedger(options: { */ const evictScopes = (at: number, force: boolean): number => { const cutoff = at - policy.retentionMs; - // Candidate selection uses one snapshot of each pool group for the whole pass. - // Re-scanning all scopes per pool member repeats work on every reservation. + // Candidate selection uses one snapshot of each proven pool group plus the shared + // unbound overlay for the whole pass. Re-scanning all scopes per pool member repeats + // work on every reservation. const pools = new Map(); + const unboundPoolTotal: ScopeState = { settled: 0, reserved: 0, unresolved: 0, lastSeenAt: 0 }; for (const [key, state] of scopes) { if (!key.startsWith("pool\0")) continue; - const group = poolContinuity.resolve(key.slice(5)); + const alias = key.slice(5); + if (!poolContinuity.known(alias)) { + if (state.settled + state.reserved + state.unresolved > 0) { + unboundPoolTotal.settled += state.settled; + unboundPoolTotal.reserved += state.reserved; + unboundPoolTotal.unresolved += state.unresolved; + } + continue; + } + const group = poolContinuity.resolve(alias); let total = pools.get(group); if (!total) pools.set(group, total = { settled: 0, reserved: 0, unresolved: 0, lastSeenAt: 0 }); total.settled += state.settled; @@ -956,7 +997,19 @@ export function createSpendReservationLedger(options: { for (const [key, state] of scopes) { const separator = key.indexOf("\0"); const scope = key.slice(0, separator) as SpendScope; - const effective = scope === "pool" ? pools.get(poolContinuity.resolve(key.slice(separator + 1)))! : state; + const alias = key.slice(separator + 1); + const unboundZero = scope === "pool" && !poolContinuity.known(alias) + && state.settled + state.reserved + state.unresolved === 0; + const proven = scope === "pool" ? pools.get(poolContinuity.resolve(alias)) : undefined; + const effective = scope === "pool" && !unboundZero + ? { + settled: (proven?.settled ?? 0) + unboundPoolTotal.settled, + reserved: (proven?.reserved ?? 0) + unboundPoolTotal.reserved, + unresolved: (proven?.unresolved ?? 0) + unboundPoolTotal.unresolved, + // The overlay affects exhaustion but does not merge retention age across identities. + lastSeenAt: proven?.lastSeenAt ?? 0, + } + : state; if (effective.reserved > 0) continue; if (scope === "pool" && state.settled + state.unresolved > 0 && !poolContinuity.known(key.slice(separator + 1))) continue; @@ -1046,7 +1099,7 @@ export function createSpendReservationLedger(options: { const preparePoolContinuity = (at: number, requested?: string): SpendDenial | undefined => { const evidence = poolContinuity.prepare(policy.poolAliases, requested, historicalPools(), - provider => aliasFor("pool", provider), at, maxTrackedScopes()); + poolAliasFor, at, maxTrackedScopes()); if (evidence === false) return { reason: "pool-history-unresolved" }; if (evidence) { // A v1 checkpoint atomically carries the unchanged balances AND salted identity @@ -1054,7 +1107,9 @@ export function createSpendReservationLedger(options: { if (!append(checkpointRecord(at, evidence))) return { reason: "reserve-not-durable", sendId: "" }; if (!poolContinuity.restore(evidence)) return { reason: "pool-history-unresolved" }; } - return unknownPoolHistory() ? { reason: "pool-history-unresolved" } : undefined; + // Unbound positive history is accounted conservatively in every candidate pool view. + // Only invalid/conflicting identity evidence or a failed durable write refuses here. + return undefined; }; /** The denial when tracking cannot fit this request, or undefined when it can. */ @@ -1089,6 +1144,11 @@ export function createSpendReservationLedger(options: { return preparePoolContinuity(now()); }, + hasUnboundPositivePoolHistory(): boolean { + assertOwnedAccounting?.(); + return hasUnboundPositivePoolHistory(); + }, + reserve(request: SpendReservationRequest): SpendReservationDecision { assertOwnedAccounting?.(); const tokens = sanitizeTokens(request.inputTokens) + sanitizeTokens(request.outputCeilingTokens); @@ -1124,7 +1184,8 @@ export function createSpendReservationLedger(options: { // it is to let the total go OVER the ceiling so the next request can be refused. const limit = request.alreadySent === true ? undefined : limitFor(ref.scope); if (limit === undefined) continue; - const state = stateFor(ref.scope, ref.alias); + const poolView = ref.scope === "pool" ? poolState(ref.alias) : undefined; + const state = ref.scope === "pool" ? poolView?.totals : scopes.get(scopeKey(ref.scope, ref.alias)); const projected = (state ? state.settled + state.reserved + state.unresolved : 0) + tokens; if (projected > limit) { const scopeId = ref.scope === "root" @@ -1132,7 +1193,10 @@ export function createSpendReservationLedger(options: { : ref.scope === "identity" ? request.scopes.identityId : request.scopes.poolId; return { reserved: false, - denial: { reason: "spend-limit-exceeded", scope: ref.scope, scopeId: scopeId ?? "", limit, projected }, + denial: { + reason: "spend-limit-exceeded", scope: ref.scope, scopeId: scopeId ?? "", limit, projected, + ...(poolView?.includesUnboundHistory ? { includesUnboundPoolHistory: true } : {}), + }, }; } } @@ -1205,7 +1269,8 @@ export function createSpendReservationLedger(options: { // Reading accounting from a handle whose ownership has ended is as wrong as writing it: // the figures describe a journal this process no longer owns. assertOwnedAccounting?.(); - const state = stateFor(scope, aliasFor(scope, scopeId)); + const alias = scope === "pool" ? poolAliasFor(scopeId) : aliasFor(scope, scopeId); + const state = stateFor(scope, alias); if (!state) return undefined; return { settled: state.settled, @@ -1217,7 +1282,7 @@ export function createSpendReservationLedger(options: { exhausted(scope: SpendScope, scopeId: string, excludingSendId?: string): boolean { assertOwnedAccounting?.(); - const alias = aliasFor(scope, scopeId); + const alias = scope === "pool" ? poolAliasFor(scopeId) : aliasFor(scope, scopeId); const state = stateFor(scope, alias); if (!state) return false; const own = excludingSendId === undefined ? undefined : reservations.get(aliasFor("send", excludingSendId)); diff --git a/src/lib/workflow-budget.ts b/src/lib/workflow-budget.ts index 0c2e518045d..b491bd0f6fa 100644 --- a/src/lib/workflow-budget.ts +++ b/src/lib/workflow-budget.ts @@ -43,6 +43,8 @@ export interface WorkflowSpendDenialDetail { readonly limit: number; /** Tokens the refused reservation would have taken the scope to, where that is known. */ readonly projected?: number; + /** A pool ceiling also includes unknown-owner history conservatively charged to every pool. */ + readonly includesUnboundPoolHistory?: boolean; } /** Operator-facing name for each scope. What an operator calls it, not what the type calls it. */ @@ -207,8 +209,8 @@ export type WorkflowDenial = * accurate for exactly one of them. Worse, the wire cannot carry the distinction on its own: * `classifyError` rewrites every 429 to `rate_limit_error` / `rate_limit_exceeded`, so the body * of a refusal this proxy made is shaped exactly like a provider rate limit. Each sentence - * therefore says which ceiling fired AND that no provider was contacted, because that is the - * first thing an operator needs and the only place left to put it. + * therefore names which ceiling fired; send refusals describe the dispatch that was stopped + * without making claims about earlier attempts in the same request. */ export function workflowDenialSummary( reason: WorkflowDenial, @@ -248,7 +250,10 @@ export function workflowDenialSummary( + (spend.projected !== undefined ? " (this send would have taken it to " + formatTokenCount(spend.projected) + ")" : "") - + ", so no provider was contacted. Spend is durable, so it does not roll forward" + + (spend.includesUnboundPoolHistory + ? ". The total includes unassigned historical provider-pool balances, conservatively counted against every provider pool until verified mappings assign them; this can overrestrict otherwise unused pools" + : "") + + ", so this send was refused before contacting a provider. Spend is durable, so it does not roll forward" + " with the send window: raise or remove spend." + spend.scope + ".maxTokens in config.json to grant more." : "This proxy refused the request locally: the task reached a configured token" @@ -301,6 +306,8 @@ export interface WorkflowBudgetEvent { readonly spendScope?: SpendScope; /** That scope's ceiling, so the event is readable without the config open beside it. */ readonly spendLimit?: number; + /** The pool ceiling conservatively included unidentified historical pool balances. */ + readonly spendIncludesUnboundPoolHistory?: boolean; /** Windowed sends at the moment of the event. */ readonly sends: number; /** Windowed distinct children at the moment of the event. */ @@ -359,6 +366,7 @@ export function recordWorkflowRefusalEvent( rootId, reason, ...(spend ? { spendScope: spend.scope, spendLimit: spend.limit } : {}), + ...(spend?.includesUnboundPoolHistory ? { spendIncludesUnboundPoolHistory: true } : {}), sends: state ? windowedSends(state, now) : 0, children: state ? windowedChildren(state, now) : 0, }); @@ -389,6 +397,8 @@ export type WorkflowDecision = spendLimit?: number; /** Tokens the refused reservation would have taken the scope to, where known. */ spendProjected?: number; + /** A pool ceiling includes unbound historical balances for every candidate pool. */ + spendIncludesUnboundPoolHistory?: boolean; }; /** @@ -521,6 +531,7 @@ export function admitWorkflowTurn( spendScope: denial.scope, spendLimit: denial.limit, ...(denial.projected !== undefined ? { spendProjected: denial.projected } : {}), + ...(denial.includesUnboundPoolHistory ? { spendIncludesUnboundPoolHistory: true } : {}), } : {}), }; @@ -595,7 +606,12 @@ export function admitWorkflowTurn( return refuse( reason, denial.reason === "spend-limit-exceeded" - ? { scope: denial.scope, limit: denial.limit, projected: denial.projected } + ? { + scope: denial.scope, + limit: denial.limit, + projected: denial.projected, + ...(denial.includesUnboundPoolHistory ? { includesUnboundPoolHistory: true } : {}), + } : undefined, ); } @@ -747,7 +763,13 @@ export function workflowSpendCeilingReached( const root = rootId ? spentRootCeiling(rootId, ledger, excludingSendId) : undefined; if (root) return root; const limit = ledger.policy.pool.maxTokens; - return poolId && limit !== undefined && ledger.exhausted("pool", poolId, excludingSendId) ? { scope: "pool", limit } : undefined; + return poolId && limit !== undefined && ledger.exhausted("pool", poolId, excludingSendId) + ? { + scope: "pool", + limit, + ...(ledger.hasUnboundPositivePoolHistory() ? { includesUnboundPoolHistory: true } : {}), + } + : undefined; } export interface WorkflowBudgetSnapshot { diff --git a/src/server/request-log.ts b/src/server/request-log.ts index 87a2bf5fbb4..341fa962bdc 100644 --- a/src/server/request-log.ts +++ b/src/server/request-log.ts @@ -63,6 +63,7 @@ import { type UsageStatus, } from "../usage/log"; import type { RequestExecutionBudget } from "../lib/request-execution-budget"; +import type { WorkflowSpendDenialDetail } from "../lib/workflow-budget"; import { attributeFinalRequest, attributeSealedAttempt } from "./request-log-failure-attribution"; import { debugAttemptDeliverySummary } from "../lib/debug"; import { @@ -205,6 +206,8 @@ export interface RequestLogContext { spendInputEstimateTokens?: number; /** Canonical provider-pool spend identity; independent of mutable, account-specific log labels. */ spendPoolId?: string; + /** Internal safe refusal detail for a request whose ceiling includes unidentified old pool spend. */ + spendRefusalDetail?: WorkflowSpendDenialDetail; /** Settles this request's durable spend entries from `addFinalRequestLog`. */ spendTracker?: RequestSpendSettlement; attempts?: PersistedUsageAttempt[]; diff --git a/src/server/responses/adapter-dispatch.ts b/src/server/responses/adapter-dispatch.ts index 6deb79f1544..4b188397c58 100644 --- a/src/server/responses/adapter-dispatch.ts +++ b/src/server/responses/adapter-dispatch.ts @@ -98,6 +98,7 @@ import { resolveClientRetryAfter } from "../../lib/retry-after"; import { cancelBodyOnAbort } from "../../lib/abort"; import { chargeWorkflowSends } from "../../lib/workflow-budget"; import { isAntigravityValidationRefusal } from "./antigravity-validation-refusal"; +import { unboundPoolSpendRefusalResponse } from "../workflow-refusal"; /** One responsibility of the Responses request pipeline; state owners are explicit. */ export async function prepareAdapterExchange( @@ -406,7 +407,8 @@ export async function prepareAdapterExchange( // blaming the provider makes the caller send the whole turn again -- the amplification this // budget exists to stop. The passthrough path has answered 429 here since #4546. if (err instanceof SendBudgetExhaustedError) { - return formatErrorResponse(429, SEND_BUDGET_EXHAUSTED_CODE, err.message); + return unboundPoolSpendRefusalResponse(logCtx) + ?? formatErrorResponse(429, SEND_BUDGET_EXHAUSTED_CODE, err.message); } const msg = describeUpstreamConnectFailure(err, connectMs); return formatErrorResponse(502, "upstream_error", msg); @@ -621,7 +623,8 @@ export async function prepareAdapterExchange( // Same rule on the recovery leg: the ladder refused to send again, so the answer names // this proxy rather than the provider it never reached. if (err instanceof SendBudgetExhaustedError) { - return { failed: formatErrorResponse(429, SEND_BUDGET_EXHAUSTED_CODE, err.message) }; + return { failed: unboundPoolSpendRefusalResponse(logCtx) + ?? formatErrorResponse(429, SEND_BUDGET_EXHAUSTED_CODE, err.message) }; } const msg = describeUpstreamConnectFailure(err, connectMs); return { failed: formatErrorResponse(502, "upstream_error", msg) }; diff --git a/src/server/responses/passthrough-dispatch.ts b/src/server/responses/passthrough-dispatch.ts index bdda62fdfb1..0300064b4d8 100644 --- a/src/server/responses/passthrough-dispatch.ts +++ b/src/server/responses/passthrough-dispatch.ts @@ -164,6 +164,7 @@ import { ambiguousResendAllowanceFor, selfContainedResponsesBody } from "./reset import { upstreamErrorMessageFromPayload, ENCRYPTED_FUNCTION_OUTPUT_REJECTION } from "../../lib/errors"; import { isTransientConsoleGoUploadRejection } from "../../providers/opencode-zen-rate-limit"; import { planReasoningEffortDowngrade } from "../../providers/reasoning-metadata"; +import { unboundPoolSpendRefusalResponse } from "../workflow-refusal"; /** Prepares and recovers one native Responses exchange before client commitment. */ export async function preparePassthroughExchange( @@ -894,7 +895,8 @@ export async function preparePassthroughExchange( releaseUpstreamHostAdmission(nativeHostState.lease); nativeHostState.lease = null; releaseCodexAuthContextProbeLease(admissionState.authCtx); - return formatErrorResponse(429, "request_send_budget_exhausted", err.message); + return unboundPoolSpendRefusalResponse(logCtx) + ?? formatErrorResponse(429, "request_send_budget_exhausted", err.message); } const refusal = unwrapUpstreamRetryEvidenceError(err); // Pacing may outlive the selected account's admission. No fetch occurred, so do diff --git a/src/server/responses/request-send-budget.ts b/src/server/responses/request-send-budget.ts index 5b4d3e26eaa..3cdfe89916b 100644 --- a/src/server/responses/request-send-budget.ts +++ b/src/server/responses/request-send-budget.ts @@ -84,8 +84,9 @@ export function createResponsesSendBudget( // returns -- a refusal an operator cannot tell from an ordinary budget exhaustion, on a // ceiling they configured themselves. Asked before dispatch, it names the scope and the // number instead. Returns undefined and touches no ledger when no ceiling is configured. - // Passthrough reports sends after they leave. Historical identity uncertainty must - // refuse here as well as in reserve(), including requests with no workflow root. + // Passthrough reports sends after they leave. Invalid or conflicting identity evidence + // still refuses here as well as in reserve(), including requests with no workflow root. + // Positive unbound balances alone are instead overlaid on each candidate pool below. const continuityRefusal = poolContinuityRefusalReason(); if (continuityRefusal) { return workflowRefusalResponse(continuityRefusal, logCtx, undefined, workflowRootId); diff --git a/src/server/responses/request-spend.ts b/src/server/responses/request-spend.ts index faab6b84099..e24490d2744 100644 --- a/src/server/responses/request-spend.ts +++ b/src/server/responses/request-spend.ts @@ -44,7 +44,7 @@ export function createRequestSpendTracker( logCtx: Pick< RequestLogContext, "provider" | "accountLogLabel" | "usageLogInputTokens" | "spendOutputCeilingTokens" | "spendInputEstimateTokens" | "spendPoolId" - > & Partial>, + > & Partial>, rootId: string | undefined, injected?: SpendReservationLedger, ): RequestSpendTracker { @@ -127,10 +127,17 @@ export function createRequestSpendTracker( ? "workflow-tracking-exhausted" : "workflow-spend-exhausted"; const detail = denial.reason === "spend-limit-exceeded" - ? { scope: denial.scope, limit: denial.limit, projected: denial.projected } : undefined; + ? { + scope: denial.scope, + limit: denial.limit, + projected: denial.projected, + ...(denial.includesUnboundPoolHistory ? { includesUnboundPoolHistory: true } : {}), + } + : undefined; const summary = workflowDenialSummary(reason, detail); markLocalRequestLogRefusal(logCtx, summary.code); logCtx.errorCode = summary.code; + if (detail?.includesUnboundPoolHistory) logCtx.spendRefusalDetail = detail; recordWorkflowRefusalEvent(rootId, reason, Date.now(), detail); return false; } diff --git a/src/server/responses/run-turn-execution.ts b/src/server/responses/run-turn-execution.ts index 463c250c4a7..2f6b8e2c77f 100644 --- a/src/server/responses/run-turn-execution.ts +++ b/src/server/responses/run-turn-execution.ts @@ -51,6 +51,7 @@ import { planWebSearch } from "../../web-search"; import { runTurnWebSearchInitialParsed, runTurnWebSearchLoop } from "../../web-search/run-turn-loop"; import { WEB_SEARCH_TOOL_NAME } from "../../web-search/synthetic-tool"; import { orderDevinMessagesOutput } from "../../claude/devin-output-order"; +import { unboundPoolSpendRefusalMessage } from "../workflow-refusal"; // LOCAL PATCH (runturn-websearch): top-level fields route binding or the // adapter itself may write during a turn. Iteration-local `turnParsed` objects @@ -330,7 +331,7 @@ export async function executeResponsesRunTurn( status: 429, errorType: "rate_limit_error", code: SEND_BUDGET_EXHAUSTED_CODE, - message: err.message, + message: unboundPoolSpendRefusalMessage(logCtx) ?? err.message, } : { type: "error", diff --git a/src/server/workflow-refusal.ts b/src/server/workflow-refusal.ts index a1457fed130..8fe6696ba86 100644 --- a/src/server/workflow-refusal.ts +++ b/src/server/workflow-refusal.ts @@ -97,6 +97,22 @@ export function workflowRefusalResponse( return refusal; } +/** Render the approved conservative-overlay explanation after a dispatch reservation refuses. */ +export function unboundPoolSpendRefusalMessage( + logCtx: Pick, +): string | undefined { + const detail = logCtx.spendRefusalDetail; + if (!detail?.includesUnboundPoolHistory) return undefined; + return workflowDenialSummary("workflow-spend-exhausted", detail).message; +} + +/** HTTP form for pre-commit send-budget catches; the tracker already recorded the event. */ +export function unboundPoolSpendRefusalResponse(logCtx: RequestLogContext): Response | undefined { + const detail = logCtx.spendRefusalDetail; + if (!detail?.includesUnboundPoolHistory) return undefined; + return workflowRefusalResponse("workflow-spend-exhausted", logCtx, undefined, undefined, detail); +} + /** * Admit one HTTP turn against its root workflow budget. * @@ -145,6 +161,7 @@ export function workflowDecisionRefusalResponse( scope: decision.spendScope, limit: decision.spendLimit, ...(decision.spendProjected !== undefined ? { projected: decision.spendProjected } : {}), + ...(decision.spendIncludesUnboundPoolHistory ? { includesUnboundPoolHistory: true } : {}), } : undefined; return workflowRefusalResponse(decision.reason, logCtx, refusalLog, undefined, spend); diff --git a/structure/transports/responses-spend.md b/structure/transports/responses-spend.md index 26bcb35d84a..666e35e593f 100644 --- a/structure/transports/responses-spend.md +++ b/structure/transports/responses-spend.md @@ -137,13 +137,18 @@ for automatically assigning old balances. Old canonical-looking aliases also nee mapping when their positive history predates identity metadata. Zero-balance history needs none. The ledger retains original scope balances and reservation targets. The canonical view adds each -member once, keeping settled, reserved and unresolved buckets separate. Explicit links can join -previously canonical groups for a verified rename; an already redirected alias cannot be assigned -to a different group. The complete proposed graph is validated atomically, so a verified merge -that repeats existing member aliases is independent of salted-key order. A removed config entry -never removes a journaled link. Group activity, -last-seen time and exhaustion govern retention; unidentified positive balances cannot be evicted. -Identity evidence is bounded and retained even when dormant under-limit scopes are evicted. +proven member once, keeping settled, reserved and unresolved buckets separate. Until ownership is +verified, every positive unbound pool balance is also counted once against each candidate provider +pool. The unbound balance stays in its original scope; a current provider name or matching salted +alias does not establish ownership. This conservative overlay may restrict an otherwise unused +provider pool until an operator supplies a verified mapping. Applying a mapping moves that balance +from the overlay into its proven group without copying it. Explicit links can join previously +canonical groups for a verified rename; an already redirected alias cannot be assigned to a +different group. The complete proposed graph is validated atomically, so a verified merge that +repeats existing member aliases is independent of salted-key order. A removed config entry never +removes a journaled link. Group activity, last-seen time and exhaustion govern retention; unbound +positive balances cannot be evicted. Identity evidence is bounded and retained even when dormant +under-limit scopes are evicted. Each eviction pass aggregates pool groups once before choosing candidates. Retention uses the group's newest activity; capacity pressure still removes only the oldest eligible individual scope per pass, and every removal journals its original scope alias. @@ -154,16 +159,18 @@ journal. New reservations still use the routed canonical pool ID. Replays and co the links; complete invalid metadata fails closed, including at the final line, and corruption is never compacted away. Unparseable torn final JSON keeps the existing conservative replay rule. -With a configured pool ceiling, any remaining unidentified positive pool history or invalid/ -conflicting mapping refuses admission. HTTP workflow admission and the Responses pre-dispatch -seam check this even without a root ID, before passthrough transports that report sends afterwards. -After routing, that seam also refuses an already-exhausted canonical pool, including mapped historical totals. -A prepaid child excludes only its own open reservation, proven by its exact permit, shared send -ledger and still-pending receipt. Proof is single-use; unrelated reservations and other pool groups -remain counted. HTTP continuity refusals record one rooted workflow event; rootless ones record none. -It is a snapshot check, not a new atomic reservation for report-only transports: crossing sends, -concurrent preflight admissions and retries reported afterwards retain their existing limitations. -`workflow_pool_history_unresolved` identifies the local 429 without disclosing aliases; storage +With a configured pool ceiling, an invalid or conflicting mapping refuses admission. Unbound +positive history alone does not block HTTP admission before a route is known; once a candidate +pool is known, its reservation check includes the proven group and every unbound positive balance. +The Responses pre-dispatch seam checks this even without a root ID, before passthrough transports +that report sends afterwards. Its local refusal explains when unknown-owner history contributed, +without exposing aliases or journal contents. A prepaid child excludes only its own open +reservation, proven by its exact permit, shared send ledger and still-pending receipt. Proof is +single-use; unrelated reservations and other proven pool groups remain counted. HTTP continuity +refusals still record one rooted workflow event; rootless ones record none. It is a snapshot check, +not a new atomic reservation for report-only transports: crossing sends, concurrent preflight +admissions and retries reported afterwards retain their existing limitations. `workflow_pool_history_unresolved` +identifies an invalid or conflicting identity-metadata refusal without disclosing aliases; storage or replay failures keep `workflow_spend_undurable`. Already-sent reports still book actual spend. Observe-only mode has no new token refusal, and root/identity accounting remains independent. @@ -173,8 +180,9 @@ salt loses newer spend and is not a supported rollback. Unmodified older binarie they can read v1 counters but do not enforce the canonical aggregate, can introduce fresh account label pools, and can discard optional identity metadata during compaction. There is no automatic downgrade barrier and no unknown-record fence. If that unsupported write has happened, the current -reader requires explicit mappings for the remaining unidentified balances rather than assuming -zero. `tests/lib/spend-pool-continuity.test.ts` exercises this using the frozen pre-change reader, +reader conservatively counts remaining unidentified positive balances against every candidate +pool until the operator verifies their mappings. `tests/lib/spend-pool-continuity.test.ts` exercises +this using the frozen pre-change reader, as well as exact aggregation, active/unresolved sends, retention, failures and rootless preflight. Budget reservations retain their exact durable proof until their own dispatch/report confirms diff --git a/tests/lib/spend-ceiling-enforcement.test.ts b/tests/lib/spend-ceiling-enforcement.test.ts index 91acd121efe..95f04f078f3 100644 --- a/tests/lib/spend-ceiling-enforcement.test.ts +++ b/tests/lib/spend-ceiling-enforcement.test.ts @@ -70,6 +70,7 @@ const watched = (inner: SpendReservationLedger, asked: string[]): SpendReservati markLost: (sendId) => inner.markLost(sendId), snapshot: (scope, scopeId) => inner.snapshot(scope, scopeId), exhausted: (scope, scopeId) => { asked.push("exhausted"); return inner.exhausted(scope, scopeId); }, + hasUnboundPositivePoolHistory: () => inner.hasUnboundPositivePoolHistory(), prune: (at) => inner.prune(at), knows: (sendId) => inner.knows(sendId), reconfigure: (next) => inner.reconfigure(next), @@ -261,7 +262,7 @@ describe("a token refusal is legible on the wire", () => { expect(summary.message).toContain("account token ceiling of 100,000"); expect(summary.message).toContain("112,500"); expect(summary.message).toContain("spend.identity.maxTokens"); - expect(summary.message).toContain("no provider was contacted"); + expect(summary.message).toContain("this send was refused before contacting a provider"); }); test("without a denial in hand the sentence is the one it always was", () => { diff --git a/tests/lib/spend-pool-continuity.test.ts b/tests/lib/spend-pool-continuity.test.ts index 1973b053b26..55e341152a8 100644 --- a/tests/lib/spend-pool-continuity.test.ts +++ b/tests/lib/spend-pool-continuity.test.ts @@ -14,17 +14,18 @@ import { } from "../../src/lib/spend-reservation-ledger"; import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; -import { admitHttpWorkflowTurn, workflowDecisionRefusalResponse } from "../../src/server/workflow-refusal"; +import { admitHttpWorkflowTurn, unboundPoolSpendRefusalMessage, unboundPoolSpendRefusalResponse } from "../../src/server/workflow-refusal"; import { createResponsesSendBudget } from "../../src/server/responses/request-send-budget"; import { claimDispatchSpendProof, createRequestExecutionBudget, deriveRequestExecutionBudget, reportDispatchSends } from "../../src/lib/request-execution-budget"; import { createPoolContinuity } from "../../src/lib/spend-pool-continuity"; -import { listWorkflowBudgetEvents, resetWorkflowBudgetsForTest, workflowSpendCeilingReached } from "../../src/lib/workflow-budget"; +import { admitWorkflowTurn, listWorkflowBudgetEvents, resetWorkflowBudgetsForTest, workflowDenialSummary, workflowSpendCeilingReached } from "../../src/lib/workflow-budget"; import { createRequestSpendTracker } from "../../src/server/responses/request-spend"; import type { RequestLogContext } from "../../src/server/request-log"; const salt = "5".repeat(64); const alias = (kind: string, id: string) => createHash("sha256").update(salt).update("\0").update(kind).update("\0").update(id).digest("hex").slice(0, 32); const pool = (id: string) => alias("pool", id); +const currentPool = (id: string) => alias("pool-current", id); const policy = (poolAliases?: unknown, overrides: Partial = {}): SpendReservationPolicy => ({ ...DEFAULT_SPEND_RESERVATION_POLICY, pool: { maxTokens: 100 }, poolAliases, ...overrides, }); @@ -65,18 +66,136 @@ describe("historical pool identity continuity", () => { } }); - test("unmapped historical debt fails closed across labels, providers, pruning and restart", () => { + test("unbound positive history is charged once against every candidate pool across pruning and restart", () => { const disk = journal([checkpoint([["provider-old-label", 100, 0]])]); for (let restart = 0; restart < 2; restart += 1) { - const ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(), now: () => 100_000 }); + const ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(), now: () => 1_000_000_000 }); + expect(ledger.checkPoolContinuity()).toBeUndefined(); + expect(ledger.hasUnboundPositivePoolHistory()).toBe(true); ledger.prune(); for (const id of ["provider", "provider-old-label", "unrelated-provider"]) { - expect(reserve(ledger, `${restart}-${id}`, id)).toMatchObject({ reserved: false, denial: { reason: "pool-history-unresolved" } }); + expect(reserve(ledger, `${restart}-${id}`, id)).toMatchObject({ + reserved: false, + denial: { reason: "spend-limit-exceeded", scope: "pool", limit: 100, projected: 101, includesUnboundPoolHistory: true }, + }); } expect(ledger.snapshot("pool", "provider-old-label")?.settled).toBe(100); } }); + test("tracking capacity pressure cannot evict unbound positive pool history", () => { + const disk = journal([checkpoint([["old-label", 10, 0]])]); + const ledger = createSpendReservationLedger({ journal: disk, salt, + policy: policy(undefined, { maxTrackedScopes: 1 }), now: () => 2 }); + expect(reserve(ledger, "cannot-forget-history", "provider", 1)).toMatchObject({ + reserved: false, denial: { reason: "tracking-capacity-exhausted", scope: "pool" }, + }); + expect(ledger.snapshot("pool", "old-label")?.settled).toBe(10); + expect(disk.lines.map(line => JSON.parse(line)).filter(record => record.kind === "drop")).toEqual([]); + }); + + test("an exhausted candidate overlay protects its under-limit proven group from retention", () => { + const record = checkpoint([["verified-label", 60, 0], ["unbound-label", 40, 0], ["empty-old-label", 0, 0]]); + const disk = journal([record]); + const ledger = createSpendReservationLedger({ journal: disk, salt, + policy: policy({ [pool("verified-label")]: "provider" }, { retentionMs: 5 }), now: () => 100 }); + expect(ledger.checkPoolContinuity()).toBeUndefined(); + expect(ledger.snapshot("pool", "provider")?.settled).toBe(100); + ledger.prune(100); // All entries are older than the retention cutoff. + expect(ledger.snapshot("pool", "provider")?.settled).toBe(100); + expect(disk.lines.map(line => JSON.parse(line)).filter(record => record.kind === "drop")).toEqual([ + { v: 1, kind: "drop", scope: "pool", alias: pool("empty-old-label"), at: 100 }, + ]); + expect(reserve(ledger, "still-exhausted", "provider", 1)).toMatchObject({ reserved: false, + denial: { reason: "spend-limit-exceeded", projected: 101, includesUnboundPoolHistory: true } }); + }); + + test("proven group plus each unbound balance admits under/exact totals and refuses over", () => { + const disk = journal([checkpoint([["verified-label", 25, 0], ["unbound-label", 10, 5]])]); + const ledger = createSpendReservationLedger({ journal: disk, salt, + policy: policy({ [pool("verified-label")]: "provider" }), now: () => 2 }); + + // Proven group: 25. Unbound history: 10 settled + 5 unresolved. Neither bucket is copied. + expect(ledger.snapshot("pool", "provider")).toEqual({ settled: 35, reserved: 0, unresolved: 5, exhausted: false }); + expect(reserve(ledger, "under", "provider", 59).reserved).toBe(true); // 99 total + expect(ledger.abandon("under")).toBe(true); + expect(reserve(ledger, "exact", "provider", 60).reserved).toBe(true); // 100 total + expect(reserve(ledger, "over", "provider", 1)).toMatchObject({ + reserved: false, + denial: { reason: "spend-limit-exceeded", projected: 101, includesUnboundPoolHistory: true }, + }); + + // The same unbound 15 is also counted once for a different candidate provider. + expect(reserve(ledger, "other-exact", "unrelated-provider", 85).reserved).toBe(true); + expect(ledger.snapshot("pool", "unrelated-provider")).toEqual({ settled: 10, reserved: 85, unresolved: 5, exhausted: true }); + expect(ledger.hasUnboundPositivePoolHistory()).toBe(true); + }); + + test("same-spelling unknown history stays unbound and is not counted twice", () => { + const disk = journal([checkpoint([["provider", 40, 0]])]); + const ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(), now: () => 2 }); + + // Matching the current provider's salted alias is not identity evidence by itself. + expect(ledger.checkPoolContinuity()).toBeUndefined(); + expect(ledger.hasUnboundPositivePoolHistory()).toBe(true); + expect(reserve(ledger, "same-spelling-exact", "provider", 60).reserved).toBe(true); + expect(ledger.snapshot("pool", "provider")).toEqual({ settled: 40, reserved: 60, unresolved: 0, exhausted: true }); + // The legacy alias stays unknown and overlays every candidate, while the new send is + // stored under the canonical-current domain and belongs only to the routed provider. + expect(JSON.parse(disk.lines.at(-1)!).targets).toEqual([{ scope: "pool", alias: currentPool("provider") }]); + expect(reserve(ledger, "unrelated-under-overlay", "unrelated-provider", 60).reserved).toBe(true); + expect(ledger.snapshot("pool", "unrelated-provider")).toEqual({ + settled: 40, reserved: 60, unresolved: 0, exhausted: true, + }); + expect(reserve(ledger, "unrelated-over", "third-provider", 61)).toMatchObject({ + reserved: false, + denial: { reason: "spend-limit-exceeded", projected: 101, includesUnboundPoolHistory: true }, + }); + + // An explicit operator mapping on restart moves only the legacy 40-token balance into + // this provider's current group; it does not copy it or assign it by spelling. + const mapped = createSpendReservationLedger({ journal: disk, salt, + policy: policy({ [pool("provider")]: "provider" }), now: () => 3 }); + expect(mapped.checkPoolContinuity()).toBeUndefined(); + expect(mapped.hasUnboundPositivePoolHistory()).toBe(false); + expect(mapped.snapshot("pool", "provider")).toMatchObject({ settled: 40, unresolved: 60 }); + expect(reserve(mapped, "unrelated-after-mapping", "unrelated-provider", 41)).toMatchObject({ + reserved: false, + denial: { reason: "spend-limit-exceeded", projected: 101 }, + }); + }); + + test("unbound history is overlaid when a pool ceiling is enabled later", () => { + const disk = journal([checkpoint([["old-label", 40, 0]])]); + const ledger = createSpendReservationLedger({ journal: disk, salt, + policy: policy(undefined, { pool: {} }), now: () => 2 }); + + expect(reserve(ledger, "observe-only", "provider", 10).reserved).toBe(true); + expect(ledger.settle("observe-only", { inputTokens: 10, outputTokens: 0 })).toBe(true); + expect(ledger.hasUnboundPositivePoolHistory()).toBe(true); + ledger.reconfigure(policy()); + + expect(reserve(ledger, "exact-after-enable", "provider", 50).reserved).toBe(true); + expect(reserve(ledger, "over-after-enable", "provider", 1)).toMatchObject({ + reserved: false, + denial: { reason: "spend-limit-exceeded", projected: 101, includesUnboundPoolHistory: true }, + }); + }); + + test("competing same-process admissions include the shared unbound balance", async () => { + const ledger = createSpendReservationLedger({ + journal: journal([checkpoint([["old-label", 40, 0]])]), salt, policy: policy(), now: () => 2, + }); + const attempts = await Promise.all([ + Promise.resolve().then(() => reserve(ledger, "contender-a", "provider", 35)), + Promise.resolve().then(() => reserve(ledger, "contender-b", "provider", 35)), + ]); + expect(attempts[0]?.reserved).toBe(true); + expect(attempts[1]).toMatchObject({ reserved: false, + denial: { reason: "spend-limit-exceeded", scope: "pool", projected: 110, includesUnboundPoolHistory: true } }); + expect(ledger.snapshot("pool", "provider")).toMatchObject({ settled: 40, reserved: 35 }); + }); + test("explicit aliases aggregate each original balance once, including canonical history", () => { const disk = journal([checkpoint([["label-a", 40, 0], ["label-b", 0, 30], ["provider", 20, 0]])]); const aliases = { [pool("label-a")]: "provider", [pool("label-b")]: "provider", [pool("provider")]: "provider" }; @@ -186,27 +305,67 @@ describe("historical pool identity continuity", () => { { v: 1, kind: "drop", scope: "pool", alias: pool("oldest"), at: 50 }, ]); expect(ledger.snapshot("pool", "provider")?.settled).toBe(20); - expect(ledger.snapshot("pool", "other")?.settled).toBe(0); + expect(ledger.snapshot("pool", "other")).toBeUndefined(); expect(ledger.snapshot("root", "new-root")?.reserved).toBe(1); const restarted = createSpendReservationLedger({ journal: disk, salt, policy: config, now: () => 50 }); expect(restarted.snapshot("pool", "provider")?.settled).toBe(20); - expect(restarted.snapshot("pool", "other")?.settled).toBe(0); + expect(restarted.snapshot("pool", "other")).toBeUndefined(); expect(restarted.snapshot("root", "new-root")?.unresolved).toBe(1); }); - test("observe-only records already-sent requests while ambiguity still blocks new dispatches", () => { + test("observe-only charges already-sent requests while unbound history still constrains admission", () => { const disk = journal([checkpoint([["unknown-label", 40, 0]])]); const ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(), now: () => 2 }); const context: RequestLogContext = { model: "fixture", provider: "provider-display", spendPoolId: "provider", usageLogInputTokens: 10 }; const tracker = createRequestSpendTracker(context, undefined, ledger); - expect(tracker.charge()).toBe(false); - expect(tracker.refusals).toBe(1); - expect(context.errorCode).toBe("workflow_pool_history_unresolved"); expect(tracker.charge({ alreadySent: true })).toBe(true); + expect(tracker.refusals).toBe(0); + expect(context.errorCode).toBeUndefined(); tracker.settle({ inputTokens: 8, outputTokens: 0 }); - expect(ledger.snapshot("pool", "provider")?.settled).toBe(8); + expect(ledger.snapshot("pool", "provider")?.settled).toBe(48); // current settled 8 plus shared unknown 40 expect(ledger.snapshot("pool", "unknown-label")?.settled).toBe(40); - expect(reserve(ledger, "new").reserved).toBe(false); + expect(reserve(ledger, "exact", "provider", 52).reserved).toBe(true); + expect(ledger.abandon("exact")).toBe(true); + const refused = reserve(ledger, "over", "provider", 53); + expect(refused).toMatchObject({ reserved: false, + denial: { reason: "spend-limit-exceeded", projected: 101, includesUnboundPoolHistory: true } }); + const rootRefusal = admitWorkflowTurn("root-with-unbound-history", "interactive", undefined, undefined, 2, + { sendId: "root-over", poolId: "provider", inputTokens: 53, outputCeilingTokens: 0 }, ledger); + expect(rootRefusal).toMatchObject({ admitted: false, reason: "workflow-spend-exhausted", + spendScope: "pool", spendIncludesUnboundPoolHistory: true }); + const message = workflowDenialSummary("workflow-spend-exhausted", { + scope: "pool", limit: 100, projected: 101, includesUnboundPoolHistory: true, + }).message; + expect(message).toContain("unassigned historical provider-pool balances"); + expect(message).not.toContain("unknown-label"); + expect(message).not.toContain(alias("pool", "unknown-label")); + expect(message).not.toContain(salt); + }); + + test("reservation crossing explains unbound-history refusal without exposing its alias", async () => { + resetWorkflowBudgetsForTest(); + const disk = journal([checkpoint([["private-old-label", 40, 0]])]); + const ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(), now: () => 2 }); + const context: RequestLogContext = { + model: "fixture", provider: "provider", spendPoolId: "provider", usageLogInputTokens: 10, + }; + const tracker = createRequestSpendTracker(context, "root-unbound", ledger); + + expect(tracker.charge({ alreadySent: true })).toBe(true); + context.usageLogInputTokens = 51; // the next physical send would cross the cap after that upstream contact + expect(tracker.charge()).toBe(false); + const message = unboundPoolSpendRefusalMessage(context); + expect(message).toContain("unassigned historical provider-pool balances"); + expect(message).toContain("this send was refused before contacting a provider"); + expect(message).not.toContain("private-old-label"); + expect(message).not.toContain(alias("pool", "private-old-label")); + expect(message).not.toContain(salt); + const response = unboundPoolSpendRefusalResponse(context); + expect(response?.status).toBe(429); + expect(await response!.text()).toContain(message!); + expect(listWorkflowBudgetEvents(1)[0]).toMatchObject({ + reason: "workflow-spend-exhausted", spendIncludesUnboundPoolHistory: true, + }); }); test("malformed or conflicting maps fail closed without clearing ceilings or saved links", () => { @@ -229,7 +388,7 @@ describe("historical pool identity continuity", () => { disk.append = () => { throw new Error("synthetic write failure"); }; const ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy({ [pool("label")]: "provider" }), now: () => 2 }); expect(reserve(ledger, "new")).toMatchObject({ reserved: false, denial: { reason: "reserve-not-durable" } }); - expect(ledger.snapshot("pool", "provider")).toBeUndefined(); + expect(ledger.snapshot("pool", "provider")?.settled).toBe(40); // the unbound view is visible without a committed link expect(ledger.snapshot("pool", "label")?.settled).toBe(40); expect(disk.lines).toHaveLength(1); }); @@ -263,7 +422,7 @@ describe("historical pool identity continuity", () => { }); -test("rootless HTTP and passthrough preflight refuse before any synthetic fetch", () => { +test("unbound history waits for routing, then limits each candidate before any synthetic fetch", async () => { const previous = process.env.OPENCODEX_HOME; const home = mkdtempSync(join(tmpdir(), "ocx-pool-history-")); process.env.OPENCODEX_HOME = home; @@ -274,24 +433,23 @@ test("rootless HTTP and passthrough preflight refuse before any synthetic fetch" configureSharedSpendLedger(policy()); resetWorkflowBudgetsForTest(); const rooted = admitHttpWorkflowTurn(new Headers({ "x-codex-parent-thread-id": "synthetic-root" })); - expect(rooted).toMatchObject({ admitted: false, reason: "workflow-pool-history-unresolved" }); - expect(listWorkflowBudgetEvents()).toMatchObject([{ rootId: "synthetic-root", reason: "workflow-pool-history-unresolved" }]); - if (rooted && !rooted.admitted) workflowDecisionRefusalResponse(rooted); - expect(listWorkflowBudgetEvents()).toHaveLength(1); + expect(rooted).toMatchObject({ admitted: true, lease: { rootId: "synthetic-root" } }); + if (rooted?.admitted) rooted.lease.release(); + expect(listWorkflowBudgetEvents()).toHaveLength(0); let syntheticFetches = 0; const decision = admitHttpWorkflowTurn(new Headers()); - expect(decision).toMatchObject({ admitted: false, reason: "workflow-pool-history-unresolved" }); - if (!decision || decision.admitted) syntheticFetches += 1; - else { - const response = workflowDecisionRefusalResponse(decision); - expect(response.status).toBe(429); - expect(response.headers.get("x-opencodex-local-refusal")).toBe("workflow_pool_history_unresolved"); - } + expect(decision).toBeUndefined(); const budget = createResponsesSendBudget({ req: new Request("https://fixture.example.test/v1/responses"), options: {}, logCtx: { model: "fixture", provider: "provider" } }); expect(budget).toBeInstanceOf(Response); - if (!(budget instanceof Response)) syntheticFetches += 1; + if (budget instanceof Response) { + expect(budget.status).toBe(429); + expect(budget.headers.get("x-opencodex-local-refusal")).toBe("workflow_spend_exhausted"); + const body = await budget.text(); + expect(body).toContain("unassigned historical provider-pool balances"); + expect(body).not.toContain("old-label"); + } else syntheticFetches += 1; expect(syntheticFetches).toBe(0); - expect(listWorkflowBudgetEvents()).toHaveLength(1); // rootless refusals add no event + expect(listWorkflowBudgetEvents()).toHaveLength(0); // rootless refusals add no event configureSharedSpendLedger(policy({ [pool("old-label")]: "provider" })); expect(admitHttpWorkflowTurn(new Headers())).toBeUndefined(); const mappedBudget = createResponsesSendBudget({ req: new Request("https://fixture.example.test/v1/responses"), options: {}, logCtx: { model: "fixture", provider: "provider-display", spendPoolId: "provider" } }); @@ -328,11 +486,18 @@ test("actual old-reader compaction preserves raw spend; compatible rollback reso old.settle("old-again", { inputTokens: 12, outputTokens: 0 }); const returned = createSpendReservationLedger({ journal: disk, salt, policy: policy({ [pool("old-label")]: "provider" }), now: () => 4 }); - expect(returned.checkPoolContinuity()?.reason).toBe("pool-history-unresolved"); - expect(returned.snapshot("pool", "new-old-label")?.settled).toBe(12); + expect(returned.checkPoolContinuity()).toBeUndefined(); + expect(returned.snapshot("pool", "new-old-label")?.settled).toBe(20); // unbound 12 plus canonical-looking 8, both still unassigned + expect(returned.snapshot("pool", "provider")?.settled).toBe(60); + expect(returned.snapshot("pool", "unrelated-provider")?.settled).toBe(20); + expect(reserve(returned, "candidate-after-old-writer", "unrelated-provider", 80).reserved).toBe(true); + expect(returned.abandon("candidate-after-old-writer")).toBe(true); returned.reconfigure(policy({ [pool("old-label")]: "provider", [pool("provider")]: "provider", [pool("new-old-label")]: "provider" })); expect(returned.checkPoolContinuity()).toBeUndefined(); expect(returned.snapshot("pool", "provider")?.settled).toBe(60); + const restarted = createSpendReservationLedger({ journal: disk, salt, policy: policy(), now: () => 5 }); + expect(restarted.checkPoolContinuity()).toBeUndefined(); + expect(restarted.snapshot("pool", "provider")?.settled).toBe(60); }); for (const kind of ["compaction", "combo"] as const) { @@ -505,19 +670,19 @@ test("preflight retains unrelated reservations, debt and mismatched scope or led }); test("prepaid exclusion follows only its canonical pool and retains settled/unresolved history", () => { - const disk = journal([checkpoint([["historical", 30, 10]])]); + const disk = journal([checkpoint([["historical", 30, 10], ["unbound", 10, 5]])]); const ledger = createSpendReservationLedger({ salt, journal: disk, policy: policy({ [pool("historical")]: "provider" }), now: () => 2 }); - const tracker = createRequestSpendTracker({ provider: "provider", usageLogInputTokens: 60 }, undefined, ledger); + const tracker = createRequestSpendTracker({ provider: "provider", usageLogInputTokens: 45 }, undefined, ledger); const budget = createRequestExecutionBudget(undefined, undefined, tracker); const decision = budget.reserveDispatch({ sendClass: "initial", targetKey: "provider", countedExternally: true }); if (!decision.allowed) throw new Error("synthetic permit refused"); const proof = claimDispatchSpendProof(budget, decision.permit)!; - expect(ledger.snapshot("pool", "provider")).toMatchObject({ settled: 30, unresolved: 10, reserved: 60 }); + expect(ledger.snapshot("pool", "provider")).toMatchObject({ settled: 40, unresolved: 15, reserved: 45 }); expect(workflowSpendCeilingReached(undefined, ledger, "provider", proof)).toBeUndefined(); ledger.reconfigure(policy({ [pool("historical")]: "renamed", [pool("provider")]: "renamed" })); expect(ledger.checkPoolContinuity()).toBeUndefined(); expect(workflowSpendCeilingReached(undefined, ledger, "renamed", proof)).toBeUndefined(); - expect(reserve(ledger, "other", "unrelated", 100).reserved).toBe(true); + expect(reserve(ledger, "other", "unrelated", 85).reserved).toBe(true); expect(workflowSpendCeilingReached(undefined, ledger, "unrelated", proof)).toMatchObject({ scope: "pool" }); ledger.reconfigure(policy(undefined, { pool: { maxTokens: 40 } })); expect(workflowSpendCeilingReached(undefined, ledger, "renamed", proof)).toMatchObject({ scope: "pool" }); diff --git a/tests/responses/responses-send-budget-errors.test.ts b/tests/responses/responses-send-budget-errors.test.ts index 6e1306980b8..dc53fa9efe9 100644 --- a/tests/responses/responses-send-budget-errors.test.ts +++ b/tests/responses/responses-send-budget-errors.test.ts @@ -78,6 +78,15 @@ describe("a spent send budget is reported as this proxy's refusal", () => { } }); + test("reservation denials preserve unbound-history detail on HTTP and SSE response paths", () => { + const adapter = source("src/server/responses/adapter-dispatch.ts"); + const passthrough = source("src/server/responses/passthrough-dispatch.ts"); + const runTurn = source("src/server/responses/run-turn-execution.ts"); + expect(adapter.match(/unboundPoolSpendRefusalResponse\(logCtx\)/g)).toHaveLength(2); + expect(passthrough).toContain("unboundPoolSpendRefusalResponse(logCtx)"); + expect(runTurn).toContain("unboundPoolSpendRefusalMessage(logCtx) ?? err.message"); + }); + test("a local 429 never rotates a credential or writes a cooldown", () => { const runTurn = source("src/server/responses/run-turn-execution.ts"); const rotate = runTurn.indexOf("const rotateRunTurnAdapterOnPreflight429"); From 82b3fd9d107d19ffbc0f8fc395d080dfcfa8fdc8 Mon Sep 17 00:00:00 2001 From: JUN Date: Thu, 8 Oct 2026 23:29:05 +0900 Subject: [PATCH 08/15] wip(spend): checkpoint contract C carry of #6370 Recovery checkpoint of the in-progress contract C implementation so the lane can continue from a managed worktree. Not review-ready; T1-T9, docs and independent review remain. Co-authored-by: Epinephrine --- scripts/test-layout/layout.json | 9 + src/config/diagnostics.ts | 23 +- src/lib/request-execution-budget.ts | 115 +- src/lib/spend-pool-alias-validation.ts | 77 + src/lib/spend-pool-continuity.ts | 108 +- src/lib/spend-reservation-ledger.ts | 301 ++-- src/lib/upstream-retry.ts | 22 +- src/lib/workflow-budget.ts | 1 + src/server/chat-native.ts | 21 +- src/server/index.ts | 2 +- src/server/index/spend-ledger-lifecycle.ts | 14 +- src/server/inference/context.ts | 2 +- src/server/messages-native.ts | 20 +- src/server/responses/adapter-continuation.ts | 8 +- src/server/responses/adapter-dispatch.ts | 8 +- src/server/responses/compact.ts | 14 +- src/server/responses/compaction-recovery.ts | 2 +- src/server/responses/passthrough-dispatch.ts | 10 +- src/server/responses/request-send-budget.ts | 16 +- src/server/responses/request-spend.ts | 281 +++- src/server/responses/request-transport.ts | 8 + src/server/workflow-refusal.ts | 2 +- src/types/config.ts | 2 +- tests/config/config-spend-ceilings.test.ts | 1 + .../config-spend-pool-collision.test.ts | 106 ++ tests/fixtures/spend-ledger-2-80-0.ts.txt | 1289 +++++++++++++++++ tests/fixtures/spend-ledger-f7a50dc3.ts.txt | 586 -------- tests/fixtures/test-layout-expected.json | 6 +- tests/helpers/legacy-spend-ledger.ts | 16 - tests/helpers/shipped-spend-ledger.ts | 36 + tests/lib/spend-contract-c.test.ts | 52 + tests/lib/spend-corruption-compat.test.ts | 31 + tests/lib/spend-experimental-history.test.ts | 26 + tests/lib/spend-pool-continuity.test.ts | 141 +- tests/lib/spend-reservation-ledger.test.ts | 6 +- tests/lib/spend-seed-overflow.test.ts | 121 ++ tests/lib/spend-unbound-expiry.test.ts | 33 + tests/lib/spend-zero-config-compat.test.ts | 20 + tests/responses/chat-native-spend.test.ts | 10 +- .../responses/responses-spend-anchor.test.ts | 130 ++ .../responses-spend-capacity-guard.test.ts | 5 +- .../responses-spend-contract-c.test.ts | 171 +++ .../responses-spend-ledger-wiring.test.ts | 100 +- 43 files changed, 2955 insertions(+), 997 deletions(-) create mode 100644 src/lib/spend-pool-alias-validation.ts create mode 100644 tests/config/config-spend-pool-collision.test.ts create mode 100644 tests/fixtures/spend-ledger-2-80-0.ts.txt delete mode 100644 tests/fixtures/spend-ledger-f7a50dc3.ts.txt delete mode 100644 tests/helpers/legacy-spend-ledger.ts create mode 100644 tests/helpers/shipped-spend-ledger.ts create mode 100644 tests/lib/spend-contract-c.test.ts create mode 100644 tests/lib/spend-corruption-compat.test.ts create mode 100644 tests/lib/spend-experimental-history.test.ts create mode 100644 tests/lib/spend-seed-overflow.test.ts create mode 100644 tests/lib/spend-unbound-expiry.test.ts create mode 100644 tests/lib/spend-zero-config-compat.test.ts create mode 100644 tests/responses/responses-spend-anchor.test.ts create mode 100644 tests/responses/responses-spend-contract-c.test.ts diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index dfb0e2e7981..a87db4601ba 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -720,6 +720,7 @@ "config-rebase-provenance-writers.test.ts": "config", "config-save-boundary.test.ts": "config", "config-spend-ceilings.test.ts": "config", + "config-spend-pool-collision.test.ts": "config", "config-user-edits.test.ts": "config", "config.test.ts": "server", "configured-native-models.test.ts": "codex-integration", @@ -1563,6 +1564,8 @@ "responses-snapshot-repair.test.ts": "responses", "responses-sparse-terminal-tool-scope.test.ts": "responses", "responses-spend-ledger-wiring.test.ts": "responses", + "responses-spend-contract-c.test.ts": "responses", + "responses-spend-anchor.test.ts": "responses", "responses-spend-capacity-guard.test.ts": "responses", "responses-spill-acl-recovery.test.ts": "responses", "responses-spill-inspection.test.ts": "responses", @@ -1701,6 +1704,12 @@ "spend-ledger-owner-startup.test.ts": "server", "spend-ledger-owner.test.ts": "lib", "spend-pool-continuity.test.ts": "lib", + "spend-contract-c.test.ts": "lib", + "spend-corruption-compat.test.ts": "lib", + "spend-unbound-expiry.test.ts": "lib", + "spend-seed-overflow.test.ts": "lib", + "spend-zero-config-compat.test.ts": "lib", + "spend-experimental-history.test.ts": "lib", "spend-reservation-ledger.test.ts": "lib", "sponsor-presets.test.ts": "providers", "sse-carriage-return.test.ts": "responses", diff --git a/src/config/diagnostics.ts b/src/config/diagnostics.ts index 14fab218952..ab622d0b73a 100644 --- a/src/config/diagnostics.ts +++ b/src/config/diagnostics.ts @@ -1,4 +1,4 @@ -import { spendPoolAliasesError } from "../lib/spend-pool-continuity"; +import { readExistingSpendSalt, spendPoolAliasesError, validatePoolAliasOwners } from "../lib/spend-pool-alias-validation"; import { isSubagentModelEntry, rawSubagentModelForce } from "./subagent-models"; import { protocolConfigSchema } from "./schema/config-schema"; import { createHash } from "node:crypto"; @@ -14,7 +14,7 @@ import { loopbackCompanionAllowed } from "../codex/loopback-target"; import { UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD } from "../codex/upstream-host-health"; import { MAX_APP_OWNED_MEMORY_BUDGET_MB, MIN_APP_OWNED_MEMORY_BUDGET_MB } from "../lib/app-owned-memory"; import { isMissingPathError } from "./atomic-write"; -import { getConfigPath } from "./paths"; +import { getConfigDir, getConfigPath } from "./paths"; import { getDefaultConfig } from "./proxy-env"; import { salvageConfigCandidate } from "./salvage"; import { @@ -82,6 +82,21 @@ export type ConfigDiagnostics = { warnings?: string[]; }; +function rawPoolAliasesError(value: unknown): string | undefined { + const record = rawConfigRecord(value); + if (!record) return undefined; + const aliases = record.spendPoolAliases; + const shapeError = spendPoolAliasesError(aliases); + if (shapeError) return shapeError; + const mapping = rawConfigRecord(aliases); + if (!mapping || Object.keys(mapping).length === 0) return undefined; + const providers = rawConfigRecord(record.providers ?? {}); + if (!providers) return "providers must be an object to validate spendPoolAliases"; + const salt = readExistingSpendSalt(getConfigDir()); + if (!salt) return "spendPoolAliases requires a safely readable existing spend salt"; + return validatePoolAliasOwners(aliases, salt, Object.keys(providers)); +} + export type ConfigFileSnapshot = { diagnostics: ConfigDiagnostics; /** Exact file contents, including a possible BOM, used as the optimistic revision. */ @@ -146,7 +161,7 @@ function validFileConfigDiagnostics(config: OcxConfig, rawParsed: unknown): Conf if (codexPoolWarning) warnings.push(codexPoolWarning); const spendWarning = malformedSpendWarning(rawParsed); if (spendWarning) warnings.push(spendWarning); - if (spendPoolAliasesError(rawConfigRecord(rawParsed)?.spendPoolAliases)) { + if (rawPoolAliasesError(rawParsed)) { warnings.push("spendPoolAliases invalid: configured pool admission is refused until the mapping is corrected"); } const plaintextWarning = malformedPlaintextV2AgentMessagesWarning(rawParsed); @@ -683,7 +698,7 @@ export function validateConfigCandidate(value: unknown): { ok: true; config: Ocx ?? agentTaskRecoveryError(value) ?? quotaResetNotifyError(value) ?? catalogAutoRefreshError(value) - ?? (spendPoolAliasesError(rawConfigRecord(value)?.spendPoolAliases) ? "schema_invalid: spendPoolAliases: invalid pool identity mapping" : null) + ?? (rawPoolAliasesError(value) ? "schema_invalid: spendPoolAliases: invalid pool identity mapping" : null) ?? spendError(value) ?? codexPoolError(value) ?? googleAntigravityStaticCatalogVersionError(value) diff --git a/src/lib/request-execution-budget.ts b/src/lib/request-execution-budget.ts index b2da4cf8b80..fbb5da5138f 100644 --- a/src/lib/request-execution-budget.ts +++ b/src/lib/request-execution-budget.ts @@ -1,3 +1,4 @@ +import { AsyncLocalStorage } from "node:async_hooks"; /** * One logical request, one execution budget (#4546). * @@ -14,7 +15,7 @@ * sends plus one alternate -- by funding the alternate from a reserve that a validated * sanitized repair can spend instead, but never both. */ -import type { SpendReservationProof } from "./spend-reservation-ledger"; +import type { SpendSeed, SpendScopes, SpendReservationProof } from "./spend-reservation-ledger"; import type { TransientSendBudget } from "./upstream-retry"; export type SendClass = @@ -146,8 +147,11 @@ export interface RequestSendObserver { * would cross a ceiling never joins the total, the total stays just under, and the ceiling * never fires for any later request either. */ - charge(options?: { alreadySent?: boolean; deferDispatch?: boolean; onReserved?: (proof: SpendReservationProof) => void }): boolean; + charge(options?: { alreadySent?: boolean; deferDispatch?: boolean; onReserved?: (proof: SpendReservationProof) => void; targetKey?: string }): boolean; /** Confirm the exact reservation once its dispatch is known. */ + readonly enforced?: boolean; + ensureSeed?(target: SpendScopes & { targetKey?: string }, proof?: SpendReservationProof): SpendSeed | undefined; + beginReporter?(): { start(seed: SpendSeed, ordinal: number): void; rebindTarget(target: SpendScopes & { targetKey?: string }, ordinal: number): boolean; report(sends: number): void; close(): void }; dispatch?(proof: SpendReservationProof): void; /** Give back a booking whose send never happened. */ refund(proof?: SpendReservationProof | null): void; @@ -159,6 +163,11 @@ export interface RequestSendObserver { * model send; the reserve is what the fourth send draws on once the base allowance is gone. */ export interface RequestExecutionBudget extends TransientSendBudget { + readonly physicalStarted?: number; + readonly physicalLimit?: number; + readonly spendEnforced?: boolean; + claimPhysicalSend?(): number | undefined; + beginSpendProducer?(): { close(): void } | undefined; readonly logicalRequestId: string; readonly policyVersion: string; readonly policy: RequestExecutionBudgetPolicy; @@ -226,6 +235,8 @@ let logicalRequestSeq = 0; */ interface SharedSendLedger { spent: number; + physicalStarted: number; + physicalLimit?: number; pendingExternalSends: Set; /** * Spend one of this logical request's replacements for an ambiguous failure. Beside `spent` @@ -269,6 +280,72 @@ export function reportDispatchSends( budget.used += count - prepaid; } +/** A helper owns its starts and can never consume another helper's receipts. */ +export type PhysicalSendReporter = ((sends: number) => void) & { + beforeSend?: () => boolean; + close?: () => void; + execute?: (run: () => Promise) => Promise; + bind?: (target: SpendScopes & { targetKey?: string }) => boolean; +}; +const activePhysicalReporters = new AsyncLocalStorage(); +/** Selection owners rebind this helper's current start before invoking the executor. */ +export function rebindPhysicalSend(budget: TransientSendBudget | undefined, target: SpendScopes & { targetKey?: string }): boolean { + if (!(budget as RequestExecutionBudget | undefined)?.spendEnforced) return true; + return activePhysicalReporters.getStore()?.bind?.(target) ?? true; +} +export function createPhysicalSendReporter( + budget: TransientSendBudget, + target: () => SpendScopes & { targetKey?: string }, + permit?: SingleUseDispatchPermit, + telemetry?: (sends: number) => void, +): PhysicalSendReporter { + const counter = sharedSendLedgers.get(budget as RequestExecutionBudget); + const observer = counter?.observer; + let producer: ReturnType> | undefined; + let closed = false; + let started = 0; + let reported = 0; + let currentOrdinal: number | undefined; + const report: PhysicalSendReporter = (sends) => { + if (closed) return; + if (!producer) { if (observer?.enforced !== true) { reportDispatchSends(budget, sends, permit); telemetry?.(sends); } return; } + const count = Math.min(Math.max(0, Math.trunc(sends)), started - reported); + producer.report(count); + reported += count; + const receipt = permit && dispatchSpendProofs.get(permit); + const prepaid = count > 0 && receipt && receipt.owner === counter && receipt.report() ? 1 : 0; + if (counter) counter.spent += count - prepaid; + telemetry?.(count); + }; + if (observer?.enforced === true) report.beforeSend = () => { + if (closed) return false; + if (observer?.enforced !== true) return true; + producer ??= observer.beginReporter?.(); + const receipt = permit && dispatchSpendProofs.get(permit); + const proof = receipt?.owner === counter ? receipt?.claim() : undefined; + const seed = observer.ensureSeed?.(target(), proof); + if (!seed) return false; + const ordinal = (budget as RequestExecutionBudget).claimPhysicalSend?.(); + if (ordinal === undefined) return false; + producer?.start(seed, ordinal); + currentOrdinal = ordinal; + started++; + return true; + }; + report.execute = run => activePhysicalReporters.run(report, run); + report.bind = (selected: SpendScopes & { targetKey?: string }) => { + if (!producer || currentOrdinal === undefined || observer?.enforced !== true) return true; + return producer.rebindTarget(selected, currentOrdinal); + }; + report.close = () => { + if (closed) return; + if (producer) report(started - reported); + closed = true; + producer?.close(); + }; + return report; +} + /** * One logical request's replacement grant: how many it has spent, and the ceiling it is held * to. @@ -313,10 +390,25 @@ function createRequestExecutionBudgetWithLedger( const targetReservations: Array<{ targetKey: string }> = []; const budget: RequestExecutionBudget = { + get physicalStarted(): number { return counter.physicalStarted; }, + get physicalLimit(): number | undefined { return counter.physicalLimit; }, + get spendEnforced(): boolean { return observer?.enforced === true; }, + beginSpendProducer() { return observer?.enforced ? observer.beginReporter?.() : undefined; }, + claimPhysicalSend(): number | undefined { + if (observer?.enforced !== true) return undefined; + if (counter.physicalLimit === undefined) { + if (!Number.isSafeInteger(policy.maxTotalModelSends) || policy.maxTotalModelSends <= 0) return undefined; + counter.physicalLimit = policy.maxTotalModelSends; + } + if (counter.physicalStarted >= counter.physicalLimit) return undefined; + return ++counter.physicalStarted; + }, get used(): number { return counter.spent; }, set used(next: number) { // A numeric report has no receipt identity. Only reportDispatchSends may settle a // prepaid permit; guessing by reservation order can refund another leg's actual send. + // Enforced sends can only be reconciled through their producer's claimed starts. + if (observer?.enforced === true) return; const charged = next - counter.spent; counter.spent = Math.max(0, next); // These sends have already left. The ledger records them even past a ceiling it would @@ -372,7 +464,7 @@ function createRequestExecutionBudgetWithLedger( // for a dispatch a cheaper check above would have refused is spend this request never // makes, and it would hold those tokens against the scope until retention expired. let spendProof: SpendReservationProof | undefined; - if (observer && !observer.charge({ deferDispatch: true, onReserved: proof => { spendProof = proof; } })) { + if (observer && !observer.charge({ targetKey: intent.targetKey, deferDispatch: true, onReserved: proof => { spendProof = proof; } })) { return { allowed: false, reason: "spend-exhausted" }; } @@ -389,17 +481,32 @@ function createRequestExecutionBudgetWithLedger( if (chargesTransition) targetTransitions += 1; lastTargetKey = intent.targetKey; + let producer: ReturnType> | undefined; + const startPhysical = (): boolean => { + if (observer?.enforced !== true) return true; + const seed = observer.ensureSeed?.({ targetKey: intent.targetKey }, spendProof); + if (!seed) return false; + const ordinal = budget.claimPhysicalSend?.(); + if (ordinal === undefined) return false; + producer = observer.beginReporter?.(); + producer?.start(seed, ordinal); + producer?.report(1); + producer?.close(); + return true; + }; let settled: "open" | "used" | "released" = "open"; const permit: SingleUseDispatchPermit = { sendClass: intent.sendClass, use(): boolean { if (settled !== "open") return false; + if (intent.countedExternally !== true && !startPhysical()) return false; settled = "used"; if (intent.countedExternally !== true && spendProof) observer?.dispatch?.(spendProof); return true; }, assumeCharge(): boolean { if (settled !== "open" || (intent.countedExternally === true && !counter.pendingExternalSends.has(receipt))) return false; + if (!startPhysical()) return false; settled = "used"; // The booking this reservation made for an external reporter is now owned by the // caller. Close only its own receipt so a later reporter cannot spend it again. @@ -453,6 +560,7 @@ export function createRequestExecutionBudget( const grant = createAmbiguousResendGrant(); return createRequestExecutionBudgetWithLedger(policy, logicalRequestId, { spent: 0, + physicalStarted: 0, pendingExternalSends: new Set(), claimAmbiguousResend: grant.claimAmbiguousResend, get ambiguousResendSpent(): boolean { return grant.ambiguousResendSpent; }, @@ -508,6 +616,7 @@ function ledgerFor(parent: RequestExecutionBudget): SharedSendLedger { get spent(): number { return parent.used; }, set spent(next: number) { parent.used = next; }, pendingExternalSends, + physicalStarted: 0, // Asked of the parent rather than counted here. A local counter is a SECOND grant: two // scopes derived from one bridged parent, or one scope beside the parent it was derived // from, each replaced an unknown-state send once. Pending bookings and the durable-spend diff --git a/src/lib/spend-pool-alias-validation.ts b/src/lib/spend-pool-alias-validation.ts new file mode 100644 index 00000000000..e116754ab07 --- /dev/null +++ b/src/lib/spend-pool-alias-validation.ts @@ -0,0 +1,77 @@ +import { createHash } from "node:crypto"; +import { constants, lstatSync, openSync, fstatSync, readFileSync, closeSync, type Stats } from "node:fs"; +import { join } from "node:path"; + +/** Read an existing single-owner salt without creating or hardening any file. */ +export function readExistingSpendSalt(home: string): string | undefined { + let fd: number | undefined; + let salt: string | undefined; + try { + const path = join(home, "spend-ledger.salt"); + const before = lstatSync(path); + if (!before.isFile() || before.nlink !== 1) return undefined; + const matches = (stat: Stats): boolean => stat.isFile() && stat.nlink === 1 + && stat.dev === before.dev && stat.ino === before.ino; + fd = openSync(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0)); + if (!matches(fstatSync(fd))) return undefined; + const contents = readFileSync(fd, "utf8").trim(); + if (matches(fstatSync(fd)) && matches(lstatSync(path)) && /^[0-9a-f]{32,}$/.test(contents)) salt = contents; + } catch { salt = undefined; } + finally { + if (fd !== undefined) { + try { closeSync(fd); } catch { salt = undefined; } + } + } + return salt; +} + +/** + * The one owner of the spend ledger's salted alias hash. The ledger, config diagnostics and + * runtime policy validation must all compute the same alias for the same (salt, domain, id), + * so the hash lives here and nowhere else. The raw id never leaves this function. + */ +export function hashSpendAlias(salt: string, kind: string, id: string): string { + return createHash("sha256").update(salt).update("\u0000").update(kind).update("\u0000").update(id) + .digest("hex").slice(0, 32); +} + +const isAlias = (value: unknown): value is string => + typeof value === "string" && /^[0-9a-f]{32}$/.test(value); + +/** Shape check. Kept outside `spend`: older strict spend schemas must keep all configured ceilings. */ +export function spendPoolAliasesError(value: unknown): string | undefined { + if (value === undefined) return undefined; + if (!value || typeof value !== "object" || Array.isArray(value)) return "spendPoolAliases must be an object"; + if (Object.keys(value).length > 4_096) return "spendPoolAliases has too many entries"; + for (const [alias, provider] of Object.entries(value)) { + if (!isAlias(alias) || typeof provider !== "string" || !provider.trim() || provider.length > 256) { + return "spendPoolAliases requires exact 32-character salted pool aliases and nonempty provider IDs"; + } + } + return undefined; +} + +/** + * Semantic owner check: an alias that is the salted pool alias of a configured provider P + * belongs to P. Mapping it to any other provider Q would move P's own history into Q's pool, + * so `aliases[h("pool", P)] = Q` with Q !== P is rejected. Re-run whenever the provider set + * changes: adding P later turns an existing mapping into a conflict. + */ +export function validatePoolAliasOwners( + aliases: unknown, + salt: string, + providerIds: Iterable, +): string | undefined { + const shape = spendPoolAliasesError(aliases); + if (shape) return shape; + if (aliases === undefined) return undefined; + const mapping = aliases as Record; + for (const provider of new Set(providerIds)) { + const owned = hashSpendAlias(salt, "pool", provider); + const target = Object.hasOwn(mapping, owned) ? mapping[owned] : undefined; + if (target !== undefined && target !== provider) { + return "spendPoolAliases maps a configured provider's own pool alias to a different provider"; + } + } + return undefined; +} diff --git a/src/lib/spend-pool-continuity.ts b/src/lib/spend-pool-continuity.ts index 89b8243afe1..f05c8aa04ec 100644 --- a/src/lib/spend-pool-continuity.ts +++ b/src/lib/spend-pool-continuity.ts @@ -1,96 +1,28 @@ -/** Explicit, salted pool identity evidence. Never infer ownership from a label prefix. */ -export interface PoolBinding { readonly alias: string; readonly canonical: string } -export interface PoolContinuityRecord { - readonly v: 1; - readonly kind: "pool-continuity"; - readonly at: number; - readonly bindings: readonly PoolBinding[]; -} - -const isAlias = (value: unknown): value is string => - typeof value === "string" && /^[0-9a-f]{32}$/.test(value); - -/** Kept outside `spend`: older strict spend schemas must keep all configured ceilings. */ -export function spendPoolAliasesError(value: unknown): string | undefined { - if (value === undefined) return undefined; - if (!value || typeof value !== "object" || Array.isArray(value)) return "spendPoolAliases must be an object"; - if (Object.keys(value).length > 4_096) return "spendPoolAliases has too many entries"; - for (const [alias, provider] of Object.entries(value)) { - if (!isAlias(alias) || typeof provider !== "string" || !provider.trim() || provider.length > 256) { - return "spendPoolAliases requires exact 32-character salted pool aliases and nonempty provider IDs"; - } - } - return undefined; -} +import { hashSpendAlias, validatePoolAliasOwners } from "./spend-pool-alias-validation"; +export { spendPoolAliasesError } from "./spend-pool-alias-validation"; -export function parsePoolContinuityRecord(value: unknown): PoolContinuityRecord | undefined { - if (!value || typeof value !== "object") return undefined; - const record = value as Record; - if (record.v !== 1 || record.kind !== "pool-continuity" - || typeof record.at !== "number" || !Number.isFinite(record.at) || record.at < 0 - || !Array.isArray(record.bindings) || record.bindings.length > 16_384) return undefined; - const bindings: PoolBinding[] = []; - const seen = new Set(); - for (const entry of record.bindings) { - if (!entry || typeof entry !== "object") return undefined; - const { alias, canonical } = entry as Partial; - if (!isAlias(alias) || !isAlias(canonical) || seen.has(alias)) return undefined; - seen.add(alias); - bindings.push({ alias, canonical }); +/** Current config supplies a read-time graph; original counters never move between buckets. */ +export function resolvePoolAliases(config: unknown, salt: string, providerIds: Iterable = []) { + const error = validatePoolAliasOwners(config, salt, providerIds); + const bindings = new Map(); + if (!error) for (const [alias, provider] of Object.entries(config ?? {})) { + bindings.set(alias, hashSpendAlias(salt, "pool", provider as string)); } - return { v: 1, kind: "pool-continuity", at: record.at, bindings }; -} - -/** A directed union: links may merge proven groups, never split or erase accounted spend. */ -export function createPoolContinuity() { - let bindings = new Map(); - const resolve = (alias: string, map = bindings): string => { + const resolve = (original: string): string => { + let alias = original; const seen = new Set(); - while (map.has(alias) && map.get(alias) !== alias) { - if (seen.has(alias)) throw new Error("cyclic pool identity evidence"); + while (bindings.has(alias) && bindings.get(alias) !== alias) { + if (seen.has(alias)) throw new Error("cyclic pool alias mapping"); seen.add(alias); - alias = map.get(alias)!; + alias = bindings.get(alias)!; } return alias; }; - const merge = (proposed: Iterable): Map | undefined => { - const next = new Map(bindings); - for (const [alias, canonical] of proposed) next.set(alias, canonical); - try { - // Validate the whole proposed graph, not an intermediate sorted prefix. A verified - // A -> B -> C rename may repeat A -> C without splitting the existing A/B group. - for (const alias of next.keys()) resolve(alias, next); - for (const [alias, canonical] of bindings) { - if (resolve(alias, next) !== resolve(canonical, next)) return undefined; - } - for (const alias of next.keys()) next.set(alias, resolve(alias, next)); - } catch { return undefined; } - return next; - }; - return { - resolve: (alias: string): string => resolve(alias), - known: (alias: string): boolean => bindings.has(alias), - size: (): number => bindings.size, - record: (at: number): PoolContinuityRecord => ({ - v: 1, kind: "pool-continuity", at, - bindings: [...bindings].map(([alias, canonical]) => ({ alias, canonical })), - }), - restore(record: PoolContinuityRecord): boolean { - const next = merge(record.bindings.map(({ alias, canonical }) => [alias, canonical] as const)); - if (!next) return false; - bindings = next; - return true; - }, - prepare(config: unknown, requested: string | undefined, historical: ReadonlySet, - hash: (provider: string) => string, at: number, capacity: number): PoolContinuityRecord | false | undefined { - if (spendPoolAliasesError(config)) return false; - const next = merge(Object.entries(config ?? {}).map(([alias, provider]) => [alias, hash(provider as string)] as const)); - if (!next) return false; - if (requested !== undefined && !next.has(requested) && !historical.has(requested)) next.set(requested, requested); - if (next.size > Math.min(16_384, capacity)) return false; - if (next.size === bindings.size && [...next].every(([key, value]) => bindings.get(key) === value)) return undefined; - return { v: 1, kind: "pool-continuity", at, - bindings: [...next].map(([alias, canonical]) => ({ alias, canonical })) }; - }, - }; + let invalid = error !== undefined; + if (!invalid) { + try { for (const alias of bindings.keys()) resolve(alias); } + catch { invalid = true; } + } + if (invalid) bindings.clear(); + return { valid: !invalid, resolve, known: (alias: string): boolean => bindings.has(alias) }; } diff --git a/src/lib/spend-reservation-ledger.ts b/src/lib/spend-reservation-ledger.ts index e513d7bf2df..6a870b401ee 100644 --- a/src/lib/spend-reservation-ledger.ts +++ b/src/lib/spend-reservation-ledger.ts @@ -54,7 +54,7 @@ */ import { appendFileSync, chmodSync, closeSync, lstatSync, mkdirSync, openSync, readFileSync, renameSync, statSync, unlinkSync, writeFileSync } from "node:fs"; -import { createHash, randomBytes } from "node:crypto"; +import { randomBytes } from "node:crypto"; import { dirname, join } from "node:path"; // Definition-site import, not the ../config barrel -- same reasoning as // src/quota/reset-seen-store.ts: the barrel pulls ~154 modules into a hot path. @@ -62,7 +62,8 @@ import { getConfigDir } from "../config/paths"; // Type-only, so it is erased before this module has a runtime import graph at all. The // config SHAPE is what this file needs; the config loader is what the note above keeps out. import type { OcxSpendConfig, OcxSpendScopeConfig } from "../types/config"; -import { createPoolContinuity, parsePoolContinuityRecord, type PoolContinuityRecord } from "./spend-pool-continuity"; +import { resolvePoolAliases } from "./spend-pool-continuity"; +import { hashSpendAlias, validatePoolAliasOwners } from "./spend-pool-alias-validation"; import { assertNotRealHomeUnderTest } from "./test-home-guard"; // Windows chmod does not remove inherited ACEs; this is the repository's icacls path. import { hardenSecretPath } from "./windows-secret-acl"; @@ -96,6 +97,7 @@ export interface SpendScopeLimit { export interface SpendReservationPolicy { /** Exact historical salted pool alias -> canonical provider. Malformed input fails closed. */ readonly poolAliases?: unknown; + readonly canonicalProviderIds?: readonly string[]; readonly root: SpendScopeLimit; readonly identity: SpendScopeLimit; readonly pool: SpendScopeLimit; @@ -264,7 +266,6 @@ type JournalRecord = v: 1; kind: "checkpoint"; at: number; - poolContinuity?: PoolContinuityRecord; scopes: { scope: SpendScope; alias: string; settled: number; unresolved: number; seenAt: number }[]; sends: { send: string; status: ReservationStatus; targets: ScopeRef[]; tokens: number; at: number; resolvedAt: number }[]; }; @@ -357,9 +358,7 @@ export function parseSpendJournalRecord(line: string): JournalRecord | undefined if (!isCountable(e.tokens) || !isCountable(e.at) || !isCountable(e.resolvedAt)) return undefined; sends.push({ send: e.send, status: e.status, targets, tokens: e.tokens, at: e.at, resolvedAt: e.resolvedAt }); } - const poolContinuity = record.poolContinuity === undefined ? undefined : parsePoolContinuityRecord(record.poolContinuity); - if (record.poolContinuity !== undefined && !poolContinuity) return undefined; - return { v: 1, kind: "checkpoint", at, scopes, sends, ...(poolContinuity ? { poolContinuity } : {}) }; + return { v: 1, kind: "checkpoint", at, scopes, sends }; } default: return undefined; @@ -621,7 +620,16 @@ export interface SpendReservationProof { readonly sendId: string; } +export interface SpendSeed { readonly sendId: string } +export type SpendSeedDecision = { reserved: true; seed: SpendSeed } | { reserved: false; denial: SpendDenial }; + export interface SpendReservationLedger { + reserveSeed(request: SpendReservationRequest): SpendSeedDecision; + reserveReportedFromSeed(seed: SpendSeed, request: { sendId: string; inputTokens: number; outputCeilingTokens: number }): boolean; + forgetResolved(sendIds: readonly string[]): boolean; + finishSeed(seed: SpendSeed): boolean; + registerReporter(): { close(): void }; + waitForReporterDrain(): Promise; reserve(request: SpendReservationRequest): SpendReservationDecision; /** Pre-dispatch guard, including transports which report their sends after dispatch. */ checkPoolContinuity(): SpendDenial | undefined; @@ -721,7 +729,15 @@ export function createSpendReservationLedger(options: { const compactAfterRecords = (): number => policy.compactAfterRecords ?? DEFAULT_COMPACT_AFTER_RECORDS; const scopes = new Map(); const reservations = new Map(); - const poolContinuity = createPoolContinuity(); + // Object identity is the capability: a copied sendId or another ledger's seed is not proof. + const seeds = new Map; abandoned: boolean }>(); + const seedForSend = (send: string) => [...seeds.values()].find(seed => seed.sends.has(send)); + const pinnedScope = (key: string): boolean => [...seeds.values()].some(seed => + seed.targets.some(ref => scopeKey(ref.scope, ref.alias) === key)); + const pendingRecords: JournalRecord[] = []; + let reporters = 0; + const drainWaiters = new Set<() => void>(); + let poolContinuity = resolvePoolAliases(policy.poolAliases, salt, policy.canonicalProviderIds); let persistFailures = 0; let corruptRecords = 0; let recordsOnDisk = 0; @@ -731,23 +747,8 @@ export function createSpendReservationLedger(options: { * credential id, a pool name -- never leaves this function, so nothing identifying is * written to disk or held in a map key. */ - const aliasFor = (kind: SpendScope | "send" | "pool-current", id: string): string => - createHash("sha256").update(salt).update("\u0000").update(kind).update("\u0000").update(id) - .digest("hex").slice(0, 32); - - // An old unbound label may hash exactly like today's provider ID. Only in that ambiguous - // case, put new routed spend in a separate hash domain so it belongs to this candidate - // without treating the old balance as verified ownership. Existing proven groups keep - // their alias, and explicit mappings of old aliases target the same current alias. - const poolAliasFor = (provider: string): string => { - const legacyAlias = aliasFor("pool", provider); - const legacy = scopes.get(scopeKey("pool", legacyAlias)); - const hasPositiveHistory = legacy !== undefined - && legacy.settled + legacy.reserved + legacy.unresolved > 0; - const ambiguous = hasPositiveHistory - && (!poolContinuity.known(legacyAlias) || poolContinuity.resolve(legacyAlias) !== legacyAlias); - return ambiguous ? aliasFor("pool-current", provider) : legacyAlias; - }; + const aliasFor = (kind: SpendScope | "send", id: string): string => hashSpendAlias(salt, kind, id); + const poolAliasFor = (provider: string): string => aliasFor("pool", provider); const scopeState = (scope: SpendScope, alias: string): ScopeState => { const key = scopeKey(scope, alias); @@ -826,6 +827,14 @@ export function createSpendReservationLedger(options: { } }; + const flushPending = (): boolean => { + while (pendingRecords.length > 0) { + if (!append(pendingRecords[0]!)) return false; + pendingRecords.shift(); + } + return true; + }; + const applyReserve = (send: string, targets: readonly ScopeRef[], tokens: number, at: number): void => { if (reservations.has(send)) return; reservations.set(send, { targets, tokens, status: "open", at, resolvedAt: at }); @@ -878,7 +887,6 @@ export function createSpendReservationLedger(options: { }; const applyCheckpoint = (record: Extract): void => { - if (record.poolContinuity && !poolContinuity.restore(record.poolContinuity)) corruptRecords += 1; scopes.clear(); reservations.clear(); for (const entry of record.scopes) { @@ -949,10 +957,16 @@ export function createSpendReservationLedger(options: { // had already fired. An exhausted scope staying exhausted across a restart is the whole // reason this store is on disk. const reconciledAt = now(); + const enforcedReplay = spendCeilingsConfigured(policy); for (const [send, reservation] of reservations) { - if (!isLive(reservation.status)) continue; - applyResolve(send, "lost", 0, reconciledAt); - append({ v: 1, kind: "lost", send, at: reconciledAt }); + if (isLive(reservation.status)) { + const record: JournalRecord = { v: 1, kind: "lost", send, at: reconciledAt }; + const durable = append(record); + applyResolve(send, "lost", 0, reconciledAt); + if (!durable && enforcedReplay) pendingRecords.push(record); + } + // No live producer or capability survives restart. Preserve totals, reclaim only IDs. + if (enforcedReplay && flushPending() && append({ v: 1, kind: "forget", send, at: reconciledAt })) applyForget(send); } } @@ -969,25 +983,40 @@ export function createSpendReservationLedger(options: { */ const evictScopes = (at: number, force: boolean): number => { const cutoff = at - policy.retentionMs; - // Candidate selection uses one snapshot of each proven pool group plus the shared - // unbound overlay for the whole pass. Re-scanning all scopes per pool member repeats - // work on every reservation. - const pools = new Map(); - const unboundPoolTotal: ScopeState = { settled: 0, reserved: 0, unresolved: 0, lastSeenAt: 0 }; + const activeKeys = new Set(); + for (const reservation of reservations.values()) if (isLive(reservation.status)) { + for (const ref of reservation.targets) activeKeys.add(scopeKey(ref.scope, ref.alias)); + } + const idle = (key: string, state: ScopeState): boolean => state.reserved === 0 + && !activeKeys.has(key) && !pinnedScope(key); + let removed = 0; + // Unknown original buckets expire at their own cutoff, even when the overlay is exhausted. + // This pass precedes group selection so a removed overlay cannot keep unrelated groups alive. + for (const [key, state] of scopes) { + if (!key.startsWith("pool\0") || poolContinuity.known(key.slice(5)) + || !idle(key, state) || state.lastSeenAt >= cutoff) continue; + if (!flushPending() || !append({ v: 1, kind: "drop", scope: "pool", alias: key.slice(5), at })) continue; + scopes.delete(key); + removed += 1; + } + // Aggregate once per pass; resolving each candidate by scanning every scope is quadratic. + const pinnedPoolGroups = new Set(); + for (const key of activeKeys) if (key.startsWith("pool\0")) pinnedPoolGroups.add(poolContinuity.resolve(key.slice(5))); + for (const seed of seeds.values()) for (const ref of seed.targets) if (ref.scope === "pool") { + pinnedPoolGroups.add(poolContinuity.resolve(ref.alias)); + } + const groups = new Map(); + const overlay: ScopeState = { settled: 0, reserved: 0, unresolved: 0, lastSeenAt: 0 }; for (const [key, state] of scopes) { if (!key.startsWith("pool\0")) continue; const alias = key.slice(5); - if (!poolContinuity.known(alias)) { - if (state.settled + state.reserved + state.unresolved > 0) { - unboundPoolTotal.settled += state.settled; - unboundPoolTotal.reserved += state.reserved; - unboundPoolTotal.unresolved += state.unresolved; - } - continue; + const known = poolContinuity.known(alias); + let total = overlay; + if (known) { + const group = poolContinuity.resolve(alias); + total = groups.get(group) ?? { settled: 0, reserved: 0, unresolved: 0, lastSeenAt: 0 }; + groups.set(group, total); } - const group = poolContinuity.resolve(alias); - let total = pools.get(group); - if (!total) pools.set(group, total = { settled: 0, reserved: 0, unresolved: 0, lastSeenAt: 0 }); total.settled += state.settled; total.reserved += state.reserved; total.unresolved += state.unresolved; @@ -995,37 +1024,35 @@ export function createSpendReservationLedger(options: { } const candidates: { key: string; scope: SpendScope; alias: string; seenAt: number }[] = []; for (const [key, state] of scopes) { + if (!idle(key, state)) continue; const separator = key.indexOf("\0"); const scope = key.slice(0, separator) as SpendScope; const alias = key.slice(separator + 1); - const unboundZero = scope === "pool" && !poolContinuity.known(alias) - && state.settled + state.reserved + state.unresolved === 0; - const proven = scope === "pool" ? pools.get(poolContinuity.resolve(alias)) : undefined; - const effective = scope === "pool" && !unboundZero - ? { - settled: (proven?.settled ?? 0) + unboundPoolTotal.settled, - reserved: (proven?.reserved ?? 0) + unboundPoolTotal.reserved, - unresolved: (proven?.unresolved ?? 0) + unboundPoolTotal.unresolved, - // The overlay affects exhaustion but does not merge retention age across identities. - lastSeenAt: proven?.lastSeenAt ?? 0, - } - : state; - if (effective.reserved > 0) continue; - if (scope === "pool" && state.settled + state.unresolved > 0 - && !poolContinuity.known(key.slice(separator + 1))) continue; - if (isExhausted(scope, effective)) continue; + const unbound = scope === "pool" && !poolContinuity.known(alias); + // Pressure cannot shorten retention for unknown history, including zero-token keys. + if (unbound && state.settled + state.reserved + state.unresolved > 0) continue; + const group = scope === "pool" ? groups.get(poolContinuity.resolve(alias)) : undefined; + const effective = scope === "pool" && !unbound ? { + settled: (group?.settled ?? 0) + overlay.settled, + reserved: (group?.reserved ?? 0) + overlay.reserved, + unresolved: (group?.unresolved ?? 0) + overlay.unresolved, + lastSeenAt: group?.lastSeenAt ?? state.lastSeenAt, + } : state; + if (effective.reserved > 0 || (scope === "pool" && pinnedPoolGroups.has(poolContinuity.resolve(alias))) + || isExhausted(scope, effective)) continue; if (!force && effective.lastSeenAt >= cutoff) continue; - candidates.push({ key, scope, alias: key.slice(separator + 1), seenAt: state.lastSeenAt }); + candidates.push({ key, scope, alias, seenAt: state.lastSeenAt }); } if (force) { candidates.sort((a, b) => a.seenAt - b.seenAt); candidates.length = Math.min(candidates.length, 1); } for (const candidate of candidates) { + if (!flushPending() || !append({ v: 1, kind: "drop", scope: candidate.scope, alias: candidate.alias, at })) continue; scopes.delete(candidate.key); - append({ v: 1, kind: "drop", scope: candidate.scope, alias: candidate.alias, at }); + removed += 1; } - return candidates.length; + return removed; }; /** @@ -1037,7 +1064,7 @@ export function createSpendReservationLedger(options: { const cutoff = at - policy.retentionMs; const candidates: { send: string; resolvedAt: number }[] = []; for (const [send, reservation] of reservations) { - if (isLive(reservation.status)) continue; + if (isLive(reservation.status) || seedForSend(send)) continue; if (!force && reservation.resolvedAt >= cutoff) continue; candidates.push({ send, resolvedAt: reservation.resolvedAt }); } @@ -1045,11 +1072,13 @@ export function createSpendReservationLedger(options: { candidates.sort((a, b) => a.resolvedAt - b.resolvedAt); candidates.length = Math.min(candidates.length, 1); } + let removed = 0; for (const candidate of candidates) { + if (!flushPending() || !append({ v: 1, kind: "forget", send: candidate.send, at })) continue; reservations.delete(candidate.send); - append({ v: 1, kind: "forget", send: candidate.send, at }); + removed += 1; } - return candidates.length; + return removed; }; /** @@ -1057,7 +1086,7 @@ export function createSpendReservationLedger(options: { * Bounded maps are not enough on their own: the file behind them is what replay reads, and * an uncompacted file grows forever on unique root and send ids. */ - const checkpointRecord = (at: number, evidence = poolContinuity.record(at)): Extract => ({ + const checkpointRecord = (at: number): Extract => ({ v: 1, kind: "checkpoint", at, @@ -1079,13 +1108,12 @@ export function createSpendReservationLedger(options: { at: reservation.at, resolvedAt: reservation.resolvedAt, })), - ...(evidence.bindings.length > 0 ? { poolContinuity: evidence } : {}), }); const compact = (at: number): void => { const rewrite = journal?.rewrite; - // A checkpoint must never erase evidence of replay corruption. - if (!journal || !rewrite || corruptRecords > 0 || recordsOnDisk < compactAfterRecords()) return; + // Restore shipped compaction semantics; corruption refusal persists until clean replay. + if (!journal || !rewrite || pendingRecords.length > 0 || recordsOnDisk < compactAfterRecords()) return; try { rewrite.call(journal, [JSON.stringify(checkpointRecord(at))]); recordsOnDisk = 1; @@ -1097,18 +1125,11 @@ export function createSpendReservationLedger(options: { } }; - const preparePoolContinuity = (at: number, requested?: string): SpendDenial | undefined => { - const evidence = poolContinuity.prepare(policy.poolAliases, requested, historicalPools(), - poolAliasFor, at, maxTrackedScopes()); - if (evidence === false) return { reason: "pool-history-unresolved" }; - if (evidence) { - // A v1 checkpoint atomically carries the unchanged balances AND salted identity - // evidence. Older parsers can still read its balances; no unknown-record fence. - if (!append(checkpointRecord(at, evidence))) return { reason: "reserve-not-durable", sendId: "" }; - if (!poolContinuity.restore(evidence)) return { reason: "pool-history-unresolved" }; + const preparePoolContinuity = (requestedProvider?: string): SpendDenial | undefined => { + const providers = [...(policy.canonicalProviderIds ?? []), ...(requestedProvider === undefined ? [] : [requestedProvider])]; + if (!poolContinuity.valid || validatePoolAliasOwners(policy.poolAliases, salt, providers)) { + return { reason: "pool-history-unresolved" }; } - // Unbound positive history is accounted conservatively in every candidate pool view. - // Only invalid/conflicting identity evidence or a failed durable write refuses here. return undefined; }; @@ -1119,9 +1140,8 @@ export function createSpendReservationLedger(options: { while (reservations.size >= maxTrackedSends()) { if (evictSends(at, true) === 0) return { reason: "tracking-capacity-exhausted" }; } - let fresh = 0; - for (const ref of refs) if (!scopes.has(scopeKey(ref.scope, ref.alias))) fresh += 1; - while (scopes.size + fresh > maxTrackedScopes()) { + const freshCount = (): number => refs.filter(ref => !scopes.has(scopeKey(ref.scope, ref.alias))).length; + while (scopes.size + freshCount() > maxTrackedScopes()) { if (evictScopes(at, true) === 0) { return { reason: "tracking-capacity-exhausted", scope: refs[0]?.scope }; } @@ -1129,7 +1149,78 @@ export function createSpendReservationLedger(options: { return undefined; }; - return { + const ledger: SpendReservationLedger = { + reserveSeed(request): SpendSeedDecision { + assertOwnedAccounting?.(); + // Initial anchors never borrow already-sent capacity or bypass token/durability checks. + const decision = ledger.reserve({ ...request, alreadySent: false }); + if (!decision.reserved) return decision; + const seed: SpendSeed = Object.freeze({ sendId: request.sendId }); + const send = aliasFor("send", seed.sendId); + seeds.set(seed, { targets: reservations.get(send)!.targets, sends: new Set([send]), abandoned: false }); + return { reserved: true, seed }; + }, + reserveReportedFromSeed(seed, request): boolean { + assertOwnedAccounting?.(); + const owned = seeds.get(seed); + const send = aliasFor("send", request.sendId); + if (!owned || owned.abandoned || reservations.has(send) + || owned.targets.some(ref => !scopes.has(scopeKey(ref.scope, ref.alias)))) return false; + const tokens = sanitizeTokens(request.inputTokens) + sanitizeTokens(request.outputCeilingTokens); + const at = now(); + const record: JournalRecord = { v: 1, kind: "reserve", send, targets: [...owned.targets], tokens, at }; + // The physical send already happened. Keep its debt even if storage is temporarily unavailable. + if (pendingRecords.length > 0 || !append(record)) pendingRecords.push(record); + applyReserve(send, owned.targets, tokens, at); + owned.sends.add(send); + compact(at); + return true; + }, + forgetResolved(sendIds): boolean { + assertOwnedAccounting?.(); + if (reporters > 0 || !flushPending()) return false; + const sends = [...new Set(sendIds.map(id => aliasFor("send", id)))]; + if (sends.some(send => { const entry = reservations.get(send); return entry && isLive(entry.status); })) return false; + for (const send of sends) { + if (!reservations.has(send)) continue; + if (!append({ v: 1, kind: "forget", send, at: now() })) return false; + applyForget(send); + } + return true; + }, + finishSeed(seed): boolean { + assertOwnedAccounting?.(); + const owned = seeds.get(seed); + if (!owned || !flushPending()) return false; + if (owned.abandoned) { + // A never-sent provisional anchor cannot have delayed physical reports. Rebinding + // during another live producer is safe only for this explicitly abandoned capability. + const send = aliasFor("send", seed.sendId); + if (reservations.has(send)) { + if (!append({ v: 1, kind: "forget", send, at: now() })) return false; + applyForget(send); + } + } else if (reporters > 0) return false; + if ([...owned.sends].some(send => reservations.has(send))) return false; + seeds.delete(seed); + return true; + }, + registerReporter() { + assertOwnedAccounting?.(); + reporters += 1; + let closed = false; + return { close() { + if (closed) return; + assertOwnedAccounting?.(); + closed = true; + reporters -= 1; + if (reporters === 0) { for (const resolve of drainWaiters) resolve(); drainWaiters.clear(); } + } }; + }, + waitForReporterDrain() { + assertOwnedAccounting?.(); + return reporters === 0 ? Promise.resolve() : new Promise(resolve => { drainWaiters.add(resolve); }); + }, // Every figure this ledger reports describes a journal it must still own. Reporting one // after ownership ended is the same error as writing then, with a quieter symptom. get persistFailures() { assertOwnedAccounting?.(); return persistFailures; }, @@ -1141,7 +1232,7 @@ export function createSpendReservationLedger(options: { assertOwnedAccounting?.(); if (policy.pool.maxTokens === undefined) return undefined; if (corruptRecords > 0) return { reason: "journal-corrupt", corruptRecords }; - return preparePoolContinuity(now()); + return preparePoolContinuity(); }, hasUnboundPositivePoolHistory(): boolean { @@ -1168,9 +1259,11 @@ export function createSpendReservationLedger(options: { if (enforced && corruptRecords > 0) { return { reserved: false, denial: { reason: "journal-corrupt", corruptRecords } }; } - const poolRef = refs.find(ref => ref.scope === "pool"); - const continuityDenial = preparePoolContinuity(at, poolRef?.alias); - if (continuityDenial && policy.pool.maxTokens !== undefined && request.alreadySent !== true) { + if (enforced && request.alreadySent !== true && !flushPending()) { + return { reserved: false, denial: { reason: "reserve-not-durable", sendId: request.sendId } }; + } + const continuityDenial = preparePoolContinuity(request.scopes.poolId); + if (continuityDenial && enforced && request.alreadySent !== true) { return { reserved: false, denial: continuityDenial }; } const capacity = makeRoom(refs, at); @@ -1220,7 +1313,9 @@ export function createSpendReservationLedger(options: { if (!reservation || reservation.status !== "open") return false; const at = now(); applyDispatch(send, at); - append({ v: 1, kind: "dispatch", send, at }); + const record: JournalRecord = { v: 1, kind: "dispatch", send, at }; + if (seedForSend(send) && pendingRecords.length > 0) pendingRecords.push(record); + else if (!append(record) && seedForSend(send)) pendingRecords.push(record); return true; }, @@ -1232,8 +1327,12 @@ export function createSpendReservationLedger(options: { // upstream the tokens may already be billed, so the caller owes settle or markLost. if (!reservation || reservation.status !== "open") return false; const at = now(); + const record: JournalRecord = { v: 1, kind: "abandon", send, at }; + const seeded = seedForSend(send); + if (seeded && (!flushPending() || !append(record))) return false; applyResolve(send, "abandoned", 0, at); - append({ v: 1, kind: "abandon", send, at }); + if (seeded) seeded.abandoned = true; + else append(record); return true; }, @@ -1244,8 +1343,11 @@ export function createSpendReservationLedger(options: { if (!reservation || !isLive(reservation.status)) return false; const tokens = sanitizeTokens(usage.inputTokens) + sanitizeTokens(usage.outputTokens); const at = now(); + const record: JournalRecord = { v: 1, kind: "settle", send, tokens, at }; + const seeded = seedForSend(send); + if (seeded && (!flushPending() || !append(record))) return false; applyResolve(send, "settled", tokens, at); - append({ v: 1, kind: "settle", send, tokens, at }); + if (!seeded) append(record); return true; }, @@ -1255,8 +1357,11 @@ export function createSpendReservationLedger(options: { const reservation = reservations.get(send); if (!reservation || !isLive(reservation.status)) return false; const at = now(); + const record: JournalRecord = { v: 1, kind: "lost", send, at }; + const seeded = seedForSend(send); + if (seeded && (!flushPending() || !append(record))) return false; applyResolve(send, "lost", 0, at); - append({ v: 1, kind: "lost", send, at }); + if (!seeded) append(record); return true; }, @@ -1287,7 +1392,7 @@ export function createSpendReservationLedger(options: { if (!state) return false; const own = excludingSendId === undefined ? undefined : reservations.get(aliasFor("send", excludingSendId)); const matches = own?.status === "open" && own.targets.some(target => target.scope === scope - && (scope === "pool" ? poolContinuity.resolve(target.alias) === poolContinuity.resolve(alias) : target.alias === alias)); + && (scope === "pool" ? (!poolContinuity.known(target.alias) || poolContinuity.resolve(target.alias) === poolContinuity.resolve(alias)) : target.alias === alias)); return isExhausted(scope, matches ? { ...state, reserved: Math.max(0, state.reserved - own.tokens) } : state); }, @@ -1303,8 +1408,15 @@ export function createSpendReservationLedger(options: { reconfigure(next: SpendReservationPolicy): void { assertOwnedAccounting?.(); policy = next; + poolContinuity = resolvePoolAliases(policy.poolAliases, salt, policy.canonicalProviderIds); }, }; + return ledger; +} + +/** Drain only an already-constructed ledger; shutdown must not open an unused journal. */ +export function waitForSharedSpendReporterDrain(): Promise { + return sharedLedger?.waitForReporterDrain() ?? Promise.resolve(); } /** Does no I/O for installations without a pool ceiling. */ @@ -1347,10 +1459,11 @@ const spendScopeLimitFromConfig = (scope: OcxSpendScopeConfig | undefined): Spen * ceiling would start refusing real traffic on the first upgrade that ran this code, against * a number nobody chose. There is deliberately no default figure here at all. */ -export function spendPolicyFromConfig(spend: OcxSpendConfig | undefined, poolAliases?: unknown): SpendReservationPolicy { +export function spendPolicyFromConfig(spend: OcxSpendConfig | undefined, poolAliases?: unknown, canonicalProviderIds?: readonly string[]): SpendReservationPolicy { const retentionDays = spend?.retentionDays; return { ...(poolAliases !== undefined ? { poolAliases } : {}), + ...(canonicalProviderIds !== undefined ? { canonicalProviderIds } : {}), root: spendScopeLimitFromConfig(spend?.root), identity: spendScopeLimitFromConfig(spend?.identity), pool: spendScopeLimitFromConfig(spend?.pool), diff --git a/src/lib/upstream-retry.ts b/src/lib/upstream-retry.ts index 62bd9d963a4..e03e4ce6846 100644 --- a/src/lib/upstream-retry.ts +++ b/src/lib/upstream-retry.ts @@ -508,7 +508,11 @@ export interface ResetRetryOptions { * `rebuildAndRefetch` recovery kind whose provider has no transient policy -- was not merely * uncounted but UNCOUNTABLE: the callback existed on a type those call sites never reach. */ - onSendsConsumed?: (sends: number) => void; + onSendsConsumed?: ((sends: number) => void) & { + beforeSend?: () => boolean; + close?: () => void; + execute?: (run: () => Promise) => Promise; + }; /** * Spend one operator-granted replacement for a pre-header reset this helper would otherwise * refuse. Absent means no operator policy, which is the fail-closed answer. @@ -629,6 +633,7 @@ export async function fetchWithResetRetry( // more send on every recovery leg, which is most of what made a bounded per-layer retry // compose into an unbounded per-request count. if (attempts === 0) throw new SendBudgetExhaustedError(opts.label); + try { let lastError: unknown; let sawReset = false; // True once this leg has spent the request's operator allowance. From that point the leg @@ -640,9 +645,13 @@ export async function fetchWithResetRetry( // Reported before the await, one physical send at a time: a send that rejects has still // been made, and this helper leaves through four exits (return, reset give-up, non-reset // rethrow, abort), so a per-send report is the only shape that is correct on all of them. - opts.onSendsConsumed?.(1); + if (opts.onSendsConsumed?.beforeSend && !opts.onSendsConsumed.beforeSend()) throw new SendBudgetExhaustedError(opts.label); + if (!opts.onSendsConsumed?.beforeSend) opts.onSendsConsumed?.(1); try { - const response = await doFetch(attempt === 0 ? firstRecovery : "connection-reset"); + let response: Response; + try { const run = () => doFetch(attempt === 0 ? firstRecovery : "connection-reset"); + response = await (opts.onSendsConsumed?.execute ? opts.onSendsConsumed.execute(run) : run()); } + finally { if (opts.onSendsConsumed?.beforeSend) opts.onSendsConsumed(1); } return spentOperatorReplacement ? settleOperatorReplacement(response) : response; } catch (err) { if (opts.abortSignal?.aborted) throw err; @@ -686,6 +695,7 @@ export async function fetchWithResetRetry( } } throw lastError ?? new Error("upstream fetch failed"); + } finally { opts.onSendsConsumed?.close?.(); } } /** @@ -717,8 +727,10 @@ export async function fetchWithTransientRetry( const countedFetch: ReplayableFetch = (recovery) => { // Incremented BEFORE the await so a rejected send still consumes budget; counting only // successes would let a reset storm loop without bound. + if (opts.onSendsConsumed?.beforeSend && !opts.onSendsConsumed.beforeSend()) throw new SendBudgetExhaustedError(opts.label); sent += 1; - return doFetch(recovery); + const run = () => doFetch(recovery); + return opts.onSendsConsumed?.execute ? opts.onSendsConsumed.execute(run) : run(); }; // No floor. A spent budget hands the inner helper 0, which refuses rather than buying one // more send -- the loop condition alone was never enough, because every later recovery leg @@ -789,7 +801,7 @@ export async function fetchWithTransientRetry( // Budget exhausted: the last response is returned with its body intact. return res; } finally { - opts.onSendsConsumed?.(sent); + try { opts.onSendsConsumed?.(sent); } finally { opts.onSendsConsumed?.close?.(); } } } diff --git a/src/lib/workflow-budget.ts b/src/lib/workflow-budget.ts index b491bd0f6fa..4950b3183a3 100644 --- a/src/lib/workflow-budget.ts +++ b/src/lib/workflow-budget.ts @@ -759,6 +759,7 @@ export function workflowSpendCeilingReached( if (!rootId && !poolId) return undefined; const ledger = spendLedger ?? (spendCeilingsConfigured() ? sharedSpendLedger() : undefined); if (!ledger) return undefined; + // This reads the current alias partition; it never publishes continuity metadata. const excludingSendId = reservation?.ledger === ledger ? reservation.sendId : undefined; const root = rootId ? spentRootCeiling(rootId, ledger, excludingSendId) : undefined; if (root) return root; diff --git a/src/server/chat-native.ts b/src/server/chat-native.ts index 2e394d23350..6a4dfa90119 100644 --- a/src/server/chat-native.ts +++ b/src/server/chat-native.ts @@ -1,3 +1,5 @@ +import { createPhysicalSendReporter } from "../lib/request-execution-budget"; +import { createInferenceSendBudget } from "./inference/context"; import { buildOpenAIChatPassthroughRequest, createOpenAIChatAdapter } from "../adapters/openai-chat"; import type { AdapterRequest, ProviderAdapter } from "../adapters/base"; import { isNativeChatRouteEligible } from "./chat-native-eligibility"; @@ -316,6 +318,7 @@ export async function runNativeChatAttempt( let activeProvider: OcxProviderConfig = route.provider; stampApiKeyAccountLabel(logCtx, route.providerName, activeProvider); const spendTracker = sendBudget ? undefined : attachRequestSpendTracker(req, logCtx); + const physicalBudget = sendBudget ?? createInferenceSendBudget(req, logCtx); let activeAdapter: ProviderAdapter = createOpenAIChatAdapter(activeProvider); let activeRequest: AdapterRequest; let retainedRequestBytes = 0; @@ -452,7 +455,15 @@ export async function runNativeChatAttempt( const encoding = new Headers(init.headers).get("accept-encoding"); if (!headers.has("accept-encoding") && encoding) headers.set("accept-encoding", encoding); if (init.signal?.aborted) throw init.signal.reason; - if (sendBudget) { + const spendReport = createPhysicalSendReporter(physicalBudget, () => ({ + poolId: logCtx.spendPoolId ?? route.providerName, identityId: logCtx.accountLogLabel, + }), execution.comboDispatchPermit); + if (physicalBudget.spendEnforced && !spendReport.beforeSend?.()) { + spendReport.close?.(); + if ((logCtx.spendTracker as { refusals?: number } | undefined)?.refusals) throw new NativeChatSpendRefusal(); + throw new SendBudgetExhaustedError(); + } + if (!physicalBudget.spendEnforced && sendBudget) { // Backstop for sends the helper cannot see coming (a reset replay). The first // report settles the combo's booking; each later one is charged and booked. if (physicalSends > 0 && sendBudget.remainingBaseSends(sharedSendCap) <= 0) { @@ -460,11 +471,12 @@ export async function runNativeChatAttempt( } physicalSends += 1; reportDispatchSends(sendBudget, 1, execution.comboDispatchPermit); - } else if (!spendTracker?.charge()) throw new NativeChatSpendRefusal(); + } else if (!physicalBudget.spendEnforced && !spendTracker?.charge()) throw new NativeChatSpendRefusal(); noteProviderAttemptSend(logCtx, route.providerName, activeProvider, logCtx.usageLogInputTokens, transportRecovery ?? recovery); // A reselected provider transport is still a physical send: the connection policy // and manual-redirect ownership wrap the selected implementation (#4992). - const dispatched = await sendWithConnectionPolicy( + let dispatched: Response; + try { dispatched = await sendWithConnectionPolicy( (activeProvider as OcxProviderTransport).fetch ?? execute, request.url, applyUpstreamRecoveryInit({ @@ -476,6 +488,7 @@ export async function runNativeChatAttempt( // because that transport wins over the executor that carries the binding. { providerName: route.providerName, provider: activeProvider }, ); + } finally { if (physicalBudget.spendEnforced) spendReport(1); spendReport.close?.(); } if (!dispatched.ok) await recordKeyAttemptFailure(logCtx, dispatched, init.signal ?? upstream.signal); return dispatched; }, @@ -560,7 +573,7 @@ export async function runNativeChatAttempt( upstream.abort(); if (req.signal.aborted) return fail(499, "Client cancelled request", "client_cancelled"); const sendError = error instanceof UpstreamRetryEvidenceError ? error.cause : error; - if (sendBudget && sendError instanceof SendBudgetExhaustedError) { + if (sendError instanceof SendBudgetExhaustedError) { // A decision this process made, answered as the Responses path answers it: 429, not 502. return fail(429, sendError.message, SEND_BUDGET_EXHAUSTED_CODE, SEND_BUDGET_EXHAUSTED_CODE); } diff --git a/src/server/index.ts b/src/server/index.ts index a82f5415a18..e5e2d0d4563 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -302,7 +302,7 @@ function startServerWithSpendLedgerOwner(port: number | undefined, deps: StartSe configureAppOwnedMemoryBudget(resolveAppOwnedMemoryBudgetBytes(config.appOwnedMemoryBudgetMb)); enforceAppOwnedMemoryBudget(); // Observe-only mode still journals physical sends, so every server owns before configuring. - spendLedgerLifecycle.configure(config.spend, config.spendPoolAliases); + spendLedgerLifecycle.configure(config.spend, config.spendPoolAliases, Object.keys(config.providers)); // After ownership: a second server on the same home is refused above, so the process running // this line is the only one appending to usage.jsonl and the only one that may compact it. setUsageLedgerRetention(config.usageLedgerMaxBytes); diff --git a/src/server/index/spend-ledger-lifecycle.ts b/src/server/index/spend-ledger-lifecycle.ts index 3fc010831da..b11045db43a 100644 --- a/src/server/index/spend-ledger-lifecycle.ts +++ b/src/server/index/spend-ledger-lifecycle.ts @@ -6,6 +6,7 @@ import { import { configureSharedSpendLedger, spendPolicyFromConfig, + waitForSharedSpendReporterDrain, } from "../../lib/spend-reservation-ledger"; import { warnIfSyncedStateDirectory } from "../../lib/synced-state-location"; @@ -26,7 +27,7 @@ export function waitForFailedStartRollback(error: unknown): Promise { } export interface SpendLedgerServerLifecycle { - configure(spend: OcxSpendConfig | undefined, poolAliases?: unknown): void; + configure(spend: OcxSpendConfig | undefined, poolAliases?: unknown, providerIds?: readonly string[]): void; track }>(server: T): T; release(): void; releaseAfterFailedStart(): Promise; @@ -49,13 +50,18 @@ export function acquireSpendLedgerServerLifecycle(configDir: string): SpendLedge owner.release(); }; return { - configure(spend, poolAliases): void { - configureSharedSpendLedger(spendPolicyFromConfig(spend, poolAliases)); + configure(spend, poolAliases, providerIds): void { + configureSharedSpendLedger(spendPolicyFromConfig(spend, poolAliases, providerIds)); }, track }>(server: T): T { // Capture the raw stop before startServer replaces the public method with full teardown. const stop = server.stop.bind(server); - failedStartStops.push(() => stop(true)); + const drainedStop = async (closeActiveConnections?: boolean): Promise => { + await stop(closeActiveConnections); + await waitForSharedSpendReporterDrain(); + }; + Object.defineProperty(server, "stop", { configurable: true, value: drainedStop }); + failedStartStops.push(() => drainedStop(true)); return server; }, release, diff --git a/src/server/inference/context.ts b/src/server/inference/context.ts index 8883a0edef3..d0c972252d8 100644 --- a/src/server/inference/context.ts +++ b/src/server/inference/context.ts @@ -34,7 +34,7 @@ export function createInferenceSendBudget( export function expandInferenceOAuthSendBudget(budget: TransientSendBudget | undefined, accounts: number): void { if (!budget) return; const policy = ingressPolicies.get(budget); - if (!policy || budget.used !== 0) return; + if (!policy || budget.used !== 0 || ("physicalStarted" in budget && Number(budget.physicalStarted) > 0)) return; ingressPolicies.delete(budget); if (accounts < 2 || !Number.isSafeInteger(accounts)) return; const sends = Math.min(accounts, GENERIC_OAUTH_MAX_ACCOUNTS_PER_REQUEST) * TRANSIENT_RETRY_MAX_ATTEMPTS; diff --git a/src/server/messages-native.ts b/src/server/messages-native.ts index bb42bd870dd..1dddf17e838 100644 --- a/src/server/messages-native.ts +++ b/src/server/messages-native.ts @@ -1,3 +1,5 @@ +import { createPhysicalSendReporter } from "../lib/request-execution-budget"; +import { createInferenceSendBudget } from "./inference/context"; /** * Managed native Messages lane (PF-08, behind `protocols.rollout.managedMessagesNative`). * @@ -43,6 +45,7 @@ import { isTranslatorBudgetExceededError, type TranslatorBudget } from "../lib/t import { applyReplayRefusalClientHeaders, applyUpstreamRecoveryInit, + SendBudgetExhaustedError, fetchWithResetRetry, fetchWithTransientRetry, isNonReplayableResponse, @@ -412,6 +415,7 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions) stampApiKeyAccountLabel(logCtx, route.providerName, activeProvider); if (oauthBinding) logCtx.provider = formatAnthropicProviderForLog(route.providerName, oauthBinding.snapshot.accountId, config); const spendTracker = attachRequestSpendTracker(req, logCtx); + const physicalBudget = createInferenceSendBudget(req, logCtx); let activeRequest: AnthropicMessagesPassthroughRequest; let retainedRequestBytes = 0; const releaseRetainedRequest = () => { @@ -527,9 +531,17 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions) const releaseFamily = snapshot ? claimAnthropicFamilyRevalidation(snapshot.accountId, route.modelId) : () => {}; if (!releaseFamily) throw new AnthropicAccountCooldownError(1); let dispatched: Response; + const spendReport = createPhysicalSendReporter(physicalBudget, () => ({ + poolId: logCtx.spendPoolId ?? route.providerName, identityId: logCtx.accountLogLabel, + })); try { if (init.signal?.aborted) throw init.signal.reason; - if (!spendTracker.charge()) throw new NativeMessagesSpendRefusal(); + if (physicalBudget.spendEnforced) { + if (!spendReport.beforeSend?.()) { + if (spendTracker.refusals) throw new NativeMessagesSpendRefusal(); + throw new SendBudgetExhaustedError(); + } + } else if (!spendTracker.charge()) throw new NativeMessagesSpendRefusal(); noteProviderAttemptSend(logCtx, route.providerName, activeProvider, logCtx.usageLogInputTokens, transportRecovery ?? recovery); dispatched = await sendWithConnectionPolicy( (activeProvider as OcxProviderTransport).fetch ?? execute, @@ -537,7 +549,10 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions) applyUpstreamRecoveryInit({ ...init, method: "POST", headers, body: wire.body }, transportRecovery), { providerName: route.providerName, provider: activeProvider }, ); - } finally { releaseFamily(); } + } finally { + if (physicalBudget.spendEnforced) spendReport(1); + spendReport.close?.(); releaseFamily(); + } if (ownsBearer && snapshot) { try { const current = getAccountCredentialWithStatus("anthropic", snapshot.accountId); @@ -653,6 +668,7 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions) upstream.abort(); if (req.signal.aborted) return fail(499, "Client cancelled request", "api_error"); const sendError = error instanceof UpstreamRetryEvidenceError ? error.cause : error; + if (sendError instanceof SendBudgetExhaustedError) return fail(429, sendError.message, "send_budget_exhausted", "send_budget_exhausted"); if (sendError instanceof NativeMessagesSpendRefusal) { const refusal = workflowRefusalResponse("workflow-spend-exhausted", logCtx); finishLog(429); diff --git a/src/server/responses/adapter-continuation.ts b/src/server/responses/adapter-continuation.ts index 5159eebc556..24f644ebfd3 100644 --- a/src/server/responses/adapter-continuation.ts +++ b/src/server/responses/adapter-continuation.ts @@ -102,6 +102,7 @@ export function createAdapterContinuations( | "noteAdapterPhysicalSend" | "noteAdapterRecoveryWithheld" | "remainingTransientSendBudget" + | "adapterSendBudget" | "transientSendReporter" | "reserveCredentialHop" | "pendingHopPermit" @@ -211,6 +212,8 @@ export function createAdapterContinuations( try { if (transportState.activeAdapter.fetchResponse) { transportState.noteRoutedAttemptSend(continuationEstimate, replayKind); + const producer = adapterDispatchBudget?.beginSpendProducer?.(); + try { return await withProviderRequestSlot(route.providerName, route.provider, nextParsed.modelId, upstream.signal, pacingSlot => transportState.activeAdapter.fetchResponse!(builtContinuationRequest, { kiroPreferAccountFailover: route.providerName === "kiro" && isGenericOAuthFailoverEnabled(config, "kiro"), @@ -228,6 +231,7 @@ export function createAdapterContinuations( modelId: nextParsed.modelId, }), })); + } finally { producer?.close(); } } // Same #1851 scope guard as the initial send: transient-5xx retry only for direct // Google AI Studio; every other adapter keeps reset-only semantics here. @@ -261,9 +265,9 @@ export function createAdapterContinuations( // Same request-scoped budget as the initial send and the 429/rotation refetches: // a terminal-guard continuation is another leg of ONE request, so handing it a // fresh `attempts` would let one request exceed the configured total-send ceiling. - ...(continuationTransientPolicy + ...(continuationTransientPolicy || sendBudgetState.adapterSendBudget?.spendEnforced ? { - attempts: remainingTransientSendBudget(continuationTransientPolicy.attempts), + attempts: remainingTransientSendBudget(continuationTransientPolicy?.attempts ?? 1), onSendsConsumed: transientSendReporter(), } : {}), diff --git a/src/server/responses/adapter-dispatch.ts b/src/server/responses/adapter-dispatch.ts index 63eab2b8af1..a59393309be 100644 --- a/src/server/responses/adapter-dispatch.ts +++ b/src/server/responses/adapter-dispatch.ts @@ -158,6 +158,7 @@ export async function prepareAdapterExchange( | "noteAdapterPhysicalSend" | "noteAdapterRecoveryWithheld" | "remainingTransientSendBudget" + | "adapterSendBudget" | "transientSendReporter" | "recoverySendAllowance" | "recoveryClassFor" @@ -334,6 +335,8 @@ export async function prepareAdapterExchange( try { if (transportState.activeAdapter.fetchResponse) { transportState.noteRoutedAttemptSend(inputTokenEstimate); + const producer = adapterDispatchBudget?.beginSpendProducer?.(); + try { upstreamResponse = await withProviderRequestSlot(route.providerName, route.provider, route.modelId, upstream.signal, pacingSlot => transportState.activeAdapter.fetchResponse!(builtInitialRequest, { kiroPreferAccountFailover: route.providerName === "kiro" && isGenericOAuthFailoverEnabled(config, "kiro"), @@ -353,6 +356,7 @@ export async function prepareAdapterExchange( ? createCodexAuthDispatchGuard(admissionState.authCtx, options.codexAuthPolicy ?? config, route.modelId, options.admission, options.visionDescribeTerminal === true) : undefined, }), })); + } finally { producer?.close(); } } else { // #1851 scope guard: transient-5xx retry on this generic adapter path is opt-in for // direct Google AI Studio only (Vertex/Antigravity use fetchResponse above). Other @@ -400,7 +404,7 @@ export async function prepareAdapterExchange( abortSignal: upstream.signal, label: safeHostLabel(builtInitialRequest.url), claimAmbiguousResend: claimPreHeaderResend, - ...(options.comboDispatchPermit || transientPolicy || resetPolicy || compactPrepaid + ...(sendBudgetState.adapterSendBudget?.spendEnforced || options.comboDispatchPermit || transientPolicy || resetPolicy || compactPrepaid ? { // A pending compaction permit already booked this leg's first physical send. ...(transientPolicy || resetPolicy || compactPrepaid @@ -590,7 +594,7 @@ export async function prepareAdapterExchange( const refetchWithPolicy = (route.provider.adapter === "google" || refetchTransientPolicy) ? fetchWithTransientRetry : fetchWithResetRetry; - const helperCountsSends = refetchTransientPolicy !== null || resetReplayPolicyFor(route.provider) !== null; + const helperCountsSends = sendBudgetState.adapterSendBudget?.spendEnforced === true || refetchTransientPolicy !== null || resetReplayPolicyFor(route.provider) !== null; const prepaid = sendBudgetState.pendingHopPermit; const configuredTotal = refetchTransientPolicy?.attempts; const refetchCap = transientSendCapFor(configuredTotal, diff --git a/src/server/responses/compact.ts b/src/server/responses/compact.ts index 592c7583093..774c74f4336 100644 --- a/src/server/responses/compact.ts +++ b/src/server/responses/compact.ts @@ -1,3 +1,5 @@ +import { createPhysicalSendReporter } from "../../lib/request-execution-budget"; +import { createInferenceSendBudget } from "../inference/context"; import { capturePoolQuotaWriter } from "../../codex/account-store"; import { previewXaiOauthWireModel } from "./core-normalize"; import { @@ -722,6 +724,7 @@ export async function handleResponsesCompact( logCtx.provider = route.codexAccountNamespace ? `${route.providerName}-${route.codexAccountNamespace}` : route.providerName; + logCtx.spendPoolId = route.providerName; logCtx.providerAdapter = route.provider.adapter; // #4940, and the same refusal the ordinary Responses path makes in request-prepare.ts. Compact has // to repeat it rather than inherit it: the native branch below dispatches straight to @@ -804,7 +807,7 @@ export async function handleResponsesCompact( // failure that hands off, continues here. The routed turn used to call handleResponses with // no budget at all, so `handleResponsesInner` minted a fresh four after the native attempt // had already spent some of the first one. - const sendBudget: RequestExecutionBudget = options.sendBudget ?? createRequestExecutionBudget(); + const sendBudget: RequestExecutionBudget = options.sendBudget ?? createInferenceSendBudget(req, logCtx); // A manual override onto another backend must not mint ciphertext the conversation model cannot replay. const manualOverrideCrossesProvider = options.compactionRoutingOverride ? !compactionRoutingKeepsProviderIdentity(config, options.compactionRoutingOverride, route) @@ -1026,8 +1029,10 @@ export async function handleResponsesCompact( doFetch: () => Promise, ): Promise => { if (sendBudget.remainingBaseSends(TRANSIENT_RETRY_MAX_ATTEMPTS) > 0) { - sendBudget.used += 1; - return doFetch(); + const report = createPhysicalSendReporter(sendBudget, () => ({ poolId: logCtx.spendPoolId ?? route.providerName, + identityId: logCtx.accountLogLabel })); + if (report.beforeSend?.() === false) { report.close?.(); return Promise.reject(new SendBudgetExhaustedError(safeHostLabel(compactUrl))); } + return doFetch().finally(() => { report(1); report.close?.(); }); } const decision = sendBudget.reserveDispatch({ sendClass: "auth-recovery", @@ -1076,7 +1081,8 @@ export async function handleResponsesCompact( // Draws the shared remainder instead of a fresh three. Compact is a native endpoint // of the same logical turn, so its sends belong to the same cap. attempts: sendBudget.remainingBaseSends(TRANSIENT_RETRY_MAX_ATTEMPTS), - onSendsConsumed: (used: number) => { sendBudget.used += Math.max(0, used); }, + onSendsConsumed: createPhysicalSendReporter(sendBudget, () => ({ poolId: logCtx.spendPoolId ?? route.providerName, + identityId: logCtx.accountLogLabel })), }); }; diff --git a/src/server/responses/compaction-recovery.ts b/src/server/responses/compaction-recovery.ts index 5503518b833..16a2b9e9182 100644 --- a/src/server/responses/compaction-recovery.ts +++ b/src/server/responses/compaction-recovery.ts @@ -47,7 +47,7 @@ function settlePhysicalSends(log: RequestLogContext, budget: RequestExecutionBud // An external report may settle a prepaid booking without changing used. Its explicit // receipt outranks the numeric delta, or the same source send would be charged twice. const unreported = Math.max(0, sent - Math.max(reported, Math.max(0, budget.used - beforeUsed))); - if (unreported > 0) budget.used += unreported; + if (unreported > 0 && !budget.spendEnforced) budget.used += unreported; return sent; } diff --git a/src/server/responses/passthrough-dispatch.ts b/src/server/responses/passthrough-dispatch.ts index 808f701c3e6..df04ac4282b 100644 --- a/src/server/responses/passthrough-dispatch.ts +++ b/src/server/responses/passthrough-dispatch.ts @@ -797,7 +797,10 @@ export async function preparePassthroughExchange( */ const sendAmbiguousReplacement = ( signal: AbortSignal = upstream.signal, - ): Promise => fetchWithHeaderTimeout( + ): Promise => { + const report = transientSendReporter(); + let started = false; + const run = () => fetchWithHeaderTimeout( request.url, applyUpstreamRecoveryInit({ method: request.method, @@ -832,11 +835,14 @@ export async function preparePassthroughExchange( // Charged to the SAME request counter every other send goes through. The // replacement is bought here rather than by a nested retry helper, so there is // one charge for one send and no per-layer counter to reconcile. - noteTransientSends(1); + if (report.beforeSend?.() === false) throw new SendBudgetExhaustedError(safeHostLabel(request.url)); + started = true; }, }), route.provider.authMode === "forward", ); + return (report.execute ? report.execute(run) : run()).finally(() => { if (started) report(1); report.close?.(); }); + }; /** * Refuse a built body that exceeds the operator's configured ceiling, before it is sent. * diff --git a/src/server/responses/request-send-budget.ts b/src/server/responses/request-send-budget.ts index ad1fbc352df..71b19153b00 100644 --- a/src/server/responses/request-send-budget.ts +++ b/src/server/responses/request-send-budget.ts @@ -1,5 +1,6 @@ +import { createInferenceSendBudget } from "../inference/context"; import type { ResponsesRequestContext } from "./core-options"; -import { claimDispatchSpendProof, createRequestExecutionBudget, isRequestExecutionBudget, reportDispatchSends } from "../../lib/request-execution-budget"; +import { claimDispatchSpendProof, createPhysicalSendReporter, createRequestExecutionBudget, isRequestExecutionBudget, reportDispatchSends } from "../../lib/request-execution-budget"; import { chargeWorkflowSends, workflowSendCeilingReached, @@ -52,7 +53,7 @@ export function createResponsesSendBudget( // fresh default of 3. It is now a holder carried on options, so a combo child inherits the // parent's spend instead of starting over per target -- both halves of the measured // amplification in #4546. - const sendBudget = options.sendBudget ?? createRequestExecutionBudget(); + const sendBudget = options.sendBudget ?? createInferenceSendBudget(req, logCtx); // The root workflow is the user-visible task. A per-request cap cannot bound a fan-out that // sends once per child seven hundred times, so every send charged to the request is charged // to the root as well (#4546). @@ -68,7 +69,11 @@ export function createResponsesSendBudget( const noteTransientSends = (used: number): void => recordTransientSends(used, pendingHopPermit ?? inheritedPermit); // Capture at helper creation, before another leg can replace the pending handoff. const transientSendReporter = (permit = pendingHopPermit ?? inheritedPermit) => - (used: number): void => recordTransientSends(used, permit); + createPhysicalSendReporter(sendBudget, () => ({ poolId: logCtx.spendPoolId ?? logCtx.provider, + identityId: logCtx.accountLogLabel }), permit, charged => { + options.onCompactionRecoverySendsReported?.(charged); + chargeWorkflowSends(workflowRootId, charged); + }); // Refused before any dispatch, and deliberately not by evicting the root's ledger entry: // dropping the record to make room would hand the fan-out a fresh allowance, which is the // laundering this ceiling exists to stop. The client is told the task needs a new grant @@ -333,6 +338,11 @@ function adapterDispatchBudgetView( refundableHop = false, ): RequestExecutionBudget { return { + get physicalStarted() { return budget.physicalStarted; }, + get physicalLimit() { return budget.physicalLimit; }, + get spendEnforced() { return budget.spendEnforced; }, + claimPhysicalSend: () => budget.claimPhysicalSend?.(), + beginSpendProducer: () => budget.beginSpendProducer?.(), get used(): number { return budget.used; }, set used(next: number) { budget.used = next; }, logicalRequestId: budget.logicalRequestId, diff --git a/src/server/responses/request-spend.ts b/src/server/responses/request-spend.ts index e24490d2744..56e25be3dda 100644 --- a/src/server/responses/request-spend.ts +++ b/src/server/responses/request-spend.ts @@ -1,6 +1,6 @@ import { randomUUID } from "node:crypto"; import type { RequestSendObserver } from "../../lib/request-execution-budget"; -import { sharedSpendLedger, type SpendReservationLedger, type SpendScopes } from "../../lib/spend-reservation-ledger"; +import { sharedSpendLedger, type SpendReservationLedger, type SpendScopes, type SpendSeed, type SpendReservationProof } from "../../lib/spend-reservation-ledger"; import { SpendLedgerOwnerError } from "../../lib/spend-ledger-owner"; import { markLocalRequestLogRefusal, type RequestLogContext } from "../request-log"; import { recordWorkflowRefusalEvent, workflowDenialSummary, type WorkflowDenial } from "../../lib/workflow-budget"; @@ -16,9 +16,22 @@ export interface RequestSpendSettlement { settle(usage: TerminalSpendUsage | undefined): void; } +export interface SpendTargetIdentity extends SpendScopes { readonly targetKey?: string } +export interface RequestSpendReporter { + start(seed: SpendSeed, ordinal: number): void; + rebindTarget(target: SpendTargetIdentity, ordinal: number): boolean; + report(sends: number): void; + close(): void; +} + export interface RequestSpendTracker extends RequestSendObserver, RequestSpendSettlement { /** Dispatches this request lost to a ledger ceiling. Zero on every ordinary request. */ readonly refusals: number; + readonly enforced: boolean; + ensureSeed(target: SpendTargetIdentity, prepaidProof?: SpendReservationProof): SpendSeed | undefined; + beginReporter(): RequestSpendReporter; + reportFromSeed(seed: SpendSeed, ordinal: number): void; + requestFinalSettlement(usage: TerminalSpendUsage | undefined): void; } /** @@ -60,6 +73,73 @@ export function createRequestSpendTracker( let refusals = 0; let resolved = false; let terminalProcessed = false; + let seededAccounting = false; + let finalRequested = false; + let finalUsage: TerminalSpendUsage | undefined; + let reporters = 0; + const seeds = new Map(); + const resolvedSendIds = new Set(); + let awaitingDrain = false; + const completedSeeds = new Set(); + const physical = new Map(); + const selectedScopes = (target: SpendTargetIdentity = {}): SpendScopes => ({ + ...(rootId !== undefined ? { rootId } : {}), + ...(target.identityId ?? logCtx.accountLogLabel ? { identityId: target.identityId ?? logCtx.accountLogLabel } : {}), + ...(target.poolId ?? logCtx.spendPoolId ?? logCtx.provider ? { poolId: target.poolId ?? logCtx.spendPoolId ?? logCtx.provider } : {}), + }); + const applicable = (scopes: SpendScopes): boolean => { + const policy = ledger().policy; + return seededAccounting || (scopes.rootId !== undefined && policy.root.maxTokens !== undefined) + || (scopes.identityId !== undefined && policy.identity.maxTokens !== undefined) + || (scopes.poolId !== undefined && policy.pool.maxTokens !== undefined); + }; + const estimates = () => ({ inputTokens: logCtx.spendInputEstimateTokens ?? logCtx.usageLogInputTokens ?? 0, + outputCeilingTokens: logCtx.spendOutputCeilingTokens ?? 0 }); + const seedState = (seed: SpendSeed) => [...seeds.values()].find(state => state.seed === seed); + const finalizeSeeds = (): void => { + if (!finalRequested || reporters !== 0 || resolved || !seededAccounting) return; + // All starts are booked before producers close. Ordinal, never callback order, owns usage. + const entries = [...physical.entries()].sort((a, b) => a[0] - b[0]); + const terminal = entries.at(-1)?.[1].sendId; + const ids: string[] = []; + for (const [, entry] of entries) { + if (!entry.sendId) return; + const reported = entry.sendId === terminal && (typeof finalUsage?.inputTokens === "number" || typeof finalUsage?.outputTokens === "number"); + if (!resolvedSendIds.has(entry.sendId)) { + const ok = reported ? ledger().settle(entry.sendId, { inputTokens: finalUsage?.inputTokens ?? 0, outputTokens: finalUsage?.outputTokens ?? 0 }) + : ledger().markLost(entry.sendId); + if (!ok) return; + resolvedSendIds.add(entry.sendId); + } + ids.push(entry.sendId); + } + for (const state of seeds.values()) if (!state.started && !completedSeeds.has(state.seed)) { + if (!resolvedSendIds.has(state.seed.sendId) && !ledger().abandon(state.seed.sendId)) return; + resolvedSendIds.add(state.seed.sendId); + ids.push(state.seed.sendId); + } + if (!ledger().forgetResolved(ids)) { + if (!awaitingDrain) { + awaitingDrain = true; + void ledger().waitForReporterDrain().then(() => { + awaitingDrain = false; + // One retry after the installation producer barrier; storage failure keeps debt. + if (!ledger().forgetResolved(ids)) return; + for (const state of seeds.values()) if (!completedSeeds.has(state.seed)) { + if (!ledger().finishSeed(state.seed)) return; + completedSeeds.add(state.seed); + } + resolved = true; + }); + } + return; + } + for (const state of seeds.values()) if (!completedSeeds.has(state.seed)) { + if (!ledger().finishSeed(state.seed)) return; + completedSeeds.add(state.seed); + } + resolved = true; + }; /** * Confirm the sends this request has already moved past. * @@ -74,7 +154,156 @@ export function createRequestSpendTracker( if (!pendingDispatch.has(sendId)) ledger().markDispatched(sendId); } }; - return { + const refusal = (denial: import("../../lib/spend-reservation-ledger").SpendDenial): void => { + refusals += 1; + // Preserve the PR's explicit unresolved-history refusal and its distinct operator code. + if (denial.reason === "pool-history-unresolved") { + const summary = workflowDenialSummary("workflow-pool-history-unresolved"); + markLocalRequestLogRefusal(logCtx, summary.code); + logCtx.errorCode = summary.code; + recordWorkflowRefusalEvent(rootId, "workflow-pool-history-unresolved", Date.now()); + return; + } + const reason: WorkflowDenial = denial.reason === "duplicate-send-id" + ? "workflow-send-replayed" + : denial.reason === "reserve-not-durable" || denial.reason === "journal-corrupt" + ? "workflow-spend-undurable" + : denial.reason === "tracking-capacity-exhausted" + ? "workflow-tracking-exhausted" + : "workflow-spend-exhausted"; + const detail = denial.reason === "spend-limit-exceeded" + ? { + scope: denial.scope, + limit: denial.limit, + projected: denial.projected, + ...(denial.includesUnboundPoolHistory ? { includesUnboundPoolHistory: true } : {}), + } + : undefined; + const summary = workflowDenialSummary(reason, detail); + markLocalRequestLogRefusal(logCtx, summary.code); + logCtx.errorCode = summary.code; + if (detail?.includesUnboundPoolHistory) logCtx.spendRefusalDetail = detail; + recordWorkflowRefusalEvent(rootId, reason, Date.now(), detail); + return; + }; + const tracker: RequestSpendTracker = { + get enforced(): boolean { return applicable(selectedScopes()); }, + ensureSeed(target, prepaidProof) { + if (resolved || finalRequested && reporters === 0) return undefined; + const scopes = selectedScopes(target); + if (!applicable(scopes)) return undefined; + const key = JSON.stringify([scopes.rootId, scopes.identityId, scopes.poolId, target.targetKey]); + const existing = seeds.get(key); + if (existing) return existing.seed; + if (prepaidProof?.ledger === ledger()) { + const prepaid = [...seeds.entries()].find(([, state]) => state.seed.sendId === prepaidProof.sendId); + if (prepaid && !prepaid[1].started) { + // The proof was reserved against the selected scopes by this tracker. + const oldIdentity = JSON.parse(prepaid[0]) as Array; + if (oldIdentity[0] === (scopes.rootId ?? null) && oldIdentity[1] === (scopes.identityId ?? null) && oldIdentity[2] === (scopes.poolId ?? null)) { + seeds.delete(prepaid[0]); seeds.set(key, prepaid[1]); return prepaid[1].seed; + } + } + } + if (target.targetKey === undefined) { + const same = [...seeds.entries()].find(([candidate]) => { + const identity = JSON.parse(candidate) as Array; + return identity[0] === (scopes.rootId ?? null) && identity[1] === (scopes.identityId ?? null) && identity[2] === (scopes.poolId ?? null); + }); + if (same) return same[1].seed; + } + // Rebind an unused provisional anchor before selection changes. Dispatched anchors stay live. + for (const [oldKey, state] of seeds) if (!state.started) { + if (!ledger().abandon(state.seed.sendId)) return undefined; + // The ledger pins abandoned anchors until its reporter drain. Keep the capability + // for final cleanup if another producer is currently registered. + if (!ledger().finishSeed(state.seed)) continue; + seeds.delete(oldKey); + const index = live.indexOf(state.seed.sendId); + if (index >= 0) live.splice(index, 1); + pendingDispatch.delete(state.seed.sendId); + } + // Exact prepaid proofs originate from this tracker and are found above; foreign receipts + // cannot enroll arbitrary scopes. Normal reservation remains the only initial admission. + if (prepaidProof && prepaidProof.ledger !== ledger()) return undefined; + const decision = ledger().reserveSeed({ sendId: randomUUID(), scopes, ...estimates() }); + if (!decision.reserved) { + // Use the established denial mapping without making another reservation. + refusal(decision.denial); + return undefined; + } + seededAccounting = true; + seeds.set(key, { seed: decision.seed, started: false, reported: false }); + return decision.seed; + }, + beginReporter() { + if (resolved || finalRequested && reporters === 0) throw new Error("Spend reporter registered after final closure"); + const lease = ledger().registerReporter(); + reporters++; + let closed = false; + let reported = 0; + const starts: Array<{ seed: SpendSeed; ordinal: number }> = []; + return { + start(seed, ordinal) { + if (closed || !seedState(seed) || physical.has(ordinal)) throw new Error("Invalid spend reporter start"); + const state = seedState(seed)!; + state.started = true; + starts.push({ seed, ordinal }); + physical.set(ordinal, { seed }); + + }, + rebindTarget(target, ordinal) { + const entry = physical.get(ordinal); + const owned = starts.find(start => start.ordinal === ordinal); + if (closed || !entry || !owned || entry.sendId) throw new Error("Invalid spend reporter rebind"); + const old = seedState(entry.seed)!; + // A helper start is provisional until selection reaches the executor. If this is + // the first start on its seed, abandon/rebind at NORMAL capacity before the wire. + physical.delete(ordinal); + old.started = [...physical.values()].some(start => start.seed === old.seed); + const seed = tracker.ensureSeed(target); + if (!seed) { starts.splice(starts.indexOf(owned), 1); return false; } + physical.set(ordinal, { seed }); + owned.seed = seed; + seedState(seed)!.started = true; + return true; + }, + report(sends) { + if (closed) return; + const count = Number.isFinite(sends) ? Math.max(0, Math.trunc(sends)) : 0; + for (let i = 0; i < count && reported < starts.length; i++) { + const entry = starts[reported++]!; + tracker.reportFromSeed(entry.seed, entry.ordinal); + } + }, + close() { + if (closed) return; + // Cancellation still books every claimed start conservatively. + this.report(starts.length - reported); + closed = true; + reporters--; + lease.close(); + finalizeSeeds(); + }, + }; + }, + reportFromSeed(seed, ordinal) { + const entry = physical.get(ordinal); + const state = seedState(seed); + if (!entry || entry.seed !== seed || entry.sendId || !state) return; + const firstOrdinal = [...physical.entries()].filter(([, send]) => send.seed === seed).map(([order]) => order).sort((a, b) => a - b)[0]; + const sendId = ordinal === firstOrdinal ? seed.sendId : randomUUID(); + if (sendId !== seed.sendId && !ledger().reserveReportedFromSeed(seed, { sendId, ...estimates() })) { + throw new Error("Already-sent spend could not be persisted"); + } + state.reported = true; + if (!ledger().markDispatched(sendId)) throw new Error("Already-sent spend dispatch could not be persisted"); + entry.sendId = sendId; + }, + requestFinalSettlement(usage) { + if (!finalRequested) { finalRequested = true; finalUsage = usage; } + finalizeSeeds(); + }, charge(options?: Parameters[0]): boolean { // A send that has already left is RECORDED, never refused: the tokens are spent, and a // booking the ledger drops is a booking the ceiling can never see. This is the reporting @@ -82,6 +311,12 @@ export function createRequestSpendTracker( // fetch -- so without it a root ceiling on the canonical Codex path would sit one send // short of its limit forever and refuse nothing. const alreadySent = options?.alreadySent === true; + if (!alreadySent && tracker.enforced) { + const seed = tracker.ensureSeed({ targetKey: options?.targetKey }); + if (!seed) return false; + options?.onReserved?.({ ledger: ledger(), sendId: seed.sendId }); + return true; + } const sendId = randomUUID(); const bookedLedger = ledger(); const scopes: SpendScopes = { @@ -109,36 +344,7 @@ export function createRequestSpendTracker( // capacity is full. Unconfigured/nonapplicable requests remain observe-only, and a // physical send reported after dispatch cannot be refused retroactively. if (alreadySent || !enforced) return true; - refusals += 1; - const denial = decision.denial; - // Preserve the PR's explicit unresolved-history refusal and its distinct operator code. - if (denial.reason === "pool-history-unresolved") { - const summary = workflowDenialSummary("workflow-pool-history-unresolved"); - markLocalRequestLogRefusal(logCtx, summary.code); - logCtx.errorCode = summary.code; - recordWorkflowRefusalEvent(rootId, "workflow-pool-history-unresolved", Date.now()); - return false; - } - const reason: WorkflowDenial = denial.reason === "duplicate-send-id" - ? "workflow-send-replayed" - : denial.reason === "reserve-not-durable" || denial.reason === "journal-corrupt" - ? "workflow-spend-undurable" - : denial.reason === "tracking-capacity-exhausted" - ? "workflow-tracking-exhausted" - : "workflow-spend-exhausted"; - const detail = denial.reason === "spend-limit-exceeded" - ? { - scope: denial.scope, - limit: denial.limit, - projected: denial.projected, - ...(denial.includesUnboundPoolHistory ? { includesUnboundPoolHistory: true } : {}), - } - : undefined; - const summary = workflowDenialSummary(reason, detail); - markLocalRequestLogRefusal(logCtx, summary.code); - logCtx.errorCode = summary.code; - if (detail?.includesUnboundPoolHistory) logCtx.spendRefusalDetail = detail; - recordWorkflowRefusalEvent(rootId, reason, Date.now(), detail); + refusal(decision.denial); return false; } if (!alreadySent) options?.onReserved?.({ ledger: bookedLedger, sendId }); @@ -151,6 +357,7 @@ export function createRequestSpendTracker( return true; }, dispatch(proof): void { + if (seededAccounting && proof.ledger === ledgerRef) return; if (proof.ledger !== ledgerRef || !pendingDispatch.has(proof.sendId)) return; ledger().markDispatched(proof.sendId); pendingDispatch.delete(proof.sendId); @@ -159,6 +366,11 @@ export function createRequestSpendTracker( if (index >= 0) live.push(...live.splice(index, 1)); }, refund(proof): void { + if (proof && proof.ledger === ledgerRef && seededAccounting) { + const match = [...seeds.entries()].find(([, state]) => state.seed.sendId === proof.sendId); + if (match && !match[1].started && ledger().abandon(proof.sendId) && ledger().finishSeed(match[1].seed)) seeds.delete(match[0]); + return; + } // null is an exact budget reservation that obtained no durable booking. if (proof === null || (proof && proof.ledger !== ledgerRef)) return; const index = proof ? live.indexOf(proof.sendId) : live.length - 1; @@ -172,6 +384,7 @@ export function createRequestSpendTracker( }, settle(usage: TerminalSpendUsage | undefined): void { if (resolved) return; + if (seededAccounting) { tracker.requestFinalSettlement(usage); return; } try { if (!terminalProcessed && live.length > 0) { const terminal = live[live.length - 1] as string; @@ -204,6 +417,7 @@ export function createRequestSpendTracker( }, get refusals(): number { return refusals; }, }; + return tracker; } /** @@ -217,7 +431,8 @@ export function attachRequestSpendTracker( req: Pick, logCtx: RequestLogContext, ledger?: SpendReservationLedger, -): RequestSendObserver { +): RequestSpendTracker { + if (logCtx.spendTracker && "ensureSeed" in logCtx.spendTracker) return logCtx.spendTracker as RequestSpendTracker; const rootId = req.headers.get("x-codex-parent-thread-id")?.trim() || undefined; const tracker = ledger === undefined ? createRequestSpendTracker(logCtx, rootId) diff --git a/src/server/responses/request-transport.ts b/src/server/responses/request-transport.ts index 7898c8e7924..e5340072728 100644 --- a/src/server/responses/request-transport.ts +++ b/src/server/responses/request-transport.ts @@ -1,3 +1,5 @@ +import { SendBudgetExhaustedError } from "../../lib/upstream-retry"; +import { rebindPhysicalSend } from "../../lib/request-execution-budget"; import { anthropicFamilyRejected, claimAnthropicFamilyRevalidation } from "../../oauth/anthropic-model-quota"; import { anthropicRatePauseUntil } from "../../oauth/anthropic-rate-limit-policy"; import { bindAnthropicRefusalCredential } from "../../oauth/anthropic-account-refusal"; @@ -459,6 +461,9 @@ export async function prepareResponsesTransport( selectedAdapter: ProviderAdapter, ...[requestParsed, incoming, emit]: Parameters> ): Promise => { + const producer = options.sendBudget && "beginSpendProducer" in options.sendBudget + ? (options.sendBudget as import("../../lib/request-execution-budget").RequestExecutionBudget).beginSpendProducer?.() : undefined; + try { for (let attempt = 0; attempt < 3; attempt++) { if (!selectionIsCurrent(adapterBindings.get(selectedAdapter))) selectedAdapter = await refreshRunTurnAdapter(requestParsed); const binding = adapterBindings.get(selectedAdapter); @@ -499,6 +504,7 @@ export async function prepareResponsesTransport( selectedAdapter = await refreshRunTurnAdapter(requestParsed); } throw new Error("Account selection changed repeatedly before turn dispatch"); + } finally { producer?.close(); } }; const oauthDispatch = (wireRequest: AdapterRequest, requestParsed = parsed): ProviderFetchOptions["dispatchOverride"] => { if (route.provider.authMode === "forward") return undefined; @@ -527,6 +533,8 @@ export async function prepareResponsesTransport( if (!releaseFamily) throw new AnthropicAccountCooldownError(1); let response: Response; try { + if (!rebindPhysicalSend(options.sendBudget, { poolId: logCtx.spendPoolId ?? route.providerName, + identityId: logCtx.accountLogLabel })) throw new SendBudgetExhaustedError(); commitKeyAttemptSend(); response = await sendWithConnectionPolicy( fetchImpl, diff --git a/src/server/workflow-refusal.ts b/src/server/workflow-refusal.ts index 8fe6696ba86..712e0b367f0 100644 --- a/src/server/workflow-refusal.ts +++ b/src/server/workflow-refusal.ts @@ -121,7 +121,7 @@ export function unboundPoolSpendRefusalResponse(logCtx: RequestLogContext): Resp * children. A request that names a parent is treated as that fan-out; a top-level request is * the conversation and may use the reserved slots. */ -/** Keep storage/integrity failures distinct from unresolved identity evidence. */ +/** Read-only alias validation: keep replay corruption distinct from conflicting pool owners. */ export function poolContinuityRefusalReason(): WorkflowDenial | undefined { const denial = sharedPoolContinuityDenial(); if (!denial) return undefined; diff --git a/src/types/config.ts b/src/types/config.ts index 912edfcf21f..050226e1fda 100644 --- a/src/types/config.ts +++ b/src/types/config.ts @@ -1052,7 +1052,7 @@ export interface OcxConfig { * upgrade against a number nobody chose. */ spend?: OcxSpendConfig; - /** Exact historical salted pool aliases mapped to canonical providers; retained verbatim for fail-closed validation. */ + /** Historical salted pool aliases; h(pool,P) cannot map to Q != P for any configured provider P. */ spendPoolAliases?: Record; /** Opt-in per-account activation of newly reset Codex quota windows. */ codexQuotaAutoRefresh?: Record { test("top-level pool aliases preserve all ceilings and malformed hand edits fail closed", () => { + writeFileSync(join(home, "spend-ledger.salt"), "7".repeat(64) + "\n", { mode: 0o600 }); const alias = "a".repeat(32); const spend = { root: { maxTokens: 200 }, identity: { maxTokens: 150 }, pool: { maxTokens: 100 } }; const valid = { ...candidate(spend), spendPoolAliases: { [alias]: "fixture-provider" } }; diff --git a/tests/config/config-spend-pool-collision.test.ts b/tests/config/config-spend-pool-collision.test.ts new file mode 100644 index 00000000000..54d20a1e24e --- /dev/null +++ b/tests/config/config-spend-pool-collision.test.ts @@ -0,0 +1,106 @@ +import { afterEach, beforeEach, expect, test } from "bun:test"; +import { createHash } from "node:crypto"; +import { existsSync, linkSync, mkdirSync, mkdtempSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { getDefaultConfig, getConfigPath, loadConfig, validateConfigCandidate } from "../../src/config"; +import { configDiagnosticsFromRaw } from "../../src/config/diagnostics"; +import { createSpendReservationLedger, DEFAULT_SPEND_RESERVATION_POLICY, type SpendReservationPolicy } from "../../src/lib/spend-reservation-ledger"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; + +const salt = "7".repeat(64); +const alias = (id: string, usedSalt = salt) => createHash("sha256").update(usedSalt).update("\0pool\0").update(id).digest("hex").slice(0, 32); +let home = ""; +let priorHome: string | undefined; +beforeEach(() => { + priorHome = process.env.OPENCODEX_HOME; + home = mkdtempSync(join(tmpdir(), "ocx-pool-collision-")); + process.env.OPENCODEX_HOME = home; +}); +afterEach(() => { + if (priorHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = priorHome; + removeTreeWithRetry(home); +}); +const provider = { adapter: "openai-responses", baseUrl: "https://pool.example.test/v1" }; +const candidate = (aliases: Record, providers = { P: provider, Q: provider }) => ({ + ...getDefaultConfig(), defaultProvider: "Q", providers, + spend: { root: { maxTokens: 200 }, identity: { maxTokens: 150 }, pool: { maxTokens: 100 } }, + spendPoolAliases: aliases, +}); +const policy = (aliases: Record, ids: string[]): SpendReservationPolicy & { canonicalProviderIds: string[] } => ({ + ...DEFAULT_SPEND_RESERVATION_POLICY, pool: { maxTokens: 100 }, poolAliases: aliases, canonicalProviderIds: ids, +}); +const reserve = (ledger: ReturnType, id = "send", poolId = "P") => + ledger.reserve({ sendId: id, scopes: { poolId }, inputTokens: 1, outputCeilingTokens: 0 }); +const installSalt = () => writeFileSync(join(home, "spend-ledger.salt"), salt + "\n", { mode: 0o600 }); + +test("salted canonical collision rejects write load and reconfigure", () => { + installSalt(); + const invalid = candidate({ [alias("P")]: "Q" }); + expect(validateConfigCandidate(invalid).ok).toBe(false); + const diagnostics = configDiagnosticsFromRaw(JSON.stringify(invalid)); + expect(diagnostics.config.spend).toEqual(invalid.spend); + expect(diagnostics.config.spendPoolAliases).toEqual(invalid.spendPoolAliases); + expect(diagnostics.warnings?.join("\n")).toContain("spendPoolAliases invalid"); + writeFileSync(getConfigPath(), JSON.stringify(invalid)); + expect(loadConfig().spend).toEqual(invalid.spend); + const loaded = createSpendReservationLedger({ salt, policy: policy(invalid.spendPoolAliases, ["P", "Q"]) }); + expect(reserve(loaded)).toMatchObject({ reserved: false, denial: { reason: "pool-history-unresolved" } }); + const ledger = createSpendReservationLedger({ salt, policy: policy({}, ["P", "Q"]) }); + expect(reserve(ledger, "before").reserved).toBe(true); + ledger.reconfigure(policy(invalid.spendPoolAliases, ["P", "Q"])); + expect(reserve(ledger, "after")).toMatchObject({ reserved: false, denial: { reason: "pool-history-unresolved" } }); + // A policy error cannot prevent settlement of an existing physical liability. + expect(ledger.settle("before", { inputTokens: 1, outputTokens: 0 })).toBe(true); +}); + +test("adding canonical provider rejects an existing foreign mapping", () => { + installSalt(); + const aliases = { [alias("P")]: "Q" }; + expect(validateConfigCandidate(candidate(aliases, { Q: provider })).ok).toBe(true); + expect(validateConfigCandidate(candidate(aliases)).ok).toBe(false); + const ledger = createSpendReservationLedger({ salt, policy: policy(aliases, ["Q"]) }); + expect(reserve(ledger, "Q-before", "Q").reserved).toBe(true); + // The actual routed provider is also checked against a stale startup roster. + expect(reserve(ledger, "P-stale", "P")).toMatchObject({ reserved: false, denial: { reason: "pool-history-unresolved" } }); + ledger.reconfigure(policy(aliases, ["P", "Q"])); + expect(ledger.checkPoolContinuity()?.reason).toBe("pool-history-unresolved"); + expect(reserve(ledger, "Q-after", "Q").reserved).toBe(false); +}); + +test("same-home self mapping succeeds and validation creates no salt", () => { + const saltPath = join(home, "spend-ledger.salt"); + expect(validateConfigCandidate(candidate({})).ok).toBe(true); + expect(validateConfigCandidate(candidate({ [alias("P")]: "P" })).ok).toBe(false); + const loaded = configDiagnosticsFromRaw(JSON.stringify(candidate({ [alias("P")]: "P" }))); + expect(loaded.config.spend?.pool?.maxTokens).toBe(100); + expect(loaded.warnings?.join("\n")).toContain("spendPoolAliases invalid"); + expect(existsSync(saltPath)).toBe(false); + expect(existsSync(join(home, "spend-ledger.jsonl"))).toBe(false); + installSalt(); + expect(validateConfigCandidate(candidate({ [alias("P")]: "P" })).ok).toBe(true); + expect(configDiagnosticsFromRaw(JSON.stringify(candidate({ [alias("P")]: "P" }))).warnings?.join("\n") ?? "").not.toContain("spendPoolAliases invalid"); + const ledger = createSpendReservationLedger({ salt, policy: policy({ [alias("P")]: "P" }, ["P", "Q"]) }); + expect(reserve(ledger).reserved).toBe(true); + // This is a same-home rule: an alias from another salt has no claimed current owner. + expect(validateConfigCandidate(candidate({ [alias("P", "8".repeat(64))]: "Q" })).ok).toBe(true); +}); + +test("unreadable malformed and linked existing salts cannot validate a mapping", () => { + const raw = candidate({ [alias("P")]: "P" }); + writeFileSync(join(home, "spend-ledger.salt"), "invalid\n"); + expect(validateConfigCandidate(raw).ok).toBe(false); + const other = mkdtempSync(join(tmpdir(), "ocx-linked-pool-salt-")); + try { + writeFileSync(join(other, "salt"), salt + "\n"); + for (const link of [linkSync, symlinkSync]) { + const linkedHome = join(other, link === linkSync ? "hard" : "symbolic"); + // A disposable home keeps the ordinary real salt above untouched. + mkdirSync(linkedHome); + link(join(other, "salt"), join(linkedHome, "spend-ledger.salt")); + process.env.OPENCODEX_HOME = linkedHome; + expect(validateConfigCandidate(raw).ok).toBe(false); + } + } finally { process.env.OPENCODEX_HOME = home; removeTreeWithRetry(other); } +}); diff --git a/tests/fixtures/spend-ledger-2-80-0.ts.txt b/tests/fixtures/spend-ledger-2-80-0.ts.txt new file mode 100644 index 00000000000..0350fb40757 --- /dev/null +++ b/tests/fixtures/spend-ledger-2-80-0.ts.txt @@ -0,0 +1,1289 @@ +/** + * Durable token spend reservation, above the send-count workflow guard (#4546). + * + * The count cap treats a 1k-token send and a 150k-token send as the same unit, and the + * in-memory ledger forgets everything on restart: an exhausted root came back with a fresh + * allowance after every relaunch, and a second process never saw the first one's spend at + * all. This ledger reserves TOKENS before dispatch and rebuilds its state from a journal + * under the opencodex home directory, so an exhausted scope is still exhausted after a + * restart. + * + * A reservation is always the request's whole input plus its ENFORCEABLE output ceiling -- + * the caller's max_output_tokens, or the model's documented cap when the caller sent none. + * Never an optimistic estimate, and never shrunk by a cache-hit expectation: a prefix that + * misses is billed in full, so the safety figure reserves as if it misses. Cache + * expectations may inform efficiency reporting; they do not move this number. + * + * Admission requires, at every scope that applies at once -- root workflow, authenticated + * identity, and account pool: + * + * settled spend + in-flight reservations + unresolved spend + this reservation <= limit + * + * Unresolved spend is the conservative residue of a send whose usage frame was lost: the + * tokens may have been billed, so the reservation is moved to unresolved rather than + * released. Minting a new root id mints no new budget because the identity and pool scopes + * still hold the spend. + * + * SUPPORTED TOPOLOGY: one live writer owns one OPENCODEX_HOME journal. Every server and + * direct shared-ledger caller must hold the state-directory SQLite lease before replay, + * append or compaction. Independent homes remain independent; multi-host shared storage + * still needs a distributed transaction boundary and is outside this local lease. + * + * Five properties this file owes its callers. Each one was absent in the first draft, and a + * budget that can be bypassed is worse than no budget because it looks like protection: + * + * 1. IDENTITY OF A SEND. A send id is either KNOWN -- and then reserving it again is refused + * rather than waved through booking nothing -- or FULLY forgotten, and then it books a + * fresh reservation. There is no third state where the ledger recognises an id and + * charges nothing for it, which is what let one id authorise unlimited physical sends. + * 2. DURABILITY BEFORE ADMISSION. Under a configured limit the reserve record must be on + * disk before the request is admitted. Failing open on a disk-full or permission error + * forgets the request across a restart, which is the exact case durability exists for. + * Observe-only mode still admits, and says so through `durable: false`. + * 3. REPLAY VALIDATES. Every journal record is checked field by field before it moves a + * counter. A corrupt record in the MIDDLE of the file would silently undercount, so it + * fails accounting closed instead; only an unparseable FINAL line -- a torn tail write -- + * is dropped quietly. + * 4. BOUNDED RETENTION. Cleanup runs automatically, writes durable tombstones so replay + * cannot resurrect what it removed, and compacts the journal to a checkpoint. When + * nothing can be evicted safely, admission is refused rather than made room for by + * forgetting an exhausted scope -- forgetting one is the laundering this layer prevents. + * 5. NOTHING IDENTIFYING ON DISK. Root ids come from a client header and identity ids are + * credential ids, so the journal stores salted aliases only, under owner-only permissions + * that are re-applied to an EXISTING file rather than trusted from its creation. + */ + +import { appendFileSync, chmodSync, closeSync, lstatSync, mkdirSync, openSync, readFileSync, renameSync, statSync, unlinkSync, writeFileSync } from "node:fs"; +import { createHash, randomBytes } from "node:crypto"; +import { dirname, join } from "node:path"; +// Definition-site import, not the ../config barrel -- same reasoning as +// src/quota/reset-seen-store.ts: the barrel pulls ~154 modules into a hot path. +import { getConfigDir } from "../config/paths"; +// Type-only, so it is erased before this module has a runtime import graph at all. The +// config SHAPE is what this file needs; the config loader is what the note above keeps out. +import type { OcxSpendConfig, OcxSpendScopeConfig } from "../types/config"; +import { assertNotRealHomeUnderTest } from "./test-home-guard"; +// Windows chmod does not remove inherited ACEs; this is the repository's icacls path. +import { hardenSecretPath } from "./windows-secret-acl"; +import { assertSpendLedgerOwnerHeld, assertStorageOwned, bindSpendLedgerOwnerHome, mintSpendLedgerStorage, onSpendLedgerOwnerReleased, resetSpendLedgerOwnerBindingForTest, SpendLedgerOwnerError, spendLedgerOwnerSnapshot, spendLedgerStoragePath, type SpendLedgerStorage } from "./spend-ledger-owner"; + +// The singleton belongs to the state directory it was built for. Releasing ownership hands that +// directory to whoever comes next, so the in-memory copy goes with it and the next construction +// replays the journal. +onSpendLedgerOwnerReleased(() => { sharedLedger = undefined; }); + +export const SPEND_LEDGER_JOURNAL_FILENAME = "spend-ledger.jsonl"; +/** + * Per-install alias salt, beside the journal. Losing it is exactly as bad as losing the + * journal -- both reset accounting, both live in the same 0700 directory -- so it is not a + * new weakness, and keeping it out of the journal stops a copied or attached journal from + * being reversible by dictionary attack on guessable pool and identity ids. + */ +export const SPEND_LEDGER_SALT_FILENAME = "spend-ledger.salt"; + +export type SpendScope = "root" | "identity" | "pool"; + +export interface SpendScopeLimit { + /** + * Approved token ceiling for the scope. Undefined means OBSERVE ONLY: spend is still + * accounted and reported, but nothing is refused. That is the unconfigured default -- + * an install that never opted in keeps the count caps and is not newly refused. + */ + readonly maxTokens?: number; +} + +export interface SpendReservationPolicy { + readonly root: SpendScopeLimit; + readonly identity: SpendScopeLimit; + readonly pool: SpendScopeLimit; + /** + * How long a dormant scope's accounting is retained. A scope may be dropped only when it + * is BOTH inactive (no open reservation) AND not exhausted inside this window; dropping + * an exhausted scope would hand it a fresh allowance on next use. + */ + readonly retentionMs: number; + /** + * Hard ceiling on tracked scopes. Retention alone bounds nothing: a caller minting a fresh + * root id per request fills the map long before the window elapses. At the ceiling the + * ledger evicts the oldest scope that is safe to forget -- idle, under its limit, past + * retention -- and if there is none it REFUSES the new scope. Refusing is the only answer + * left: the alternative is evicting an exhausted scope, which hands it a fresh allowance. + */ + readonly maxTrackedScopes?: number; + /** Hard ceiling on remembered send ids, with the same evict-or-refuse rule. */ + readonly maxTrackedSends?: number; + /** + * Journal records after which the file is compacted into a single checkpoint. Without + * this the file grows forever even while the in-memory maps stay bounded, and replay + * resurrects every entry cleanup removed. + */ + readonly compactAfterRecords?: number; +} + +const DEFAULT_MAX_TRACKED_SCOPES = 4_096; +const DEFAULT_MAX_TRACKED_SENDS = 16_384; +const DEFAULT_COMPACT_AFTER_RECORDS = 8_192; + +/** + * Unconfigured default: every limit undefined, so token accounting runs in observe-only + * mode and the count caps remain the only enforcement. Real numbers belong behind + * explicit operator configuration. + */ +export const DEFAULT_SPEND_RESERVATION_POLICY: SpendReservationPolicy = { + root: {}, + identity: {}, + pool: {}, + retentionMs: 7 * 24 * 60 * 60_000, +}; + +export interface SpendScopes { + readonly rootId?: string; + readonly identityId?: string; + readonly poolId?: string; +} + +export interface SpendUsage { + readonly inputTokens: number; + readonly outputTokens: number; +} + +export interface SpendReservationRequest { + /** Stable id of the physical send. Settlement is idempotent on this key. */ + readonly sendId: string; + readonly scopes: SpendScopes; + readonly inputTokens: number; + /** Enforceable output ceiling -- max_output_tokens or the model's documented cap. */ + readonly outputCeilingTokens: number; + readonly at?: number; + /** + * This send has ALREADY left for upstream and is being recorded rather than admitted. + * + * Some transports report their physical sends after the fact -- the passthrough ladder + * reports through `onSendsConsumed`, and an adapter's inner retries are counted when they + * finish. For those, a ceiling cannot refuse anything: the tokens are spent. Refusing to + * BOOK them is the worse answer, and it is not hypothetical -- it is a fixpoint. The send + * that would cross the ceiling gets dropped from the total, the total stays just under the + * limit forever, the scope never reads as exhausted, and the ceiling never fires again for + * any request. So a recorded send skips the limit check and takes the scope over its + * ceiling, which is what makes the NEXT request refusable. + * + * It skips the durability refusal for the same reason: a journal that could not be written + * is a reason to report degradation, never a reason to forget spend that really happened. + * Identity, capacity and journal-integrity denials still apply -- those say the ledger + * cannot account for the send at all, which no flag here can change. + */ + readonly alreadySent?: boolean; +} + +/** + * Why a reservation was refused. Every member refuses a DISPATCH: none of them is an + * "already fine, carry on" answer, because that is precisely how a duplicate send id used + * to buy an unlimited number of physical sends while the scope totals never moved. + */ +export type SpendDenial = + | { + readonly reason: "spend-limit-exceeded"; + readonly scope: SpendScope; + readonly scopeId: string; + readonly limit: number; + readonly projected: number; + } + /** This send id is already known -- open, settled, lost or abandoned. */ + | { readonly reason: "duplicate-send-id"; readonly sendId: string } + /** The reserve record could not be written, and a configured limit needs it to survive. */ + | { readonly reason: "reserve-not-durable"; readonly sendId: string } + /** Replay rejected records mid-file, so no scope total can be proven complete. */ + | { readonly reason: "journal-corrupt"; readonly corruptRecords: number } + /** Tracking is full and nothing may be forgotten safely. */ + | { readonly reason: "tracking-capacity-exhausted"; readonly scope?: SpendScope }; + +export type SpendReservationDecision = + | { + readonly reserved: true; + readonly sendId: string; + readonly tokens: number; + /** + * False only in observe-only mode, where the reservation was admitted although its + * journal record did not reach disk. A restart will not remember this spend; the flag + * is how a caller learns that instead of discovering it after the fact. + */ + readonly durable: boolean; + } + | { readonly reserved: false; readonly denial: SpendDenial }; + +interface ScopeState { + settled: number; + reserved: number; + unresolved: number; + lastSeenAt: number; +} + +/** + * `open` means admitted but not yet handed to a transport: it may still be abandoned for + * free. `dispatched` means bytes left for upstream, so from there a missing usage frame is + * unresolved SPEND rather than a release -- it may have been billed. Only a dispatched send + * can become `lost`; only an undispatched one can become `abandoned`. + */ +type ReservationStatus = "open" | "dispatched" | "settled" | "lost" | "abandoned"; + +interface ScopeRef { + readonly scope: SpendScope; + readonly alias: string; +} + +interface Reservation { + readonly targets: readonly ScopeRef[]; + readonly tokens: number; + status: ReservationStatus; + readonly at: number; + /** When the status last changed; drives eviction of resolved entries. */ + resolvedAt: number; +} + +/** + * Journal shape. Every id on disk is a salted alias, never a root header value, credential + * id or pool name. `forget` and `drop` are the tombstones that make bounded cleanup + * durable -- without them replay rebuilds exactly what cleanup removed -- and `checkpoint` + * is a whole-state snapshot that lets the file be compacted instead of growing forever. + */ +type JournalRecord = + | { v: 1; kind: "reserve"; send: string; targets: ScopeRef[]; tokens: number; at: number } + | { v: 1; kind: "dispatch"; send: string; at: number } + | { v: 1; kind: "settle"; send: string; tokens: number; at: number } + | { v: 1; kind: "lost"; send: string; at: number } + | { v: 1; kind: "abandon"; send: string; at: number } + | { v: 1; kind: "forget"; send: string; at: number } + | { v: 1; kind: "drop"; scope: SpendScope; alias: string; at: number } + | { + v: 1; + kind: "checkpoint"; + at: number; + scopes: { scope: SpendScope; alias: string; settled: number; unresolved: number; seenAt: number }[]; + sends: { send: string; status: ReservationStatus; targets: ScopeRef[]; tokens: number; at: number; resolvedAt: number }[]; + }; + +const isCountable = (value: unknown): value is number => + typeof value === "number" && Number.isFinite(value) && value >= 0; + +const isAlias = (value: unknown): value is string => + typeof value === "string" && value.length > 0 && value.length <= 256; + +const isScopeName = (value: unknown): value is SpendScope => + value === "root" || value === "identity" || value === "pool"; + +const isStatus = (value: unknown): value is ReservationStatus => + value === "open" || value === "dispatched" || value === "settled" + || value === "lost" || value === "abandoned"; + +const parseTargets = (value: unknown): ScopeRef[] | undefined => { + if (!Array.isArray(value) || value.length > 3) return undefined; + const targets: ScopeRef[] = []; + for (const entry of value) { + if (typeof entry !== "object" || entry === null) return undefined; + const { scope, alias } = entry as { scope?: unknown; alias?: unknown }; + if (!isScopeName(scope) || !isAlias(alias)) return undefined; + targets.push({ scope, alias }); + } + return targets; +}; + +/** + * Validate one journal line into a record, or reject it. + * + * Exported because this is the boundary where a hostile or damaged file meets the accounting: + * `JSON.parse(line) as JournalRecord` type-asserts a lie, and a bare `null` line or a + * `{"v":1,"kind":"reserve"}` with no fields crashed the rebuild rather than being rejected. + * Every field is checked, including that numbers are finite and non-negative. + */ +export function parseSpendJournalRecord(line: string): JournalRecord | undefined { + let raw: unknown; + try { + raw = JSON.parse(line); + } catch { + return undefined; + } + if (typeof raw !== "object" || raw === null || Array.isArray(raw)) return undefined; + const record = raw as Record; + if (record.v !== 1) return undefined; + if (!isCountable(record.at)) return undefined; + const at = record.at; + switch (record.kind) { + case "reserve": { + const targets = parseTargets(record.targets); + if (!isAlias(record.send) || targets === undefined || !isCountable(record.tokens)) return undefined; + return { v: 1, kind: "reserve", send: record.send, targets, tokens: record.tokens, at }; + } + case "settle": + if (!isAlias(record.send) || !isCountable(record.tokens)) return undefined; + return { v: 1, kind: "settle", send: record.send, tokens: record.tokens, at }; + case "dispatch": + if (!isAlias(record.send)) return undefined; + return { v: 1, kind: "dispatch", send: record.send, at }; + case "lost": + if (!isAlias(record.send)) return undefined; + return { v: 1, kind: "lost", send: record.send, at }; + case "abandon": + if (!isAlias(record.send)) return undefined; + return { v: 1, kind: "abandon", send: record.send, at }; + case "forget": + if (!isAlias(record.send)) return undefined; + return { v: 1, kind: "forget", send: record.send, at }; + case "drop": + if (!isScopeName(record.scope) || !isAlias(record.alias)) return undefined; + return { v: 1, kind: "drop", scope: record.scope, alias: record.alias, at }; + case "checkpoint": { + if (!Array.isArray(record.scopes) || !Array.isArray(record.sends)) return undefined; + const scopes: { scope: SpendScope; alias: string; settled: number; unresolved: number; seenAt: number }[] = []; + for (const entry of record.scopes) { + if (typeof entry !== "object" || entry === null) return undefined; + const e = entry as Record; + if (!isScopeName(e.scope) || !isAlias(e.alias)) return undefined; + if (!isCountable(e.settled) || !isCountable(e.unresolved) || !isCountable(e.seenAt)) return undefined; + scopes.push({ scope: e.scope, alias: e.alias, settled: e.settled, unresolved: e.unresolved, seenAt: e.seenAt }); + } + const sends: { send: string; status: ReservationStatus; targets: ScopeRef[]; tokens: number; at: number; resolvedAt: number }[] = []; + for (const entry of record.sends) { + if (typeof entry !== "object" || entry === null) return undefined; + const e = entry as Record; + const targets = parseTargets(e.targets); + if (!isAlias(e.send) || !isStatus(e.status) || targets === undefined) return undefined; + if (!isCountable(e.tokens) || !isCountable(e.at) || !isCountable(e.resolvedAt)) return undefined; + sends.push({ send: e.send, status: e.status, targets, tokens: e.tokens, at: e.at, resolvedAt: e.resolvedAt }); + } + return { v: 1, kind: "checkpoint", at, scopes, sends }; + } + default: + return undefined; + } +} + +/** + * Append-mostly persistence. `read` returns raw lines so replay can tell a torn TAIL write + * from corruption earlier in the file; only the former is safe to drop quietly. `append` + * THROWS when the record did not reach storage -- that signal is what lets admission refuse + * rather than admit a request a restart would forget. `rewrite` is optional: a store that + * cannot replace its contents atomically simply never compacts. + */ +export interface SpendJournal { + read(): string[]; + append(line: string): void; + rewrite?(lines: string[]): void; +} + +/** + * Re-apply owner-only permissions to a file that already exists. + * + * `mode` in a write option is honoured only when the file is CREATED, so a journal that was + * created loose -- by an older build, a restored backup, or a lax umask -- would keep its + * mode forever. Best-effort by design: a non-owner cannot chmod, and failing every append + * over it would be worse than the loose mode it is fixing. + * + * `force` marks the points where the WINDOWS ACL can actually be wrong: creation, compaction, + * and each process's replay. Windows chmod cannot drop inherited ACEs, so icacls is the real + * boundary there, and its memo keys on the file's ctime -- which every append changes. Running + * it per reservation would therefore spawn a process per send while protecting nothing an + * append can alter. On POSIX the mode is checked on every write and repaired the moment it + * drifts, which costs one stat. + */ +function hardenLedgerFile(path: string, options: { readonly force?: boolean } = {}): void { + if (process.platform === "win32") { + if (options.force) hardenSecretPath(path, { required: false }); + return; + } + try { + if ((statSync(path).mode & 0o777) === 0o600) return; + chmodSync(path, 0o600); + } catch { /* best-effort: a non-owner cannot chmod */ } +} + +/** + * Fault injection for the journal's own filesystem steps. Internal test contract, not config. + * + * The compaction cleanup only runs when a step after the exclusive create fails, and there is no + * portable way to make a validate, harden or rename fail on demand. Without a seam the cleanup + * would ship asserted by reading alone, which is how a failure path stays broken. + */ +export type SpendJournalFaultStep = "stat" | "create" | "write" | "validate" | "harden" | "rename"; + +let journalFaultForTests: ((step: SpendJournalFaultStep, temp: string) => void) | undefined; + +export function setSpendJournalFaultForTests( + fault: ((step: SpendJournalFaultStep, temp: string) => void) | undefined, +): void { + journalFaultForTests = fault; +} + +/** + * Does a directory entry exist here, whatever it points at? + * + * `existsSync` follows the link, so a symlink whose target is absent reads as "no file" and an + * append then creates that target somewhere else entirely. The entry itself is what decides + * whether the safety check runs. + */ +function ledgerEntryExists(path: string): boolean { + try { + journalFaultForTests?.("stat", path); + lstatSync(path); + return true; + } catch (error) { + // Only a genuinely absent entry is absent. Treating every failure as "no file" meant a + // permission denial or an I/O error skipped assertSafeLedgerFile entirely and let the + // append proceed against whatever is actually there, which is the case that check exists + // for. An entry we cannot inspect is a refusal, not an empty slot. + // + // Raised as the module's typed error rather than the raw fs error: the path and errno of a + // state file are not something a client should be handed. + if ((error as NodeJS.ErrnoException | null)?.code === "ENOENT") return false; + throw new SpendLedgerOwnerError( + "SPEND_LEDGER_OWNER_UNAVAILABLE", + "Spend-ledger storage could not be inspected safely.", + { cause: error }, + ); + } +} + +/** Which ledger file a refusal is about. A role, never a path: the path names the user's home. */ +export type SpendLedgerFileRole = "journal" | "journal-compaction" | "salt"; + +/** The one condition that refused the file, so a report can say which guard fired. */ +export type SpendLedgerFileRefusal = + | "not-regular-file" + | "symbolic-link" + | "extra-hard-link" + | "foreign-owner" + | "invalid-salt"; + +/** + * A ledger file failed a safety condition. + * + * The guard is unchanged; this only says which file role and which condition refused it. Issue + * #6314 sat for days on "could not be opened safely" because the same sentence covered five + * conditions and two files, and the one that fired (a second hard link to the journal, most + * likely held briefly by a cloud-sync daemon) could only be found with an instrumented build. + * Role and condition are fixed vocabulary, so the message carries no path, salt, alias or + * request content. + */ +export class SpendLedgerFileRefusedError extends SpendLedgerOwnerError { + constructor(readonly role: SpendLedgerFileRole, readonly refusal: SpendLedgerFileRefusal) { + super( + "SPEND_LEDGER_OWNER_UNAVAILABLE", + `Spend-ledger storage could not be opened safely (${role}: ${refusal}).` + + (refusal === "extra-hard-link" + ? " Another directory entry links to this file; if the opencodex state directory is inside an iCloud Drive or other synced folder, move it out." + : ""), + ); + this.name = "SpendLedgerFileRefusedError"; + } +} + +function ledgerFileRefusal(path: string): SpendLedgerFileRefusal | undefined { + const stat = lstatSync(path); + if (stat.isSymbolicLink()) return "symbolic-link"; + if (!stat.isFile()) return "not-regular-file"; + if (stat.nlink !== 1) return "extra-hard-link"; + if (process.platform !== "win32" && stat.uid !== process.getuid!()) return "foreign-owner"; + return undefined; +} + +function assertSafeLedgerFile(path: string, role: SpendLedgerFileRole): void { + const refusal = ledgerFileRefusal(path); + if (refusal !== undefined) throw new SpendLedgerFileRefusedError(role, refusal); +} + +/** + * The production journal. Its location comes from the owned state directory and every touch + * proves that ownership, so there is no entrypoint here that writes a caller-chosen path. + */ +export function createOwnedFileSpendJournal(storage: SpendLedgerStorage): SpendJournal { + assertStorageOwned(storage); + const path = spendLedgerStoragePath(storage); + const ensureDir = (): string => { + const dir = dirname(path); + // The guard runs before any mutation so a rejected write leaves nothing behind. + assertNotRealHomeUnderTest(dir); + mkdirSync(dir, { recursive: true, mode: 0o700 }); + return dir; + }; + return { + read(): string[] { + assertStorageOwned(storage); + if (!ledgerEntryExists(path)) return []; + assertSafeLedgerFile(path, "journal"); + // Replay is once per process and is the moment a journal inherited from an older build + // or a restored backup first passes through here. + hardenLedgerFile(path, { force: true }); + return readFileSync(path, "utf8").split("\n").filter((line) => line.length > 0); + }, + append(line: string): void { + assertStorageOwned(storage); + ensureDir(); + const created = !ledgerEntryExists(path); + if (!created) assertSafeLedgerFile(path, "journal"); + appendFileSync(path, line + "\n", { encoding: "utf8", mode: 0o600 }); + assertSafeLedgerFile(path, "journal"); + hardenLedgerFile(path, { force: created }); + }, + rewrite(lines: string[]): void { + assertStorageOwned(storage); + ensureDir(); + // Same directory, so the rename is atomic on the same filesystem: a crash mid-compaction + // leaves either the old journal or the new one, never a half-written ledger. + if (ledgerEntryExists(path)) assertSafeLedgerFile(path, "journal"); + const temp = `${path}.compact-${process.pid}-${randomBytes(6).toString("hex")}`; + // The creation is INSIDE the cleanup, not before it. The name carries random bytes, so a + // failure anywhere after the entry exists used to leave a uniquely named file and the next + // attempt made another: repeated failures accumulated instead of overwriting one fixed + // name. Only an entry this call created is removed, so a name that turned out to belong to + // something else is left alone, and the original journal and the primary error survive. + let fd: number | undefined; + let created = false; + let renamed = false; + try { + // Exclusive create FIRST, so "this entry is ours" is a fact rather than a guess about + // which error a combined write threw. EEXIST leaves created false and the name is left + // alone; every failure after this point is cleaned because the entry is provably ours, + // including a write that stopped partway through. + journalFaultForTests?.("create", temp); + fd = openSync(temp, "wx", 0o600); + created = true; + journalFaultForTests?.("write", temp); + writeFileSync(fd, lines.map((line) => line + "\n").join(""), { encoding: "utf8" }); + closeSync(fd); + fd = undefined; + journalFaultForTests?.("validate", temp); + assertSafeLedgerFile(temp, "journal-compaction"); + journalFaultForTests?.("harden", temp); + hardenLedgerFile(temp, { force: true }); + journalFaultForTests?.("rename", temp); + renameSync(temp, path); + renamed = true; + } finally { + if (fd !== undefined) { + try { closeSync(fd); } catch { /* the compaction failure is the one to report */ } + } + if (created && !renamed) { + try { unlinkSync(temp); } catch { /* same */ } + } + } + assertSafeLedgerFile(path, "journal"); + hardenLedgerFile(path, { force: true }); + }, + }; +} + +/** + * Load the per-install alias salt, minting it on first use. + * + * The salt must be STABLE across restarts or replay cannot match a live request to its own + * recorded spend, which would hand every scope a fresh allowance -- so it is a file, not a + * per-process value. + */ +export function loadOrCreateSpendLedgerSalt(storage: SpendLedgerStorage): string { + assertStorageOwned(storage); + const path = spendLedgerStoragePath(storage); + if (ledgerEntryExists(path)) { + assertSafeLedgerFile(path, "salt"); + hardenLedgerFile(path, { force: true }); + const existing = readFileSync(path, "utf8").trim(); + if (/^[0-9a-f]{32,}$/.test(existing)) return existing; + throw new SpendLedgerFileRefusedError("salt", "invalid-salt"); + } + const dir = dirname(path); + assertStorageOwned(storage); + assertNotRealHomeUnderTest(dir); + mkdirSync(dir, { recursive: true, mode: 0o700 }); + const salt = randomBytes(32).toString("hex"); + writeFileSync(path, salt + "\n", { encoding: "utf8", mode: 0o600, flag: "wx" }); + assertSafeLedgerFile(path, "salt"); + hardenLedgerFile(path, { force: true }); + return salt; +} + +export interface ScopeSpendSnapshot { + readonly settled: number; + readonly reserved: number; + readonly unresolved: number; + readonly exhausted: boolean; +} + +export interface SpendReservationLedger { + reserve(request: SpendReservationRequest): SpendReservationDecision; + /** + * The send left for upstream. Until this is called the reservation may be abandoned for + * free; after it, a missing usage frame becomes unresolved spend. Returns false when the + * send is unknown or no longer open. + */ + markDispatched(sendId: string): boolean; + /** + * The send never happened -- local validation, routing, or a refusal before any byte left + * this process. The reservation is RELEASED and books nothing, because inventing debt the + * account never incurred is its own way of breaking the budget. Refused once the send is + * dispatched: from there only settle or markLost is honest. + */ + abandon(sendId: string): boolean; + /** + * Settle with real usage. Returns false when the send is unknown or already resolved -- + * double settlement is as wrong as none, so a repeat call changes nothing. + */ + settle(sendId: string, usage: SpendUsage): boolean; + /** + * Usage never arrived. The reservation moves to unresolved spend -- it may have been + * billed -- rather than being released. Idempotent on the same key as settle. + */ + markLost(sendId: string): boolean; + snapshot(scope: SpendScope, scopeId: string): ScopeSpendSnapshot | undefined; + exhausted(scope: SpendScope, scopeId: string): boolean; + /** + * Drop dormant scopes per the retention rule in SpendReservationPolicy. Cleanup also runs + * automatically on every reservation, so nothing depends on a caller remembering this. + */ + prune(now?: number): void; + /** Whether this send id is already known, and therefore refused. */ + knows(sendId: string): boolean; + /** + * Replace the live policy. + * + * Every figure already accounted survives: raising, lowering or clearing a ceiling changes + * what is REFUSED from here on and never what was spent. Rebuilding the ledger instead + * would replay the journal into a second set of maps while the first still holds this + * process's open reservations, and the two would then disagree about what is in flight. + */ + reconfigure(next: SpendReservationPolicy): void; + /** + * The policy in force. A live read, not a copy: a caller that formats a refusal has to name + * the ceiling this ledger would enforce on the NEXT request, not the one it was built with. + */ + readonly policy: SpendReservationPolicy; + /** Journal writes that failed; a nonzero count means durability is degraded. */ + readonly persistFailures: number; + /** + * Records replay rejected in the MIDDLE of the journal. Nonzero means no scope total can + * be proven complete, so configured limits refuse rather than undercount. + */ + readonly corruptRecords: number; + /** True when durability is degraded in either direction: failed writes or a corrupt file. */ + readonly degraded: boolean; +} + +const scopeKey = (scope: SpendScope, alias: string): string => scope + "\0" + alias; + +const sanitizeTokens = (value: number): number => + Number.isFinite(value) ? Math.max(0, Math.trunc(value)) : 0; + +export function createSpendReservationLedger(options: { + readonly journal?: SpendJournal; + readonly policy?: SpendReservationPolicy; + readonly now?: () => number; + /** + * Per-install alias salt. Production passes the file-backed value from + * `loadOrCreateSpendLedgerSalt`; an empty default is for in-memory journals, which have + * no file anyone could correlate. + */ + readonly salt?: string; + /** + * Shared production ledgers prove their exact ownership before reading or changing + * accounting. Identity, not just the directory name: a handle kept across a release and a + * reacquire describes a journal another writer may have changed in between. + */ + readonly assertOwnedAccounting?: () => void; +} = {}): SpendReservationLedger { + // Mutable because the ceilings are operator configuration, and configuration is reloadable. + // The three bounds below are read through functions for the same reason: a value captured + // at construction would answer for the policy this ledger was BUILT with, and an operator + // who raised a bound would keep the old one until the process restarted. + let policy = options.policy ?? DEFAULT_SPEND_RESERVATION_POLICY; + const journal = options.journal; + const now = options.now ?? (() => Date.now()); + const salt = options.salt ?? ""; + const assertOwnedAccounting = options.assertOwnedAccounting; + const maxTrackedScopes = (): number => policy.maxTrackedScopes ?? DEFAULT_MAX_TRACKED_SCOPES; + const maxTrackedSends = (): number => policy.maxTrackedSends ?? DEFAULT_MAX_TRACKED_SENDS; + const compactAfterRecords = (): number => policy.compactAfterRecords ?? DEFAULT_COMPACT_AFTER_RECORDS; + const scopes = new Map(); + const reservations = new Map(); + let persistFailures = 0; + let corruptRecords = 0; + let recordsOnDisk = 0; + + /** + * Salted alias for one identifier. The raw value -- a client-supplied root header, a + * credential id, a pool name -- never leaves this function, so nothing identifying is + * written to disk or held in a map key. + */ + const aliasFor = (kind: SpendScope | "send", id: string): string => + createHash("sha256").update(salt).update("\u0000").update(kind).update("\u0000").update(id) + .digest("hex").slice(0, 32); + + const scopeState = (scope: SpendScope, alias: string): ScopeState => { + const key = scopeKey(scope, alias); + let state = scopes.get(key); + if (!state) { + state = { settled: 0, reserved: 0, unresolved: 0, lastSeenAt: 0 }; + scopes.set(key, state); + } + return state; + }; + + const limitFor = (scope: SpendScope): number | undefined => policy[scope].maxTokens; + + const isExhausted = (scope: SpendScope, state: ScopeState): boolean => { + const limit = limitFor(scope); + return limit !== undefined && state.settled + state.reserved + state.unresolved >= limit; + }; + + /** The scopes a request touches, as aliases. Creates no state: a refusal must leave none. */ + const refsFor = (targets: SpendScopes): ScopeRef[] => { + const refs: ScopeRef[] = []; + if (targets.rootId !== undefined) refs.push({ scope: "root", alias: aliasFor("root", targets.rootId) }); + if (targets.identityId !== undefined) refs.push({ scope: "identity", alias: aliasFor("identity", targets.identityId) }); + if (targets.poolId !== undefined) refs.push({ scope: "pool", alias: aliasFor("pool", targets.poolId) }); + return refs; + }; + + /** + * Returns whether the record reached storage. With no journal there is nothing to fail, + * and the caller's durability question is vacuously satisfied. + */ + const append = (record: JournalRecord): boolean => { + if (!journal) return true; + try { + journal.append(JSON.stringify(record)); + recordsOnDisk += 1; + return true; + } catch (error) { + if (error instanceof SpendLedgerOwnerError) throw error; + // In-memory state still bounds this process; the counter is how a caller learns the + // restart guarantee degraded instead of discovering it after the fact. + persistFailures += 1; + return false; + } + }; + + const applyReserve = (send: string, targets: readonly ScopeRef[], tokens: number, at: number): void => { + if (reservations.has(send)) return; + reservations.set(send, { targets, tokens, status: "open", at, resolvedAt: at }); + for (const ref of targets) { + const state = scopeState(ref.scope, ref.alias); + state.reserved += tokens; + state.lastSeenAt = Math.max(state.lastSeenAt, at); + } + }; + + const isLive = (status: ReservationStatus): boolean => status === "open" || status === "dispatched"; + + /** + * Resolve a live reservation. `settled` books the real figure, `lost` keeps the whole + * reservation as unresolved spend because it may have been billed, and `abandoned` + * releases it because no byte ever left this process. + */ + const applyResolve = (send: string, outcome: "settled" | "lost" | "abandoned", tokens: number, at: number): void => { + const reservation = reservations.get(send); + if (!reservation || !isLive(reservation.status)) return; + reservation.status = outcome; + reservation.resolvedAt = at; + for (const ref of reservation.targets) { + const state = scopeState(ref.scope, ref.alias); + state.reserved = Math.max(0, state.reserved - reservation.tokens); + if (outcome === "lost") state.unresolved += reservation.tokens; + else if (outcome === "settled") state.settled += tokens; + state.lastSeenAt = Math.max(state.lastSeenAt, at); + } + }; + + const applyDispatch = (send: string, at: number): void => { + const reservation = reservations.get(send); + if (!reservation || reservation.status !== "open") return; + reservation.status = "dispatched"; + reservation.resolvedAt = at; + }; + + /** Tombstone replay: the entry is gone, so a later reuse of the id books a fresh charge. */ + const applyForget = (send: string): void => { + const reservation = reservations.get(send); + if (!reservation || isLive(reservation.status)) return; + reservations.delete(send); + }; + + const applyDrop = (scope: SpendScope, alias: string): void => { + const state = scopes.get(scopeKey(scope, alias)); + if (!state || state.reserved > 0) return; + scopes.delete(scopeKey(scope, alias)); + }; + + const applyCheckpoint = (record: Extract): void => { + scopes.clear(); + reservations.clear(); + for (const entry of record.scopes) { + scopes.set(scopeKey(entry.scope, entry.alias), { + settled: entry.settled, + reserved: 0, + unresolved: entry.unresolved, + lastSeenAt: entry.seenAt, + }); + } + for (const entry of record.sends) { + // `reserved` is rebuilt from the live entries rather than trusted from the snapshot, + // so the two can never disagree about the same tokens. + if (isLive(entry.status)) { + applyReserve(entry.send, entry.targets, entry.tokens, entry.at); + if (entry.status === "dispatched") applyDispatch(entry.send, entry.resolvedAt); + continue; + } + reservations.set(entry.send, { + targets: entry.targets, + tokens: entry.tokens, + status: entry.status, + at: entry.at, + resolvedAt: entry.resolvedAt, + }); + } + }; + + // Rebuild from the journal before serving: an exhausted scope must still be exhausted + // after a restart, which is the whole reason this store exists. + if (journal) { + const lines = journal.read(); + recordsOnDisk = lines.length; + for (let index = 0; index < lines.length; index += 1) { + const line = lines[index] as string; + const record = parseSpendJournalRecord(line); + if (!record) { + // A rejected FINAL line is a torn tail write -- the process died between the write + // and its newline -- and is dropped quietly, because that record never completed and + // therefore never authorised anything. A rejected line ANYWHERE ELSE is different: + // the records after it did complete, so skipping it silently undercounts a scope and + // hands back budget. It is counted, and a configured limit refuses on it below. + if (index < lines.length - 1) corruptRecords += 1; + continue; + } + switch (record.kind) { + case "reserve": applyReserve(record.send, record.targets, sanitizeTokens(record.tokens), record.at); break; + case "dispatch": applyDispatch(record.send, record.at); break; + case "settle": applyResolve(record.send, "settled", sanitizeTokens(record.tokens), record.at); break; + case "lost": applyResolve(record.send, "lost", 0, record.at); break; + case "abandon": applyResolve(record.send, "abandoned", 0, record.at); break; + case "forget": applyForget(record.send); break; + case "drop": applyDrop(record.scope, record.alias); break; + case "checkpoint": applyCheckpoint(record); break; + } + } + // A reservation that survived replay has no owner left. The process that made it is gone, + // so nothing in this one can ever settle it, and leaving it live means the send stays + // pending forever against a scope that can never resolve it. Deleting the entry is not the + // alternative either: that would hand the same send id a second reservation. + // + // Both live states resolve to UNRESOLVED, including an undispatched one. The tempting + // distinction -- open never reached the wire, so give its tokens back -- assumes the + // journal is complete up to the crash, and the torn-tail handling above says it is not: a + // send can dispatch and die before its dispatch record lands. Abandoning that reservation + // returns tokens for a send that may have been billed, and worse, it RESETS a ceiling that + // had already fired. An exhausted scope staying exhausted across a restart is the whole + // reason this store is on disk. + const reconciledAt = now(); + for (const [send, reservation] of reservations) { + if (!isLive(reservation.status)) continue; + applyResolve(send, "lost", 0, reconciledAt); + append({ v: 1, kind: "lost", send, at: reconciledAt }); + } + } + + /** + * Bounded cleanup. It runs before every admission, so nothing depends on a caller + * remembering `prune()` -- the first draft exported one and no production path called it. + * Every removal writes a tombstone: without one, replay rebuilds precisely what cleanup + * removed and the file keeps growing while the maps look bounded. + * + * `force` is the at-capacity pass. It ignores the retention window but never the safety + * rule: an ACTIVE or EXHAUSTED scope is not a candidate at any pressure, because dropping + * one hands it a fresh allowance under the same id. When that leaves nothing to remove, + * the caller refuses admission rather than making room by forgetting a spent scope. + */ + const evictScopes = (at: number, force: boolean): number => { + const cutoff = at - policy.retentionMs; + const candidates: { key: string; scope: SpendScope; alias: string; seenAt: number }[] = []; + for (const [key, state] of scopes) { + const separator = key.indexOf("\0"); + const scope = key.slice(0, separator) as SpendScope; + if (state.reserved > 0) continue; + if (isExhausted(scope, state)) continue; + if (!force && state.lastSeenAt >= cutoff) continue; + candidates.push({ key, scope, alias: key.slice(separator + 1), seenAt: state.lastSeenAt }); + } + if (force) { + candidates.sort((a, b) => a.seenAt - b.seenAt); + candidates.length = Math.min(candidates.length, 1); + } + for (const candidate of candidates) { + scopes.delete(candidate.key); + append({ v: 1, kind: "drop", scope: candidate.scope, alias: candidate.alias, at }); + } + return candidates.length; + }; + + /** + * Forget resolved send ids. A forgotten id is forgotten COMPLETELY: reusing it later books + * a fresh reservation against every scope, which is conservative. The state this must never + * produce is the middle one -- an id the ledger recognises but charges nothing for. + */ + const evictSends = (at: number, force: boolean): number => { + const cutoff = at - policy.retentionMs; + const candidates: { send: string; resolvedAt: number }[] = []; + for (const [send, reservation] of reservations) { + if (isLive(reservation.status)) continue; + if (!force && reservation.resolvedAt >= cutoff) continue; + candidates.push({ send, resolvedAt: reservation.resolvedAt }); + } + if (force) { + candidates.sort((a, b) => a.resolvedAt - b.resolvedAt); + candidates.length = Math.min(candidates.length, 1); + } + for (const candidate of candidates) { + reservations.delete(candidate.send); + append({ v: 1, kind: "forget", send: candidate.send, at }); + } + return candidates.length; + }; + + /** + * Replace the journal with a single checkpoint once it has grown past its record budget. + * Bounded maps are not enough on their own: the file behind them is what replay reads, and + * an uncompacted file grows forever on unique root and send ids. + */ + const compact = (at: number): void => { + const rewrite = journal?.rewrite; + if (!journal || !rewrite || recordsOnDisk < compactAfterRecords()) return; + const checkpoint: JournalRecord = { + v: 1, + kind: "checkpoint", + at, + scopes: [...scopes].map(([key, state]) => { + const separator = key.indexOf("\0"); + return { + scope: key.slice(0, separator) as SpendScope, + alias: key.slice(separator + 1), + settled: state.settled, + unresolved: state.unresolved, + seenAt: state.lastSeenAt, + }; + }), + sends: [...reservations].map(([send, reservation]) => ({ + send, + status: reservation.status, + targets: [...reservation.targets], + tokens: reservation.tokens, + at: reservation.at, + resolvedAt: reservation.resolvedAt, + })), + }; + try { + rewrite.call(journal, [JSON.stringify(checkpoint)]); + recordsOnDisk = 1; + } catch (error) { + if (error instanceof SpendLedgerOwnerError) throw error; + // Compaction is maintenance, not accounting: a failed rewrite leaves the previous + // journal intact and every figure in it still replayable. + persistFailures += 1; + } + }; + + /** The denial when tracking cannot fit this request, or undefined when it can. */ + const makeRoom = (refs: readonly ScopeRef[], at: number): SpendDenial | undefined => { + evictSends(at, false); + evictScopes(at, false); + while (reservations.size >= maxTrackedSends()) { + if (evictSends(at, true) === 0) return { reason: "tracking-capacity-exhausted" }; + } + let fresh = 0; + for (const ref of refs) if (!scopes.has(scopeKey(ref.scope, ref.alias))) fresh += 1; + while (scopes.size + fresh > maxTrackedScopes()) { + if (evictScopes(at, true) === 0) { + return { reason: "tracking-capacity-exhausted", scope: refs[0]?.scope }; + } + } + return undefined; + }; + + return { + // Every figure this ledger reports describes a journal it must still own. Reporting one + // after ownership ended is the same error as writing then, with a quieter symptom. + get persistFailures() { assertOwnedAccounting?.(); return persistFailures; }, + get corruptRecords() { assertOwnedAccounting?.(); return corruptRecords; }, + get degraded() { assertOwnedAccounting?.(); return persistFailures > 0 || corruptRecords > 0; }, + get policy() { assertOwnedAccounting?.(); return policy; }, + + reserve(request: SpendReservationRequest): SpendReservationDecision { + assertOwnedAccounting?.(); + const tokens = sanitizeTokens(request.inputTokens) + sanitizeTokens(request.outputCeilingTokens); + const at = request.at ?? now(); + const send = aliasFor("send", request.sendId); + const refs = refsFor(request.scopes); + const enforced = refs.some((ref) => limitFor(ref.scope) !== undefined); + + // A send id this ledger already knows is REFUSED. Returning success while booking + // nothing -- the old behaviour -- let one id authorise an unlimited number of physical + // sends with the scope totals never moving. + if (reservations.has(send)) { + return { reserved: false, denial: { reason: "duplicate-send-id", sendId: request.sendId } }; + } + // Replay could not prove these totals are complete, so a configured ceiling cannot be + // enforced on them. Observe-only accounting continues and reports the degradation. + if (enforced && corruptRecords > 0) { + return { reserved: false, denial: { reason: "journal-corrupt", corruptRecords } }; + } + const capacity = makeRoom(refs, at); + if (capacity) return { reserved: false, denial: capacity }; + + // Check every scope before mutating any: a refusal must not leave a partial + // reservation booked on the scopes that would have passed. Reading state without + // creating it matters here -- a denied request must not leave a tracked scope behind. + for (const ref of refs) { + // A recorded send has no limit to fail: it already happened, and the point of booking + // it is to let the total go OVER the ceiling so the next request can be refused. + const limit = request.alreadySent === true ? undefined : limitFor(ref.scope); + if (limit === undefined) continue; + const state = scopes.get(scopeKey(ref.scope, ref.alias)); + const projected = (state ? state.settled + state.reserved + state.unresolved : 0) + tokens; + if (projected > limit) { + const scopeId = ref.scope === "root" + ? request.scopes.rootId + : ref.scope === "identity" ? request.scopes.identityId : request.scopes.poolId; + return { + reserved: false, + denial: { reason: "spend-limit-exceeded", scope: ref.scope, scopeId: scopeId ?? "", limit, projected }, + }; + } + } + + // Durability BEFORE admission. The record goes to disk first, and under a configured + // limit a failed write refuses the request rather than admitting one that a restart + // would forget -- which is exactly the disk-full and permission case durability is for. + const durable = append({ v: 1, kind: "reserve", send, targets: refs, tokens, at }); + if (!durable && enforced && request.alreadySent !== true) { + return { reserved: false, denial: { reason: "reserve-not-durable", sendId: request.sendId } }; + } + applyReserve(send, refs, tokens, at); + compact(at); + return { reserved: true, sendId: request.sendId, tokens, durable }; + }, + + markDispatched(sendId: string): boolean { + assertOwnedAccounting?.(); + const send = aliasFor("send", sendId); + const reservation = reservations.get(send); + if (!reservation || reservation.status !== "open") return false; + const at = now(); + applyDispatch(send, at); + append({ v: 1, kind: "dispatch", send, at }); + return true; + }, + + abandon(sendId: string): boolean { + assertOwnedAccounting?.(); + const send = aliasFor("send", sendId); + const reservation = reservations.get(send); + // Only an UNDISPATCHED reservation may be released for free. Once bytes have left for + // upstream the tokens may already be billed, so the caller owes settle or markLost. + if (!reservation || reservation.status !== "open") return false; + const at = now(); + applyResolve(send, "abandoned", 0, at); + append({ v: 1, kind: "abandon", send, at }); + return true; + }, + + settle(sendId: string, usage: SpendUsage): boolean { + assertOwnedAccounting?.(); + const send = aliasFor("send", sendId); + const reservation = reservations.get(send); + if (!reservation || !isLive(reservation.status)) return false; + const tokens = sanitizeTokens(usage.inputTokens) + sanitizeTokens(usage.outputTokens); + const at = now(); + applyResolve(send, "settled", tokens, at); + append({ v: 1, kind: "settle", send, tokens, at }); + return true; + }, + + markLost(sendId: string): boolean { + assertOwnedAccounting?.(); + const send = aliasFor("send", sendId); + const reservation = reservations.get(send); + if (!reservation || !isLive(reservation.status)) return false; + const at = now(); + applyResolve(send, "lost", 0, at); + append({ v: 1, kind: "lost", send, at }); + return true; + }, + + knows(sendId: string): boolean { + assertOwnedAccounting?.(); + return reservations.has(aliasFor("send", sendId)); + }, + + snapshot(scope: SpendScope, scopeId: string): ScopeSpendSnapshot | undefined { + // Reading accounting from a handle whose ownership has ended is as wrong as writing it: + // the figures describe a journal this process no longer owns. + assertOwnedAccounting?.(); + const state = scopes.get(scopeKey(scope, aliasFor(scope, scopeId))); + if (!state) return undefined; + return { + settled: state.settled, + reserved: state.reserved, + unresolved: state.unresolved, + exhausted: isExhausted(scope, state), + }; + }, + + exhausted(scope: SpendScope, scopeId: string): boolean { + assertOwnedAccounting?.(); + const state = scopes.get(scopeKey(scope, aliasFor(scope, scopeId))); + return state !== undefined && isExhausted(scope, state); + }, + + prune(at: number = now()): void { + assertOwnedAccounting?.(); + // Removal requires BOTH inactive and not exhausted inside the window. An + // exhausted-but-idle scope that was dropped would be recreated fresh under the + // same id -- the exact laundering the ceiling exists to stop. + evictSends(at, false); + evictScopes(at, false); + }, + + reconfigure(next: SpendReservationPolicy): void { + assertOwnedAccounting?.(); + policy = next; + }, + }; +} + +let sharedLedger: SpendReservationLedger | undefined; +/** + * The operator policy in effect. Held beside the ledger rather than inside it because the + * ledger is built lazily: a configured ceiling has to be remembered from startup until the + * first request that actually reserves, and an install that configures nothing must still + * open no journal. + */ +let sharedPolicy: SpendReservationPolicy = DEFAULT_SPEND_RESERVATION_POLICY; + +/** Whether any scope carries a ceiling -- that is, whether anything at all can be refused. */ +export function spendCeilingsConfigured(policy: SpendReservationPolicy = sharedPolicy): boolean { + return policy.root.maxTokens !== undefined + || policy.identity.maxTokens !== undefined + || policy.pool.maxTokens !== undefined; +} + +/** The policy the process-wide ledger enforces right now. */ +export function sharedSpendPolicy(): SpendReservationPolicy { + return sharedPolicy; +} + +const spendScopeLimitFromConfig = (scope: OcxSpendScopeConfig | undefined): SpendScopeLimit => + scope?.maxTokens !== undefined && Number.isFinite(scope.maxTokens) && scope.maxTokens > 0 + ? { maxTokens: Math.trunc(scope.maxTokens) } + : {}; + +/** + * The ledger policy an operator's `spend` section asks for. + * + * An absent section, an empty one, and one whose every ceiling is absent all produce the + * unconfigured default: observe-only accounting that refuses nothing. That equivalence is the + * load-bearing part. This ledger is on and journaling by default, so shipping a default + * ceiling would start refusing real traffic on the first upgrade that ran this code, against + * a number nobody chose. There is deliberately no default figure here at all. + */ +export function spendPolicyFromConfig(spend: OcxSpendConfig | undefined): SpendReservationPolicy { + const retentionDays = spend?.retentionDays; + return { + root: spendScopeLimitFromConfig(spend?.root), + identity: spendScopeLimitFromConfig(spend?.identity), + pool: spendScopeLimitFromConfig(spend?.pool), + retentionMs: retentionDays !== undefined && Number.isFinite(retentionDays) && retentionDays > 0 + ? Math.trunc(retentionDays) * 24 * 60 * 60_000 + : DEFAULT_SPEND_RESERVATION_POLICY.retentionMs, + }; +} + +/** + * Apply an operator policy to the process-wide ledger. + * + * Startup calls this with the loaded config, and a reload may call it again: the ledger keeps + * every figure it has already accounted, so changing a ceiling changes what is refused from + * here on and never what was spent. It does not CREATE the ledger -- an install that + * configures no ceiling must not open a journal merely because the server started. + */ +export function configureSharedSpendLedger(policy: SpendReservationPolicy): void { + // Recording a policy value touches no journal, so it needs no ownership. Changing a ledger + // that already exists does, because that ledger is a live view of an owned directory. + if (sharedLedger) { + assertSpendLedgerOwnerHeld(); + bindSpendLedgerOwnerHome(); + } + sharedPolicy = policy; + sharedLedger?.reconfigure(policy); +} + +/** + * Process-wide ledger backed by the journal under OPENCODEX_HOME. Created lazily so + * importing the module -- or running a request path that never reserves -- touches no + * disk. One directory at a time, not one directory for the life of the process: the singleton + * is discarded when its ownership ends, so a later directory replays its own journal rather + * than inheriting figures from the previous one. + */ +export function sharedSpendLedger(): SpendReservationLedger { + assertSpendLedgerOwnerHeld(); + bindSpendLedgerOwnerHome(); + if (!sharedLedger) { + // Minted by the owner module from the directory it actually owns, and carrying the exact + // ownership they were minted under. Nothing here chooses a path or supplies its own guard. + const journalStorage = mintSpendLedgerStorage(SPEND_LEDGER_JOURNAL_FILENAME); + const saltStorage = mintSpendLedgerStorage(SPEND_LEDGER_SALT_FILENAME); + const journalPath = spendLedgerStoragePath(journalStorage); + const saltPath = spendLedgerStoragePath(saltStorage); + const assertOwnedAccounting = (): void => { + assertStorageOwned(journalStorage); + if (ledgerEntryExists(journalPath)) assertSafeLedgerFile(journalPath, "journal"); + if (ledgerEntryExists(saltPath)) assertSafeLedgerFile(saltPath, "salt"); + }; + sharedLedger = createSpendReservationLedger({ + journal: createOwnedFileSpendJournal(journalStorage), + salt: loadOrCreateSpendLedgerSalt(saltStorage), + policy: sharedPolicy, + assertOwnedAccounting, + }); + } + return sharedLedger; +} + +const MAX_DIAGNOSTIC_ERROR_COUNT = 1_000_000; + +/** Scalar-only and side-effect-free: reading diagnostics never constructs or replays. */ +export function spendLedgerDiagnosticsSnapshot(): { + readonly ownership: "held" | "unheld"; + readonly initialized: boolean; + readonly configured: boolean; + readonly degraded: boolean; + readonly persistFailures: number; + readonly corruptRecords: number; +} { + const ledger = sharedLedger; + const bounded = (value: number): number => Math.min(MAX_DIAGNOSTIC_ERROR_COUNT, Math.max(0, value)); + return { + ...spendLedgerOwnerSnapshot(), + initialized: ledger !== undefined, + configured: spendCeilingsConfigured(sharedPolicy), + degraded: ledger?.degraded ?? false, + persistFailures: bounded(ledger?.persistFailures ?? 0), + corruptRecords: bounded(ledger?.corruptRecords ?? 0), + }; +} + +/** + * Test seam for discarding the singleton outright. + * + * Production discards it too, but only with the ownership it belongs to, which is what + * `onSpendLedgerOwnerReleased` does. Neither path resets a spent budget: the journal is the + * durable record and the next construction replays it. + */ +export function resetSharedSpendLedgerForTest(): void { + sharedLedger = undefined; + sharedPolicy = DEFAULT_SPEND_RESERVATION_POLICY; + resetSpendLedgerOwnerBindingForTest(); +} diff --git a/tests/fixtures/spend-ledger-f7a50dc3.ts.txt b/tests/fixtures/spend-ledger-f7a50dc3.ts.txt deleted file mode 100644 index 48c5dae3d18..00000000000 --- a/tests/fixtures/spend-ledger-f7a50dc3.ts.txt +++ /dev/null @@ -1,586 +0,0 @@ -// Unchanged parser and ledger function excerpts from lidge-jun/opencodex -// f7a50dc35b493d29f7a5c6bf2a459e5441549580:src/lib/spend-reservation-ledger.ts -// Frozen deliberately: tests execute the old implementation, never emulate it. -const isCountable = (value: unknown): value is number => - typeof value === "number" && Number.isFinite(value) && value >= 0; - -const isAlias = (value: unknown): value is string => - typeof value === "string" && value.length > 0 && value.length <= 256; - -const isScopeName = (value: unknown): value is SpendScope => - value === "root" || value === "identity" || value === "pool"; - -const isStatus = (value: unknown): value is ReservationStatus => - value === "open" || value === "dispatched" || value === "settled" - || value === "lost" || value === "abandoned"; - -const parseTargets = (value: unknown): ScopeRef[] | undefined => { - if (!Array.isArray(value) || value.length > 3) return undefined; - const targets: ScopeRef[] = []; - for (const entry of value) { - if (typeof entry !== "object" || entry === null) return undefined; - const { scope, alias } = entry as { scope?: unknown; alias?: unknown }; - if (!isScopeName(scope) || !isAlias(alias)) return undefined; - targets.push({ scope, alias }); - } - return targets; -}; - -/** - * Validate one journal line into a record, or reject it. - * - * Exported because this is the boundary where a hostile or damaged file meets the accounting: - * `JSON.parse(line) as JournalRecord` type-asserts a lie, and a bare `null` line or a - * `{"v":1,"kind":"reserve"}` with no fields crashed the rebuild rather than being rejected. - * Every field is checked, including that numbers are finite and non-negative. - */ -export function parseSpendJournalRecord(line: string): JournalRecord | undefined { - let raw: unknown; - try { - raw = JSON.parse(line); - } catch { - return undefined; - } - if (typeof raw !== "object" || raw === null || Array.isArray(raw)) return undefined; - const record = raw as Record; - if (record.v !== 1) return undefined; - if (!isCountable(record.at)) return undefined; - const at = record.at; - switch (record.kind) { - case "reserve": { - const targets = parseTargets(record.targets); - if (!isAlias(record.send) || targets === undefined || !isCountable(record.tokens)) return undefined; - return { v: 1, kind: "reserve", send: record.send, targets, tokens: record.tokens, at }; - } - case "settle": - if (!isAlias(record.send) || !isCountable(record.tokens)) return undefined; - return { v: 1, kind: "settle", send: record.send, tokens: record.tokens, at }; - case "dispatch": - if (!isAlias(record.send)) return undefined; - return { v: 1, kind: "dispatch", send: record.send, at }; - case "lost": - if (!isAlias(record.send)) return undefined; - return { v: 1, kind: "lost", send: record.send, at }; - case "abandon": - if (!isAlias(record.send)) return undefined; - return { v: 1, kind: "abandon", send: record.send, at }; - case "forget": - if (!isAlias(record.send)) return undefined; - return { v: 1, kind: "forget", send: record.send, at }; - case "drop": - if (!isScopeName(record.scope) || !isAlias(record.alias)) return undefined; - return { v: 1, kind: "drop", scope: record.scope, alias: record.alias, at }; - case "checkpoint": { - if (!Array.isArray(record.scopes) || !Array.isArray(record.sends)) return undefined; - const scopes: { scope: SpendScope; alias: string; settled: number; unresolved: number; seenAt: number }[] = []; - for (const entry of record.scopes) { - if (typeof entry !== "object" || entry === null) return undefined; - const e = entry as Record; - if (!isScopeName(e.scope) || !isAlias(e.alias)) return undefined; - if (!isCountable(e.settled) || !isCountable(e.unresolved) || !isCountable(e.seenAt)) return undefined; - scopes.push({ scope: e.scope, alias: e.alias, settled: e.settled, unresolved: e.unresolved, seenAt: e.seenAt }); - } - const sends: { send: string; status: ReservationStatus; targets: ScopeRef[]; tokens: number; at: number; resolvedAt: number }[] = []; - for (const entry of record.sends) { - if (typeof entry !== "object" || entry === null) return undefined; - const e = entry as Record; - const targets = parseTargets(e.targets); - if (!isAlias(e.send) || !isStatus(e.status) || targets === undefined) return undefined; - if (!isCountable(e.tokens) || !isCountable(e.at) || !isCountable(e.resolvedAt)) return undefined; - sends.push({ send: e.send, status: e.status, targets, tokens: e.tokens, at: e.at, resolvedAt: e.resolvedAt }); - } - return { v: 1, kind: "checkpoint", at, scopes, sends }; - } - default: - return undefined; - } -} - -const scopeKey = (scope: SpendScope, alias: string): string => scope + "\0" + alias; - -const sanitizeTokens = (value: number): number => - Number.isFinite(value) ? Math.max(0, Math.trunc(value)) : 0; - -export function createSpendReservationLedger(options: { - readonly journal?: SpendJournal; - readonly policy?: SpendReservationPolicy; - readonly now?: () => number; - /** - * Per-install alias salt. Production passes the file-backed value from - * `loadOrCreateSpendLedgerSalt`; an empty default is for in-memory journals, which have - * no file anyone could correlate. - */ - readonly salt?: string; - /** - * Shared production ledgers prove their exact ownership before reading or changing - * accounting. Identity, not just the directory name: a handle kept across a release and a - * reacquire describes a journal another writer may have changed in between. - */ - readonly assertOwnedAccounting?: () => void; -} = {}): SpendReservationLedger { - // Mutable because the ceilings are operator configuration, and configuration is reloadable. - // The three bounds below are read through functions for the same reason: a value captured - // at construction would answer for the policy this ledger was BUILT with, and an operator - // who raised a bound would keep the old one until the process restarted. - let policy = options.policy ?? DEFAULT_SPEND_RESERVATION_POLICY; - const journal = options.journal; - const now = options.now ?? (() => Date.now()); - const salt = options.salt ?? ""; - const assertOwnedAccounting = options.assertOwnedAccounting; - const maxTrackedScopes = (): number => policy.maxTrackedScopes ?? DEFAULT_MAX_TRACKED_SCOPES; - const maxTrackedSends = (): number => policy.maxTrackedSends ?? DEFAULT_MAX_TRACKED_SENDS; - const compactAfterRecords = (): number => policy.compactAfterRecords ?? DEFAULT_COMPACT_AFTER_RECORDS; - const scopes = new Map(); - const reservations = new Map(); - let persistFailures = 0; - let corruptRecords = 0; - let recordsOnDisk = 0; - - /** - * Salted alias for one identifier. The raw value -- a client-supplied root header, a - * credential id, a pool name -- never leaves this function, so nothing identifying is - * written to disk or held in a map key. - */ - const aliasFor = (kind: SpendScope | "send", id: string): string => - createHash("sha256").update(salt).update("\u0000").update(kind).update("\u0000").update(id) - .digest("hex").slice(0, 32); - - const scopeState = (scope: SpendScope, alias: string): ScopeState => { - const key = scopeKey(scope, alias); - let state = scopes.get(key); - if (!state) { - state = { settled: 0, reserved: 0, unresolved: 0, lastSeenAt: 0 }; - scopes.set(key, state); - } - return state; - }; - - const limitFor = (scope: SpendScope): number | undefined => policy[scope].maxTokens; - - const isExhausted = (scope: SpendScope, state: ScopeState): boolean => { - const limit = limitFor(scope); - return limit !== undefined && state.settled + state.reserved + state.unresolved >= limit; - }; - - /** The scopes a request touches, as aliases. Creates no state: a refusal must leave none. */ - const refsFor = (targets: SpendScopes): ScopeRef[] => { - const refs: ScopeRef[] = []; - if (targets.rootId !== undefined) refs.push({ scope: "root", alias: aliasFor("root", targets.rootId) }); - if (targets.identityId !== undefined) refs.push({ scope: "identity", alias: aliasFor("identity", targets.identityId) }); - if (targets.poolId !== undefined) refs.push({ scope: "pool", alias: aliasFor("pool", targets.poolId) }); - return refs; - }; - - /** - * Returns whether the record reached storage. With no journal there is nothing to fail, - * and the caller's durability question is vacuously satisfied. - */ - const append = (record: JournalRecord): boolean => { - if (!journal) return true; - try { - journal.append(JSON.stringify(record)); - recordsOnDisk += 1; - return true; - } catch (error) { - if (error instanceof SpendLedgerOwnerError) throw error; - // In-memory state still bounds this process; the counter is how a caller learns the - // restart guarantee degraded instead of discovering it after the fact. - persistFailures += 1; - return false; - } - }; - - const applyReserve = (send: string, targets: readonly ScopeRef[], tokens: number, at: number): void => { - if (reservations.has(send)) return; - reservations.set(send, { targets, tokens, status: "open", at, resolvedAt: at }); - for (const ref of targets) { - const state = scopeState(ref.scope, ref.alias); - state.reserved += tokens; - state.lastSeenAt = Math.max(state.lastSeenAt, at); - } - }; - - const isLive = (status: ReservationStatus): boolean => status === "open" || status === "dispatched"; - - /** - * Resolve a live reservation. `settled` books the real figure, `lost` keeps the whole - * reservation as unresolved spend because it may have been billed, and `abandoned` - * releases it because no byte ever left this process. - */ - const applyResolve = (send: string, outcome: "settled" | "lost" | "abandoned", tokens: number, at: number): void => { - const reservation = reservations.get(send); - if (!reservation || !isLive(reservation.status)) return; - reservation.status = outcome; - reservation.resolvedAt = at; - for (const ref of reservation.targets) { - const state = scopeState(ref.scope, ref.alias); - state.reserved = Math.max(0, state.reserved - reservation.tokens); - if (outcome === "lost") state.unresolved += reservation.tokens; - else if (outcome === "settled") state.settled += tokens; - state.lastSeenAt = Math.max(state.lastSeenAt, at); - } - }; - - const applyDispatch = (send: string, at: number): void => { - const reservation = reservations.get(send); - if (!reservation || reservation.status !== "open") return; - reservation.status = "dispatched"; - reservation.resolvedAt = at; - }; - - /** Tombstone replay: the entry is gone, so a later reuse of the id books a fresh charge. */ - const applyForget = (send: string): void => { - const reservation = reservations.get(send); - if (!reservation || isLive(reservation.status)) return; - reservations.delete(send); - }; - - const applyDrop = (scope: SpendScope, alias: string): void => { - const state = scopes.get(scopeKey(scope, alias)); - if (!state || state.reserved > 0) return; - scopes.delete(scopeKey(scope, alias)); - }; - - const applyCheckpoint = (record: Extract): void => { - scopes.clear(); - reservations.clear(); - for (const entry of record.scopes) { - scopes.set(scopeKey(entry.scope, entry.alias), { - settled: entry.settled, - reserved: 0, - unresolved: entry.unresolved, - lastSeenAt: entry.seenAt, - }); - } - for (const entry of record.sends) { - // `reserved` is rebuilt from the live entries rather than trusted from the snapshot, - // so the two can never disagree about the same tokens. - if (isLive(entry.status)) { - applyReserve(entry.send, entry.targets, entry.tokens, entry.at); - if (entry.status === "dispatched") applyDispatch(entry.send, entry.resolvedAt); - continue; - } - reservations.set(entry.send, { - targets: entry.targets, - tokens: entry.tokens, - status: entry.status, - at: entry.at, - resolvedAt: entry.resolvedAt, - }); - } - }; - - // Rebuild from the journal before serving: an exhausted scope must still be exhausted - // after a restart, which is the whole reason this store exists. - if (journal) { - const lines = journal.read(); - recordsOnDisk = lines.length; - for (let index = 0; index < lines.length; index += 1) { - const line = lines[index] as string; - const record = parseSpendJournalRecord(line); - if (!record) { - // A rejected FINAL line is a torn tail write -- the process died between the write - // and its newline -- and is dropped quietly, because that record never completed and - // therefore never authorised anything. A rejected line ANYWHERE ELSE is different: - // the records after it did complete, so skipping it silently undercounts a scope and - // hands back budget. It is counted, and a configured limit refuses on it below. - if (index < lines.length - 1) corruptRecords += 1; - continue; - } - switch (record.kind) { - case "reserve": applyReserve(record.send, record.targets, sanitizeTokens(record.tokens), record.at); break; - case "dispatch": applyDispatch(record.send, record.at); break; - case "settle": applyResolve(record.send, "settled", sanitizeTokens(record.tokens), record.at); break; - case "lost": applyResolve(record.send, "lost", 0, record.at); break; - case "abandon": applyResolve(record.send, "abandoned", 0, record.at); break; - case "forget": applyForget(record.send); break; - case "drop": applyDrop(record.scope, record.alias); break; - case "checkpoint": applyCheckpoint(record); break; - } - } - // A reservation that survived replay has no owner left. The process that made it is gone, - // so nothing in this one can ever settle it, and leaving it live means the send stays - // pending forever against a scope that can never resolve it. Deleting the entry is not the - // alternative either: that would hand the same send id a second reservation. - // - // Both live states resolve to UNRESOLVED, including an undispatched one. The tempting - // distinction -- open never reached the wire, so give its tokens back -- assumes the - // journal is complete up to the crash, and the torn-tail handling above says it is not: a - // send can dispatch and die before its dispatch record lands. Abandoning that reservation - // returns tokens for a send that may have been billed, and worse, it RESETS a ceiling that - // had already fired. An exhausted scope staying exhausted across a restart is the whole - // reason this store is on disk. - const reconciledAt = now(); - for (const [send, reservation] of reservations) { - if (!isLive(reservation.status)) continue; - applyResolve(send, "lost", 0, reconciledAt); - append({ v: 1, kind: "lost", send, at: reconciledAt }); - } - } - - /** - * Bounded cleanup. It runs before every admission, so nothing depends on a caller - * remembering `prune()` -- the first draft exported one and no production path called it. - * Every removal writes a tombstone: without one, replay rebuilds precisely what cleanup - * removed and the file keeps growing while the maps look bounded. - * - * `force` is the at-capacity pass. It ignores the retention window but never the safety - * rule: an ACTIVE or EXHAUSTED scope is not a candidate at any pressure, because dropping - * one hands it a fresh allowance under the same id. When that leaves nothing to remove, - * the caller refuses admission rather than making room by forgetting a spent scope. - */ - const evictScopes = (at: number, force: boolean): number => { - const cutoff = at - policy.retentionMs; - const candidates: { key: string; scope: SpendScope; alias: string; seenAt: number }[] = []; - for (const [key, state] of scopes) { - const separator = key.indexOf("\0"); - const scope = key.slice(0, separator) as SpendScope; - if (state.reserved > 0) continue; - if (isExhausted(scope, state)) continue; - if (!force && state.lastSeenAt >= cutoff) continue; - candidates.push({ key, scope, alias: key.slice(separator + 1), seenAt: state.lastSeenAt }); - } - if (force) { - candidates.sort((a, b) => a.seenAt - b.seenAt); - candidates.length = Math.min(candidates.length, 1); - } - for (const candidate of candidates) { - scopes.delete(candidate.key); - append({ v: 1, kind: "drop", scope: candidate.scope, alias: candidate.alias, at }); - } - return candidates.length; - }; - - /** - * Forget resolved send ids. A forgotten id is forgotten COMPLETELY: reusing it later books - * a fresh reservation against every scope, which is conservative. The state this must never - * produce is the middle one -- an id the ledger recognises but charges nothing for. - */ - const evictSends = (at: number, force: boolean): number => { - const cutoff = at - policy.retentionMs; - const candidates: { send: string; resolvedAt: number }[] = []; - for (const [send, reservation] of reservations) { - if (isLive(reservation.status)) continue; - if (!force && reservation.resolvedAt >= cutoff) continue; - candidates.push({ send, resolvedAt: reservation.resolvedAt }); - } - if (force) { - candidates.sort((a, b) => a.resolvedAt - b.resolvedAt); - candidates.length = Math.min(candidates.length, 1); - } - for (const candidate of candidates) { - reservations.delete(candidate.send); - append({ v: 1, kind: "forget", send: candidate.send, at }); - } - return candidates.length; - }; - - /** - * Replace the journal with a single checkpoint once it has grown past its record budget. - * Bounded maps are not enough on their own: the file behind them is what replay reads, and - * an uncompacted file grows forever on unique root and send ids. - */ - const compact = (at: number): void => { - const rewrite = journal?.rewrite; - if (!journal || !rewrite || recordsOnDisk < compactAfterRecords()) return; - const checkpoint: JournalRecord = { - v: 1, - kind: "checkpoint", - at, - scopes: [...scopes].map(([key, state]) => { - const separator = key.indexOf("\0"); - return { - scope: key.slice(0, separator) as SpendScope, - alias: key.slice(separator + 1), - settled: state.settled, - unresolved: state.unresolved, - seenAt: state.lastSeenAt, - }; - }), - sends: [...reservations].map(([send, reservation]) => ({ - send, - status: reservation.status, - targets: [...reservation.targets], - tokens: reservation.tokens, - at: reservation.at, - resolvedAt: reservation.resolvedAt, - })), - }; - try { - rewrite.call(journal, [JSON.stringify(checkpoint)]); - recordsOnDisk = 1; - } catch (error) { - if (error instanceof SpendLedgerOwnerError) throw error; - // Compaction is maintenance, not accounting: a failed rewrite leaves the previous - // journal intact and every figure in it still replayable. - persistFailures += 1; - } - }; - - /** The denial when tracking cannot fit this request, or undefined when it can. */ - const makeRoom = (refs: readonly ScopeRef[], at: number): SpendDenial | undefined => { - evictSends(at, false); - evictScopes(at, false); - while (reservations.size >= maxTrackedSends()) { - if (evictSends(at, true) === 0) return { reason: "tracking-capacity-exhausted" }; - } - let fresh = 0; - for (const ref of refs) if (!scopes.has(scopeKey(ref.scope, ref.alias))) fresh += 1; - while (scopes.size + fresh > maxTrackedScopes()) { - if (evictScopes(at, true) === 0) { - return { reason: "tracking-capacity-exhausted", scope: refs[0]?.scope }; - } - } - return undefined; - }; - - return { - // Every figure this ledger reports describes a journal it must still own. Reporting one - // after ownership ended is the same error as writing then, with a quieter symptom. - get persistFailures() { assertOwnedAccounting?.(); return persistFailures; }, - get corruptRecords() { assertOwnedAccounting?.(); return corruptRecords; }, - get degraded() { assertOwnedAccounting?.(); return persistFailures > 0 || corruptRecords > 0; }, - get policy() { assertOwnedAccounting?.(); return policy; }, - - reserve(request: SpendReservationRequest): SpendReservationDecision { - assertOwnedAccounting?.(); - const tokens = sanitizeTokens(request.inputTokens) + sanitizeTokens(request.outputCeilingTokens); - const at = request.at ?? now(); - const send = aliasFor("send", request.sendId); - const refs = refsFor(request.scopes); - const enforced = refs.some((ref) => limitFor(ref.scope) !== undefined); - - // A send id this ledger already knows is REFUSED. Returning success while booking - // nothing -- the old behaviour -- let one id authorise an unlimited number of physical - // sends with the scope totals never moving. - if (reservations.has(send)) { - return { reserved: false, denial: { reason: "duplicate-send-id", sendId: request.sendId } }; - } - // Replay could not prove these totals are complete, so a configured ceiling cannot be - // enforced on them. Observe-only accounting continues and reports the degradation. - if (enforced && corruptRecords > 0) { - return { reserved: false, denial: { reason: "journal-corrupt", corruptRecords } }; - } - const capacity = makeRoom(refs, at); - if (capacity) return { reserved: false, denial: capacity }; - - // Check every scope before mutating any: a refusal must not leave a partial - // reservation booked on the scopes that would have passed. Reading state without - // creating it matters here -- a denied request must not leave a tracked scope behind. - for (const ref of refs) { - // A recorded send has no limit to fail: it already happened, and the point of booking - // it is to let the total go OVER the ceiling so the next request can be refused. - const limit = request.alreadySent === true ? undefined : limitFor(ref.scope); - if (limit === undefined) continue; - const state = scopes.get(scopeKey(ref.scope, ref.alias)); - const projected = (state ? state.settled + state.reserved + state.unresolved : 0) + tokens; - if (projected > limit) { - const scopeId = ref.scope === "root" - ? request.scopes.rootId - : ref.scope === "identity" ? request.scopes.identityId : request.scopes.poolId; - return { - reserved: false, - denial: { reason: "spend-limit-exceeded", scope: ref.scope, scopeId: scopeId ?? "", limit, projected }, - }; - } - } - - // Durability BEFORE admission. The record goes to disk first, and under a configured - // limit a failed write refuses the request rather than admitting one that a restart - // would forget -- which is exactly the disk-full and permission case durability is for. - const durable = append({ v: 1, kind: "reserve", send, targets: refs, tokens, at }); - if (!durable && enforced && request.alreadySent !== true) { - return { reserved: false, denial: { reason: "reserve-not-durable", sendId: request.sendId } }; - } - applyReserve(send, refs, tokens, at); - compact(at); - return { reserved: true, sendId: request.sendId, tokens, durable }; - }, - - markDispatched(sendId: string): boolean { - assertOwnedAccounting?.(); - const send = aliasFor("send", sendId); - const reservation = reservations.get(send); - if (!reservation || reservation.status !== "open") return false; - const at = now(); - applyDispatch(send, at); - append({ v: 1, kind: "dispatch", send, at }); - return true; - }, - - abandon(sendId: string): boolean { - assertOwnedAccounting?.(); - const send = aliasFor("send", sendId); - const reservation = reservations.get(send); - // Only an UNDISPATCHED reservation may be released for free. Once bytes have left for - // upstream the tokens may already be billed, so the caller owes settle or markLost. - if (!reservation || reservation.status !== "open") return false; - const at = now(); - applyResolve(send, "abandoned", 0, at); - append({ v: 1, kind: "abandon", send, at }); - return true; - }, - - settle(sendId: string, usage: SpendUsage): boolean { - assertOwnedAccounting?.(); - const send = aliasFor("send", sendId); - const reservation = reservations.get(send); - if (!reservation || !isLive(reservation.status)) return false; - const tokens = sanitizeTokens(usage.inputTokens) + sanitizeTokens(usage.outputTokens); - const at = now(); - applyResolve(send, "settled", tokens, at); - append({ v: 1, kind: "settle", send, tokens, at }); - return true; - }, - - markLost(sendId: string): boolean { - assertOwnedAccounting?.(); - const send = aliasFor("send", sendId); - const reservation = reservations.get(send); - if (!reservation || !isLive(reservation.status)) return false; - const at = now(); - applyResolve(send, "lost", 0, at); - append({ v: 1, kind: "lost", send, at }); - return true; - }, - - knows(sendId: string): boolean { - assertOwnedAccounting?.(); - return reservations.has(aliasFor("send", sendId)); - }, - - snapshot(scope: SpendScope, scopeId: string): ScopeSpendSnapshot | undefined { - // Reading accounting from a handle whose ownership has ended is as wrong as writing it: - // the figures describe a journal this process no longer owns. - assertOwnedAccounting?.(); - const state = scopes.get(scopeKey(scope, aliasFor(scope, scopeId))); - if (!state) return undefined; - return { - settled: state.settled, - reserved: state.reserved, - unresolved: state.unresolved, - exhausted: isExhausted(scope, state), - }; - }, - - exhausted(scope: SpendScope, scopeId: string): boolean { - assertOwnedAccounting?.(); - const state = scopes.get(scopeKey(scope, aliasFor(scope, scopeId))); - return state !== undefined && isExhausted(scope, state); - }, - - prune(at: number = now()): void { - assertOwnedAccounting?.(); - // Removal requires BOTH inactive and not exhausted inside the window. An - // exhausted-but-idle scope that was dropped would be recreated fresh under the - // same id -- the exact laundering the ceiling exists to stop. - evictSends(at, false); - evictScopes(at, false); - }, - - reconfigure(next: SpendReservationPolicy): void { - assertOwnedAccounting?.(); - policy = next; - }, - }; -} diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index afbf30fa553..795496588f6 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -467,7 +467,7 @@ "protocol-settings.test.ts": "config", "config-mutation-lock.test.ts": "config", "config-non-object-backup.test.ts": "config", "config-ownership-uninstall.test.ts": "config", "config-rebase-provenance-writers.test.ts": "config", "config-save-boundary.test.ts": "config", - "config-spend-ceilings.test.ts": "config", "config-user-edits.test.ts": "config", + "config-spend-ceilings.test.ts": "config", "config-spend-pool-collision.test.ts": "config", "config-user-edits.test.ts": "config", "config.test.ts": "server", "configured-native-models.test.ts": "codex-integration", "gui-desktop-sidecar-signing.test.ts": "gui", "linux-desktop-packaged-ci.test.ts": "ci-workflows", "probe-timeout-env.test.ts": "server", "subagent-roster-migration.test.ts": "routing", @@ -928,7 +928,7 @@ "responses-send-budget-errors.test.ts": "responses", "responses-memory-models.test.ts": "responses", "responses-shadow-intercept.test.ts": "responses", "responses-show-thinking-summary.test.ts": "responses", "responses-snapshot-repair-server.test.ts": "responses", "responses-snapshot-repair.test.ts": "responses", - "responses-sparse-terminal-tool-scope.test.ts": "responses", "responses-spend-ledger-wiring.test.ts": "responses", + "responses-sparse-terminal-tool-scope.test.ts": "responses", "responses-spend-ledger-wiring.test.ts": "responses", "responses-spend-contract-c.test.ts": "responses", "responses-spend-anchor.test.ts": "responses", "responses-spend-capacity-guard.test.ts": "responses", "responses-spill-acl-recovery.test.ts": "responses", "responses-spill-inspection.test.ts": "responses", "responses-spill-orphan-sweep.test.ts": "responses", "responses-spill-shutdown-clock.test.ts": "responses", "responses-state-snapshot-select.test.ts": "responses", @@ -1000,7 +1000,7 @@ "spend-ceiling-enforcement.test.ts": "lib", "spend-instrumentation-log.test.ts": "server", "spend-ledger-file-journal.test.ts": "lib", "spend-ledger-lifecycle.test.ts": "server", "spend-ledger-owner-startup.test.ts": "server", "spend-ledger-owner.test.ts": "lib", - "spend-pool-continuity.test.ts": "lib", "spend-reservation-ledger.test.ts": "lib", + "spend-pool-continuity.test.ts": "lib", "spend-contract-c.test.ts": "lib", "spend-corruption-compat.test.ts": "lib", "spend-unbound-expiry.test.ts": "lib", "spend-seed-overflow.test.ts": "lib", "spend-zero-config-compat.test.ts": "lib", "spend-experimental-history.test.ts": "lib", "spend-reservation-ledger.test.ts": "lib", "sponsor-presets.test.ts": "providers", "sse-carriage-return.test.ts": "responses", "sse-client-frame-bounds.test.ts": "responses", "sse-decoder.test.ts": "responses", "sse-failed-tail.test.ts": "responses", diff --git a/tests/helpers/legacy-spend-ledger.ts b/tests/helpers/legacy-spend-ledger.ts deleted file mode 100644 index 2de285d36ed..00000000000 --- a/tests/helpers/legacy-spend-ledger.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { readFileSync } from "node:fs"; -import { createHash } from "node:crypto"; -import { fixturePath } from "./repo-root"; -import { DEFAULT_SPEND_RESERVATION_POLICY, type SpendJournal, type SpendReservationLedger, type SpendReservationPolicy } from "../../src/lib/spend-reservation-ledger"; - -/** Execute immutable pre-continuity production code, with only its ordinary imports supplied. */ -const source = readFileSync(fixturePath("spend-ledger-f7a50dc3.ts.txt"), "utf8"); -const code = new Bun.Transpiler({ loader: "ts", target: "bun" }) - .transformSync(source.replaceAll("export function", "function")); -export const createLegacySpendLedger = new Function( - "createHash", "DEFAULT_SPEND_RESERVATION_POLICY", "DEFAULT_MAX_TRACKED_SCOPES", - "DEFAULT_MAX_TRACKED_SENDS", "DEFAULT_COMPACT_AFTER_RECORDS", "SpendLedgerOwnerError", - `${code}\nreturn createSpendReservationLedger;`, -)(createHash, DEFAULT_SPEND_RESERVATION_POLICY, 4_096, 16_384, 8_192, class SpendLedgerOwnerError extends Error {}) as - (options: { journal: SpendJournal; salt?: string; policy: SpendReservationPolicy; now: () => number }) => - Pick; diff --git a/tests/helpers/shipped-spend-ledger.ts b/tests/helpers/shipped-spend-ledger.ts new file mode 100644 index 00000000000..4027897c06a --- /dev/null +++ b/tests/helpers/shipped-spend-ledger.ts @@ -0,0 +1,36 @@ +import { readFileSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { fixturePath, repoPath } from "./repo-root"; +import { DEFAULT_SPEND_RESERVATION_POLICY, type SpendJournal, type SpendReservationPolicy } from "../../src/lib/spend-reservation-ledger"; + +// Freeze the complete shipped module; adapt relative imports only, never accounting code. +const source = readFileSync(fixturePath("spend-ledger-2-80-0.ts.txt"), "utf8"); +if (createHash("sha256").update(source).digest("hex") !== "fccd58e3e70fde8c4e0b0df919b0718659f5e18893934583cd65338fedaf5b0e") { + throw new Error("Shipped spend-ledger fixture changed"); +} +const adapted = source.replace(/from "(\.[^"]+)"/g, (_, specifier: string) => + `from ${JSON.stringify(pathToFileURL(resolve(repoPath("src/lib"), specifier + ".ts")).href)}`); +const code = new Bun.Transpiler({ loader: "ts", target: "bun" }).transformSync(adapted); +type ShippedModule = Pick; +const shipped = await import(`data:text/javascript;base64,${Buffer.from(code).toString("base64")}`) as ShippedModule; +export const loadShippedSpendLedger = (): ShippedModule => shipped; +export const createShippedSpendLedger = shipped.createSpendReservationLedger; +export const testSpendSalt = "5".repeat(64); +// Independent oracle for the specified salt/NUL/domain wire contract. +export const spendAlias = (kind: string, id: string): string => createHash("sha256") + .update(testSpendSalt + "\0" + kind + "\0" + id).digest("hex").slice(0, 32); +export const spendTestPolicy = (overrides: Partial = {}): SpendReservationPolicy => ({ + ...DEFAULT_SPEND_RESERVATION_POLICY, pool: { maxTokens: 100 }, ...overrides, +}); +export function spendTestJournal(records: unknown[] = []): SpendJournal & { lines: string[]; failAppend: boolean; failRewrite: boolean } { + const lines = records.map(record => typeof record === "string" ? record : JSON.stringify(record)); + return { lines, failAppend: false, failRewrite: false, read() { return [...lines]; }, + append(line) { if (this.failAppend) throw new Error("append unavailable"); lines.push(line); }, + rewrite(next) { if (this.failRewrite) throw new Error("rewrite unavailable"); lines.splice(0, lines.length, ...next); } }; +} +export const spendCheckpoint = (entries: Array<[string, number, number]>, seenAt = 1) => ({ + v: 1, kind: "checkpoint", at: seenAt, + scopes: entries.map(([id, settled, unresolved]) => ({ scope: "pool", alias: spendAlias("pool", id), settled, unresolved, seenAt })), sends: [], +}); diff --git a/tests/lib/spend-contract-c.test.ts b/tests/lib/spend-contract-c.test.ts new file mode 100644 index 00000000000..daf0b82bb26 --- /dev/null +++ b/tests/lib/spend-contract-c.test.ts @@ -0,0 +1,52 @@ +import { expect, test } from "bun:test"; +import { createSpendReservationLedger } from "../../src/lib/spend-reservation-ledger"; +import { createShippedSpendLedger, loadShippedSpendLedger, spendAlias, spendCheckpoint, spendTestJournal, spendTestPolicy, testSpendSalt as salt } from "../helpers/shipped-spend-ledger"; + +const request = (sendId: string, poolId = "P", tokens = 10) => ({ sendId, scopes: { poolId }, inputTokens: tokens, outputCeilingTokens: 0 }); +test("2.80.0 parses every emitted v1 kind and checkpoint status", () => { + const disk = spendTestJournal(); + const ledger = createSpendReservationLedger({ journal: disk, salt, policy: spendTestPolicy({ retentionMs: 10 }), now: () => 2 }); + for (const id of ["open", "dispatched", "settled", "lost", "abandoned"]) expect(ledger.reserve(request(id)).reserved).toBe(true); + ledger.markDispatched("dispatched"); ledger.settle("settled", { inputTokens: 7, outputTokens: 0 }); + ledger.markDispatched("lost"); ledger.markLost("lost"); ledger.abandon("abandoned"); + ledger.reconfigure(spendTestPolicy({ compactAfterRecords: 1, retentionMs: 10 })); + ledger.reserve(request("checkpoint", "P", 0)); + const cp = JSON.parse(disk.lines[0]!); + expect(cp.kind).toBe("checkpoint"); expect(cp.poolContinuity).toBeUndefined(); + expect(new Set(cp.sends.map((s: {status: string}) => s.status))).toEqual(new Set(["open", "dispatched", "settled", "lost", "abandoned"])); + expect(cp.scopes.every((s: {alias: string}) => s.alias === spendAlias("pool", "P"))).toBe(true); + // Collect all eight kinds through ordinary public operations and cleanup. + const all = spendTestJournal(); + const ordinary = createSpendReservationLedger({ journal: all, salt, now: () => 1, policy: spendTestPolicy({ pool: {}, retentionMs: 1 }) }); + for (const id of ["settle", "lost", "abandon"]) { ordinary.reserve(request(id)); if (id !== "abandon") ordinary.markDispatched(id); } + ordinary.settle("settle", { inputTokens: 3, outputTokens: 0 }); ordinary.markLost("lost"); ordinary.abandon("abandon"); ordinary.prune(3); + const lines = [...all.lines, ...disk.lines]; + expect(new Set(lines.map(line => JSON.parse(line).kind))).toEqual(new Set(["reserve", "dispatch", "settle", "lost", "abandon", "forget", "drop", "checkpoint"])); + for (const line of lines) expect(loadShippedSpendLedger().parseSpendJournalRecord(line)).toEqual(JSON.parse(line)); + const old = createShippedSpendLedger({ journal: spendTestJournal(disk.lines), salt, policy: spendTestPolicy(), now: () => 2 }); + expect(old.corruptRecords).toBe(0); expect(old.snapshot("pool", "P")).toMatchObject({ settled: 7, unresolved: 30 }); +}); +test("old compact retain and reupgrade keep legacy per-label totals", () => { + const disk = spendTestJournal([spendCheckpoint([["A", 80, 0], ["P", 20, 0]])]); + const policy = spendTestPolicy({ poolAliases: { [spendAlias("pool", "A")]: "P" }, retentionMs: 10 }); + const next = createSpendReservationLedger({ journal: disk, salt, policy, now: () => 2 }); + expect(next.snapshot("pool", "P")?.settled).toBe(100); + const old = createShippedSpendLedger({ journal: disk, salt, policy: spendTestPolicy({ compactAfterRecords: 1, retentionMs: 10 }), now: () => 2 }); + expect(old.snapshot("pool", "A")?.settled).toBe(80); expect(old.snapshot("pool", "P")?.settled).toBe(20); + old.reserve(request("old", "P", 0)); old.abandon("old"); + expect(createSpendReservationLedger({ journal: disk, salt, policy, now: () => 2 }).snapshot("pool", "P")?.settled).toBe(100); + old.prune(13); old.reserve(request("compact-after-retention", "P", 0)); + expect(createSpendReservationLedger({ journal: disk, salt, policy, now: () => 13 }).snapshot("pool", "P")?.settled ?? 0).toBe(0); +}); +test("same traffic has no promised downgrade allowance equivalence", () => { + const disk = spendTestJournal(); + const next = createSpendReservationLedger({ journal: disk, salt, policy: spendTestPolicy(), now: () => 2 }); + next.reserve(request("physical", "P", 40)); next.settle("physical", { inputTokens: 40, outputTokens: 0 }); + const old = createShippedSpendLedger({ journal: disk, salt, policy: spendTestPolicy(), now: () => 2 }); + const neverUpgraded = createShippedSpendLedger({ journal: spendTestJournal(), salt, policy: spendTestPolicy(), now: () => 2 }); + neverUpgraded.reserve(request("physical", "account-label", 40)); neverUpgraded.settle("physical", { inputTokens: 40, outputTokens: 0 }); + expect(old.snapshot("pool", "P")?.settled).toBe(40); + expect(neverUpgraded.snapshot("pool", "P")).toBeUndefined(); + expect(old.reserve(request("extra", "P", 61)).reserved).toBe(false); + expect(neverUpgraded.reserve(request("extra", "P", 61)).reserved).toBe(true); +}); diff --git a/tests/lib/spend-corruption-compat.test.ts b/tests/lib/spend-corruption-compat.test.ts new file mode 100644 index 00000000000..caebac6151f --- /dev/null +++ b/tests/lib/spend-corruption-compat.test.ts @@ -0,0 +1,31 @@ +import { expect, test } from "bun:test"; +import { createSpendReservationLedger } from "../../src/lib/spend-reservation-ledger"; +import { spendCheckpoint, spendTestJournal, spendTestPolicy, testSpendSalt as salt } from "../helpers/shipped-spend-ledger"; +const request = (sendId: string, poolId = "P") => ({ sendId, scopes: { poolId }, inputTokens: 1, outputCeilingTokens: 0 }); +const corrupt = () => spendTestJournal([spendCheckpoint([["P", 40, 0]]), "broken middle", {v:1,kind:"forget",send:"missing",at:2}]); +const policy = spendTestPolicy({ compactAfterRecords: 1 }); +test("corrupt middle line compacts valid state without clearing live refusal", () => { + const disk = corrupt(); const ledger = createSpendReservationLedger({ journal: disk, salt, policy, now: () => 3 }); + expect(ledger.reserve(request("refuse"))).toMatchObject({ reserved: false, denial: { reason: "journal-corrupt" } }); + // Unconfigured applicable root traffic can trigger ordinary scheduled compaction. + ledger.reserve({ ...request("observe"), scopes: { rootId: "root" } }); + expect(disk.lines).toHaveLength(1); expect(JSON.parse(disk.lines[0]!).poolContinuity).toBeUndefined(); + expect(ledger.corruptRecords).toBe(1); expect(ledger.reserve(request("still-refuse")).reserved).toBe(false); +}); +test("restart after compaction restores dev enforcement", () => { + const disk = corrupt(); const ledger = createSpendReservationLedger({ journal: disk, salt, policy, now: () => 3 }); + ledger.reserve({ ...request("observe"), scopes: {} }); + const replay = createSpendReservationLedger({ journal: disk, salt, policy, now: () => 3 }); + expect(replay.corruptRecords).toBe(0); expect(replay.snapshot("pool", "P")?.settled).toBe(40); + expect(replay.reserve(request("now-admits")).reserved).toBe(true); +}); +test("complete final null remains corruption before compaction", () => { + const ledger = createSpendReservationLedger({ journal: spendTestJournal([spendCheckpoint([["P", 40, 0]]), "null"]), salt, policy }); + expect(ledger.corruptRecords).toBe(1); expect(ledger.reserve(request("refuse")).reserved).toBe(false); +}); +test("rewrite failure retains prior journal", () => { + const disk = corrupt(); disk.failRewrite = true; + const before = [...disk.lines]; const ledger = createSpendReservationLedger({ journal: disk, salt, policy, now: () => 3 }); + ledger.reserve({ ...request("observe"), scopes: {} }); + expect(disk.lines.slice(0, before.length)).toEqual(before); expect(ledger.persistFailures).toBe(1); expect(ledger.corruptRecords).toBe(1); +}); diff --git a/tests/lib/spend-experimental-history.test.ts b/tests/lib/spend-experimental-history.test.ts new file mode 100644 index 00000000000..68617bdc95c --- /dev/null +++ b/tests/lib/spend-experimental-history.test.ts @@ -0,0 +1,26 @@ +import { expect, test } from "bun:test"; +import { createSpendReservationLedger } from "../../src/lib/spend-reservation-ledger"; +import { createShippedSpendLedger, spendAlias, spendTestJournal, spendTestPolicy, testSpendSalt as salt } from "../helpers/shipped-spend-ledger"; +const experimental = () => ({v:1,kind:"checkpoint",at:1,poolContinuity:{v:1,kind:"pool-continuity",at:1,bindings:[{alias:spendAlias("pool-current","P"),canonical:spendAlias("pool-current","P")}]},scopes:[{scope:"pool",alias:spendAlias("pool-current","P"),settled:30,unresolved:0,seenAt:1}],sends:[{send:spendAlias("send","live"),status:"dispatched",targets:[{scope:"pool",alias:spendAlias("pool-current","P")}],tokens:10,at:1,resolvedAt:1}]}); +test("experimental aliases and live targets remain conservative across compaction",()=>{ + const disk=spendTestJournal([experimental()]);const policy=spendTestPolicy({compactAfterRecords:1,retentionMs:10}); + const ledger=createSpendReservationLedger({journal:disk,salt,policy,now:()=>2}); + expect(ledger.snapshot("pool","Q")).toMatchObject({settled:30,unresolved:10}); + ledger.reserve({sendId:"compact",scopes:{poolId:"P"},inputTokens:0,outputCeilingTokens:0}); + expect(JSON.parse(disk.lines[0]!).poolContinuity).toBeUndefined(); + expect(JSON.parse(disk.lines[0]!).scopes.some((r:{alias:string})=>r.alias===spendAlias("pool-current","P"))).toBe(true); + expect(createSpendReservationLedger({journal:disk,salt,policy,now:()=>2}).snapshot("pool","Q")).toMatchObject({settled:30,unresolved:10}); +}); +test("experimental journal is excluded from C without immediate deletion",()=>{ + const disk=spendTestJournal([experimental()]);const ledger=createSpendReservationLedger({journal:disk,salt,policy:spendTestPolicy({retentionMs:10}),now:()=>2}); + expect(ledger.corruptRecords).toBe(0);expect(ledger.snapshot("pool","P")).toMatchObject({settled:30,unresolved:10}); + // The shipped reader does not look up the unpublished pool-current domain. + expect(createShippedSpendLedger({journal:spendTestJournal(disk.lines),salt,now:()=>2}).snapshot("pool","P")).toBeUndefined(); + expect(disk.lines.some(line=>JSON.parse(line).kind==="drop")).toBe(false); +}); +test("dormant experimental liability expires only through ordinary durable drop",()=>{ + const disk=spendTestJournal([experimental()]);const policy=spendTestPolicy({retentionMs:10});const ledger=createSpendReservationLedger({journal:disk,salt,policy,now:()=>2}); + ledger.prune(12);expect(ledger.snapshot("pool","P")?.unresolved).toBe(10);ledger.prune(13);expect(ledger.snapshot("pool","P")).toBeUndefined(); + expect(JSON.parse(disk.lines.at(-1)!)).toMatchObject({kind:"drop",alias:spendAlias("pool-current","P")}); + expect(createSpendReservationLedger({journal:disk,salt,policy,now:()=>13}).snapshot("pool","P")).toBeUndefined(); +}); diff --git a/tests/lib/spend-pool-continuity.test.ts b/tests/lib/spend-pool-continuity.test.ts index 55e341152a8..52e0d4631bb 100644 --- a/tests/lib/spend-pool-continuity.test.ts +++ b/tests/lib/spend-pool-continuity.test.ts @@ -2,7 +2,7 @@ import { executeComboResponses } from "../../src/server/responses/core-combo"; import { clearComboSelectionState, clearComboTargetCooldowns } from "../../src/combos"; import { createTranslatorBudget } from "../../src/lib/translator-budget"; import type { OcxConfig } from "../../src/types"; -import { createLegacySpendLedger } from "../helpers/legacy-spend-ledger"; +import { createShippedSpendLedger } from "../helpers/shipped-spend-ledger"; import { describe, expect, test } from "bun:test"; import { mkdtempSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; @@ -17,7 +17,7 @@ import { removeTreeWithRetry } from "../helpers/remove-tree"; import { admitHttpWorkflowTurn, unboundPoolSpendRefusalMessage, unboundPoolSpendRefusalResponse } from "../../src/server/workflow-refusal"; import { createResponsesSendBudget } from "../../src/server/responses/request-send-budget"; import { claimDispatchSpendProof, createRequestExecutionBudget, deriveRequestExecutionBudget, reportDispatchSends } from "../../src/lib/request-execution-budget"; -import { createPoolContinuity } from "../../src/lib/spend-pool-continuity"; +import { resolvePoolAliases } from "../../src/lib/spend-pool-continuity"; import { admitWorkflowTurn, listWorkflowBudgetEvents, resetWorkflowBudgetsForTest, workflowDenialSummary, workflowSpendCeilingReached } from "../../src/lib/workflow-budget"; import { createRequestSpendTracker } from "../../src/server/responses/request-spend"; import type { RequestLogContext } from "../../src/server/request-log"; @@ -25,7 +25,7 @@ import type { RequestLogContext } from "../../src/server/request-log"; const salt = "5".repeat(64); const alias = (kind: string, id: string) => createHash("sha256").update(salt).update("\0").update(kind).update("\0").update(id).digest("hex").slice(0, 32); const pool = (id: string) => alias("pool", id); -const currentPool = (id: string) => alias("pool-current", id); + const policy = (poolAliases?: unknown, overrides: Partial = {}): SpendReservationPolicy => ({ ...DEFAULT_SPEND_RESERVATION_POLICY, pool: { maxTokens: 100 }, poolAliases, ...overrides, }); @@ -43,33 +43,23 @@ const reserve = (ledger: ReturnType, sendId describe("historical pool identity continuity", () => { test("verified group merges are atomic and independent of salted key order", () => { - const low = "1".repeat(32), high = "2".repeat(32), target = "3".repeat(32); - for (const [a, b] of [[low, high], [high, low]]) { - const continuity = createPoolContinuity(); - expect(continuity.restore({ v: 1, kind: "pool-continuity", at: 1, - bindings: [{ alias: a!, canonical: b! }] })).toBe(true); - const record = continuity.prepare({ [a!]: target, [b!]: target }, undefined, new Set(), id => id, 2, 100); - expect(record).not.toBe(false); - if (!record) throw new Error("merge was refused"); - expect(continuity.resolve(a!)).toBe(b!); // prepare does not publish evidence - expect(continuity.restore(record)).toBe(true); - expect(continuity.resolve(a!)).toBe(target); - expect(continuity.resolve(b!)).toBe(target); - const before = continuity.record(3); - expect(continuity.prepare({ [a!]: "4".repeat(32) }, undefined, new Set(), id => id, 3, 100)).toBe(false); - expect(continuity.record(3)).toEqual(before); - expect(continuity.restore({ v: 1, kind: "pool-continuity", at: 3, - bindings: [{ alias: a!, canonical: "4".repeat(32) }] })).toBe(false); - expect(continuity.record(3)).toEqual(before); - expect(continuity.prepare({ [target]: a!, [a!]: target }, undefined, new Set(), id => id, 3, 100)).toBe(false); - expect(continuity.record(3)).toEqual(before); + for (const [a, b] of [["A", "B"], ["B", "A"]]) { + const config = { [pool(a!)]: b!, [pool(b!)]: "target" }; + const continuity = resolvePoolAliases(config, salt); + expect(continuity.valid).toBe(true); + expect(continuity.resolve(pool(a!))).toBe(pool("target")); + expect(continuity.resolve(pool(b!))).toBe(pool("target")); + expect(resolvePoolAliases({ [pool(a!)]: b!, [pool(b!)]: a! }, salt).valid).toBe(false); + // Resolution is read-only and cannot change another previously resolved view. + expect(resolvePoolAliases({ [pool(a!)]: "other" }, salt).resolve(pool(a!))).toBe(pool("other")); + expect(continuity.resolve(pool(a!))).toBe(pool("target")); } }); test("unbound positive history is charged once against every candidate pool across pruning and restart", () => { const disk = journal([checkpoint([["provider-old-label", 100, 0]])]); for (let restart = 0; restart < 2; restart += 1) { - const ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(), now: () => 1_000_000_000 }); + const ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(), now: () => 2 }); expect(ledger.checkPoolContinuity()).toBeUndefined(); expect(ledger.hasUnboundPositivePoolHistory()).toBe(true); ledger.prune(); @@ -94,26 +84,24 @@ describe("historical pool identity continuity", () => { expect(disk.lines.map(line => JSON.parse(line)).filter(record => record.kind === "drop")).toEqual([]); }); - test("an exhausted candidate overlay protects its under-limit proven group from retention", () => { - const record = checkpoint([["verified-label", 60, 0], ["unbound-label", 40, 0], ["empty-old-label", 0, 0]]); - const disk = journal([record]); + test("dormant unbound overlay expires before under-limit proven group retention", () => { + const disk = journal([checkpoint([["verified-label", 60, 0], ["unbound-label", 40, 0], ["empty-old-label", 0, 0]])]); const ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy({ [pool("verified-label")]: "provider" }, { retentionMs: 5 }), now: () => 100 }); - expect(ledger.checkPoolContinuity()).toBeUndefined(); - expect(ledger.snapshot("pool", "provider")?.settled).toBe(100); - ledger.prune(100); // All entries are older than the retention cutoff. expect(ledger.snapshot("pool", "provider")?.settled).toBe(100); + ledger.prune(100); + expect(ledger.snapshot("pool", "provider")).toBeUndefined(); expect(disk.lines.map(line => JSON.parse(line)).filter(record => record.kind === "drop")).toEqual([ + { v: 1, kind: "drop", scope: "pool", alias: pool("unbound-label"), at: 100 }, { v: 1, kind: "drop", scope: "pool", alias: pool("empty-old-label"), at: 100 }, + { v: 1, kind: "drop", scope: "pool", alias: pool("verified-label"), at: 100 }, ]); - expect(reserve(ledger, "still-exhausted", "provider", 1)).toMatchObject({ reserved: false, - denial: { reason: "spend-limit-exceeded", projected: 101, includesUnboundPoolHistory: true } }); }); test("proven group plus each unbound balance admits under/exact totals and refuses over", () => { const disk = journal([checkpoint([["verified-label", 25, 0], ["unbound-label", 10, 5]])]); const ledger = createSpendReservationLedger({ journal: disk, salt, - policy: policy({ [pool("verified-label")]: "provider" }), now: () => 2 }); + policy: policy({ [pool("verified-label")]: "provider", [pool("provider")]: "provider" }), now: () => 2 }); // Proven group: 25. Unbound history: 10 settled + 5 unresolved. Neither bucket is copied. expect(ledger.snapshot("pool", "provider")).toEqual({ settled: 35, reserved: 0, unresolved: 5, exhausted: false }); @@ -140,29 +128,17 @@ describe("historical pool identity continuity", () => { expect(ledger.hasUnboundPositivePoolHistory()).toBe(true); expect(reserve(ledger, "same-spelling-exact", "provider", 60).reserved).toBe(true); expect(ledger.snapshot("pool", "provider")).toEqual({ settled: 40, reserved: 60, unresolved: 0, exhausted: true }); - // The legacy alias stays unknown and overlays every candidate, while the new send is - // stored under the canonical-current domain and belongs only to the routed provider. - expect(JSON.parse(disk.lines.at(-1)!).targets).toEqual([{ scope: "pool", alias: currentPool("provider") }]); - expect(reserve(ledger, "unrelated-under-overlay", "unrelated-provider", 60).reserved).toBe(true); - expect(ledger.snapshot("pool", "unrelated-provider")).toEqual({ - settled: 40, reserved: 60, unresolved: 0, exhausted: true, - }); - expect(reserve(ledger, "unrelated-over", "third-provider", 61)).toMatchObject({ - reserved: false, - denial: { reason: "spend-limit-exceeded", projected: 101, includesUnboundPoolHistory: true }, + // New bookings use the ordinary pool domain; the unbound original counts once. + expect(JSON.parse(disk.lines.at(-1)!).targets).toEqual([{ scope: "pool", alias: pool("provider") }]); + expect(reserve(ledger, "unrelated-under-overlay", "unrelated-provider", 1)).toMatchObject({ + reserved: false, denial: { reason: "spend-limit-exceeded", projected: 101 }, }); - - // An explicit operator mapping on restart moves only the legacy 40-token balance into - // this provider's current group; it does not copy it or assign it by spelling. const mapped = createSpendReservationLedger({ journal: disk, salt, policy: policy({ [pool("provider")]: "provider" }), now: () => 3 }); expect(mapped.checkPoolContinuity()).toBeUndefined(); expect(mapped.hasUnboundPositivePoolHistory()).toBe(false); expect(mapped.snapshot("pool", "provider")).toMatchObject({ settled: 40, unresolved: 60 }); - expect(reserve(mapped, "unrelated-after-mapping", "unrelated-provider", 41)).toMatchObject({ - reserved: false, - denial: { reason: "spend-limit-exceeded", projected: 101 }, - }); + expect(reserve(mapped, "unrelated-after-mapping", "unrelated-provider", 100).reserved).toBe(true); }); test("unbound history is overlaid when a pool ceiling is enabled later", () => { @@ -205,10 +181,9 @@ describe("historical pool identity continuity", () => { expect(reserve(ledger, "new", "provider", 10).reserved).toBe(true); expect(ledger.settle("new", { inputTokens: 10, outputTokens: 0 })).toBe(true); expect(ledger.settle("new", { inputTokens: 10, outputTokens: 0 })).toBe(false); - // Clearing config never removes durable evidence. Repeated replay/compaction never adds - // a migrated copy of a balance or charges a canonical self-alias twice. + // Read-time mappings must be supplied after restart; original counters never move. for (let restart = 0; restart < 3; restart += 1) { - ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(undefined, { compactAfterRecords: 1 }), now: () => 3 }); + ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(aliases, { compactAfterRecords: 1 }), now: () => 3 }); expect(ledger.checkPoolContinuity()).toBeUndefined(); expect(ledger.snapshot("pool", "provider")).toEqual({ settled: 70, unresolved: 30, reserved: 0, exhausted: true }); expect(reserve(ledger, `denied-${restart}`)).toMatchObject({ reserved: false, denial: { reason: "spend-limit-exceeded", projected: 101 } }); @@ -217,7 +192,7 @@ describe("historical pool identity continuity", () => { expect(disk.lines.join("\n")).not.toContain("provider"); }); - test("old open/dispatched sends become unresolved exactly once; duplicate send IDs remain refused", () => { + test("old open/dispatched sends become unresolved exactly once; replay reclaims IDs durably", () => { const old = ["a", "b"].flatMap(id => [ { v: 1, kind: "reserve", send: alias("send", id), targets: [{ scope: "pool", alias: pool(`label-${id}`) }], tokens: 30, at: 1 }, ...(id === "b" ? [{ v: 1, kind: "dispatch", send: alias("send", id), at: 1 }] : []), @@ -228,7 +203,10 @@ describe("historical pool identity continuity", () => { const ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(aliases), now: () => 2 }); expect(ledger.checkPoolContinuity()).toBeUndefined(); expect(ledger.snapshot("pool", "provider")?.unresolved).toBe(60); - expect(reserve(ledger, "a")).toMatchObject({ reserved: false, denial: { reason: "duplicate-send-id" } }); + expect(ledger.knows("a")).toBe(false); + expect(reserve(ledger, "a").reserved).toBe(true); + expect(ledger.snapshot("pool", "provider")?.reserved).toBe(1); + expect(ledger.abandon("a")).toBe(true); } }); @@ -323,7 +301,7 @@ describe("historical pool identity continuity", () => { expect(context.errorCode).toBeUndefined(); tracker.settle({ inputTokens: 8, outputTokens: 0 }); expect(ledger.snapshot("pool", "provider")?.settled).toBe(48); // current settled 8 plus shared unknown 40 - expect(ledger.snapshot("pool", "unknown-label")?.settled).toBe(40); + expect(ledger.snapshot("pool", "unknown-label")?.settled).toBe(48); // both unmapped originals overlay every candidate expect(reserve(ledger, "exact", "provider", 52).reserved).toBe(true); expect(ledger.abandon("exact")).toBe(true); const refused = reserve(ledger, "over", "provider", 53); @@ -368,12 +346,12 @@ describe("historical pool identity continuity", () => { }); }); - test("malformed or conflicting maps fail closed without clearing ceilings or saved links", () => { + test("malformed or conflicting maps fail closed without clearing ceilings or original balances", () => { const disk = journal([checkpoint([["label", 40, 0]])]); const ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy({ [pool("label")]: "provider" }), now: () => 2 }); expect(ledger.checkPoolContinuity()).toBeUndefined(); - for (const aliases of [null, [], { label: "provider" }, { [pool("label")]: "different-provider" }]) { - ledger.reconfigure(policy(aliases)); + for (const aliases of [null, [], { label: "provider" }, { [pool("provider")]: "different-provider" }]) { + ledger.reconfigure(policy(aliases, { canonicalProviderIds: ["provider"] })); expect(ledger.policy.pool.maxTokens).toBe(100); expect(ledger.checkPoolContinuity()?.reason).toBe("pool-history-unresolved"); expect(reserve(ledger, "new").reserved).toBe(false); @@ -383,7 +361,7 @@ describe("historical pool identity continuity", () => { expect(ledger.checkPoolContinuity()).toBeUndefined(); }); - test("evidence write failure cannot publish a mapping or erase historical balances", () => { + test("reserve write failure cannot authorize a send or erase historical balances", () => { const disk = journal([checkpoint([["label", 40, 0]])]); disk.append = () => { throw new Error("synthetic write failure"); }; const ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy({ [pool("label")]: "provider" }), now: () => 2 }); @@ -393,16 +371,14 @@ describe("historical pool identity continuity", () => { expect(disk.lines).toHaveLength(1); }); - test("complete invalid metadata at the final line fails closed and survives attempted compaction", () => { - const invalid = { ...checkpoint([["label", 40, 0]]), poolContinuity: { v: 1, kind: "pool-continuity", at: 1, bindings: [{ alias: "bad", canonical: pool("provider") }] } }; - const disk = journal([invalid]); - let ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(), now: () => 2 }); - expect(ledger.corruptRecords).toBe(1); - expect(ledger.checkPoolContinuity()?.reason).toBe("journal-corrupt"); - ledger.reconfigure(policy(undefined, { pool: {}, compactAfterRecords: 1 })); - reserve(ledger, "observed", "provider", 1, true); - ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(), now: () => 3 }); - expect(ledger.corruptRecords).toBe(1); + test("experimental checkpoint metadata is ignored while valid original counters survive", () => { + const experimental = { ...checkpoint([["label", 40, 0]]), poolContinuity: { unexpected: true } }; + const disk = journal([experimental]); + const ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(), now: () => 2 }); + expect(ledger.corruptRecords).toBe(0); + expect(ledger.checkPoolContinuity()).toBeUndefined(); + expect(ledger.snapshot("pool", "provider")?.settled).toBe(40); + expect(parseSpendJournalRecord(JSON.stringify(experimental))).toEqual(checkpoint([["label", 40, 0]])); }); test("a v1 checkpoint remains parseable when compatibility metadata is omitted by an old reader", () => { @@ -477,11 +453,10 @@ test("actual old-reader compaction preserves raw spend; compatible rollback reso expect(reserve(modern, "modern", "provider", 8).reserved).toBe(true); modern.settle("modern", { inputTokens: 8, outputTokens: 0 }); expect(modern.snapshot("pool", "provider")?.settled).toBe(48); - const old = createLegacySpendLedger({ journal: disk, salt, + const old = createShippedSpendLedger({ journal: disk, salt, policy: { ...DEFAULT_SPEND_RESERVATION_POLICY, compactAfterRecords: 1 }, now: () => 3 }); expect(old.corruptRecords).toBe(0); - // Unsupported old binaries can still write a newly account-qualified label and strip - // compatibility metadata. Do not claim an automatic downgrade barrier or its enforcement. + // Contract C restores the shipped per-label interpretation; aggregate allowance is not promised. expect(old.reserve({ sendId: "old-again", scopes: { poolId: "new-old-label" }, inputTokens: 12, outputCeilingTokens: 0 }).reserved).toBe(true); old.settle("old-again", { inputTokens: 12, outputTokens: 0 }); const returned = createSpendReservationLedger({ journal: disk, salt, @@ -530,7 +505,11 @@ for (const kind of ["compaction", "combo"] as const) { const dispatch = child.adapterDispatchBudget!.reserveDispatch({ sendClass: "initial", targetKey: "provider/fixture" }); expect(dispatch.allowed).toBe(true); if (dispatch.allowed) expect(dispatch.permit.use()).toBe(true); - } else child.noteTransientSends(1); // synthetic report-only combo dispatch, no fetch + } else { + const report = child.transientSendReporter(); + expect(report.beforeSend?.()).toBe(true); // synthetic executor reaches its before-wire seam + report(1); report.close?.(); + } expect(sendBudget.used).toBe(1); tracker.settle({ inputTokens: 100, outputTokens: 0 }); expect(sharedSpendLedger().snapshot("pool", "provider")).toMatchObject({ settled: 100, reserved: 0 }); @@ -637,6 +616,7 @@ test("receipt identity survives out-of-order handoff and reports cannot authoriz expect(first.budget.used).toBe(2); // report consumed A; B was already assumed const second = prepaidFixture(10, 100); + second.ledger.reconfigure(policy(undefined, { root: {}, pool: {} })); const reported = second.reservePermit(), pending = second.reservePermit(); reportDispatchSends(second.budget, 1, reported); expect(claimDispatchSpendProof(second.budget, reported)).toBeUndefined(); @@ -717,7 +697,9 @@ test("actual combo dispatch forwards only its own prepaid permit into the child expect(child).not.toBeInstanceOf(Response); if (child instanceof Response) return child; sends += 1; - child.noteTransientSends(1); + const report = child.transientSendReporter(); + expect(report.beforeSend?.()).toBe(true); + report(1); report.close?.(); return Response.json({ id: "synthetic-response", output: [] }); }, handleComboResponses: async () => { throw new Error("unexpected nested combo"); }, @@ -740,7 +722,8 @@ test("actual combo dispatch forwards only its own prepaid permit into the child }); test("a later child report preserves the earlier receipt and refunds its exact pool", () => { - const ledger = createSpendReservationLedger({ salt }); + const ledger = createSpendReservationLedger({ salt, + policy: policy({ [pool("earlier")]: "earlier", [pool("later")]: "later" }, { pool: {} }) }); const logCtx = { provider: "earlier", usageLogInputTokens: 10 }; const tracker = createRequestSpendTracker(logCtx, "root", ledger); const budget = createRequestExecutionBudget(undefined, undefined, tracker); @@ -781,19 +764,22 @@ test("captured reporters retain receipt ownership across handoffs, cancellation owner.pendingHopPermit = b; const reportB = owner.transientSendReporter(); reportB(0); // A cancelled/no-send helper cannot settle a receipt. + expect(reportB.beforeSend?.()).toBe(true); reportB(1); b.release(); expect(budget.used).toBe(2); expect(claimDispatchSpendProof(budget, b)).toBeUndefined(); expect(claimDispatchSpendProof(budget, a)).toBeDefined(); - if (finish === "report") reportA(1); + if (finish === "report") { expect(reportA.beforeSend?.()).toBe(true); reportA(1); } if (finish === "assume") expect(a.assumeCharge()).toBe(true); a.release(); a.release(); expect(budget.used).toBe(finish === "release" ? 1 : 2); // The same reporter's next count is a real retry, never another prepaid receipt. + expect(reportB.beforeSend?.()).toBe(true); reportB(1); expect(budget.used).toBe(finish === "release" ? 2 : 3); + reportA.close?.(); reportB.close?.(); tracker.settle(undefined); expect(ledger.snapshot("pool", "provider")).toMatchObject({ reserved: 0, settled: 0, unresolved: finish === "release" ? 20 : 30, @@ -803,6 +789,7 @@ test("captured reporters retain receipt ownership across handoffs, cancellation test("unnamed and foreign reports cannot consume a pending receipt", () => { const { ledger, budget, reservePermit } = prepaidFixture(10, 100); + ledger.reconfigure(policy(undefined, { root: {}, pool: {} })); // legacy observe-only numeric reporting const a = reservePermit(), b = reservePermit(); const foreign = prepaidFixture(10, 100).reservePermit(); budget.used += 1; diff --git a/tests/lib/spend-reservation-ledger.test.ts b/tests/lib/spend-reservation-ledger.test.ts index 9440504bea2..65f59630c08 100644 --- a/tests/lib/spend-reservation-ledger.test.ts +++ b/tests/lib/spend-reservation-ledger.test.ts @@ -191,8 +191,10 @@ describe("spend reservation ledger, send identity", () => { // ...and after a restart rebuilds the ledger from the journal. const restarted = createSpendReservationLedger({ journal, policy: policy(1_000), now: () => 2_000 }); - expect(restarted.knows("s1")).toBe(true); - expect(restarted.reserve(request).reserved).toBe(false); + // Durable replay forgets resolved IDs, never their totals; reuse books a fresh charge. + expect(restarted.knows("s1")).toBe(false); + expect(restarted.reserve(request).reserved).toBe(true); + expect(restarted.snapshot("root", "r1")).toMatchObject({ settled: 20, reserved: 20 }); }); test("an undispatched reservation is released and only a dispatched one becomes unresolved", () => { diff --git a/tests/lib/spend-seed-overflow.test.ts b/tests/lib/spend-seed-overflow.test.ts new file mode 100644 index 00000000000..3ddeaf3fcb8 --- /dev/null +++ b/tests/lib/spend-seed-overflow.test.ts @@ -0,0 +1,121 @@ +import { expect, test } from "bun:test"; +import { createSpendReservationLedger, sharedSpendLedger, type SpendReservationLedger, type SpendSeed } from "../../src/lib/spend-reservation-ledger"; +import { createShippedSpendLedger, spendAlias, spendTestJournal, spendTestPolicy, testSpendSalt as salt } from "../helpers/shipped-spend-ledger"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { acquireSpendLedgerServerLifecycle } from "../../src/server/index/spend-ledger-lifecycle"; +import { spendLedgerOwnerSnapshot } from "../../src/lib/spend-ledger-owner"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; +const request = (sendId: string, scopes = {poolId:"P"}, tokens=0) => ({sendId,scopes,inputTokens:tokens,outputCeilingTokens:0}); +const seed = (ledger: SpendReservationLedger, id="seed", scopes={poolId:"P"}, tokens=0): SpendSeed => { + const decision=ledger.reserveSeed(request(id,scopes,tokens)); expect(decision.reserved).toBe(true); + if(!decision.reserved) throw new Error("seed denied"); return decision.seed; +}; +const reported = (ledger: SpendReservationLedger, anchor: SpendSeed, id: string, tokens=0) => ledger.reserveReportedFromSeed(anchor,{sendId:id,inputTokens:tokens,outputCeilingTokens:0}); +const factory = (disk=spendTestJournal(), scopeCap=1, sendCap=1) => createSpendReservationLedger({journal:disk,salt,policy:spendTestPolicy({maxTrackedScopes:scopeCap,maxTrackedSends:sendCap}),now:()=>2}); +test("normal initial seed refuses before wire at full send capacity",()=>{ + const ledger=factory(); const first=seed(ledger); expect(ledger.reserveSeed(request("blocked"))).toMatchObject({reserved:false,denial:{reason:"tracking-capacity-exhausted"}}); + expect(ledger.knows(first.sendId)).toBe(true); +}); +test("40 retained plus 70 delayed usage survives old replay as 110",()=>{ + const disk=spendTestJournal(); const ledger=factory(disk); const anchor=seed(ledger,"first",{poolId:"P"},40); + ledger.markDispatched(anchor.sendId); ledger.settle(anchor.sendId,{inputTokens:40,outputTokens:0}); + expect(reported(ledger,anchor,"delayed",70)).toBe(true); ledger.markDispatched("delayed"); ledger.settle("delayed",{inputTokens:70,outputTokens:0}); + expect(ledger.forgetResolved([anchor.sendId,"delayed"])).toBe(true); expect(ledger.finishSeed(anchor)).toBe(true); + const old=createShippedSpendLedger({journal:disk,salt,policy:spendTestPolicy(),now:()=>2}); + expect(old.corruptRecords).toBe(0); expect(old.snapshot("pool","P")?.settled).toBe(110); + expect(old.reserve(request("next",{poolId:"P"},1)).reserved).toBe(false); +}); +test("concurrent reporters stay within Msend times L with zero new scope keys", async()=>{ + const disk=spendTestJournal(); const ledger=factory(disk,1,2); const anchors=[seed(ledger,"a"),seed(ledger,"b")]; + const reporters=anchors.map(()=>ledger.registerReporter()); + await Promise.all(anchors.map(async(anchor,index)=>{ for(let n=1;n<4;n++) expect(reported(ledger,anchor,`${index}-${n}`,1)).toBe(true); })); + expect(disk.lines.filter(line=>JSON.parse(line).kind==="reserve")).toHaveLength(8); + expect(new Set(disk.lines.flatMap(line=>JSON.parse(line).targets?.map((r:{scope:string;alias:string})=>r.scope+":"+r.alias)??[])).size).toBe(1); + expect(ledger.reserveSeed(request("new" )).reserved).toBe(false); reporters.forEach(r=>r.close()); await ledger.waitForReporterDrain(); +}); +test("sequential distinct scope seeds cannot bypass normal scope cap",()=>{ + const ledger=factory(); const anchor=seed(ledger,"first",{poolId:"P"},100); + ledger.markDispatched(anchor.sendId); ledger.settle(anchor.sendId,{inputTokens:100,outputTokens:0}); ledger.forgetResolved([anchor.sendId]); ledger.finishSeed(anchor); + ledger.reconfigure(spendTestPolicy({maxTrackedScopes:1,maxTrackedSends:1,poolAliases:{[spendAlias("pool","P")]:"P"}})); + expect(ledger.reserveSeed(request("next",{poolId:"Q"},1))).toMatchObject({reserved:false,denial:{reason:"tracking-capacity-exhausted"}}); +}); +test("live zero-token seed pins every target",()=>{ + const disk=spendTestJournal();const ledger=factory(disk,3); const anchor=seed(ledger,"zero",{rootId:"R",identityId:"I",poolId:"P"}); + ledger.prune(1e12); expect(reported(ledger,anchor,"report",7)).toBe(true); + for(const [scope,id] of [["root","R"],["identity","I"],["pool","P"]] as const) expect(ledger.snapshot(scope,id)?.reserved).toBe(7); + expect(disk.lines.some(line=>JSON.parse(line).kind==="drop")).toBe(false); +}); +test("settle lost then durable forget preserves shipped-reader liability",()=>{ + const disk=spendTestJournal();const ledger=factory(disk); const anchor=seed(ledger,"s",{poolId:"P"},10); + ledger.markDispatched("s"); expect(reported(ledger,anchor,"lost",20)).toBe(true);ledger.markDispatched("lost"); + ledger.settle("s",{inputTokens:5,outputTokens:0}); ledger.markLost("lost"); expect(ledger.forgetResolved(["s","lost"])).toBe(true); + expect(ledger.finishSeed(anchor)).toBe(true); expect(ledger.knows("s")).toBe(false); + expect(createShippedSpendLedger({journal:disk,salt,now:()=>2}).snapshot("pool","P")).toMatchObject({settled:5,unresolved:20}); +}); +test("orphan replay forgets send IDs without reducing totals",()=>{ + const disk=spendTestJournal();const ledger=factory(disk);seed(ledger,"orphan",{poolId:"P"},10);ledger.markDispatched("orphan"); + const replay=factory(disk); expect(replay.knows("orphan")).toBe(false); expect(replay.snapshot("pool","P")?.unresolved).toBe(10); + expect(JSON.parse(disk.lines.at(-1)!).kind).toBe("forget"); +}); +test("foreign closed and abandoned seeds cannot authorize reported overflow",()=>{ + const ledger=factory();const foreign=seed(factory());expect(reported(ledger,foreign,"foreign")).toBe(false); + const anchor=seed(ledger); expect(reported(ledger,{sendId:anchor.sendId},"forged")).toBe(false); + ledger.abandon(anchor.sendId);expect(reported(ledger,anchor,"abandoned")).toBe(false);expect(ledger.finishSeed(anchor)).toBe(true); + expect(reported(ledger,anchor,"closed")).toBe(false); +}); +test("terminal waits for reporters and failed persistence cannot reclaim a seed",async()=>{ + const disk=spendTestJournal();const ledger=factory(disk);const anchor=seed(ledger,"seed",{poolId:"P"},10);ledger.markDispatched("seed"); + const reporter=ledger.registerReporter(); let drained=false; const drain=ledger.waitForReporterDrain().then(()=>{drained=true;}); + await Promise.resolve();expect(drained).toBe(false);expect(ledger.finishSeed(anchor)).toBe(false); + reporter.close();reporter.close();await drain;expect(drained).toBe(true); + disk.failAppend=true; expect(ledger.settle("seed",{inputTokens:7,outputTokens:0})).toBe(false); + expect(ledger.forgetResolved(["seed"])).toBe(false);expect(ledger.finishSeed(anchor)).toBe(false);expect(ledger.knows("seed")).toBe(true); + disk.failAppend=false;expect(ledger.settle("seed",{inputTokens:7,outputTokens:0})).toBe(true); + disk.failAppend=true;expect(ledger.forgetResolved(["seed"])).toBe(false);expect(ledger.finishSeed(anchor)).toBe(false); + disk.failAppend=false;expect(ledger.forgetResolved(["seed"])).toBe(true);expect(ledger.finishSeed(anchor)).toBe(true); + const delayedDisk = spendTestJournal(); + const delayedLedger = factory(delayedDisk); + const delayedSeed = seed(delayedLedger, "initial", { poolId: "P" }, 40); + delayedLedger.markDispatched("initial"); + delayedDisk.failAppend = true; + expect(reported(delayedLedger, delayedSeed, "physical-but-unwritten", 70)).toBe(true); + delayedLedger.markDispatched("physical-but-unwritten"); + expect(delayedLedger.snapshot("pool", "P")?.reserved).toBe(110); + expect(delayedLedger.markLost("physical-but-unwritten")).toBe(false); + expect(delayedLedger.finishSeed(delayedSeed)).toBe(false); + delayedDisk.failAppend = false; + expect(delayedLedger.markLost("physical-but-unwritten")).toBe(true); + expect(delayedLedger.markLost("initial")).toBe(true); + expect(delayedLedger.forgetResolved(["initial", "physical-but-unwritten"])).toBe(true); + expect(delayedLedger.finishSeed(delayedSeed)).toBe(true); + expect(createShippedSpendLedger({ journal: delayedDisk, salt, now: () => 2 }).snapshot("pool", "P")?.unresolved).toBe(110); + // Listener shutdown retains the home owner until enforced reporters finish. + const previous = process.env.OPENCODEX_HOME; + const home = mkdtempSync(join(tmpdir(), "ocx-seed-drain-")); + process.env.OPENCODEX_HOME = home; + const lifecycle = acquireSpendLedgerServerLifecycle(home); + let shutdownReporter: ReturnType | undefined; + try { + lifecycle.configure({ pool: { maxTokens: 100 } }, undefined, ["P"]); + const shared = sharedSpendLedger(); + shutdownReporter = shared.registerReporter(); + const listener = lifecycle.track({ stop: async () => {} }); + let stopped = false; + const stopping = Promise.resolve(listener.stop()).then(() => { lifecycle.release(); stopped = true; }); + await Promise.resolve(); await Promise.resolve(); + expect(stopped).toBe(false); expect(spendLedgerOwnerSnapshot().ownership).toBe("held"); + shutdownReporter.close(); await stopping; + expect(stopped).toBe(true); expect(spendLedgerOwnerSnapshot().ownership).toBe("unheld"); + } finally { + shutdownReporter?.close(); lifecycle.release(); + if (previous === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previous; + removeTreeWithRetry(home); + } +}); +test("ceiling removal does not drop seeded accounting",()=>{ + const disk=spendTestJournal();const ledger=factory(disk);const anchor=seed(ledger);ledger.reconfigure(spendTestPolicy({pool:{}})); + expect(reported(ledger,anchor,"delayed",70)).toBe(true);ledger.markDispatched("delayed");ledger.settle("delayed",{inputTokens:70,outputTokens:0}); + expect(createShippedSpendLedger({journal:disk,salt,now:()=>2}).snapshot("pool","P")?.settled).toBe(70); +}); diff --git a/tests/lib/spend-unbound-expiry.test.ts b/tests/lib/spend-unbound-expiry.test.ts new file mode 100644 index 00000000000..75e796652e8 --- /dev/null +++ b/tests/lib/spend-unbound-expiry.test.ts @@ -0,0 +1,33 @@ +import { expect, test } from "bun:test"; +import { createSpendReservationLedger } from "../../src/lib/spend-reservation-ledger"; +import { spendAlias, spendCheckpoint, spendTestJournal, spendTestPolicy, testSpendSalt as salt } from "../helpers/shipped-spend-ledger"; +const request = (sendId: string, poolId = "P", tokens = 1) => ({sendId, scopes: {poolId}, inputTokens: tokens, outputCeilingTokens: 0}); +const policy = spendTestPolicy({ retentionMs: 10 }); +test("30 settled 10 unresolved expires strictly after retention cutoff", () => { + const disk = spendTestJournal([spendCheckpoint([["unknown", 30, 10]])]); + const ledger = createSpendReservationLedger({journal:disk,salt,policy,now:()=>2}); + ledger.prune(11); expect(ledger.snapshot("pool", "P")).toMatchObject({settled:30,unresolved:10}); + ledger.prune(12); expect(ledger.snapshot("pool", "P")).toBeUndefined(); + expect(JSON.parse(disk.lines.at(-1)!)).toEqual({v:1,kind:"drop",scope:"pool",alias:spendAlias("pool","unknown"),at:12}); + expect(createSpendReservationLedger({journal:disk,salt,policy,now:()=>12}).snapshot("pool","P")).toBeUndefined(); +}); +test("capacity pressure never expires unknown history early", () => { + const disk = spendTestJournal([spendCheckpoint([["unknown",30,10]])]); + const ledger = createSpendReservationLedger({journal:disk,salt,policy:{...policy,maxTrackedScopes:1},now:()=>11}); + expect(ledger.reserve(request("full"))).toMatchObject({reserved:false,denial:{reason:"tracking-capacity-exhausted"}}); + expect(ledger.snapshot("pool","P")).toMatchObject({settled:30,unresolved:10}); expect(disk.lines).toHaveLength(1); +}); +test("failed drop cannot reduce enforced admission total", () => { + const disk = spendTestJournal([spendCheckpoint([["unknown",30,10]])]); disk.failAppend=true; + const ledger = createSpendReservationLedger({journal:disk,salt,policy,now:()=>12}); ledger.prune(); + expect(ledger.snapshot("pool","P")).toMatchObject({settled:30,unresolved:10}); + expect(ledger.reserve(request("cannot-launder","P",61)).reserved).toBe(false); + expect(ledger.persistFailures).toBeGreaterThan(0); expect(disk.lines).toHaveLength(1); +}); +test("expiry rebuilds unrelated candidate overlay before admission", () => { + const disk=spendTestJournal([spendCheckpoint([["unknown",40,0]]),spendCheckpoint([["unknown",40,0],["verified",60,0]])]); + const ledger=createSpendReservationLedger({journal:disk,salt,policy:{...policy,poolAliases:{[spendAlias("pool","verified")]:"P"}},now:()=>12}); + expect(ledger.reserve(request("Q-admits","Q",100)).reserved).toBe(true); + expect(ledger.snapshot("pool","P")?.settled ?? 0).toBe(0); + expect(disk.lines.filter(line=>JSON.parse(line).kind==="drop")).toHaveLength(2); +}); diff --git a/tests/lib/spend-zero-config-compat.test.ts b/tests/lib/spend-zero-config-compat.test.ts new file mode 100644 index 00000000000..2c41cd41d30 --- /dev/null +++ b/tests/lib/spend-zero-config-compat.test.ts @@ -0,0 +1,20 @@ +import { expect, test } from "bun:test"; +import { createSpendReservationLedger } from "../../src/lib/spend-reservation-ledger"; +import { createShippedSpendLedger, spendTestJournal, spendTestPolicy, testSpendSalt as salt } from "../helpers/shipped-spend-ledger"; +const req=(id:string,tokens=10)=>({sendId:id,scopes:{poolId:"P"},inputTokens:tokens,outputCeilingTokens:0}); +test("three unconfigured sends match shipped bytes for equal pool inputs",()=>{ + const oldDisk=spendTestJournal(),newDisk=spendTestJournal();const policy=spendTestPolicy({pool:{}}); + for(const ledger of [createShippedSpendLedger({journal:oldDisk,salt,policy,now:()=>2}),createSpendReservationLedger({journal:newDisk,salt,policy,now:()=>2})]) { + for(const id of ["a","b","c"]) {expect(ledger.reserve(req(id)).reserved).toBe(true);ledger.markDispatched(id);ledger.settle(id,{inputTokens:8,outputTokens:2});} + } + expect(newDisk.lines).toEqual(oldDisk.lines);expect(newDisk.lines).toHaveLength(9); +}); +test("unconfigured full capacity remains permissive with shipped omitted booking",()=>{ + const policy=spendTestPolicy({pool:{},maxTrackedSends:1});const old=createShippedSpendLedger({journal:spendTestJournal(),salt,policy,now:()=>2});const next=createSpendReservationLedger({journal:spendTestJournal(),salt,policy,now:()=>2}); + for(const ledger of [old,next]) {expect(ledger.reserve(req("first")).reserved).toBe(true);expect(ledger.reserve({...req("omitted"),alreadySent:true}).reserved).toBe(false);expect(ledger.snapshot("pool","P")?.reserved).toBe(10);} +}); +test("later ceiling sees retained ordinary history without continuity checkpoints",()=>{ + const disk=spendTestJournal();const ledger=createSpendReservationLedger({journal:disk,salt,policy:spendTestPolicy({pool:{}}),now:()=>2}); + ledger.reserve(req("retained",100));ledger.settle("retained",{inputTokens:100,outputTokens:0});ledger.reconfigure(spendTestPolicy()); + expect(ledger.reserve(req("blocked",1)).reserved).toBe(false);expect(disk.lines.some(line=>JSON.parse(line).kind==="checkpoint")).toBe(false); +}); diff --git a/tests/responses/chat-native-spend.test.ts b/tests/responses/chat-native-spend.test.ts index 7ee598d32b3..2a4690feee6 100644 --- a/tests/responses/chat-native-spend.test.ts +++ b/tests/responses/chat-native-spend.test.ts @@ -125,7 +125,7 @@ test("native Chat refuses a physical send that exceeds the configured pool spend } }); -test("native Chat reports a spend refusal on a transient retry leg as local 429", async () => { +test("native Chat reuses its normal seed on a stable transient retry and retains both liabilities", async () => { const messages = [{ role: "user", content: "hello" }]; let upstreamSends = 0; const upstream = Bun.serve({ @@ -148,11 +148,11 @@ test("native Chat reports a spend refusal on a transient retry leg as local 429" headers: { "content-type": "application/json" }, body: JSON.stringify({ model: "mock/test-model", messages, max_tokens: 1 }), }); - expect(response.status).toBe(429); - expect(response.headers.get("x-opencodex-local-refusal")).toBe("workflow_spend_exhausted"); - expect(upstreamSends).toBe(1); + expect(response.status).toBe(503); + expect(response.headers.get("x-opencodex-local-refusal")).toBeNull(); + expect(upstreamSends).toBe(2); expect(getRequestLogEntries().findLast(row => row.inboundProtocol === "chat")).toMatchObject({ - status: 429, errorCode: "workflow_spend_exhausted", + status: 503, }); }); diff --git a/tests/responses/responses-spend-anchor.test.ts b/tests/responses/responses-spend-anchor.test.ts new file mode 100644 index 00000000000..cace04e532a --- /dev/null +++ b/tests/responses/responses-spend-anchor.test.ts @@ -0,0 +1,130 @@ +import { expect, test } from "bun:test"; +import { createSpendReservationLedger, DEFAULT_SPEND_RESERVATION_POLICY } from "../../src/lib/spend-reservation-ledger"; +import { createRequestExecutionBudget, deriveRequestExecutionBudget, CODEX_TEXT_GUARDED_BUDGET_POLICY, createPhysicalSendReporter } from "../../src/lib/request-execution-budget"; +import { createRequestSpendTracker } from "../../src/server/responses/request-spend"; +import { fetchWithResetRetry, fetchWithTransientRetry } from "../../src/lib/upstream-retry"; + +function fixture(enforced = true, capacity = 8) { + const lines: string[] = []; + const ledger = createSpendReservationLedger({ salt: "anchor-fixture", policy: { + ...DEFAULT_SPEND_RESERVATION_POLICY, maxTrackedSends: capacity, + ...(enforced ? { pool: { maxTokens: 10000 } } : {}), + }, journal: { read: () => [], append: line => { lines.push(line); } } }); + const ctx = { provider: "P", spendPoolId: "P", accountLogLabel: "A", spendInputEstimateTokens: 10, spendOutputCeilingTokens: 20 }; + const tracker = createRequestSpendTracker(ctx, undefined, ledger); + const budget = createRequestExecutionBudget({ ...CODEX_TEXT_GUARDED_BUDGET_POLICY }, undefined, tracker); + const reporter = () => createPhysicalSendReporter(budget, () => ({ poolId: ctx.spendPoolId, identityId: ctx.accountLogLabel })); + return { lines, ledger, ctx, tracker, budget, reporter }; +} + +test("ordinary rootless passthrough seeds before its first fetch", async () => { + const f = fixture(true, 1); + f.ledger.reserve({ sendId: "busy", scopes: { poolId: "P" }, inputTokens: 1, outputCeilingTokens: 0 }); + let wires = 0; + await expect(fetchWithTransientRetry(async () => { wires++; return new Response(); }, { attempts: 1, onSendsConsumed: f.reporter() })).rejects.toThrow(); + expect(wires).toBe(0); + expect(f.tracker.refusals).toBe(1); +}); + +test("generic adapter initial send adopts one normal seed", async () => { + const f = fixture(); + const decision = f.budget.reserveDispatch({ sendClass: "initial", targetKey: "P/A", countedExternally: true }); + expect(decision.allowed).toBe(true); + if (!decision.allowed) throw new Error("dispatch refused"); + await fetchWithResetRetry(async () => { expect(decision.permit.use()).toBe(true); return new Response(); }, { + attempts: 1, onSendsConsumed: createPhysicalSendReporter(f.budget, () => ({ poolId: "P", identityId: "A", targetKey: "P/A" }), decision.permit), + }); + expect(f.lines.filter(line => JSON.parse(line).kind === "reserve")).toHaveLength(1); + f.tracker.settle({ inputTokens: 2, outputTokens: 3 }); + expect(f.ledger.snapshot("pool", "P")?.settled).toBe(5); +}); + +test("stable reset transient and continuation retries never create new scope entries", async () => { + const f = fixture(true, 1); + for (const helper of [fetchWithResetRetry, fetchWithTransientRetry, fetchWithResetRetry]) { + await helper(async () => new Response(), { attempts: 1, onSendsConsumed: f.reporter() }); + } + f.tracker.settle({ inputTokens: 4 }); + expect(f.ledger.snapshot("pool", "P")?.unresolved).toBe(60); + expect(f.ledger.snapshot("pool", "P")?.settled).toBe(4); + expect(f.budget.physicalStarted).toBe(3); +}); + +test("delayed key A batch remains on A after key B reselection", () => { + const f = fixture(); + const a = f.reporter(); + expect(a.beforeSend?.()).toBe(true); + f.ctx.accountLogLabel = "B"; + const b = f.reporter(); + expect(b.beforeSend?.()).toBe(true); + b(1); b.close?.(); + a(1); a.close?.(); + f.tracker.settle({ inputTokens: 7 }); + expect(f.ledger.snapshot("identity", "A")?.unresolved).toBe(30); + expect(f.ledger.snapshot("identity", "B")?.settled).toBe(7); +}); + +test("combo B adopts B seed on a shared parent observer", () => { + const f = fixture(); + const child = deriveRequestExecutionBudget(f.budget, CODEX_TEXT_GUARDED_BUDGET_POLICY); + const report = createPhysicalSendReporter(child, () => ({ poolId: "B", identityId: "account-B" })); + expect(report.beforeSend?.()).toBe(true); report(1); report.close?.(); + expect(f.budget.physicalStarted).toBe(1); + f.tracker.settle({ inputTokens: 11 }); + expect(f.ledger.snapshot("pool", "B")?.settled).toBe(11); + const reserve = f.lines.map(line => JSON.parse(line)).find(record => record.kind === "reserve"); + expect(reserve.targets.filter((target: { scope: string }) => target.scope === "pool")).toHaveLength(1); + expect(f.ledger.snapshot("identity", "account-B")?.settled).toBe(11); +}); + +test("all helper native adapter WS and compact executors enforce frozen L", async () => { + const f = fixture(); + let wires = 0; + for (let i = 0; i < 4; i++) await fetchWithResetRetry(async () => { wires++; return new Response(); }, { attempts: 1, onSendsConsumed: f.reporter() }); + Object.assign(f.budget.policy, { maxTotalModelSends: 100 }); + await expect(fetchWithResetRetry(async () => { wires++; return new Response(); }, { attempts: 1, onSendsConsumed: f.reporter() })).rejects.toThrow(); + expect(wires).toBe(4); + expect(f.budget.physicalLimit).toBe(4); +}); + +test("native Messages without a budget shares the finite enforced ingress limit", () => { + const f = fixture(); + const report = f.reporter(); + for (let i = 0; i < 4; i++) expect(report.beforeSend?.()).toBe(true); + expect(report.beforeSend?.()).toBe(false); + report(4); report.close?.(); + expect(f.budget.physicalStarted).toBe(4); +}); + +test("async terminal cancel and shutdown drain reporters before forgetting", async () => { + const f = fixture(); + const report = f.reporter(); + expect(report.beforeSend?.()).toBe(true); + f.tracker.settle(undefined); + let drained = false; + const drain = f.ledger.waitForReporterDrain().then(() => { drained = true; }); + await Promise.resolve(); + expect(drained).toBe(false); + expect(f.ledger.snapshot("pool", "P")?.reserved).toBe(30); + report(1); report.close?.(); + await drain; + expect(f.ledger.snapshot("pool", "P")?.unresolved).toBe(30); +}); + +test("recovery telemetry cannot mint unclaimed sends", () => { + const f = fixture(); + const report = f.reporter(); + report(70); report.close?.(); + f.budget.used += 70; + expect(f.budget.physicalStarted).toBe(0); + expect(f.lines.filter(line => JSON.parse(line).kind === "reserve")).toHaveLength(0); +}); + +test("unconfigured helper keeps shipped no-refusal behavior", async () => { + const f = fixture(false, 1); + f.ledger.reserve({ sendId: "busy", scopes: { poolId: "P" }, inputTokens: 1, outputCeilingTokens: 0 }); + let wires = 0; + await fetchWithResetRetry(async () => { wires++; return new Response(); }, { attempts: 1, onSendsConsumed: f.reporter() }); + expect(wires).toBe(1); + expect(f.tracker.refusals).toBe(0); +}); diff --git a/tests/responses/responses-spend-capacity-guard.test.ts b/tests/responses/responses-spend-capacity-guard.test.ts index 1cafcfdfa30..2207678f0ea 100644 --- a/tests/responses/responses-spend-capacity-guard.test.ts +++ b/tests/responses/responses-spend-capacity-guard.test.ts @@ -59,11 +59,11 @@ describe("configured spend cannot dispatch without a capacity booking", () => { }); } - test("an already-sent capacity failure preserves physical count without a false refusal", () => { + test("unclaimed enforced numeric reports cannot bypass normal seed capacity", () => { const f = fixture({ pool: { maxTokens: 1000 }, maxTrackedSends: 1 }); f.ledger.reserve({ sendId: "busy", scopes: { poolId: "fixture-pool" }, inputTokens: 1, outputCeilingTokens: 0 }); f.budget.used += 1; - expect(f.budget.used).toBe(1); + expect(f.budget.used).toBe(0); expect(f.tracker.refusals).toBe(0); expect(f.context.errorCode).toBeUndefined(); expect(f.context.localTerminalReason).toBeUndefined(); @@ -104,6 +104,7 @@ for (const [denial, code] of [ // and actual journal failures/corruption are covered by the ledger regression suite. const ledger = Object.create(f.ledger) as typeof f.ledger; ledger.reserve = () => ({ reserved: false, denial }); + ledger.reserveSeed = () => ({ reserved: false, denial }); const tracker = createRequestSpendTracker(f.context, undefined, ledger); expect(tracker.charge({ alreadySent })).toBe(alreadySent || !enforced); expect(tracker.refusals).toBe(enforced && !alreadySent ? 1 : 0); diff --git a/tests/responses/responses-spend-contract-c.test.ts b/tests/responses/responses-spend-contract-c.test.ts new file mode 100644 index 00000000000..d0db6bb7680 --- /dev/null +++ b/tests/responses/responses-spend-contract-c.test.ts @@ -0,0 +1,171 @@ +import { expect, test } from "bun:test"; +import { createHash } from "node:crypto"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createSpendReservationLedger, configureSharedSpendLedger, sharedSpendLedger, DEFAULT_SPEND_RESERVATION_POLICY, type SpendJournal, type SpendReservationPolicy } from "../../src/lib/spend-reservation-ledger"; +import { createRequestSpendTracker } from "../../src/server/responses/request-spend"; +import { createRequestExecutionBudget, claimDispatchSpendProof, createPhysicalSendReporter } from "../../src/lib/request-execution-budget"; +import { workflowSpendCeilingReached } from "../../src/lib/workflow-budget"; +import { parseRequest } from "../../src/responses/parser"; +import { routeModel } from "../../src/router"; +import { applyFinalRouteRequestNormalization } from "../../src/server/responses/core-normalize"; +import { handleNativeMessages } from "../../src/server/messages-native"; +import { executeComboResponses } from "../../src/server/responses/core-combo"; +import { clearComboSelectionState, clearComboTargetCooldowns } from "../../src/combos"; +import { createTranslatorBudget } from "../../src/lib/translator-budget"; +import type { RequestLogContext } from "../../src/server/request-log"; +import type { OcxConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; + +const salt = "6".repeat(64); +const pool = (id: string) => createHash("sha256").update(salt).update("\0pool\0").update(id).digest("hex").slice(0, 32); +const policy = (poolAliases?: unknown, limit = 100): SpendReservationPolicy => ({ + ...DEFAULT_SPEND_RESERVATION_POLICY, pool: { maxTokens: limit }, poolAliases, +}); +const journal = (records: unknown[] = []): SpendJournal & { lines: string[] } => { + const lines = records.map(record => JSON.stringify(record)); + return { lines, read: () => [...lines], append: line => { lines.push(line); }, + rewrite: next => { lines.splice(0, lines.length, ...next); } }; +}; +const checkpoint = (entries: Array<[string, number, number]>) => ({ + v: 1, kind: "checkpoint", at: 1, sends: [], + scopes: entries.map(([id, settled, unresolved]) => ({ scope: "pool", alias: pool(id), settled, unresolved, seenAt: 1 })), +}); +const reserve = (ledger: ReturnType, sendId: string, poolId: string, tokens: number) => + ledger.reserve({ sendId, scopes: { poolId }, inputTokens: tokens, outputCeilingTokens: 0 }); +const log = (provider = "P"): RequestLogContext => ({ model: "model", provider, spendPoolId: provider, + usageLogInputTokens: 10, spendOutputCeilingTokens: 0 }); + +test("Responses A B and Messages share canonical ceiling", async () => { + const home = mkdtempSync(join(tmpdir(), "ocx-contract-c-messages-")); + const priorHome = process.env.OPENCODEX_HOME; + process.env.OPENCODEX_HOME = home; + const release = acquireOwnedSpendHome(); + const translatorBudget = createTranslatorBudget(); + let sends = 0; + const config: OcxConfig = { port: 0, defaultProvider: "P", providers: { P: { + adapter: "anthropic", apiKey: "fixture-key", baseUrl: "https://messages.example.test/v1", + fetch: async () => { + sends += 1; + return Response.json({ id: "msg_fixture", type: "message", role: "assistant", model: "model", + content: [{ type: "text", text: "ok" }], stop_reason: "end_turn", stop_sequence: null, + usage: { input_tokens: 10, output_tokens: 0 } }); + }, + } } }; + try { + writeFileSync(join(home, "spend-ledger.salt"), salt + "\n", { mode: 0o600 }); + configureSharedSpendLedger(policy(undefined, 31)); + const ledger = sharedSpendLedger(); + for (const account of ["A", "B"]) { + const ctx = log(); + const parsed = parseRequest({ model: "P/model", input: [] }); + await applyFinalRouteRequestNormalization({ parsed, route: routeModel(config, parsed.modelId), config, + req: new Request("http://localhost/v1/responses"), logCtx: ctx, inboundWire: "responses" }); + ctx.provider = `P-${account}`; + ctx.accountLogLabel = account; + const tracker = createRequestSpendTracker(ctx, undefined, ledger); + const budget = createRequestExecutionBudget(undefined, undefined, tracker); + const report = createPhysicalSendReporter(budget, () => ({ poolId: "P", identityId: account })); + expect(report.beforeSend?.()).toBe(true); + report(1); report.close?.(); + tracker.settle({ inputTokens: 10, outputTokens: 0 }); + } + const ctx = log("stale"); + const body = { model: "model", messages: [{ role: "user", content: "hello" }], max_tokens: 1, stream: false }; + const response = await handleNativeMessages({ req: new Request("http://localhost/v1/messages", { method: "POST" }), + config, logCtx: ctx, route: routeModel(config, "P/model"), body, requestedModel: "P/model", translatorBudget }); + expect(response.status).toBe(200); + await response.text(); + ctx.spendTracker?.settle({ inputTokens: 10, outputTokens: 0 }); + expect(sends).toBe(1); + expect(ctx.spendPoolId).toBe("P"); + expect(ledger.snapshot("pool", "P")).toMatchObject({ settled: 30, reserved: 0, unresolved: 0 }); + const over = log("P-C"); over.spendPoolId = "P"; + expect(createRequestSpendTracker(over, undefined, ledger).charge()).toBe(false); + const records = (await Bun.file(join(home, "spend-ledger.jsonl")).text()).trim().split("\n").map(line => JSON.parse(line)); + expect(records.filter(record => record.kind === "reserve").flatMap(record => record.targets) + .filter(target => target.scope === "pool").map(target => target.alias)).toEqual([pool("P"), pool("P"), pool("P")]); + expect(records.some(record => record.poolContinuity !== undefined)).toBe(false); + } finally { + translatorBudget.dispose(); release(); + if (priorHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = priorHome; + removeTreeWithRetry(home); + } +}); + +test("own unbound canonical bucket is counted once for absent partial complete maps", () => { + for (const aliases of [undefined, { [pool("old-A")]: "P" }, + { [pool("old-A")]: "P", [pool("old-B")]: "P", [pool("P")]: "P" }]) { + const disk = journal([checkpoint([["P", 20, 0], ["old-A", 30, 0], ["old-B", 0, 10]])]); + const ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(aliases), now: () => 2 }); + expect(ledger.snapshot("pool", "P")).toMatchObject({ settled: 50, unresolved: 10, reserved: 0 }); + expect(reserve(ledger, "exact", "P", 40).reserved).toBe(true); + const written = disk.lines.map(line => JSON.parse(line)).findLast(record => record.kind === "reserve"); + expect(written.targets).toEqual([{ scope: "pool", alias: pool("P") }]); + expect(ledger.snapshot("pool", "P")).toMatchObject({ settled: 50, unresolved: 10, reserved: 40 }); + expect(reserve(ledger, "over", "P", 1)).toMatchObject({ reserved: false, denial: { projected: 101 } }); + } +}); + +test("restart does not self-bind ambiguous canonical history", () => { + const disk = journal(); + let ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(), now: () => 2 }); + expect(reserve(ledger, "initial", "P", 40).reserved).toBe(true); + expect(ledger.settle("initial", { inputTokens: 40, outputTokens: 0 })).toBe(true); + ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(), now: () => 3 }); + expect(ledger.hasUnboundPositivePoolHistory()).toBe(true); + expect(ledger.snapshot("pool", "Q")?.settled).toBe(40); + expect(reserve(ledger, "other-over", "Q", 61)).toMatchObject({ reserved: false, denial: { projected: 101 } }); + ledger.reconfigure(policy({ [pool("P")]: "P" })); + expect(ledger.hasUnboundPositivePoolHistory()).toBe(false); + expect(ledger.snapshot("pool", "Q")).toBeUndefined(); + ledger.reconfigure(policy()); + expect(ledger.snapshot("pool", "Q")?.settled).toBe(40); + expect(disk.lines.some(line => JSON.parse(line).poolContinuity !== undefined)).toBe(false); +}); + +test("combo A to B preserves exact current permit exclusion", async () => { + const config: OcxConfig = { port: 0, defaultProvider: "A", providers: { + A: { adapter: "openai-chat", apiKey: "fixture-a", baseUrl: "https://a.example.test/v1" }, + B: { adapter: "openai-chat", apiKey: "fixture-b", baseUrl: "https://b.example.test/v1" }, + }, combos: { spend: { strategy: "failover", targets: [{ provider: "A", model: "m" }, { provider: "B", model: "m" }] } } }; + const disk = journal(); + const ledger = createSpendReservationLedger({ journal: disk, salt, + policy: policy({ [pool("A")]: "A", [pool("B")]: "B" }, 10) }); + const ctx = log("stale"); + const tracker = createRequestSpendTracker(ctx, undefined, ledger); + const budget = createRequestExecutionBudget(undefined, "combo-contract-c", tracker); + const translatorBudget = createTranslatorBudget(); + clearComboSelectionState(); clearComboTargetCooldowns(); + const visited: string[] = []; + try { + const body = { model: "combo/spend", input: [], max_output_tokens: 10 }; + const result = await executeComboResponses(new Request("http://localhost/v1/responses", { + method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body), + }), body, "spend", config, ctx, { sendBudget: budget, translatorBudget }, { + handleResponses: async (_req, _config, child, options) => { + const provider = ctx.spendPoolId!; + visited.push(provider); + const proof = claimDispatchSpendProof(options!.sendBudget!, options!.comboDispatchPermit); + expect(proof).toBeDefined(); + expect(workflowSpendCeilingReached(undefined, ledger, provider, proof)).toBeUndefined(); + expect(ledger.snapshot("pool", provider)?.reserved).toBe(10); + if (provider === "B") expect(workflowSpendCeilingReached(undefined, ledger, "A", proof)?.scope).toBe("pool"); + const report = createPhysicalSendReporter(options!.sendBudget!, () => ({ poolId: provider }), options!.comboDispatchPermit); + expect(report.beforeSend?.()).toBe(true); + report(1); report.close?.(); + return provider === "A" ? Response.json({ error: { message: "fixture outage" } }, { status: 503 }) + : Response.json({ id: "fixture", output: [] }); + }, + handleComboResponses: async () => { throw new Error("unexpected nested combo"); }, + }); + expect(result.status).toBe(200); + expect(visited).toEqual(["A", "B"]); + tracker.settle({ inputTokens: 10, outputTokens: 0 }); + expect(ledger.snapshot("pool", "A")).toMatchObject({ unresolved: 10, reserved: 0 }); + expect(ledger.snapshot("pool", "B")).toMatchObject({ settled: 10, reserved: 0 }); + expect(disk.lines.some(line => JSON.parse(line).poolContinuity !== undefined)).toBe(false); + } finally { translatorBudget.dispose(); clearComboSelectionState(); clearComboTargetCooldowns(); } +}); diff --git a/tests/responses/responses-spend-ledger-wiring.test.ts b/tests/responses/responses-spend-ledger-wiring.test.ts index 152e69f0f6e..ec13c7085ca 100644 --- a/tests/responses/responses-spend-ledger-wiring.test.ts +++ b/tests/responses/responses-spend-ledger-wiring.test.ts @@ -1,3 +1,4 @@ +import { hashSpendAlias } from "../../src/lib/spend-pool-alias-validation"; import { describe, expect, test } from "bun:test"; import { mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; @@ -7,7 +8,7 @@ import { DEFAULT_SPEND_RESERVATION_POLICY, type SpendJournal, } from "../../src/lib/spend-reservation-ledger"; -import { createRequestExecutionBudget, reportDispatchSends } from "../../src/lib/request-execution-budget"; +import { createRequestExecutionBudget, reportDispatchSends, createPhysicalSendReporter } from "../../src/lib/request-execution-budget"; import { createRequestSpendTracker } from "../../src/server/responses/request-spend"; import { SpendLedgerOwnerError, type SpendLedgerOwnerErrorCode } from "../../src/lib/spend-ledger-owner"; import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; @@ -97,7 +98,8 @@ describe("the request path books every physical send on the durable ledger", () }); test("a canonical pool identity is independent of an account-specific display label", () => { - const ledger = createSpendReservationLedger({ journal: memoryJournal() }); + const journal = memoryJournal(); + const ledger = createSpendReservationLedger({ journal, salt: "wiring-fixture" }); const tracker = createRequestSpendTracker(logContext({ provider: "anthropic-p123abc", spendPoolId: "anthropic", @@ -105,7 +107,8 @@ describe("the request path books every physical send on the durable ledger", () expect(tracker.charge()).toBe(true); expect(ledger.snapshot("pool", "anthropic")?.reserved).toBe(500); - expect(ledger.snapshot("pool", "anthropic-p123abc")).toBeUndefined(); + const record = journal.lines.map(line => JSON.parse(line)).find(record => record.kind === "reserve"); + expect(record.targets.find((target: { scope: string }) => target.scope === "pool").alias).toBe(hashSpendAlias("wiring-fixture", "pool", "anthropic")); }); test("resolved Responses routes share a pool ceiling across account display labels", async () => { @@ -124,9 +127,13 @@ describe("the request path books every physical send on the durable ledger", () // Credential resolution gives each request its own account-qualified log label. logCtx.provider = `pool-${account}`; const tracker = createRequestSpendTracker(logCtx, `root-${account}`, ledger); - expect(tracker.charge()).toBe(allowed); + const budget = createRequestExecutionBudget(undefined, undefined, tracker); + const reporter = createPhysicalSendReporter(budget, () => ({ poolId: logCtx.spendPoolId, identityId: logCtx.accountLogLabel })); + expect(reporter.beforeSend?.()).toBe(allowed); + if (allowed) reporter(1); + reporter.close?.(); if (allowed) tracker.settle({ inputTokens: 100, outputTokens: 300 }); - expect(ledger.snapshot("pool", logCtx.provider)).toBeUndefined(); + expect(ledger.snapshot("pool", logCtx.provider)?.reserved ?? 0).toBe(0); } expect(ledger.snapshot("pool", "pool")?.settled).toBe(400); expect(ledger.snapshot("identity", "account-a")?.settled).toBe(400); @@ -142,16 +149,19 @@ describe("the request path books every physical send on the durable ledger", () } }; const ledger = createSpendReservationLedger({ journal: memoryJournal(), policy: { ...DEFAULT_SPEND_RESERVATION_POLICY, pool: { maxTokens: 500 }, - } }); + poolAliases: { [hashSpendAlias("wiring-fixture", "pool", "first")]: "first", [hashSpendAlias("wiring-fixture", "pool", "second")]: "second" }, + }, salt: "wiring-fixture" }); const logCtx: RequestLogContext = { model: "", provider: "", spendPoolId: "stale-route", usageLogInputTokens: 100, spendOutputCeilingTokens: 300 }; const tracker = createRequestSpendTracker(logCtx, "fallback-root", ledger); + const budget = createRequestExecutionBudget(undefined, undefined, tracker); for (const provider of ["first", "second"]) { const parsed = parseRequest({ model: `${provider}/model`, input: [] }); await applyFinalRouteRequestNormalization({ parsed, route: routeModel(config, parsed.modelId), config, req: new Request("http://localhost/v1/responses"), logCtx, inboundWire: "responses" }); logCtx.provider = `${provider}-account`; - expect(tracker.charge()).toBe(true); + const reporter = createPhysicalSendReporter(budget, () => ({ poolId: logCtx.spendPoolId })); + expect(reporter.beforeSend?.()).toBe(true); reporter(1); reporter.close?.(); expect(ledger.snapshot("pool", provider)?.reserved).toBe(400); } tracker.settle({ inputTokens: 100, outputTokens: 300 }); @@ -171,7 +181,8 @@ describe("the request path books every physical send on the durable ledger", () ] } } }; const ledger = createSpendReservationLedger({ journal: memoryJournal(), policy: { ...DEFAULT_SPEND_RESERVATION_POLICY, pool: { maxTokens: 500 }, - } }); + poolAliases: { [hashSpendAlias("wiring-fixture", "pool", "first")]: "first", [hashSpendAlias("wiring-fixture", "pool", "second")]: "second" }, + }, salt: "wiring-fixture" }); const logCtx: RequestLogContext = { model: "", provider: "", spendPoolId: "stale-route", usageLogInputTokens: 100, spendOutputCeilingTokens: 300 }; const tracker = createRequestSpendTracker(logCtx, "combo-root", ledger); @@ -189,7 +200,9 @@ describe("the request path books every physical send on the durable ledger", () }), body, "spend", config, logCtx, { sendBudget, translatorBudget }, { handleResponses: async (_req, _config, childLog, options) => { children += 1; - reportDispatchSends(options!.sendBudget!, 1, options!.comboDispatchPermit); + const reporter = createPhysicalSendReporter(options!.sendBudget!, () => ({ poolId: childLog.spendPoolId, + identityId: childLog.accountLogLabel }), options!.comboDispatchPermit); + expect(reporter.beforeSend?.()).toBe(true); reporter(1); reporter.close?.(); childLog.provider += `-account-${children}`; return children === 1 ? Response.json({ error: { message: "fixture outage" } }, { status: 503 }) @@ -230,21 +243,24 @@ describe("the request path books every physical send on the durable ledger", () expect(root?.settled).toBe(0); }); - test("a ledger ceiling refuses the dispatch instead of describing it afterwards", () => { - const ledger = createSpendReservationLedger({ - journal: memoryJournal(), - policy: { ...DEFAULT_SPEND_RESERVATION_POLICY, root: { maxTokens: 900 } }, - }); + test("a normal first target refuses while stable retries reuse its anchor", () => { + const ledger = createSpendReservationLedger({ journal: memoryJournal(), policy: { + ...DEFAULT_SPEND_RESERVATION_POLICY, root: { maxTokens: 900 }, + } }); const tracker = createRequestSpendTracker(logContext(), "root-d", ledger); const budget = createRequestExecutionBudget(undefined, "lr-ceiling", tracker); - - expect(budget.reserveDispatch({ sendClass: "initial", targetKey: "p|m" }).allowed).toBe(true); - const refused = budget.reserveDispatch({ sendClass: "transient", targetKey: "p|m" }); + for (const sendClass of ["initial", "transient"] as const) { + const decision = budget.reserveDispatch({ sendClass, targetKey: "p|m" }); + expect(decision.allowed).toBe(true); + if (!decision.allowed) throw new Error("stable retry refused"); + expect(decision.permit.use()).toBe(true); + } + expect(ledger.snapshot("root", "root-d")?.reserved).toBe(1000); + const refused = budget.reserveDispatch({ sendClass: "account-failover", targetKey: "other|m" }); expect(refused.allowed).toBe(false); - if (refused.allowed) throw new Error("unreachable"); + if (refused.allowed) throw new Error("new target admitted"); expect(refused.reason).toBe("spend-exhausted"); - // Refused before the budget charged it, so the send is not counted either. - expect(budget.used).toBe(1); + expect(budget.used).toBe(2); expect(tracker.refusals).toBe(1); }); @@ -275,42 +291,24 @@ describe("the request path books every physical send on the durable ledger", () expect(third.snapshot("root", "root-e")?.reserved).toBe(0); }); - test("a send that already left is recorded past the ceiling, so the next one can be refused", () => { - // The canonical passthrough ladder does not reserve its physical sends; it reports them - // after the fetch through onSendsConsumed, which assigns through budget.used. A ceiling - // cannot refuse those -- the tokens are spent -- and DROPPING them is a fixpoint: the send - // that would cross the limit never joins the total, the total sits one send short of the - // ceiling forever, and nothing is ever refused. Recording it is what arms the refusal. - const ledger = createSpendReservationLedger({ - journal: memoryJournal(), - policy: { ...DEFAULT_SPEND_RESERVATION_POLICY, root: { maxTokens: 900 } }, - }); + test("seeded already-sent usage crosses the ceiling without dropping liability", () => { + const ledger = createSpendReservationLedger({ journal: memoryJournal(), policy: { + ...DEFAULT_SPEND_RESERVATION_POLICY, root: { maxTokens: 900 }, + } }); const tracker = createRequestSpendTracker(logContext(), "root-f", ledger); const budget = createRequestExecutionBudget(undefined, "lr-reported", tracker); - - budget.used += 1; - expect(ledger.snapshot("root", "root-f")?.reserved).toBe(500); - expect(ledger.exhausted("root", "root-f")).toBe(false); - - // This one projects 1000 against a ceiling of 900 and is booked anyway, because it left. - budget.used += 1; + const reporter = createPhysicalSendReporter(budget, () => ({ poolId: "test-pool" })); + expect(reporter.beforeSend?.()).toBe(true); + expect(reporter.beforeSend?.()).toBe(true); + reporter(2); reporter.close?.(); expect(ledger.snapshot("root", "root-f")?.reserved).toBe(1000); expect(ledger.exhausted("root", "root-f")).toBe(true); - // Nothing was refused after the fact -- there was nothing left to refuse. expect(tracker.refusals).toBe(0); - - // The ceiling is now armed: the next send that asks BEFORE dispatching is refused. - const refused = budget.reserveDispatch({ sendClass: "transient", targetKey: "p|m" }); - expect(refused.allowed).toBe(false); - if (refused.allowed) throw new Error("unreachable"); - expect(refused.reason).toBe("spend-exhausted"); - - // And a send that already left cannot be handed back for free afterwards: it is marked - // dispatched when it is recorded, so a refund keeps the tokens as unresolved spend. - tracker.refund(); - const root = ledger.snapshot("root", "root-f"); - expect(root?.reserved).toBe(500); - expect(root?.unresolved).toBe(500); + tracker.settle({ inputTokens: 600 }); + expect(ledger.snapshot("root", "root-f")?.unresolved).toBe(500); + expect(ledger.snapshot("root", "root-f")?.settled).toBe(600); + const next = createRequestExecutionBudget(undefined, undefined, createRequestSpendTracker(logContext(), "root-f", ledger)); + expect(next.reserveDispatch({ sendClass: "initial", targetKey: "p|m" }).allowed).toBe(false); }); test("under a configured ceiling a reservation that cannot be made durable refuses the send", () => { From e1f2de2a7b38137c283b595a9e74666da6c52db1 Mon Sep 17 00:00:00 2001 From: JUN Date: Thu, 8 Oct 2026 23:54:11 +0900 Subject: [PATCH 09/15] fix(spend): bind contract C accounting to physical executors Preserve v1 ledger interpretation, durable seed settlement, actual transport coverage and one-invocation CLI accounting. Co-authored-by: Epinephrine --- .../docs/ja/reference/configuration/server.md | 89 ++--- .../docs/ko/reference/configuration/server.md | 90 ++--- .../docs/reference/configuration/server.md | 131 +++---- .../docs/ru/reference/configuration/server.md | 105 +++--- .../zh-cn/reference/configuration/server.md | 79 ++--- src/adapters/coding-agent/turn.ts | 25 +- src/adapters/kiro-retry.ts | 13 +- src/adapters/physical-send.ts | 13 +- src/adapters/zed.ts | 17 +- src/lib/request-execution-budget.ts | 21 +- src/lib/spend-reservation-ledger.ts | 7 +- src/server/responses/adapter-dispatch.ts | 3 + src/server/responses/compact.ts | 9 +- src/server/responses/request-send-budget.ts | 13 +- src/server/responses/request-spend.ts | 10 +- structure/config.md | 2 +- structure/providers-and-adapters.md | 2 +- structure/providers/kiro.md | 2 + structure/runtime.md | 4 +- structure/transports/responses-spend.md | 158 +++++---- tests/lib/execution-budget-permits.test.ts | 53 +++ tests/lib/spend-seed-overflow.test.ts | 71 +++- tests/providers/claude-cli-adapter.test.ts | 78 ++++- tests/providers/zed-hosted-provider.test.ts | 72 +++- .../responses/responses-spend-anchor.test.ts | 319 +++++++++++++++++- tests/server/inference-send-budget.test.ts | 21 ++ 26 files changed, 1005 insertions(+), 402 deletions(-) diff --git a/docs-site/src/content/docs/ja/reference/configuration/server.md b/docs-site/src/content/docs/ja/reference/configuration/server.md index dab41bbda05..135fdfe4cd4 100644 --- a/docs-site/src/content/docs/ja/reference/configuration/server.md +++ b/docs-site/src/content/docs/ja/reference/configuration/server.md @@ -202,63 +202,32 @@ The dashboard warns about old or unknown CLI versions, unavailable targets, and ## 過去のプール使用量の継承 -プロバイダープールの上限にはルーティング先の正規プロバイダーを使い、リクエストログにはアカウント別の -表示ラベルを残します。古いジャーナルでは、その表示ラベルに使用量が計上されている場合があります。 -ジャーナルに保存されるのはソルト付きのエイリアスであり、元のプロバイダー名やアカウント名ではありません。 -そのため、OpenCodex は現在のアカウント一覧、ラベルの接頭辞、短縮されたアカウント ID から対応関係を推測しません。 - -`spend.pool.maxTokens` が設定され、正の過去残高の所有先が不明な場合、元のスコープに残したまま、 -各残高を候補となるすべてのプロバイダープールに一度ずつ加算し、そのプロバイダーの確認済みグループと合算します。 -この合計と予約分が上限に収まる場合だけリクエストを許可します。そのため、未使用のプールも一時的に制限されることがあります。 -ローカル拒否では、エイリアスやジャーナル内容を公開せず、この保守的な加算を説明します。 -ルートが判明する前は、不明な履歴だけを理由に HTTP admission を拒否しません。 -ルーティング後の事前確認では、保守的に合算した使用量がすでに上限に達している正規プールを拒否します。 -予約によってすでに許可されたリカバリー送信やコンボ送信では、その予約を自分自身に対して再度計上しません。 -それ以外の予約はすべて計上されます。この確認により、送信後に使用量を報告するパススルー送信が -アトミックな予約に変わるわけではありません。上限をまたぐ送信、同時に許可されるリクエスト、 -事後報告される再試行には、従来の制約が残ります。監視のみの構成は引き続き監視のみです。 -ルートと ID の上限もそれぞれ独立して適用されます。 - -解決するには、運用者自身が保持している証拠を使い、過去の各ソルト付きプールエイリアスが -どの正規プロバイダーに属するかを確認する必要があります。`config.json` のトップレベルに -`spendPoolAliases` オブジェクトを追加してください。各キーには、その同じインストールのジャーナルにある -32 文字の小文字 16 進数のプールエイリアスを正確に指定し、各値には確認済みの正規プロバイダー ID を指定します。 -古いエイリアスがすでに正規プロバイダーを表していても、ID メタデータがなければ明示的なエントリが必要です。 -正の使用量を持ち、識別できないエイリアスは、すべての候補プールに保守的に加算されます。 -似た名前、アカウントの削除、現在のプロバイダー名やハッシュ、短縮ラベルの衝突から対応関係を推測しないでください。 -ジャーナルやソルトを公開しないでください。 - -`spendPoolAliases` は `spend` の外に置いてください。古いバージョンは `spend` 内の未知のキーを拒否し、 -セクション全体を無効にする場合があります。無効なトップレベルのマッピングは設定の書き込み時に拒否されます。 -手動編集で `spendPoolAliases` が不正になった場合、既存の上限を維持したままプールへのリクエスト受け入れを拒否します。 -通常の設定手順でマッピングを修正し、再起動してください。マッピングを空にしたり削除したりしても、 -すでに永続的に記録された対応関係は消えません。一度リダイレクトされたエイリアスを別のグループへ再割り当てすることはできません。 -確認済みの正規プロバイダー名の変更では、既存の使用量を分割せずにグループを統合できます。 -マッピング全体をまとめて検証するため、有効なグループ統合はエイリアスキーの順序に依存しません。 - -元の使用量はそれぞれ一度だけ計上されます。確定済みの使用量、処理中の予約、未解決の使用量はすべて計上し、 -不明な使用量を払い戻しとして扱うことはありません。元の予約先は保持されます。 -新しいリクエストでは引き続き正規プロバイダーのプールを記録し、チェックポイントには生のアカウント名や -プロバイダー名を追加せず、ソルト付きの ID 識別情報を保持します。識別できない正の使用履歴と、稼働中または -上限に達したグループはクリーンアップから保護されます。休眠中で上限未満の項目には従来の保持ルールが引き続き適用されます。 -ID 識別情報の保存容量には上限があり、容量を使い切った場合は情報を忘れるのではなくプールへの受け入れを停止します。 -検証できない履歴を復元する自動ツールはありません。 - -### 安全なロールバックの要件 - -サポートされるロールバックでは、リクエストを正規プロバイダーに帰属させる処理と、プール間の継承に対応した -台帳の読み書き処理の両方を維持するかバックポートし、同じジャーナル、ソルト、確認済みマッピングを使う必要があります。 -最新のジャーナルを保持してください。アップグレード前のコピーを復元すると、それ以降の使用量が欠落します。 -ダウングレードを互換性があるように見せるために、記録の削除、ソルトの変更、上限の引き上げ、制限の無効化をしないでください。 - -変更を加えていない古いバイナリへの切り戻しは、**サポート対象のロールバックではありません**。 -古いバイナリは v1 の生の計上値を読めても、エイリアスをまたぐプロバイダー合計に上限を適用せず、 -新たなアカウントラベルのプールを作成したり、圧縮時に任意の ID メタデータを破棄したりする場合があります。 -ダウングレードを自動的に防ぐ仕組みはありません。そのような古い書き込み処理が実行された場合、新しい読み取り処理は、 -運用者が残っているマッピングを確認するまで、識別できない正の使用履歴を各候補プールに保守的に加算します。 -ストレージやジャーナルの整合性による拒否はエイリアスの問題とは別で、`workflow_spend_undurable` のままです。 -安全でないファイルや所有権の問題は、代わりにストレージエラーとして伝播する場合がありますが、リクエストは許可されません。 - -## トークン予約と上限 - -リクエストに `spend.root.maxTokens`、`spend.identity.maxTokens`、`spend.pool.maxTokens` のいずれかが適用され、追跡容量の不足や送信 ID の重複などで予約を記録できない場合、新しい送信はディスパッチ前に拒否されます。適用される上限がないリクエストは引き続き観測のみです。 +プロバイダープールの上限は正規のルーティング先に適用し、ログのアカウント表示名とは分けて扱います。 +`spendPoolAliases` は `config.json` のトップレベルに置きます。キーは同じインストールの過去のプール別名 +（32 文字の小文字 16 進数）、値は確認済みの正規プロバイダー ID です。現在の名前やアカウント一覧から推測しません。 + +対応関係は読み取り時だけ適用し、元の残高を移動したり、ジャーナルにリンクを保存したりしません。 +削除した対応関係の履歴は再び所有先不明になります。各元残高は確認済みグループか、全候補への保守的な加算の +どちらかで一度だけ計上します。現在の正規別名と一致していても、未登録の履歴を二重計上しません。 +設定済みプロバイダー自身のソルト付き別名を別のプロバイダーに割り当てる設定は拒否します。 +プロバイダー追加時にも再検証し、検証だけでソルトを作成することはありません。 + +休眠中の所有先不明の履歴は、最終活動時刻が `spend.retentionDays` の期限より古くなってから失効します。 +容量不足を理由に正の履歴を早く削除しません。送信中の予約と seed はゼロトークンでも保持し、削除記録の保存後に残高を減らします。 +適用される上限がある送信は、各送信先・キーの初回に通常容量の予約を必要とします。 +再試行は同じ範囲を使い、要求全体の有限の送信回数上限（通常 4、既存の OAuth 拡張では最大 18）を共有します。 +最終精算は報告完了を待ち、送信 ID の削除後も残高を保持します。上限未設定では従来の監視のみの動作を保ち、容量満杯時の記録省略も変わりません。 + +Claude CLI、CodeBuddy、Qoder は CLI 呼び出し 1 回を送信 1 回と数え、起動前に通常の予約を必要とします。 +CLI 内部の再試行・ツールターンは L に個別計上しませんが、初期見積もりを超えた分も実際に報告された使用量で精算します。 + +### 2.80.0 へのロールバック（契約 C） + +新しい記録は通常の v1 形式と `pool` 別名を使います。未改変の 2.80.0 は自身のラベル別規則で読み取り、圧縮・保持処理を行えます。 +最新のジャーナルとソルトを維持してください。ダウングレード後の正規プール合算や、同じ通信を最初から 2.80.0 で処理した場合の残り枠は保証しません。 +再アップグレード時には、残存する元残高に現在の設定を適用します。起動フェンスや追加の照合コマンドは不要です。 +未公開の実験版の `pool-current` / `poolContinuity` 履歴は契約 C の対象外です。残高は所有先不明の履歴として保持し、通常の期限でのみ失効させます。 + +完全だが不正な記録（最終行の `null` も含む）は上限付きの受け入れを拒否します。圧縮は有効な残高を保持して続行し、 +実行中の拒否はクリーンなジャーナルを再起動で読み直すまで残ります。途中で切れた最終 JSON 行は従来の復旧規則に従います。 +詳しい境界とエラーは[英語の正本](/reference/configuration/server/#historical-pool-continuity)を参照してください。 diff --git a/docs-site/src/content/docs/ko/reference/configuration/server.md b/docs-site/src/content/docs/ko/reference/configuration/server.md index 32dc569716b..cfde815855e 100644 --- a/docs-site/src/content/docs/ko/reference/configuration/server.md +++ b/docs-site/src/content/docs/ko/reference/configuration/server.md @@ -259,59 +259,37 @@ The dashboard warns about old or unknown CLI versions, unavailable targets, and ## 과거 풀 사용량의 연속성 -제공자 풀 상한은 라우팅된 정규 제공자를 기준으로 적용하고, 요청 로그에는 계정별 표시 레이블을 유지합니다. -이전 저널에는 이 표시 레이블에 사용량이 기록되어 있을 수 있습니다. 저널은 원래 제공자·계정 이름이 아닌 -솔트가 적용된 별칭을 저장하므로, OpenCodex는 현재 계정 목록, 레이블 접두사, 축약된 계정 ID로 매핑을 추측하지 않습니다. - -`spend.pool.maxTokens`가 설정되어 있고 양수인 과거 잔액의 소유자를 알 수 없으면, -각 잔액은 원래 스코프에 보존되며 후보 제공자 풀마다 한 번씩 더해져 확인된 그룹과 합산됩니다. -합계와 예약량이 한도에 들어오는 경우에만 요청을 허용합니다. 이 방식은 사용하지 않는 풀도 일시적으로 제한할 수 있습니다. -로컬 거부 메시지는 별칭이나 저널 내용을 공개하지 않고 이러한 보수적 계산을 설명합니다. -라우트를 알기 전에는 불명확한 이력만으로 HTTP admission을 거부하지 않습니다. -라우팅 후 사전 검사에서도 보수적으로 합산한 사용량이 이미 한도에 도달한 정규 풀을 거부합니다. -예약을 통해 이미 허용된 복구 또는 콤보 전송에서는 그 예약을 자신에게 다시 계산하지 않습니다. -다른 예약은 모두 계산합니다. 이 검사가 사용량을 사후 보고하는 패스스루 전송을 원자적 예약으로 바꾸지는 않습니다. -한도를 넘기는 전송, 동시 요청 허용, 사후 보고되는 재시도에는 기존 제약이 그대로 남습니다. -관찰 전용 구성은 계속 관찰 전용으로 동작합니다. 루트와 ID 상한도 각각 독립적으로 적용됩니다. - -해결하려면 운영자가 보관한 근거를 사용해 과거의 각 솔트 적용 풀 별칭이 어느 정규 제공자에 속하는지 확인해야 합니다. -`config.json`의 최상위에 `spendPoolAliases` 객체를 추가하세요. 각 키는 동일 설치의 저널에 있는 -32자리 소문자 16진수 풀 별칭과 정확히 일치해야 하고, 각 값은 확인된 정규 제공자 ID여야 합니다. -이전 별칭이 이미 정규 제공자를 나타내더라도 ID 메타데이터가 없으면 명시적 항목이 필요합니다. -식별할 수 없는 양수 사용량의 별칭은 모든 후보 풀에 보수적으로 계산됩니다. -비슷한 이름, 계정 삭제, 현재 제공자 이름이나 해시, 축약 레이블 충돌로 매핑을 추측하지 말고, -저널이나 솔트를 공개하지 마세요. - -`spendPoolAliases`는 `spend` 밖에 두세요. 이전 버전은 `spend` 안의 알 수 없는 키를 거부하며 -섹션 전체를 비활성화할 수 있습니다. 잘못된 최상위 매핑은 설정 쓰기 시 거부됩니다. -수동 편집으로 `spendPoolAliases`가 잘못된 형식이 되면 기존 상한을 유지한 채 풀 요청 허용을 차단합니다. -일반적인 설정 절차로 매핑을 수정하고 재시작하세요. 매핑을 비우거나 삭제해도 이미 영구 기록된 연결은 지워지지 않습니다. -이전에 다른 대상으로 연결한 별칭은 다른 그룹에 재할당할 수 없습니다. 확인된 정규 제공자 이름 변경은 -기존 사용량을 나누지 않고 그룹을 합칠 수 있습니다. 전체 매핑을 함께 검증하므로 유효한 그룹 병합은 별칭 키 순서에 의존하지 않습니다. - -각 원래 사용량은 한 번만 계산합니다. 확정된 사용량, 진행 중 예약, 미해결 사용량을 모두 계산하며, -알 수 없는 사용량을 환급으로 취급하지 않습니다. 원래 예약 대상은 유지됩니다. -새 요청은 계속 정규 제공자 풀을 기록하고, 체크포인트는 원래 계정·제공자 이름을 추가하지 않은 채 -솔트가 적용된 ID 식별 근거를 보존합니다. 알 수 없는 양수 사용 이력과 활성 상태 또는 한도에 도달한 그룹은 -정리 대상에서 보호됩니다. 휴면 상태이고 한도 미만인 항목에는 기존 보존 규칙이 계속 적용됩니다. -ID 식별 근거의 저장 용량에는 한도가 있으며, 용량을 소진하면 근거를 잊는 대신 풀 요청 허용을 중단합니다. -검증할 수 없는 이력을 복원하는 자동 도구는 없습니다. - -### 안전한 롤백 요건 - -지원되는 롤백은 요청을 정규 제공자에 귀속시키는 처리와 연속성을 인식하는 원장 읽기·쓰기 처리를 모두 유지하거나 -백포트해야 하며, 동일한 저널, 솔트, 검증된 매핑을 사용해야 합니다. -최신 저널을 유지하세요. 업그레이드 전 사본을 복원하면 이후 사용량이 누락됩니다. -다운그레이드가 호환되는 것처럼 보이게 하려고 기록을 삭제하거나, 솔트를 바꾸거나, 상한을 높이거나, 제한 적용을 끄지 마세요. - -수정하지 않은 이전 바이너리로 되돌리는 것은 **지원되는 롤백이 아닙니다**. -이전 바이너리는 v1 원시 사용량을 읽을 수 있지만 별칭 간 제공자 합계에 상한을 적용하지 않으며, -새 계정 레이블 풀을 만들거나 압축할 때 선택적 ID 메타데이터를 버릴 수 있습니다. -다운그레이드를 자동으로 막는 장치는 없습니다. 이런 이전 쓰기 처리가 실행되었다면, 새 읽기 처리는 -운영자가 남은 매핑을 확인할 때까지 식별되지 않은 양수 사용 이력을 모든 후보 제공자 풀에 보수적으로 계산합니다. -스토리지 또는 저널 무결성에 따른 거부는 별칭 문제와 별개이며 `workflow_spend_undurable`을 유지합니다. -안전하지 않은 파일이나 소유권 문제는 대신 스토리지 오류로 전파될 수 있으며, 이 경우에도 요청을 허용하지 않습니다. - -## 토큰 예약 및 한도 - -요청에 `spend.root.maxTokens`, `spend.identity.maxTokens`, `spend.pool.maxTokens` 중 하나가 적용되고 추적 용량 부족이나 중복된 전송 ID 등으로 예약을 기록할 수 없으면 새 전송을 디스패치 전에 거부합니다. 적용되는 한도가 없는 요청은 계속 관측만 합니다. +제공자 풀 한도는 실제로 선택된 제공자에 적용합니다. 요청 로그의 계정 표시 이름과는 구분합니다. +`config.json` 최상위의 `spendPoolAliases`에 검증된 매핑을 넣으세요. 키는 같은 설치의 과거 풀 별칭인 +32자리 소문자 16진수이고, 값은 확인된 제공자 ID입니다. 현재 이름이나 계정 목록만 보고 추측하지 마세요. + +매핑은 잔액을 읽을 때만 적용합니다. 원래 잔액을 옮기거나 저널에 연결 정보를 저장하지 않습니다. +매핑을 삭제하면 그 이력은 다시 소유자 미확인 상태가 됩니다. 각 잔액은 확인된 그룹에 넣거나 모든 후보 풀에 +보수적으로 더하며, 한 번만 계산합니다. 현재 제공자의 해시와 같은 미확인 잔액도 두 번 세지 않습니다. +설정된 제공자 자신의 솔트 별칭을 다른 제공자에 연결하면 거부합니다. 제공자를 추가할 때도 검사하며, +검증 과정에서 새 솔트를 만들지는 않습니다. 매핑이 잘못돼도 기존 한도는 유지합니다. + +휴면 상태인 미확인 이력은 마지막 활동이 `spend.retentionDays`보다 오래됐을 때만 만료됩니다. +용량이 부족해도 양수 이력을 먼저 지우지 않습니다. 진행 중인 예약과 seed는 0토큰이어도 보존하며, +삭제 기록이 저장된 뒤에만 허용 여부를 판단하는 잔액을 줄입니다. +한도가 적용되는 요청은 대상·키별 첫 전송 전에 일반 추적 용량으로 예약해야 합니다. 재시도는 같은 범위를 쓰며, +요청 전체가 첫 전송 때 고정한 유한한 전송 횟수 한도(기본 4회, 기존 OAuth 확장은 최대 18회)를 공유합니다. +최종 정산은 보고가 끝날 때까지 기다립니다. 전송 ID를 지워도 잔액은 남습니다. +적용되는 한도가 없으면 기존 관측 동작을 유지하며, 추적 용량이 찼을 때 기록을 생략하는 동작도 그대로입니다. + +Claude CLI·CodeBuddy·Qoder는 CLI 호출 한 번을 전송 한 번으로 셉니다. 실행 전 일반 예약이 필요하며, 예약하지 못하면 실행을 거부합니다. +CLI 내부 재시도·도구 턴은 L에 따로 세지 않지만 비용은 보고된 실제 사용량으로 정산하며, 최초 추정량을 넘은 비용도 기록합니다. + +### 2.80.0으로 롤백: 계약 C + +새 기록은 기존 v1 형식과 `pool` 별칭을 씁니다. 수정하지 않은 2.80.0도 자신의 라벨별 규칙대로 읽고 압축·보존할 수 있습니다. +최신 저널과 솔트를 유지하세요. 롤백 후 제공자별 통합 집계가 유지되거나, 같은 요청을 처음부터 2.80.0에서 처리했을 때와 +남은 한도가 같다는 보장은 없습니다. 다시 업그레이드하면 남아 있는 원래 잔액에 현재 매핑을 적용합니다. +별도의 실행 차단 장치나 조정 명령은 필요하지 않습니다. +미공개 실험 버전의 `pool-current` / `poolContinuity` 이력은 계약 C에서 제외합니다. +원래 잔액을 미확인 이력으로 보존하고 일반 보존 기간이 지난 뒤 만료시키며, 즉시 변환하거나 삭제하지 않습니다. + +완전하지만 잘못된 기록(마지막 줄의 `null` 포함)이 있으면 한도가 적용되는 요청을 거부합니다. +압축은 유효한 잔액을 보존하며 계속할 수 있습니다. 실행 중 거부 상태는 재시작 뒤 깨끗한 저널을 읽을 때까지 남습니다. +쓰다 끊긴 마지막 JSON 줄은 기존 복구 규칙을 따릅니다. 세부 조건과 오류 코드는 +[영문 원문](/reference/configuration/server/#historical-pool-continuity)을 참조하세요. diff --git a/docs-site/src/content/docs/reference/configuration/server.md b/docs-site/src/content/docs/reference/configuration/server.md index b36f7f426e0..bdb3b22a286 100644 --- a/docs-site/src/content/docs/reference/configuration/server.md +++ b/docs-site/src/content/docs/reference/configuration/server.md @@ -944,62 +944,75 @@ The dashboard warns about old or unknown CLI versions, unavailable targets, and ## Historical pool continuity -Provider-pool ceilings use the canonical routed provider, while request logs keep account-specific -display labels. Older journals may hold spend under those display labels. A journal stores salted -aliases, not the original provider/account names, so OpenCodex does not guess a mapping from a -current account roster, a label prefix, or a shortened account ID. - -If `spend.pool.maxTokens` is configured and positive historical pool balances remain unidentified, -the ledger keeps each balance in its original scope and conservatively counts it once against every -candidate provider pool, in addition to that provider's verified group. A request is admitted only -when that total plus its reservation fits the ceiling. This may restrict an otherwise unused pool -until the operator verifies mappings; the request-local refusal explains this overrestriction -without exposing aliases or journal contents. Unidentified history alone does not block admission -before a route is known. After routing, preflight also refuses a canonical pool whose conservative -total is already exhausted. A recovery or combo send already admitted by a reservation does not -count that same reservation against itself a second time; all other reservations remain counted. -This check does not turn post-reported passthrough sends into atomic reservations: -a crossing send, concurrent admissions or retries reported afterwards retain their existing limits. -Observe-only installs remain observe-only. Root and identity ceilings remain in force independently. - -To resolve it, an operator must verify which canonical provider each historical salted pool alias -belongs to, using their own retained evidence. Verified mappings keep an old balance from being -charged to every candidate pool. Add a top-level `spendPoolAliases` object in -`config.json`: each key is the exact 32-character lowercase hexadecimal pool alias from that same -installation's journal; each value is its verified canonical provider ID. An old alias that already -represents the canonical provider still needs an explicit entry when it lacks identity metadata. -A positive alias that cannot be identified remains conservatively charged to every candidate pool. -Do not infer a match from similar names, account deletion, a current provider name/hash, or a -short-label collision, and do not share the journal or salt publicly. - -Keep `spendPoolAliases` outside `spend`: older versions reject unknown keys inside `spend` and can -disable the entire section. Invalid top-level mappings are rejected on configuration writes; -malformed `spendPoolAliases` hand edits retain existing ceilings and fail pool admission closed. Correct the mapping -and restart through the ordinary configuration workflow. Empty/removing mappings does not erase -links already recorded durably. A previously redirected alias cannot be reassigned to a different -group; verified canonical renames can join groups without splitting existing spend. The complete -mapping is checked together, so a valid group merge does not depend on alias-key order. - -Each original balance is counted once. Settled usage, in-flight reservations and unresolved usage -all count; unknown usage is never treated as a refund. Original reservation targets are retained. -New requests continue to record the canonical provider pool, and checkpoints retain salted identity -evidence without adding raw account/provider names. Unknown positive history and active or exhausted -groups are protected from cleanup; the existing dormant, under-limit retention rule still applies. -Identity evidence remains bounded; if its capacity is exhausted, pool admission stops rather than -forgetting it. No automatic tool reconstructs unverifiable history. - -### Safe rollback requirements - -A supported rollback must retain or backport both canonical request attribution and the -continuity-aware ledger reader/writer, with the same journal, salt and verified mappings. -Keep the latest journal: restoring a pre-upgrade copy would omit later spend. Do not remove records, -change the salt, raise ceilings, or disable enforcement to make a downgrade appear compatible. - -An unmodified older binary is **not a supported rollback**. It can read the v1 raw balances but -does not enforce the cross-alias provider total, may create a new account-label pool, and may discard -optional identity metadata when compacting. There is no automatic downgrade barrier. If such an old -writer has run, the new reader conservatively counts unidentified positive history against every -candidate provider pool until the operator verifies the remaining mappings. Storage or -journal-integrity denials are separate from an alias problem and keep `workflow_spend_undurable`; -unsafe-file/ownership failures may instead propagate as storage errors, without admitting the -request. +Provider-pool ceilings use the canonical routed provider. Request logs still use account-specific +display labels. Earlier journals may record spend under those labels; salted aliases alone cannot +prove which provider owned them. OpenCodex does not infer ownership from current accounts or names. + +With `spend.pool.maxTokens`, each original balance contributes once to its verified provider group +or, when unmapped and positive, to every candidate pool. This includes an unmapped bucket whose +hash matches the current canonical provider; it is never counted twice. Settled, reserved and +unresolved usage all count. Unknown usage is not a refund. This conservative view may restrict an +unused pool until its historical mappings are verified. Root and identity ceilings remain independent. + +Add verified mappings to the top-level `spendPoolAliases` object in `config.json`. Each key is an +exact 32-character lowercase hexadecimal pool alias from this installation's journal; each value +is its verified canonical provider ID. Keep evidence for the mapping and keep the journal and salt +private. A canonical-looking historical alias also needs an explicit mapping to establish ownership. + +Mappings apply when balances are read. They do not move original balances or persist links in the +journal: removing a mapping makes that history unbound again. Keep this object outside `spend`, +because older versions reject unknown fields inside that strict section. Invalid mappings are +rejected on writes; invalid hand edits retain ceilings and refuse pool admission. A configured +provider's own salted pool alias cannot map to another provider. This is checked again when the +provider set changes. Validation reads an existing salt without creating one. + +Dormant unbound history expires only after its last activity is strictly older than +`spend.retentionDays`. Capacity pressure cannot shorten that interval for positive unknown history. +Live reservations and seed targets remain protected, even at zero tokens. Cleanup persists a normal +journal deletion record before admission uses the reduced total. Known groups keep the ordinary +active/exhausted protections. No automatic tool reconstructs unverifiable mappings. + +### Token reservations and reporting + +When a root, identity or pool ceiling applies, the first physical send for each selected target/key +requires a normal-capacity reservation. If that cannot be recorded, the send is refused before it +reaches upstream. Stable retries reuse that reservation's scopes; a different target/key needs a +new one. Additional sends are counted against a finite request-wide limit fixed at the first start +(default four; the existing bounded OAuth profile can expand to eighteen before sending). + +Delayed reports retain already-started usage. Terminal settlement waits for reporters, and send IDs +are forgotten only after their accounting is durable; forgetting an ID does not remove its balance. +The limit bounds retries, not the final bill: actual usage or already-started retries can exceed a +token estimate, and their full liability remains counted. With no applicable ceiling, behavior stays +observe-only, including omitted bookings when tracking capacity is full. No identity checkpoints are +added for unconfigured traffic. + +Claude CLI, CodeBuddy and Qoder count each CLI invocation as one send. Their first invocation +still needs a normal reservation and is refused if that reservation cannot be obtained. Internal +CLI retries and tool turns do not separately consume L; their cost settles from actual reported +usage, including amounts above the initial estimate. This exception preserves CLI use with configured +spend ceilings while limiting the number of child invocations. + +### Rollback to 2.80.0: contract C + +New records use the existing v1 format and `pool` alias domain. **Unmodified 2.80.0 reads them using +its own per-label rules**, including ordinary compaction and retention. There is no required +backport, launcher fence or reconciliation command. Keep the current journal and salt: restoring +an older copy loses the spend recorded since that copy. + +This does not preserve canonical aggregation after downgrade, nor promise the same remaining +allowance that 2.80.0 would have calculated for identical traffic without the upgrade. It may book +new traffic under account labels. On re-upgrade, the current alias configuration applies to the +original balances still retained by 2.80.0. New checkpoints contain no identity-link metadata that +an older reader must preserve. + +Journals written by unpublished experimental builds with `pool-current` aliases or `poolContinuity` +metadata are excluded from this compatibility contract, including later checkpoints carrying those +aliases. Their original balances and send targets remain conservative unbound history until normal +retention expires them; they are not converted or immediately deleted. + +A complete invalid journal record (including a final `null`) causes configured admission to refuse. +Compaction can still preserve valid accounting in a clean checkpoint; the live refusal remains until +a clean restart replays it. A torn, unparseable final line keeps the existing recovery behavior. +Storage/corruption denials use `workflow_spend_undurable`; unsafe files and ownership failures remain +storage errors. Alias validation failures use `workflow_pool_history_unresolved`. diff --git a/docs-site/src/content/docs/ru/reference/configuration/server.md b/docs-site/src/content/docs/ru/reference/configuration/server.md index b65b39c977f..378dc4a1350 100644 --- a/docs-site/src/content/docs/ru/reference/configuration/server.md +++ b/docs-site/src/content/docs/ru/reference/configuration/server.md @@ -250,69 +250,42 @@ The dashboard warns about old or unknown CLI versions, unavailable targets, and ## Непрерывный учёт истории пулов -Лимиты пулов провайдеров применяются к каноническому провайдеру, выбранному маршрутизацией, а журналы -запросов сохраняют отображаемые метки отдельных аккаунтов. В старых журналах расход мог учитываться -под этими метками. Журнал хранит псевдонимы с солью, а не исходные имена провайдеров и аккаунтов, -поэтому OpenCodex не выводит соответствия из текущего списка аккаунтов, префикса метки или сокращённого ID аккаунта. - -Если задан `spend.pool.maxTokens`, положительные исторические балансы с неизвестным владельцем -сохраняются в исходных областях и один раз учитываются для каждого возможного пула провайдера, -дополнительно к подтверждённой группе этого провайдера. Запрос допускается, только если сумма и резерв -укладываются в лимит. Это может временно ограничить даже неиспользуемые пулы. Локальный отказ объясняет -такой консервативный учёт, не раскрывая псевдонимы или содержимое журнала. Неизвестная история сама по себе -не блокирует HTTP-допуск до определения маршрута. После маршрутизации предварительная проверка отклоняет -канонический пул, консервативная сумма которого уже исчерпала лимит. Отправка при восстановлении или через combo, уже допущенная по резервированию, -не учитывает то же резервирование повторно против самой себя; все остальные резервирования учитываются. -Эта проверка не превращает passthrough-отправки с последующим отчётом об использовании в атомарные резервирования: -отправка, пересекающая лимит, одновременные допуски и повторы с последующим отчётом сохраняют существующие ограничения. -Установки в режиме только наблюдения остаются в этом режиме. Лимиты корня и идентичности продолжают действовать независимо. - -Для устранения блокировки оператор должен по собственным сохранённым подтверждениям установить, -какому каноническому провайдеру принадлежит каждый исторический псевдоним пула с солью. -Добавьте объект `spendPoolAliases` на верхнем уровне `config.json`: каждый ключ должен точно совпадать -с 32-символьным шестнадцатеричным псевдонимом пула в нижнем регистре из журнала той же установки, -а каждое значение должно быть подтверждённым каноническим ID провайдера. Даже старый псевдоним, -уже обозначающий канонического провайдера, требует явной записи, если у него нет метаданных идентичности. -Неопознанный псевдоним с положительным расходом консервативно учитывается для каждого возможного пула. -Не выводите соответствие из похожих имён, удаления аккаунта, текущего имени или хэша провайдера, -либо совпадения коротких меток; не публикуйте журнал или соль. - -Размещайте `spendPoolAliases` вне `spend`: старые версии отклоняют неизвестные ключи внутри `spend` -и могут отключить весь раздел. Некорректные соответствия верхнего уровня отклоняются при записи конфигурации; -если ручная правка нарушила формат `spendPoolAliases`, существующие лимиты сохраняются, а допуск к пулам блокируется. -Исправьте соответствия и перезапустите сервис обычным способом изменения конфигурации. -Очистка или удаление соответствий не стирает уже надёжно записанные связи. Ранее перенаправленный псевдоним -нельзя назначить другой группе; подтверждённые переименования канонического провайдера могут объединять -группы без разделения существующего расхода. Все соответствия проверяются вместе, поэтому корректное -объединение групп не зависит от порядка ключей псевдонимов. - -Каждая исходная величина расхода учитывается ровно один раз. Учитываются подтверждённое использование, -резервирования в процессе выполнения и неразрешённое использование; неизвестное использование никогда -не считается возвратом. Исходные цели резервирования сохраняются. Новые запросы продолжают записываться -в пул канонического провайдера, а контрольные точки сохраняют сведения об идентичности с солью, -не добавляя исходные имена аккаунтов и провайдеров. Неопознанная история с положительным расходом, -активные группы и группы с исчерпанным лимитом защищены от очистки; существующее правило хранения -неактивных записей ниже лимита продолжает действовать. Объём сведений об идентичности ограничен: -при его исчерпании допуск к пулам прекращается, а сведения не забываются. -Автоматического инструмента для восстановления неподтверждаемой истории нет. - -### Требования к безопасному откату - -Поддерживаемый откат должен сохранять или переносить в старую версию как отнесение запросов к каноническому -провайдеру, так и чтение и запись реестра с учётом непрерывности истории, используя тот же журнал, -соль и подтверждённые соответствия. Сохраните актуальный журнал: восстановление копии до обновления -исключит более поздний расход. Не удаляйте записи, не меняйте соль, не повышайте лимиты и не отключайте -их применение, чтобы создать видимость совместимости при понижении версии. - -Неизменённый старый бинарный файл **не является поддерживаемым вариантом отката**. Он может читать -исходные величины расхода v1, но не применяет лимит к сумме по всем псевдонимам провайдера, -может создать новый пул с меткой аккаунта и отбросить необязательные метаданные идентичности при уплотнении. -Автоматического запрета понижения версии нет. Если такая старая версия выполняла запись, новая версия -при чтении консервативно учитывает неопознанную историю с положительным расходом для каждого возможного -пула провайдера, пока оператор не подтвердит оставшиеся соответствия. Отказы из-за хранилища или целостности журнала не связаны с проблемой -псевдонимов и сохраняют `workflow_spend_undurable`; проблемы небезопасных файлов или прав владения -могут вместо этого передаваться как ошибки хранилища, при этом запрос не допускается. - -## Резервирование токенов и лимиты - -Если к запросу применяется `spend.root.maxTokens`, `spend.identity.maxTokens` или `spend.pool.maxTokens`, а резервирование нельзя записать, например из-за заполнения ёмкости отслеживания или повторного ID отправки, новая отправка отклоняется до диспетчеризации. Запросы без применимого лимита остаются в режиме наблюдения. +Лимиты пула относятся к каноническому провайдеру маршрута, а не к отображаемой метке аккаунта. +Верхнеуровневый `spendPoolAliases` в `config.json` связывает точный исторический псевдоним пула +(32 строчных шестнадцатеричных символа из журнала этой установки) с проверенным ID провайдера. +Текущие имена и список аккаунтов не доказывают владельца истории. + +Соответствия применяются при чтении: исходные балансы не перемещаются, связи в журнал не записываются. +Удаление соответствия снова делает историю неопознанной. Каждый баланс учитывается один раз: +в подтверждённой группе либо консервативно для каждого возможного пула. Совпадение с текущим +каноническим хешем не приводит к двойному учёту. Псевдоним самого настроенного провайдера нельзя +назначить другому провайдеру. Проверка повторяется при добавлении провайдеров и не создаёт соль. +Некорректное соответствие сохраняет лимиты и блокирует допуск к пулу. + +Неактивная неопознанная история истекает только когда её последняя активность строго старше +`spend.retentionDays`. Нехватка места не сокращает этот срок для положительных балансов. +Активные резервирования и seed сохраняются даже при нулевой оценке; уменьшение баланса требует +сначала записать удаление в журнал. При применимом лимите первая отправка для каждой цели/ключа +требует обычного резервирования. Повторы используют те же области и общий конечный лимит отправок, +фиксируемый при первом старте (обычно 4, существующее расширение OAuth — до 18). +Окончательный расчёт ждёт отчётов; удаление ID отправки не удаляет баланс. Без применимого лимита +сохраняется режим наблюдения, включая пропуск записей при заполненной ёмкости отслеживания. + +Для Claude CLI, CodeBuddy и Qoder один вызов CLI считается одной отправкой и требует обычного резерва до запуска. +Внутренние повторы и инструментальные ходы CLI отдельно не расходуют L, но стоимость учитывается по фактически сообщённому использованию, включая превышение первоначальной оценки. + +### Откат к 2.80.0: контракт C + +Новые записи используют обычный формат v1 и псевдонимы `pool`. Неизменённая 2.80.0 читает их, +уплотняет журнал и применяет срок хранения по своим правилам отдельных меток. Сохраните текущие +журнал и соль. После отката не гарантируется каноническая агрегация или тот же остаток лимита, +который 2.80.0 вычислила бы для того же трафика без обновления. При повторном обновлении текущая +конфигурация применяется к сохранившимся исходным балансам. Барьер запуска и команда сверки не требуются. +История неопубликованных экспериментальных сборок с `pool-current` / `poolContinuity` исключена +из контракта C. Исходные балансы сохраняются как неопознанная история до обычного истечения срока; +немедленного преобразования или удаления нет. + +Полная, но некорректная запись (включая последний `null`) блокирует запросы с лимитом. +Уплотнение продолжает сохранять корректные балансы; отказ живого процесса сохраняется до перезапуска +с чистым журналом. Оборванная последняя строка JSON обрабатывается по прежним правилам восстановления. +Подробности и коды ошибок: [английский оригинал](/reference/configuration/server/#historical-pool-continuity). diff --git a/docs-site/src/content/docs/zh-cn/reference/configuration/server.md b/docs-site/src/content/docs/zh-cn/reference/configuration/server.md index 190a8fe6580..bb731ecb98c 100644 --- a/docs-site/src/content/docs/zh-cn/reference/configuration/server.md +++ b/docs-site/src/content/docs/zh-cn/reference/configuration/server.md @@ -216,51 +216,34 @@ The dashboard warns about old or unknown CLI versions, unavailable targets, and ## 历史池用量的连续性 -提供方池上限按路由选定的规范提供方计算，而请求日志保留各账户的显示标签。 -旧日志可能将用量记在这些显示标签下。日志保存的是加盐别名，而不是原始提供方或账户名称， -因此 OpenCodex 不会根据当前账户列表、标签前缀或缩短的账户 ID 推测映射。 - -如果配置了 `spend.pool.maxTokens`，无法确认所有者的正历史余额会保留在原始范围内， -并且会对每个候选提供方池各计入一次，再与该提供方已验证的组相加。 -只有合计用量加上本次预留不超过额度时才会准入。这可能暂时限制原本未使用的池。 -本地拒绝会说明这种保守计算，不会公开别名或日志内容。路由确定前，仅有未识别历史不会阻止 HTTP 准入。 -路由后的预检也会拒绝保守合计已耗尽额度的规范池。 -已通过预留获准的恢复或组合发送，不会再次将同一预留计入自身；其他所有预留仍会计入。 -这项检查不会将事后报告用量的透传发送变成原子预留：跨越上限的发送、并发准入或事后报告的重试， -仍受现有机制的限制。仅观察模式的安装仍保持仅观察模式。根节点和身份上限继续独立生效。 - -要解除阻止，操作员必须使用自己保留的证据，确认每个历史加盐池别名所属的规范提供方。 -在 `config.json` 顶层添加 `spendPoolAliases` 对象：每个键必须与同一安装实例日志中的 -32 位小写十六进制池别名完全一致，每个值则是已经核实的规范提供方 ID。 -旧别名即使已代表规范提供方，只要缺少身份元数据，仍然需要显式条目。 -用量大于零且无法识别的别名会保守地计入每个候选池。不要根据名称相似、账户删除、 -当前提供方名称或哈希、短标签冲突推测映射，也不要公开日志或盐值。 - -请将 `spendPoolAliases` 放在 `spend` 之外：旧版本会拒绝 `spend` 内的未知键，并可能禁用整个部分。 -无效的顶层映射会在写入配置时被拒绝；手动编辑导致 `spendPoolAliases` 格式错误时， -现有上限会被保留，池准入会关闭并拒绝请求。请按正常配置流程修正映射并重启。 -清空或删除映射不会抹去已经持久记录的关联。先前已重定向的别名不能重新分配给其他组； -经核实的规范提供方重命名可以合并组，而不会拆分已有用量。 -系统会一起检查完整映射，因此有效的分组合并不依赖别名键的顺序。 - -每份原始用量只计算一次。已结算用量、进行中的预留和未解决的用量都会计入；未知用量绝不会被视为退款。 -原始预留目标会被保留。新请求继续记录规范提供方池，检查点保留加盐身份依据， -而不加入原始账户或提供方名称。未知且用量大于零的历史记录、活跃组以及额度耗尽的组都受保护，不会被清理； -现有的休眠且未达上限记录保留规则仍然适用。身份依据的存储容量仍有上限；容量耗尽时， -系统会停止池准入，而不会遗忘这些依据。没有自动工具可以重建无法核实的历史记录。 - -### 安全回滚要求 - -受支持的回滚必须保留或回移两项功能：按规范提供方归属请求，以及识别历史连续性的账本读写， -并使用相同的日志、盐值和已核实的映射。请保留最新日志：恢复升级前的副本会遗漏之后的用量。 -不要通过删除记录、更改盐值、提高上限或关闭限制，来让降级看起来兼容。 - -未经修改的旧二进制文件**不是受支持的回滚方式**。它可以读取 v1 原始用量，但不会对提供方的跨别名合计用量实施限制， -可能创建新的账户标签池，也可能在压缩时丢弃可选身份元数据。没有自动阻止降级的机制。 -如果这样的旧写入程序已经运行，新读取程序会将无法识别且用量大于零的历史记录保守地计入 -每个候选提供方池，直到操作员核实剩余映射。存储或日志完整性导致的拒绝与别名问题分开，仍使用 -`workflow_spend_undurable`；不安全文件或所有权问题可能改为作为存储错误向上传递，同样不会允许请求通过。 - -## 令牌预留与额度 - -如果请求受 `spend.root.maxTokens`、`spend.identity.maxTokens` 或 `spend.pool.maxTokens` 限制，但因跟踪容量已满或发送 ID 重复等原因无法记录预留，则会在分派前拒绝新的发送。没有适用额度限制的请求仍保持仅观察模式。 +提供方池额度按路由选定的规范提供方计算，请求日志仍保留账户显示标签。 +在 `config.json` 顶层配置 `spendPoolAliases`：键是本安装日志中的历史池加盐别名 +（32 位小写十六进制），值是已核实的提供方 ID。当前名称或账户列表不能证明历史归属。 + +映射仅在读取时应用，不移动原始余额，也不把关联写入日志。删除映射会使该历史重新成为归属未知。 +每份原始余额只计算一次：计入已确认的组，或保守地计入每个候选池。 +即使未知别名与当前规范提供方的哈希相同，也不会重复计算。 +已配置提供方自己的池别名不能映射到另一个提供方；新增提供方时会重新验证。验证只读取现有盐值，不创建盐值。 +无效映射保留原有额度并拒绝池准入。 + +休眠的未知历史只有在最后活动时间严格早于 `spend.retentionDays` 的截止时间后才过期。 +容量压力不会提前删除正余额的未知历史。活动预留和 seed 即使为零令牌也会保留； +只有删除记录持久化后，准入计算才使用减少后的余额。 +有适用额度的请求必须在每个目标/密钥首次发送前取得普通容量的预留。 +重试沿用相同范围，并共享首次发送时固定的有限发送次数上限（通常 4，现有 OAuth 扩展最多 18）。 +最终结算等待报告完成；遗忘发送 ID 不会删除余额。无适用额度时保留原有仅观察行为，包括容量已满时省略记账。 + +Claude CLI、CodeBuddy 和 Qoder 将一次 CLI 调用计为一次发送，启动前仍必须取得普通预留；无法预留就拒绝启动。 +CLI 内部重试和工具轮次不单独计入 L，但费用按报告的实际使用量结算，包括超出初始估算的部分。 + +### 回滚至 2.80.0：契约 C + +新记录使用普通 v1 格式和 `pool` 别名。未经修改的 2.80.0 可以按自身的逐标签规则读取、压缩并执行保留策略。 +请保留最新日志和盐值。降级后不保证规范提供方聚合，也不保证与从未升级、始终运行 2.80.0 的相同流量具有相同剩余额度。 +再次升级时，当前映射应用于仍然保留的原始余额。不需要启动屏障或对账命令。 +未发布实验版本的 `pool-current` / `poolContinuity` 历史不属于契约 C；原始余额作为未知历史保留到正常过期， +不会立即转换或删除。 + +完整但无效的记录（包括末行 `null`）会拒绝有额度限制的请求。压缩仍可保留有效余额， +当前进程的拒绝状态持续到重启并读取干净日志。写入中断的最后一行 JSON 沿用原有恢复规则。 +详细边界及错误码见[英文原文](/reference/configuration/server/#historical-pool-continuity)。 diff --git a/src/adapters/coding-agent/turn.ts b/src/adapters/coding-agent/turn.ts index c46a53ad6ef..82255bbae18 100644 --- a/src/adapters/coding-agent/turn.ts +++ b/src/adapters/coding-agent/turn.ts @@ -4,6 +4,8 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import type { AdapterEvent, OcxParsedRequest, OcxProviderConfig } from "../../types"; import { isTranslatorBudgetExceededError } from "../../lib/translator-budget"; +import { SendBudgetExhaustedError } from "../../lib/upstream-retry"; +import type { SingleUseDispatchPermit } from "../../lib/request-execution-budget"; import { commandInvocation } from "../../lib/win-exec"; import { isStandaloneBinary } from "../../lib/standalone"; import { modelRecordValue } from "../../reasoning-effort"; @@ -287,6 +289,18 @@ export async function runCodingAgentTurn(input: CodingAgentTurnInput): Promise undefined); return; } @@ -665,9 +676,6 @@ export async function runCodingAgentTurn(input: CodingAgentTurnInput): Promise undefined); - } } // Reap the process so no zombie is left behind (§三十): wait for the real `close`, and @@ -748,6 +756,11 @@ export async function runCodingAgentTurn(input: CodingAgentTurnInput): Promise undefined); + } } function boundedStderr(chunks: string[]): string { diff --git a/src/adapters/kiro-retry.ts b/src/adapters/kiro-retry.ts index 1e020f92bcb..39a03ac3719 100644 --- a/src/adapters/kiro-retry.ts +++ b/src/adapters/kiro-retry.ts @@ -183,12 +183,12 @@ async function fetchWithResetRecovery( sendClass: "transient", targetKey: destination, rebasedTarget: destinationRebased, }); destinationRebased = false; - if (decision && (!decision.allowed || !decision.permit.use())) { + if (decision && !decision.allowed) { throw new SendBudgetExhaustedError(destination); } // Reported after admission and before dispatch, so a refused send is never counted and an // admitted one is counted exactly once whichever way the fetch below settles. - notePhysicalSend(attempt > 0); + const permit = decision?.allowed ? decision.permit : undefined; const requestUrl = request.url; try { const headers = new Headers(request.headers); @@ -201,7 +201,12 @@ async function fetchWithResetRecovery( ...(recovered ? { keepalive: false } : {}), }, timeoutMs, ctx.abortSignal, ctx.stream, (async (input, init) => { try { - return await (executor.unpacedFetch ?? executor)(input, init); + const run = async (): Promise => { + if (permit && !permit.use()) throw new SendBudgetExhaustedError(destination); + notePhysicalSend(attempt > 0); + return (executor.unpacedFetch ?? executor)(input, init); + }; + return await (permit?.execute ? permit.execute(run) : run()); } catch (error) { if (ctx.abortSignal?.aborted) throw abortError(ctx.abortSignal); const signal = init?.signal; @@ -224,7 +229,7 @@ async function fetchWithResetRecovery( baseDelayMs: RESET_RETRY_BASE_MS, maxDelayMs: RESET_RETRY_MAX_MS, }), ctx.abortSignal); - } + } finally { permit?.release(); } } finally { releaseProviderRequestSlot(slot); } diff --git a/src/adapters/physical-send.ts b/src/adapters/physical-send.ts index 669129019ee..b7cf47c574b 100644 --- a/src/adapters/physical-send.ts +++ b/src/adapters/physical-send.ts @@ -36,11 +36,14 @@ export function createAdapterPhysicalSend(ctx: AdapterFetchContext = {}, fallbac const physicalExecutor = (async (input, init) => { if (ctx.abortSignal?.aborted) throw abortError(ctx.abortSignal); if (init?.signal?.aborted) throw abortError(init.signal); - if (dispatched || (permit && !permit.use())) throw new SendBudgetExhaustedError(options.url); - dispatched = true; - ordinal += 1; - ctx.onPhysicalSend?.({ ordinal, ...(options.recovery ? { recovery: options.recovery } : {}) }); - return (executor.unpacedFetch ?? executor)(input, init); + const run = async (): Promise => { + if (dispatched || (permit && !permit.use())) throw new SendBudgetExhaustedError(options.url); + dispatched = true; + ordinal += 1; + ctx.onPhysicalSend?.({ ordinal, ...(options.recovery ? { recovery: options.recovery } : {}) }); + return (executor.unpacedFetch ?? executor)(input, init); + }; + return permit?.execute ? permit.execute(run) : run(); }) as typeof globalThis.fetch; let slot: ProviderRequestSlot | undefined; try { diff --git a/src/adapters/zed.ts b/src/adapters/zed.ts index 106d73769d0..61f748d0449 100644 --- a/src/adapters/zed.ts +++ b/src/adapters/zed.ts @@ -4,6 +4,7 @@ import { createAnthropicAdapter } from "./anthropic"; import { createGoogleAdapter } from "./google"; import { createOpenAIChatAdapter } from "./openai-chat"; import { createResponsesPassthroughAdapter } from "./openai-responses"; +import { createAdapterPhysicalSend } from "./physical-send"; import type { AdapterEvent, OcxParsedRequest, OcxProviderConfig } from "../types"; import { createTranslatorBudget, releaseTranslatedEvent, retainTranslatedEvent, type TranslatorBudget } from "../lib/translator-budget"; import { releaseCompactionCiphertextLease } from "../responses/compaction"; @@ -13,6 +14,7 @@ import { resolveZedModels, scrubZedCredentials, zedLlmFetch, + ZED_CLOUD_BASE_URL, ZED_HEADERS, type ZedCredentials, } from "../providers/zed"; @@ -340,8 +342,21 @@ export function createZedAdapter(provider: OcxProviderConfig): ProviderAdapter { async fetchResponse(request, ctx) { if (!credentials) throw new Error("Zed request credentials were not initialized"); const fetchFn = ctx?.executor ?? globalThis.fetch; + const send = createAdapterPhysicalSend(ctx); + let completionAttempts = 0; return zedLlmFetch(credentials, "/completions", { - fetchFn, + fetchFn: (async (input, init) => { + const url = input instanceof Request ? input.url : new URL(String(input)).href; + // Token exchange/account lookup are support traffic; both completion attempts + // share admission and physical ordinals, including the provider's 401 replay. + if (url === `${ZED_CLOUD_BASE_URL}/client/users/me` + || url === `${ZED_CLOUD_BASE_URL}/client/llm_tokens`) return fetchFn(input, init); + const recovery = completionAttempts++ > 0; + return send({ url, sendClass: recovery ? "auth-recovery" : "initial", + ...(recovery ? { recovery: "oauth-401" } : {}), + dispatch: executor => executor(input, init), + }); + }) as typeof globalThis.fetch, signal: ctx?.abortSignal, baseUrl: provider.baseUrl, fetchInit: { diff --git a/src/lib/request-execution-budget.ts b/src/lib/request-execution-budget.ts index fbb5da5138f..795801ea0bb 100644 --- a/src/lib/request-execution-budget.ts +++ b/src/lib/request-execution-budget.ts @@ -99,6 +99,8 @@ export interface SingleUseDispatchPermit { * false, which is what keeps a retry thunk from sending twice on one permit. */ use(): boolean; + /** Keep the selected start rebindable until its executor finishes. */ + execute?(run: () => Promise): Promise; /** * Hand back a reservation that never dispatched -- a credential move that found no alternate, * a rebuild abandoned before the send. Idempotent, and a no-op once the permit was used or @@ -482,6 +484,8 @@ function createRequestExecutionBudgetWithLedger( lastTargetKey = intent.targetKey; let producer: ReturnType> | undefined; + let executing = false; + let physicalOrdinal: number | undefined; const startPhysical = (): boolean => { if (observer?.enforced !== true) return true; const seed = observer.ensureSeed?.({ targetKey: intent.targetKey }, spendProof); @@ -490,13 +494,26 @@ function createRequestExecutionBudgetWithLedger( if (ordinal === undefined) return false; producer = observer.beginReporter?.(); producer?.start(seed, ordinal); - producer?.report(1); - producer?.close(); + physicalOrdinal = ordinal; + if (!executing) { producer?.report(1); producer?.close(); } return true; }; let settled: "open" | "used" | "released" = "open"; const permit: SingleUseDispatchPermit = { sendClass: intent.sendClass, + async execute(run: () => Promise): Promise { + if (executing) throw new Error("Dispatch permit executor already active"); + executing = true; + const binding: PhysicalSendReporter = () => {}; + binding.bind = (target: SpendScopes & { targetKey?: string }) => physicalOrdinal === undefined || !producer + ? true : producer.rebindTarget(target, physicalOrdinal); + try { return await activePhysicalReporters.run(binding, run); } + finally { + executing = false; + producer?.report(1); + producer?.close(); + } + }, use(): boolean { if (settled !== "open") return false; if (intent.countedExternally !== true && !startPhysical()) return false; diff --git a/src/lib/spend-reservation-ledger.ts b/src/lib/spend-reservation-ledger.ts index 6a870b401ee..9ef81145556 100644 --- a/src/lib/spend-reservation-ledger.ts +++ b/src/lib/spend-reservation-ledger.ts @@ -651,6 +651,7 @@ export interface SpendReservationLedger { /** * Settle with real usage. Returns false when the send is unknown or already resolved -- * double settlement is as wrong as none, so a repeat call changes nothing. + * Seeded terminal debt applies immediately; failed writes queue before durable forgetting. */ settle(sendId: string, usage: SpendUsage): boolean; /** @@ -1345,7 +1346,9 @@ export function createSpendReservationLedger(options: { const at = now(); const record: JournalRecord = { v: 1, kind: "settle", send, tokens, at }; const seeded = seedForSend(send); - if (seeded && (!flushPending() || !append(record))) return false; + // Terminal usage cannot be refused: the tracker may never report it again, and + // the estimate can be smaller than the actual bill. Preserve ordered retryable debt. + if (seeded && (pendingRecords.length > 0 || !append(record))) pendingRecords.push(record); applyResolve(send, "settled", tokens, at); if (!seeded) append(record); return true; @@ -1359,7 +1362,7 @@ export function createSpendReservationLedger(options: { const at = now(); const record: JournalRecord = { v: 1, kind: "lost", send, at }; const seeded = seedForSend(send); - if (seeded && (!flushPending() || !append(record))) return false; + if (seeded && (pendingRecords.length > 0 || !append(record))) pendingRecords.push(record); applyResolve(send, "lost", 0, at); if (!seeded) append(record); return true; diff --git a/src/server/responses/adapter-dispatch.ts b/src/server/responses/adapter-dispatch.ts index a59393309be..2e1276598de 100644 --- a/src/server/responses/adapter-dispatch.ts +++ b/src/server/responses/adapter-dispatch.ts @@ -554,6 +554,8 @@ export async function prepareAdapterExchange( try { if (transportState.activeAdapter.fetchResponse) { transportState.noteRoutedAttemptSend(retryEstimate, recovery); + const producer = adapterDispatchBudget?.beginSpendProducer?.(); + try { return await withProviderRequestSlot(route.providerName, route.provider, route.modelId, upstream.signal, pacingSlot => { // The dispatch boundary is HERE, not before the pacing wait: that wait can reject for // an abort, a saturated queue, an expired slot or a removed provider, and none of @@ -583,6 +585,7 @@ export async function prepareAdapterExchange( }), }); }); + } finally { producer?.close(); } } // #2643 review: this leg used to call fetchWithHeaderTimeout directly, so an // opted-in provider's transient-5xx policy applied to the initial send and to diff --git a/src/server/responses/compact.ts b/src/server/responses/compact.ts index 774c74f4336..83b04bb230e 100644 --- a/src/server/responses/compact.ts +++ b/src/server/responses/compact.ts @@ -1090,9 +1090,12 @@ export async function handleResponsesCompact( // actually happens, so every recorder call names the context that produced it. let outcomeCtx = authCtx; const localDispatchRefusal = (error: unknown): Response | undefined => { - const response = mapCodexAuthContextErrorToResponse(unwrapUpstreamRetryEvidenceError(error), { - now: Date.now(), accountSelector: route.codexAccountNamespace, - }); + const cause = unwrapUpstreamRetryEvidenceError(error); + const response = cause instanceof SendBudgetExhaustedError + ? formatErrorResponse(429, "request_send_budget_exhausted", cause.message) + : mapCodexAuthContextErrorToResponse(cause, { + now: Date.now(), accountSelector: route.codexAccountNamespace, + }); if (response) { releaseUpstreamHostAdmission(compactHostAdmissionLease); compactHostAdmissionLease = null; diff --git a/src/server/responses/request-send-budget.ts b/src/server/responses/request-send-budget.ts index 71b19153b00..9cca3ec2386 100644 --- a/src/server/responses/request-send-budget.ts +++ b/src/server/responses/request-send-budget.ts @@ -356,6 +356,7 @@ function adapterDispatchBudgetView( claimAmbiguousResend: (limit: number): boolean => budget.claimAmbiguousResend?.(limit) === true, get ambiguousResendSpent(): boolean { return budget.ambiguousResendSpent === true; }, reserveDispatch(intent: DispatchIntent): DispatchDecision { + const refundable = refundableHop || budget.spendEnforced === true; // A dispatch whose upstream state is unknown is refused on its own merits. A hop that // already paid does not make an unsafe replay safe, so that check stays with the budget. if (intent.replaySafe !== false) { @@ -367,26 +368,30 @@ function adapterDispatchBudgetView( hopPermit.release(); return budget.reserveDispatch({ ...intent, sendClass: hopPermit.sendClass }); } - if (hopPermit !== undefined && (refundableHop || hopPermit.assumeCharge())) { + if (hopPermit !== undefined && (refundable || hopPermit.assumeCharge())) { let settled = false; + let executionPermit = hopPermit; const confirm = (): boolean => { if (settled) return false; settled = true; // Admission can still reject this replay. Close its external booking only when // the adapter invokes its executor, leaving release() able to refund it until then. - if (!refundableHop || hopPermit.assumeCharge()) return true; + if (!refundable || hopPermit.assumeCharge()) return true; // Preserve the old fallback for a permit another leg already settled. const fresh = budget.reserveDispatch(intent); - return fresh.allowed && fresh.permit.assumeCharge(); + if (!fresh.allowed) return false; + executionPermit = fresh.permit; + return fresh.permit.assumeCharge(); }; return { allowed: true, permit: { sendClass: hopPermit.sendClass, + execute: run => executionPermit.execute ? executionPermit.execute(run) : run(), use: confirm, assumeCharge: confirm, release: (): void => { - if (!refundableHop || settled) return; + if (!refundable || settled) return; settled = true; hopPermit.release(); }, diff --git a/src/server/responses/request-spend.ts b/src/server/responses/request-spend.ts index 56e25be3dda..81cf6fbc884 100644 --- a/src/server/responses/request-spend.ts +++ b/src/server/responses/request-spend.ts @@ -1,6 +1,6 @@ import { randomUUID } from "node:crypto"; import type { RequestSendObserver } from "../../lib/request-execution-budget"; -import { sharedSpendLedger, type SpendReservationLedger, type SpendScopes, type SpendSeed, type SpendReservationProof } from "../../lib/spend-reservation-ledger"; +import { sharedSpendLedger, sharedSpendPolicy, type SpendReservationLedger, type SpendScopes, type SpendSeed, type SpendReservationProof } from "../../lib/spend-reservation-ledger"; import { SpendLedgerOwnerError } from "../../lib/spend-ledger-owner"; import { markLocalRequestLogRefusal, type RequestLogContext } from "../request-log"; import { recordWorkflowRefusalEvent, workflowDenialSummary, type WorkflowDenial } from "../../lib/workflow-budget"; @@ -88,7 +88,7 @@ export function createRequestSpendTracker( ...(target.poolId ?? logCtx.spendPoolId ?? logCtx.provider ? { poolId: target.poolId ?? logCtx.spendPoolId ?? logCtx.provider } : {}), }); const applicable = (scopes: SpendScopes): boolean => { - const policy = ledger().policy; + const policy = ledgerRef?.policy ?? sharedSpendPolicy(); return seededAccounting || (scopes.rootId !== undefined && policy.root.maxTokens !== undefined) || (scopes.identityId !== undefined && policy.identity.maxTokens !== undefined) || (scopes.poolId !== undefined && policy.pool.maxTokens !== undefined); @@ -427,16 +427,20 @@ export function createRequestSpendTracker( * request passes exactly once, whatever transport served it and however it ended, and it is * where the terminal usage is already known. */ +const trackerRequests = new WeakMap>(); + export function attachRequestSpendTracker( req: Pick, logCtx: RequestLogContext, ledger?: SpendReservationLedger, ): RequestSpendTracker { - if (logCtx.spendTracker && "ensureSeed" in logCtx.spendTracker) return logCtx.spendTracker as RequestSpendTracker; + const existing = logCtx.spendTracker as RequestSpendTracker | undefined; + if (existing && trackerRequests.get(existing) === req) return existing; const rootId = req.headers.get("x-codex-parent-thread-id")?.trim() || undefined; const tracker = ledger === undefined ? createRequestSpendTracker(logCtx, rootId) : createRequestSpendTracker(logCtx, rootId, ledger); logCtx.spendTracker = tracker; + trackerRequests.set(tracker, req); return tracker; } diff --git a/structure/config.md b/structure/config.md index 50983c3c225..c6af3364d9a 100644 --- a/structure/config.md +++ b/structure/config.md @@ -101,7 +101,7 @@ A schema-invalid top-level JSON value is repairable only when it is a non-array | Compaction and memory routing | `compactionRouting.model`, optional `compactionRouting.reasoningEffort`, optional `compactionRouting.triggers`, optional `compactionRouting.sourceModels`; `memoryModels.extract`, `memoryModels.consolidation` | Explicit Codex compaction metadata whose `compaction.trigger` is one the block names activates a request-local override; `triggers` defaults to `["manual"]`. Optional source selectors restrict the override to exact incoming models or `provider/*`. See [Responses compaction](transports/responses-failover.md#compaction-routing-overrides). Invalid hand edits disable the block with a load warning without discarding providers; candidate writes reject invalid blocks. Each optional memory phase names a nonblank model and optional declared `reasoningEffort`; absence preserves the current route, candidate writes reject malformed values, and load degrades only the invalid phase with a warning. See [memory phase routing](transports/responses-failover.md#memory-phase-routing). | | Catalog | `disabledModels`, `customModels`, `modelCacheTtlMs`, `providerContextCaps`, `contextCapValue`, per-provider `modelDisplayNames`, `codexAccountNamespaces`, `codexAccountPickerEnabled` | Catalog state is derived; config only records intent. Exact provider model display names are durable display only overlays. The picker flag is an explicit visibility override, while selector mappings remain the durable exact-routing contract. | | Retained state | `appOwnedMemoryBudgetMb` | Process-wide eviction target for app-owned logs, caches, blobs, and continuation payloads. Default 256 MiB, valid 64..4096; pinned state may temporarily exceed the target, but every pin-capable store has a finite local cap and their documented aggregate stays below `APP_OWNED_WORST_CASE_PINNED_BYTES` (512 MiB). Neither value caps RSS or native runtime memory. | -| Spend | `spend.root`, `spend.identity`, `spend.pool`, `spend.retentionDays`, top-level `spendPoolAliases` | Durable token ceilings for the spend-reservation ledger. Absent is the default and means observe-only accounting: spend is still journaled and nothing is refused, so observe-only and enforced servers take the same state-directory writer lease. One live process may write one directory; explicit sibling instances need separate `OPENCODEX_HOME` directories. There is no default figure for any scope — the ledger is on by default, so a shipped ceiling would refuse real traffic on upgrade against a number nobody chose. Strictly validated and positive-integer only, because 0 would read as a budget and refuse everything; a malformed section degrades to no ceiling, which is why the write path rejects it and load diagnostics report it. Top-level `spendPoolAliases` is preserved verbatim on load; malformed mappings leave ceilings intact and refuse pool admission, while candidate writes reject them. It stays outside the legacy strict `spend` object. Resolution and application live in `src/lib/spend-reservation-ledger.ts`; see [historical continuity and rollback](transports/responses-spend.md#historical-pool-continuity-and-rollback). | +| Spend | `spend.root`, `spend.identity`, `spend.pool`, `spend.retentionDays`, top-level `spendPoolAliases` | Durable token ceilings for the spend-reservation ledger. Absent is the default and means observe-only accounting: spend is still journaled and nothing is refused, so observe-only and enforced servers take the same state-directory writer lease. One live process may write one directory; explicit sibling instances need separate `OPENCODEX_HOME` directories. There is no default figure for any scope — the ledger is on by default, so a shipped ceiling would refuse real traffic on upgrade against a number nobody chose. Strictly validated and positive-integer only, because 0 would read as a budget and refuse everything; a malformed section degrades to no ceiling, which is why the write path rejects it and load diagnostics report it. Top-level `spendPoolAliases` is preserved verbatim on load; malformed mappings leave ceilings intact and refuse pool admission, while candidate writes reject them. Mappings apply at read time and are not persisted as journal links. `src/lib/spend-pool-alias-validation.ts` checks the existing home salt without creating it and rejects a configured provider’s own pool alias mapped to a different provider, including when providers are added later. It stays outside the legacy strict `spend` object. Resolution and application live in `src/lib/spend-reservation-ledger.ts`; see [historical continuity and rollback](transports/responses-spend.md#historical-pool-continuity-and-rollback). | | Transport | stream mode, timeouts, proxy settings, `websockets`, `emptyCompletionRetry` | `streamMode` persists in config.json; Windows services need a persisted input, and macOS uses it for explicit eager-relay opt-in. Empty-completion replay is an explicit top-level opt-in because its second upstream request may be billable. | | Canonical ChatGPT upstream transport | `providers.openai.upstreamWebsocket` | Omitted uses upstream WebSocket when eligible; explicit `false` selects HTTP/SSE without changing the canonical provider identity. `true` is rejected on the canonical row. This is independent of the client-facing `websockets` setting. | | Provider egress | `providers..proxy`, `providers..noProxy` | An absent `proxy` inherits global egress; `"direct"` or `null` forces direct egress; HTTP(S) and SOCKS5(H) URLs select a provider-owned proxy. `noProxy` uses NO_PROXY syntax and sends a matching destination direct across either a provider-owned or inherited global proxy. `src/lib/provider-egress.ts` owns parsing and request-local resolution. `forwardClientHeaders` opts into `originator`, `x-client-request-id`, `x-codex-app-version`, or `user-agent` on Responses inference only; provider headers win and all other names are rejected on load/write and filtered at runtime. | diff --git a/structure/providers-and-adapters.md b/structure/providers-and-adapters.md index ca6d53f18da..aba459b2fe1 100644 --- a/structure/providers-and-adapters.md +++ b/structure/providers-and-adapters.md @@ -1,7 +1,7 @@ # Providers And Adapters +Configured spend ceilings bind adapter inference executors to the [seed and physical-start contract](transports/responses-spend.md#historical-pool-continuity-and-rollback). Zed completion and auth-recovery replays claim separately; token acquisition stays outside inference accounting. Claude CLI, CodeBuddy and Qoder each count one invocation toward the frozen request limit, seed before spawn and retain a producer through completion. Child-internal retries/tool turns are not separately counted; reported actual usage settles in full, including amounts above the seed estimate. Devin combines only [consecutive same-ID tool results](adapters/registry.md#devin-consecutive-tool-results), using linear accumulation while preserving intervening message slots, image parts and error markers without mutating the parsed request. - Anthropic account pause, model routes, and quota labels follow the [Anthropic account-pool contract](providers/anthropic-account-pool.md). Devin Messages follows the [per-turn output ordering contract](clients/claude-desktop.md#devin-messages-output-ordering), preserving late signatures before text/tools without changing Responses or Chat ordering. Managed native Anthropic serving UUID and observed CLI header continuity follow [native Messages](data-planes/protocol-paths.md#managed-native-messages); generated Responses retain the adapter's compatibility fingerprint. diff --git a/structure/providers/kiro.md b/structure/providers/kiro.md index 47981213ec1..514ddfd41ee 100644 --- a/structure/providers/kiro.md +++ b/structure/providers/kiro.md @@ -1,5 +1,7 @@ # Kiro Provider +Kiro retry permits retain their [physical executor span](../transports/responses-spend.md#historical-pool-continuity-and-rollback) through credential selection and rebuild. A newly selected identity needs its own normal-capacity seed before inference I/O; a refused permit leaves the executor untouched. + Native steering follows [the shared WebSocket contract](../transports/streaming-health.md#experimental-native-mid-turn-steering); this surface's defaults remain unchanged. The configuration-only [plaintext V2 contract](../subagents.md#plaintext-v2-agent-messages) diff --git a/structure/runtime.md b/structure/runtime.md index 9e69b4482a4..0dc41e446b9 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -216,8 +216,8 @@ refusal is about two directories owned at once, not forever: releasing the final discards the singleton with its binding, so the same process may then own a different directory and build a ledger by replaying that directory's own journal. A second process on the same directory is refused even for observe-only spend configuration, while a separate directory is -independent. Ordinary stop releases the final reference after listener teardown, and every thrown -startup path releases its reference. SQLite and the OS release a crashed owner; no PID, timestamp, +independent. Ordinary stop drains the spend reporters after listener teardown before releasing the final +reference. `src/server/index/spend-ledger-lifecycle.ts` also holds the lease through failed-start listener rollback. Enforced seeds remain live until reporters close and settlement plus send-ID forgetting are durable; scope totals survive forgetting. SQLite and the OS release a crashed owner; no PID, timestamp, TTL or lock-file deletion participates in recovery. Startup passes top-level pool-alias evidence with the spend policy. HTTP admission checks [historical continuity](transports/responses-spend.md#historical-pool-continuity-and-rollback) even without a root; failed accounting reads release the active-turn lease. Rooted continuity denials record exactly one workflow refusal event before response formatting. An explicit Codex integration OFF skips startup cache invalidation before the user-scoped catalog diff --git a/structure/transports/responses-spend.md b/structure/transports/responses-spend.md index aae29cb8eac..2faf0c19316 100644 --- a/structure/transports/responses-spend.md +++ b/structure/transports/responses-spend.md @@ -44,8 +44,8 @@ replace it. `reportDispatchSends` verifies shared-ledger ownership and consumes Native Chat combo children carry the same exact permit to their physical-send boundary. Reset-only generic combo helpers report their named prepaid receipt too, without changing the selected retry cap; compaction reconciliation therefore does not count that source again. -Numeric `used` updates and unnamed, foreign, released or already-reported permits charge actual -sends without consuming another reservation. Extra retries remain full charges. Reports may arrive +Without enforced spend, numeric `used` updates and unnamed, foreign, released or already-reported permits charge +sends without consuming another reservation. Enforced spend accepts only claimed physical starts; extra retries remain full charges. Reports may arrive out of reservation order; no FIFO ordering is required. An adapter that owns its transport — Kiro's reset ladder, Cursor's transport ladder, or Devin's bounded pre-output stated-reset replay — reserves once per physical send instead, so no reporter ever @@ -54,7 +54,7 @@ arrives. Those ladders are handed through `pendingHopPermit` on the adapter's first reservation and closes the booking through `permit.assumeCharge()`. The Antigravity fetch web-search path opts into `refundableAdapterDispatchBudget`: its booking stays refundable until executor invocation confirms it through `permit.use()` or -`permit.assumeCharge()`, and failed admission releases it. Other callers keep early settlement. +`permit.assumeCharge()`, and failed admission releases it. Other callers keep early settlement only when spend enforcement is inactive; enforced permits stay refundable until execution. Letting both charge is how one physical send became two charges, and how a spent allowance answered a 429 with a synthetic error instead of the rate limit it was recovering from (#4709). @@ -115,11 +115,10 @@ what those sends may cost, and it is the only bound here that survives a restart caller is `request-spend.ts`, installed on the execution budget at genuine ingress in `core.ts` and parked on the log context so `addFinalRequestLog` can settle it. Native Chat installs the same tracker before its independent physical-send ladder and charges it immediately before each dispatch, so that fast path cannot bypass root, identity or provider-pool ceilings. -The Responses path books by observing the budget's own send counter rather than calling each dispatch site; Native Chat directly charges messages, tool definitions and the output ceiling. That counter moves exactly once per physical send — a reservation increments it, a -refund decrements it, and an externally reported send settles against a booking already counted — -so one ledger entry per increment is one entry per send, and a dispatch path added later cannot -forget to book. The previous attempt at this wiring shipped the whole reserve/dispatch/settle -vocabulary with no caller at all (#4707), which is the failure mode this shape rules out. +Unconfigured requests preserve the legacy counter observer: reservations increment the send +counter, refunds decrement it, and named reports reconcile prepaid sends. Enforced requests use +the seed and physical-start protocol below across Responses, native Messages and native Chat. +Each transport claims immediately before inference I/O and retains a producer through completion. Provider-pool accounting uses the canonical routed provider identity, not the mutable display label that may identify an OAuth account in request logs. Responses final-route normalization captures @@ -132,60 +131,89 @@ target provider; child account labels and the logical `combo` label do not creat ### Historical pool continuity and rollback -`src/lib/spend-pool-continuity.ts` accepts only explicit operator mappings from exact salted -historical pool aliases to canonical provider IDs, configured in top-level `spendPoolAliases`. -Current account rosters, label prefixes, short IDs, and renamed/deleted providers are not evidence -for automatically assigning old balances. Old canonical-looking aliases also need explicit -mapping when their positive history predates identity metadata. Zero-balance history needs none. - -The ledger retains original scope balances and reservation targets. The canonical view adds each -proven member once, keeping settled, reserved and unresolved buckets separate. Until ownership is -verified, every positive unbound pool balance is also counted once against each candidate provider -pool. The unbound balance stays in its original scope; a current provider name or matching salted -alias does not establish ownership. This conservative overlay may restrict an otherwise unused -provider pool until an operator supplies a verified mapping. Applying a mapping moves that balance -from the overlay into its proven group without copying it. Explicit links can join previously -canonical groups for a verified rename; an already redirected alias cannot be assigned to a -different group. The complete proposed graph is validated atomically, so a verified merge that -repeats existing member aliases is independent of salted-key order. A removed config entry never -removes a journaled link. Group activity, last-seen time and exhaustion govern retention; unbound -positive balances cannot be evicted. Identity evidence is bounded and retained even when dormant -under-limit scopes are evicted. -Each eviction pass aggregates pool groups once before choosing candidates. Retention uses the -group's newest activity; capacity pressure still removes only the oldest eligible individual -scope per pass, and every removal journals its original scope alias. - -Before a mapping authorizes admission, a v1 checkpoint durably carries both unchanged accounting -and optional salted `poolContinuity` metadata. No raw provider/account names are added to the -journal. New reservations still use the routed canonical pool ID. Replays and compaction preserve -the links; complete invalid metadata fails closed, including at the final line, and corruption is -never compacted away. Unparseable torn final JSON keeps the existing conservative replay rule. - -With a configured pool ceiling, an invalid or conflicting mapping refuses admission. Unbound -positive history alone does not block HTTP admission before a route is known; once a candidate -pool is known, its reservation check includes the proven group and every unbound positive balance. -The Responses pre-dispatch seam checks this even without a root ID, before passthrough transports -that report sends afterwards. Its local refusal explains when unknown-owner history contributed, -without exposing aliases or journal contents. A prepaid child excludes only its own open -reservation, proven by its exact permit, shared send ledger and still-pending receipt. Proof is -single-use; unrelated reservations and other proven pool groups remain counted. HTTP continuity -refusals still record one rooted workflow event; rootless ones record none. It is a snapshot check, -not a new atomic reservation for report-only transports: crossing sends, concurrent preflight -admissions and retries reported afterwards retain their existing limitations. `workflow_pool_history_unresolved` -identifies an invalid or conflicting identity-metadata refusal without disclosing aliases; storage -or replay failures keep `workflow_spend_undurable`. Already-sent reports still book actual spend. -Observe-only mode has no new token refusal, and root/identity accounting remains independent. - -Supported rollback retains/backports **both** canonical route attribution and the continuity-aware -reader/writer, using the same journal, salt and verified bindings. Restoring an older journal or -salt loses newer spend and is not a supported rollback. Unmodified older binaries are unsupported: -they can read v1 counters but do not enforce the canonical aggregate, can introduce fresh account -label pools, and can discard optional identity metadata during compaction. There is no automatic -downgrade barrier and no unknown-record fence. If that unsupported write has happened, the current -reader conservatively counts remaining unidentified positive balances against every candidate -pool until the operator verifies their mappings. `tests/lib/spend-pool-continuity.test.ts` exercises -this using the frozen pre-change reader, -as well as exact aggregation, active/unresolved sends, retention, failures and rootless preflight. +`src/lib/spend-pool-continuity.ts` applies the current top-level `spendPoolAliases` mapping +at read time. Keys are exact salted historical pool aliases; values are verified canonical +provider IDs. Original scope counters and send targets stay unchanged. Every original bucket +contributes once: either to its mapped group or, when unmapped and positive, to every candidate +pool as conservative unbound history. A candidate's own unmapped canonical bucket participates +in that overlay once, never again as a group member. Account rosters, label prefixes and a +matching canonical hash do not establish historical ownership, including after restart. + +Mappings are configuration, not journal metadata. Removing or changing a mapping changes the +next read-time view; it does not rewrite original counters or persist an identity link. +`src/lib/spend-pool-alias-validation.ts` owns the salted hash and validates that a configured +provider P's own pool alias cannot map to another provider Q. Configuration reads and writes, +provider-set changes, live reconfiguration and selected-provider admission enforce this rule. +Validation reads an existing home salt without creating one. A malformed or conflicting mapping +keeps existing ceilings and refuses applicable pool admission with `workflow_pool_history_unresolved`. + +Dormant unbound history expires only when its last activity is strictly before the configured +retention cutoff. Capacity pressure cannot expire positive unknown history early. Live reservations +and seed targets remain pinned, including zero-token targets. An ordinary v1 `drop` must persist +before admission uses a reduced total; the overlay is then rebuilt before group eviction decisions. +Mapped groups retain the existing active/exhausted protection and group-activity retention rules. + +New bookings use the existing `pool` hash domain for the canonical routed provider. Writers emit +only ordinary v1 records and checkpoints, without `poolContinuity` metadata or a `pool-current` +domain. Contract C is compatibility with the shipped 2.80.0 reader's own record interpretation: +2.80.0 reads these counters, compacts them and expires them using its existing per-label rules. +Downgrade does not preserve canonical cross-alias aggregation or promise the allowance 2.80.0 +would have computed for the same traffic without an upgrade. Keep the current journal and salt; +restoring an older copy omits subsequent spend. Re-upgrade applies current configured mappings +to the original balances that survived ordinary old-version retention. There is no launch fence, +wrapper enrollment or reconciliation command. `tests/lib/spend-contract-c.test.ts` uses the complete +frozen 2.80.0 reader in `tests/fixtures/spend-ledger-2-80-0.ts.txt` for this contract. + +Unpublished experimental journals containing `pool-current` aliases or `poolContinuity` bindings +are excluded from contract C, including later checkpoints that retain those opaque aliases. The +reader ignores the deprecated bindings but preserves original balances and send targets as +unbound history; it neither converts nor immediately deletes them. Ordinary durable retention +applies. `tests/lib/spend-experimental-history.test.ts` covers replay, compaction and expiry. + +Compaction retains valid accounting even after a corrupt complete record. Configured admission +remains refused while corruption is recorded in the live ledger; replay of the clean compacted +journal restores enforcement, matching the shipped behavior. Complete final invalid JSON values +such as `null` are corruption. An unparseable torn final line retains the existing conservative +replay rule. Storage/corruption refusals use `workflow_spend_undurable`; unsafe-file and ownership +errors remain storage failures. `tests/lib/spend-corruption-compat.test.ts` checks these boundaries. + +For requests with an applicable root, identity or pool ceiling, each selected target/key first +obtains a normal-capacity pre-send seed through `reserveSeed`. A failed initial seed refuses the +wire call, including rootless passthrough and generic adapter ingress. Stable retries reuse that +seed; a real identity change must acquire its own normal seed. A prepaid combo child adopts only +its exact permit on the same shared send ledger, excluding that reservation once from preflight. +Reported additional sends reuse the seed's exact scope references through `reserveReportedFromSeed` +and allocate no new scope keys. Already-started sends remain booked even during persistence failure. + +`src/lib/request-execution-budget.ts` shares a physical-start counter across derived budgets and +freezes the finite positive limit L at the first claim. The default guarded limit is four; bounded +OAuth expansion happens before the first start and allows at most eighteen for that profile. +`createPhysicalSendReporter` owns selected-seed start receipts, reports only claimed sends and +closes its producer after reconciliation. Adapter permits claim once at their physical boundary; their executor span keeps the start +rebindable until OAuth selection reaches the wire, then reports and closes it. Zed completion +requests and token-refresh replays use `src/adapters/physical-send.ts`; authentication exchanges +remain outside the inference count. Kiro retains this span through its executor-side rebuild. +Telemetry and raw numeric budget updates cannot create extra enforced send records. +`request-spend.ts` waits for producers before terminal settlement, marks unknown usage lost, then +persists ordinary `forget` records before releasing seeds. Forgotten send IDs do not subtract +scope totals. Persistence failure retains the accounting debt. Shutdown drains reporters before +releasing ledger ownership; restart resolves orphan reservations conservatively without reviving +request capabilities. + +Claude CLI, CodeBuddy and Qoder use one CLI invocation as one physical-count unit. The shared +`src/adapters/coding-agent/turn.ts` runner reserves a normal seed before spawning and keeps its +producer until completion. Child-internal retries and tool turns do not each consume L because +the proxy cannot observe those wire boundaries. Terminal actual usage, including usage above the +initial estimate, still settles in full. This preserves configured CLI use while bounding launches. + +Within a fixed policy epoch, every live enforced request retains at least one normal-capacity +seed. For normal send capacity M and largest admitted physical limit Lmax, retained sends are +bounded by M × Lmax, with no overflow scope allocation. Legacy or experimental over-cap history +is retained and can prevent new seeds; lowering policy limits does not erase existing accounting. +`tests/lib/spend-seed-overflow.test.ts` covers capacity, delayed reports, persistence and finalization. +Unconfigured traffic keeps ordinary record shapes and capacity omission behavior; it does not +create seed instrumentation or identity checkpoints. `tests/lib/spend-zero-config-compat.test.ts` +compares exact bytes with the shipped reader for equal pool inputs. Budget reservations retain their exact durable proof until their own dispatch/report confirms them. A later reservation does not confirm an earlier pending send. Refunds remove the exact @@ -214,9 +242,9 @@ describing it afterwards, and the restart. The request tracker checks the current policy against the exact root, identity and pool scopes on each charge. With an applicable ceiling, any refused booking prevents a new dispatch, including full tracking capacity or a duplicate send id; the log uses the existing specific -workflow refusal reason. Requests without an applicable ceiling remain observe-only. Reports -of sends that already left stay permissive and do not count as local refusals; this guard does -not promise complete accounting when those post-dispatch bookings fail. +workflow refusal reason. Requests without an applicable ceiling remain observe-only. Enforced reports use their live seed and retain already-started liability without a new +capacity admission. Observe-only reports keep the shipped permissive behavior, including omitted +bookings at capacity. `tests/responses/responses-spend-capacity-guard.test.ts` covers these boundaries. The default policy still sets no token ceiling on any scope, so an unconfigured install accounts diff --git a/tests/lib/execution-budget-permits.test.ts b/tests/lib/execution-budget-permits.test.ts index a5973d1c231..edce9f1f980 100644 --- a/tests/lib/execution-budget-permits.test.ts +++ b/tests/lib/execution-budget-permits.test.ts @@ -576,3 +576,56 @@ test("a reported external receipt cannot be taken over by an adapter", () => { decision.permit.release(); expect(budget.used).toBe(1); }); + +test("adapter executor rebinds its live permit to the selected account before wire", async () => { + const { createSpendReservationLedger, DEFAULT_SPEND_RESERVATION_POLICY } = await import("../../src/lib/spend-reservation-ledger"); + const { createRequestSpendTracker } = await import("../../src/server/responses/request-spend"); + const { createAdapterPhysicalSend } = await import("../../src/adapters/physical-send"); + const { rebindPhysicalSend } = await import("../../src/lib/request-execution-budget"); + const ledger = createSpendReservationLedger({ salt: "permit-selected-account", policy: { + ...DEFAULT_SPEND_RESERVATION_POLICY, identity: { maxTokens: 30 }, pool: { maxTokens: 1000 }, + } }); + const ctx = { provider: "P", spendPoolId: "P", accountLogLabel: "A", spendInputEstimateTokens: 1, spendOutputCeilingTokens: 0 }; + const tracker = createRequestSpendTracker(ctx, undefined, ledger); + const budget = createRequestExecutionBudget(undefined, undefined, tracker); + let wires = 0; + const send = createAdapterPhysicalSend({ sendBudget: budget, executor: (async () => { + ctx.accountLogLabel = "B"; + expect(rebindPhysicalSend(budget, { poolId: "P", identityId: "B" })).toBe(true); + wires++; + tracker.settle({ inputTokens: 7 }); + expect(ledger.snapshot("identity", "B")?.reserved).toBe(1); + return new Response(); + }) as typeof fetch }); + await send({ url: "https://permit.example.test/", dispatch: execute => execute("https://permit.example.test/") }); + expect(wires).toBe(1); + expect(budget.physicalStarted).toBe(1); + expect(ledger.snapshot("identity", "A")?.settled ?? 0).toBe(0); + expect(ledger.snapshot("identity", "A")?.reserved ?? 0).toBe(0); + expect(ledger.snapshot("identity", "B")).toMatchObject({ settled: 7, reserved: 0, unresolved: 0 }); +}); + +test("adapter executor refuses a reselection whose identity cannot obtain a normal seed", async () => { + const { createSpendReservationLedger, DEFAULT_SPEND_RESERVATION_POLICY } = await import("../../src/lib/spend-reservation-ledger"); + const { createRequestSpendTracker } = await import("../../src/server/responses/request-spend"); + const { createAdapterPhysicalSend } = await import("../../src/adapters/physical-send"); + const { rebindPhysicalSend } = await import("../../src/lib/request-execution-budget"); + const { SendBudgetExhaustedError } = await import("../../src/lib/upstream-retry"); + const ledger = createSpendReservationLedger({ salt: "permit-selected-account", policy: { + ...DEFAULT_SPEND_RESERVATION_POLICY, identity: { maxTokens: 30 }, pool: { maxTokens: 1000 }, + } }); + ledger.reserve({ sendId: "prior", scopes: { identityId: "B" }, inputTokens: 30, outputCeilingTokens: 0 }); + ledger.settle("prior", { inputTokens: 30, outputTokens: 0 }); + const tracker = createRequestSpendTracker({ provider: "P", accountLogLabel: "A", spendInputEstimateTokens: 1, spendOutputCeilingTokens: 0 }, undefined, ledger); + const budget = createRequestExecutionBudget(undefined, undefined, tracker); + let wires = 0; + const send = createAdapterPhysicalSend({ sendBudget: budget, executor: (async () => { + if (!rebindPhysicalSend(budget, { poolId: "P", identityId: "B" })) throw new SendBudgetExhaustedError(); + wires++; + return new Response(); + }) as typeof fetch }); + await expect(send({ url: "https://permit.example.test/", dispatch: execute => execute("https://permit.example.test/") })).rejects.toThrow(SendBudgetExhaustedError); + tracker.settle(undefined); + expect(wires).toBe(0); + expect(ledger.snapshot("identity", "B")).toMatchObject({ settled: 30, reserved: 0, unresolved: 0 }); +}); diff --git a/tests/lib/spend-seed-overflow.test.ts b/tests/lib/spend-seed-overflow.test.ts index 3ddeaf3fcb8..ff524cf4cf8 100644 --- a/tests/lib/spend-seed-overflow.test.ts +++ b/tests/lib/spend-seed-overflow.test.ts @@ -7,6 +7,7 @@ import { join } from "node:path"; import { acquireSpendLedgerServerLifecycle } from "../../src/server/index/spend-ledger-lifecycle"; import { spendLedgerOwnerSnapshot } from "../../src/lib/spend-ledger-owner"; import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { createRequestSpendTracker } from "../../src/server/responses/request-spend"; const request = (sendId: string, scopes = {poolId:"P"}, tokens=0) => ({sendId,scopes,inputTokens:tokens,outputCeilingTokens:0}); const seed = (ledger: SpendReservationLedger, id="seed", scopes={poolId:"P"}, tokens=0): SpendSeed => { const decision=ledger.reserveSeed(request(id,scopes,tokens)); expect(decision.reserved).toBe(true); @@ -70,9 +71,9 @@ test("terminal waits for reporters and failed persistence cannot reclaim a seed" const reporter=ledger.registerReporter(); let drained=false; const drain=ledger.waitForReporterDrain().then(()=>{drained=true;}); await Promise.resolve();expect(drained).toBe(false);expect(ledger.finishSeed(anchor)).toBe(false); reporter.close();reporter.close();await drain;expect(drained).toBe(true); - disk.failAppend=true; expect(ledger.settle("seed",{inputTokens:7,outputTokens:0})).toBe(false); + disk.failAppend=true; expect(ledger.settle("seed",{inputTokens:7,outputTokens:0})).toBe(true); expect(ledger.forgetResolved(["seed"])).toBe(false);expect(ledger.finishSeed(anchor)).toBe(false);expect(ledger.knows("seed")).toBe(true); - disk.failAppend=false;expect(ledger.settle("seed",{inputTokens:7,outputTokens:0})).toBe(true); + disk.failAppend=false;expect(ledger.settle("seed",{inputTokens:7,outputTokens:0})).toBe(false); disk.failAppend=true;expect(ledger.forgetResolved(["seed"])).toBe(false);expect(ledger.finishSeed(anchor)).toBe(false); disk.failAppend=false;expect(ledger.forgetResolved(["seed"])).toBe(true);expect(ledger.finishSeed(anchor)).toBe(true); const delayedDisk = spendTestJournal(); @@ -83,10 +84,10 @@ test("terminal waits for reporters and failed persistence cannot reclaim a seed" expect(reported(delayedLedger, delayedSeed, "physical-but-unwritten", 70)).toBe(true); delayedLedger.markDispatched("physical-but-unwritten"); expect(delayedLedger.snapshot("pool", "P")?.reserved).toBe(110); - expect(delayedLedger.markLost("physical-but-unwritten")).toBe(false); + expect(delayedLedger.markLost("physical-but-unwritten")).toBe(true); expect(delayedLedger.finishSeed(delayedSeed)).toBe(false); delayedDisk.failAppend = false; - expect(delayedLedger.markLost("physical-but-unwritten")).toBe(true); + expect(delayedLedger.markLost("physical-but-unwritten")).toBe(false); expect(delayedLedger.markLost("initial")).toBe(true); expect(delayedLedger.forgetResolved(["initial", "physical-but-unwritten"])).toBe(true); expect(delayedLedger.finishSeed(delayedSeed)).toBe(true); @@ -119,3 +120,65 @@ test("ceiling removal does not drop seeded accounting",()=>{ expect(reported(ledger,anchor,"delayed",70)).toBe(true);ledger.markDispatched("delayed");ledger.settle("delayed",{inputTokens:70,outputTokens:0}); expect(createShippedSpendLedger({journal:disk,salt,now:()=>2}).snapshot("pool","P")?.settled).toBe(70); }); + +for (const sends of [1, 2]) test(`terminal usage survives append failure without another tracker settlement (${sends} sends)`, async () => { + const disk = spendTestJournal(); + const ledger = createSpendReservationLedger({ journal: disk, salt, now: () => 2, + policy: spendTestPolicy({ pool: { maxTokens: 50 } }) }); + const tracker = createRequestSpendTracker({ provider: "P", spendInputEstimateTokens: 1 }, undefined, ledger); + const anchor = tracker.ensureSeed({ poolId: "P" }); + if (!anchor) throw new Error("seed denied"); + const reporter = tracker.beginReporter(); + for (let ordinal = 1; ordinal <= sends; ordinal++) reporter.start(anchor, ordinal); + reporter.report(sends); + reporter.close(); + disk.failAppend = true; + tracker.settle({ inputTokens: 70, outputTokens: 0 }); + // Let the tracker's one drain callback run while storage is still unavailable. + await Promise.resolve(); + expect(ledger.snapshot("pool", "P")).toMatchObject({ settled: 70, reserved: 0, unresolved: sends - 1 }); + expect(ledger.knows(anchor.sendId)).toBe(true); + expect(disk.lines.some(line => JSON.parse(line).kind === "forget")).toBe(false); + disk.failAppend = false; + expect(ledger.reserve(request("next", { poolId: "P" }, 1))).toMatchObject({ + reserved: false, denial: { reason: "spend-limit-exceeded", projected: 70 + sends }, + }); + const old = createShippedSpendLedger({ journal: disk, salt, now: () => 2 }); + expect(old.snapshot("pool", "P")).toMatchObject({ settled: 70, unresolved: sends - 1 }); + expect(old.corruptRecords).toBe(0); +}); + +test("partial pending flush preserves reserve dispatch and terminal order without duplicate liability", () => { + const disk = spendTestJournal(); + const append = disk.append.bind(disk); + let writesLeft = Number.POSITIVE_INFINITY; + disk.append = line => { + if (writesLeft-- <= 0) throw new Error("partial flush unavailable"); + append(line); + }; + const ledger = createSpendReservationLedger({ journal: disk, salt, now: () => 2, + policy: spendTestPolicy({ pool: { maxTokens: 50 } }) }); + const anchor = seed(ledger, "initial", { poolId: "P" }, 1); + ledger.markDispatched("initial"); + writesLeft = 0; + expect(reported(ledger, anchor, "terminal", 1)).toBe(true); + expect(ledger.markDispatched("terminal")).toBe(true); + expect(ledger.markLost("initial")).toBe(true); + expect(ledger.settle("terminal", { inputTokens: 70, outputTokens: 0 })).toBe(true); + expect(ledger.settle("terminal", { inputTokens: 700, outputTokens: 0 })).toBe(false); + expect(ledger.markLost("terminal")).toBe(false); + expect(ledger.snapshot("pool", "P")).toMatchObject({ settled: 70, reserved: 0, unresolved: 1 }); + writesLeft = 1; + expect(ledger.forgetResolved(["initial", "terminal"])).toBe(false); + expect(disk.lines.map(line => JSON.parse(line).kind)).toEqual(["reserve", "dispatch", "reserve"]); + expect(ledger.knows("terminal")).toBe(true); + writesLeft = Number.POSITIVE_INFINITY; + expect(ledger.reserve(request("next", { poolId: "P" }, 1))).toMatchObject({ + reserved: false, denial: { reason: "spend-limit-exceeded", projected: 72 }, + }); + expect(disk.lines.map(line => JSON.parse(line).kind)).toEqual(["reserve", "dispatch", "reserve", "dispatch", "lost", "settle"]); + expect(ledger.forgetResolved(["initial", "terminal"])).toBe(true); + expect(ledger.finishSeed(anchor)).toBe(true); + expect(createShippedSpendLedger({ journal: disk, salt, now: () => 2 }).snapshot("pool", "P")) + .toMatchObject({ settled: 70, unresolved: 1 }); +}); diff --git a/tests/providers/claude-cli-adapter.test.ts b/tests/providers/claude-cli-adapter.test.ts index 86d2c164b4f..563610dbf9c 100644 --- a/tests/providers/claude-cli-adapter.test.ts +++ b/tests/providers/claude-cli-adapter.test.ts @@ -18,6 +18,10 @@ import { PROVIDER_REGISTRY } from "../../src/providers/registry"; import { deriveProviderPresets, providerConfigSeed } from "../../src/providers/derive"; import type { AdapterEvent, OcxParsedRequest, OcxProviderConfig } from "../../src/types"; import { createTestTranslatorBudget } from "../helpers/translator-budget"; +import { createRequestExecutionBudget, CODEX_TEXT_GUARDED_BUDGET_POLICY } from "../../src/lib/request-execution-budget"; +import { createSpendReservationLedger } from "../../src/lib/spend-reservation-ledger"; +import { createRequestSpendTracker } from "../../src/server/responses/request-spend"; +import { createShippedSpendLedger, spendTestJournal, spendTestPolicy, testSpendSalt } from "../helpers/shipped-spend-ledger"; const enc = new TextEncoder(); @@ -35,7 +39,7 @@ interface FakeChild extends EventEmitter { written: string[]; } -function fakeChild(stdout: Uint8Array[], opts: { stderr?: string; exitCode?: number } = {}): FakeChild { +function fakeChild(stdout: Uint8Array[], opts: { stderr?: string; exitCode?: number; emitClose?: boolean } = {}): FakeChild { const child = new EventEmitter() as FakeChild; child.stdout = Readable.from(stdout); child.stderr = Readable.from(opts.stderr ? [enc.encode(opts.stderr)] : []); @@ -44,7 +48,7 @@ function fakeChild(stdout: Uint8Array[], opts: { stderr?: string; exitCode?: num child.killed = false; child.exitCode = null; child.kill = () => { child.killed = true; return true; }; - setTimeout(() => { child.exitCode = opts.exitCode ?? 0; child.emit("close", opts.exitCode ?? 0); }, 3); + if (opts.emitClose !== false) setTimeout(() => { child.exitCode = opts.exitCode ?? 0; child.emit("close", opts.exitCode ?? 0); }, 3); return child; } @@ -122,6 +126,76 @@ describe("claude-cli is an official-harness provider, not a Messages relay", () }); }); +describe("CLI invocation spend admission", () => { + function accounting(maxTrackedSends = 10) { + const journal = spendTestJournal(); + const ledger = createSpendReservationLedger({ journal, salt: testSpendSalt, now: () => 2, + policy: spendTestPolicy({ pool: { maxTokens: 50 }, maxTrackedSends }) }); + const tracker = createRequestSpendTracker({ provider: "claude-cli", spendInputEstimateTokens: 1 }, undefined, ledger); + const policy = { ...CODEX_TEXT_GUARDED_BUDGET_POLICY }; + const budget = createRequestExecutionBudget(policy, undefined, tracker); + return { journal, ledger, tracker, policy, budget }; + } + const terminal = enc.encode(JSON.stringify({ type: "result", subtype: "success", is_error: false, + usage: { input_tokens: 70, output_tokens: 0 } }) + "\n"); + + test("full NORMAL seed capacity refuses before spawning the CLI", async () => { + const f = accounting(1); + f.ledger.reserve({ sendId: "busy", scopes: { poolId: "claude-cli" }, inputTokens: 1, outputCeilingTokens: 0 }); + let spawns = 0; + const adapter = createClaudeCliAdapter(provider(), { which: () => "/test/claude", + spawn: () => { spawns++; return fakeChild([terminal]) as unknown as ChildProcess; } }); + await expect(adapter.runTurn!(parsed(), { ...incoming(), sendBudget: f.budget }, () => {})) + .rejects.toMatchObject({ code: "request_send_budget_exhausted" }); + expect(spawns).toBe(0); + expect(f.budget.physicalStarted).toBe(0); + }); + + test("a frozen invocation limit refuses the next CLI spawn after policy expansion", async () => { + const f = accounting(); + Object.assign(f.policy, { maxTotalModelSends: 1, baseSendAllowance: 1 }); + let spawns = 0; + const adapter = createClaudeCliAdapter(provider(), { which: () => "/test/claude", + spawn: () => { spawns++; return fakeChild([terminal]) as unknown as ChildProcess; } }); + await adapter.runTurn!(parsed(), { ...incoming(), sendBudget: f.budget }, () => {}); + Object.assign(f.policy, { maxTotalModelSends: 100, baseSendAllowance: 100 }); + await expect(adapter.runTurn!(parsed(), { ...incoming(), sendBudget: f.budget }, () => {})) + .rejects.toMatchObject({ code: "request_send_budget_exhausted" }); + expect(spawns).toBe(1); + expect(f.budget.physicalStarted).toBe(1); + expect(f.budget.physicalLimit).toBe(1); + f.tracker.settle({ inputTokens: 70 }); + }); + + test("CLI terminal usage above its estimate settles only after the child is reaped", async () => { + const f = accounting(); + const child = fakeChild([terminal], { emitClose: false }); + const adapter = createClaudeCliAdapter(provider(), { which: () => "/test/claude", + spawn: () => child as unknown as ChildProcess }); + let observedTerminal!: () => void; + const terminalSeen = new Promise(resolve => { observedTerminal = resolve; }); + const turn = adapter.runTurn!(parsed(), { ...incoming(), sendBudget: f.budget }, event => { + if (event.type !== "done") return; + f.tracker.settle(event.usage); + observedTerminal(); + }); + await terminalSeen; + let drained = false; + const drain = f.ledger.waitForReporterDrain().then(() => { drained = true; }); + await Promise.resolve(); + expect(drained).toBe(false); + expect(f.journal.lines.some(line => JSON.parse(line).kind === "forget")).toBe(false); + child.exitCode = 0; + child.emit("close", 0); + await turn; + await drain; + expect(f.budget.physicalStarted).toBe(1); + expect(f.ledger.snapshot("pool", "claude-cli")).toMatchObject({ settled: 70, reserved: 0 }); + expect(createShippedSpendLedger({ journal: f.journal, salt: testSpendSalt, now: () => 2 }) + .snapshot("pool", "claude-cli")?.settled).toBe(70); + }); +}); + describe("claude-cli headless arguments keep tool ownership with the client", () => { test("disables built-in tools and every MCP source, and never requests a bypass", () => { const args = buildArgs(CLAUDE_CLI_PROFILE, parsed(), provider()); diff --git a/tests/providers/zed-hosted-provider.test.ts b/tests/providers/zed-hosted-provider.test.ts index cbe4e1bf20a..f7178289175 100644 --- a/tests/providers/zed-hosted-provider.test.ts +++ b/tests/providers/zed-hosted-provider.test.ts @@ -2,7 +2,7 @@ import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test import { mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { constants, createPublicKey, publicEncrypt } from "node:crypto"; +import { constants, createHash, createPublicKey, publicEncrypt } from "node:crypto"; import { createZedAdapter, zedEventStream, type ZedProvider } from "../../src/adapters/zed"; import { setIcaclsRunnerForTests, resetHardenedStateForTests } from "../../src/lib/windows-secret-acl"; import { getValidAccessTokenSnapshot } from "../../src/oauth"; @@ -22,6 +22,10 @@ import { } from "../../src/providers/zed"; import { parseRequest } from "../../src/responses/parser"; import type { OcxProviderConfig } from "../../src/types"; +import { createRequestExecutionBudget } from "../../src/lib/request-execution-budget"; +import { createSpendReservationLedger, DEFAULT_SPEND_RESERVATION_POLICY } from "../../src/lib/spend-reservation-ledger"; +import { createRequestSpendTracker } from "../../src/server/responses/request-spend"; +import { SendBudgetExhaustedError } from "../../src/lib/upstream-retry"; const originalFetch = globalThis.fetch; @@ -298,6 +302,72 @@ describe("Zed Hosted AI provider", () => { }); }); +describe("Zed inference send accounting", () => { + async function fixture(limit: number, full = false) { + const catalog = [jsonResponse({ default_organization_id: "org-1" }), + jsonResponse({ token: "llm-token-1" }), jsonResponse({ models: [{ id: "gpt-5.6", provider: "open_ai" }] })]; + globalThis.fetch = (async () => { + const response = catalog.shift(); + if (!response) throw new Error("unexpected Zed catalog fetch"); + return response; + }) as typeof globalThis.fetch; + const adapter = withTestTranslatorBudget(createZedAdapter({ adapter: "zed", authMode: "oauth", + baseUrl: "https://zed-proxy.example.test/v1/", apiKey: "access-token" })); + const parsed = parseRequest({ model: "gpt-5.6", input: "hello", stream: true }); + parsed._zedAuthContext = { userId: "user-1" }; + const request = await adapter.buildRequest(parsed); + const salt = "5".repeat(64); + const poolAlias = createHash("sha256").update(`${salt}\0pool\0zed-canonical`).digest("hex").slice(0, 32); + const ledger = createSpendReservationLedger({ salt, policy: { + ...DEFAULT_SPEND_RESERVATION_POLICY, pool: { maxTokens: 100 }, maxTrackedSends: full ? 1 : 10, + canonicalProviderIds: ["zed-canonical"], poolAliases: { [poolAlias]: "zed-canonical" }, + } }); + if (full) expect(ledger.reserveSeed({ sendId: "occupied", scopes: { poolId: "zed-canonical" }, + inputTokens: 1, outputCeilingTokens: 0 }).reserved).toBe(true); + const tracker = createRequestSpendTracker({ provider: "Zed display label", spendPoolId: "zed-canonical", + spendInputEstimateTokens: 10 }, undefined, ledger); + const budget = createRequestExecutionBudget({ maxTotalModelSends: limit, baseSendAllowance: limit, + finalRecoveryAllowance: 0, maxAlternateTargetSends: 0, maxTargetTransitions: 0 }, undefined, tracker); + return { adapter, request, ledger, tracker, budget }; + } + + test("NORMAL seed refusal makes zero completion executor calls", async () => { + const { adapter, request, budget } = await fixture(2, true); + let calls = 0; + await expect(adapter.fetchResponse!(request, { sendBudget: budget, + executor: (async () => { calls++; return new Response("unexpected"); }) as typeof globalThis.fetch, + })).rejects.toBeInstanceOf(SendBudgetExhaustedError); + expect(calls).toBe(0); + expect(budget.physicalStarted).toBe(0); + }); + + for (const limit of [1, 2]) test(`401 replay respects physical L=${limit} and canonical pool accounting`, async () => { + const { adapter, request, ledger, tracker, budget } = await fixture(limit); + const calls: string[] = []; + const physical: Array<{ ordinal: number; recovery?: string }> = []; + let completions = 0; + const result = adapter.fetchResponse!(request, { sendBudget: budget, onPhysicalSend: send => physical.push(send), + executor: (async (input) => { + const url = String(input); + calls.push(url); + if (new URL(url).pathname === "/client/llm_tokens") return jsonResponse({ token: "llm-token-2" }); + expect(url).toBe("https://zed-proxy.example.test/v1/completions"); + return ++completions === 1 ? new Response("expired", { status: 401 }) : new Response("ok"); + }) as typeof globalThis.fetch, + }); + if (limit === 1) await expect(result).rejects.toBeInstanceOf(SendBudgetExhaustedError); + else expect(await (await result).text()).toBe("ok"); + expect(completions).toBe(limit); + expect(calls.filter(url => new URL(url).pathname === "/client/llm_tokens")).toHaveLength(1); + expect(physical).toEqual(limit === 1 ? [{ ordinal: 1 }] : [{ ordinal: 1 }, { ordinal: 2, recovery: "oauth-401" }]); + expect(budget.used).toBe(limit); + expect(budget.physicalStarted).toBe(limit); + tracker.requestFinalSettlement(limit === 2 ? { inputTokens: 7, outputTokens: 0 } : undefined); + expect(ledger.snapshot("pool", "zed-canonical")).toMatchObject({ settled: limit === 2 ? 7 : 0, unresolved: 10 }); + expect(ledger.snapshot("pool", "Zed display label")?.settled ?? 0).toBe(0); + }); +}); + describe("Zed stream framing", () => { async function translate(chunks: string[], provider: ZedProvider = "anthropic"): Promise>> { const encoder = new TextEncoder(); diff --git a/tests/responses/responses-spend-anchor.test.ts b/tests/responses/responses-spend-anchor.test.ts index cace04e532a..e9acb5ec3c7 100644 --- a/tests/responses/responses-spend-anchor.test.ts +++ b/tests/responses/responses-spend-anchor.test.ts @@ -3,6 +3,23 @@ import { createSpendReservationLedger, DEFAULT_SPEND_RESERVATION_POLICY } from " import { createRequestExecutionBudget, deriveRequestExecutionBudget, CODEX_TEXT_GUARDED_BUDGET_POLICY, createPhysicalSendReporter } from "../../src/lib/request-execution-budget"; import { createRequestSpendTracker } from "../../src/server/responses/request-spend"; import { fetchWithResetRetry, fetchWithTransientRetry } from "../../src/lib/upstream-retry"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { configureSharedSpendLedger, sharedSpendLedger } from "../../src/lib/spend-reservation-ledger"; +import { handleNativeMessages } from "../../src/server/messages-native"; +import { handleNativeChatCompletions, runNativeChatAttempt } from "../../src/server/chat-native"; +import { handleResponses, handleResponsesCompact } from "../../src/server/responses"; +import { beginInferenceAttempt } from "../../src/server/inference/attempt"; +import { routeModel } from "../../src/router"; +import { createTranslatorBudget } from "../../src/lib/translator-budget"; +import type { OcxConfig } from "../../src/types"; +import type { RequestLogContext } from "../../src/server/request-log"; +import type { RequestExecutionBudget } from "../../src/lib/request-execution-budget"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { resetProviderRequestPacingForTest } from "../../src/providers/request-pacing"; +import { acquireSpendLedgerServerLifecycle } from "../../src/server/index/spend-ledger-lifecycle"; function fixture(enforced = true, capacity = 8) { const lines: string[] = []; @@ -17,7 +34,7 @@ function fixture(enforced = true, capacity = 8) { return { lines, ledger, ctx, tracker, budget, reporter }; } -test("ordinary rootless passthrough seeds before its first fetch", async () => { +test("retry helper refuses a normal seed before its first fetch at capacity", async () => { const f = fixture(true, 1); f.ledger.reserve({ sendId: "busy", scopes: { poolId: "P" }, inputTokens: 1, outputCeilingTokens: 0 }); let wires = 0; @@ -26,7 +43,7 @@ test("ordinary rootless passthrough seeds before its first fetch", async () => { expect(f.tracker.refusals).toBe(1); }); -test("generic adapter initial send adopts one normal seed", async () => { +test("retry helper adopts one prepaid normal seed", async () => { const f = fixture(); const decision = f.budget.reserveDispatch({ sendClass: "initial", targetKey: "P/A", countedExternally: true }); expect(decision.allowed).toBe(true); @@ -39,7 +56,7 @@ test("generic adapter initial send adopts one normal seed", async () => { expect(f.ledger.snapshot("pool", "P")?.settled).toBe(5); }); -test("stable reset transient and continuation retries never create new scope entries", async () => { +test("stable reset and transient helper invocations reuse one scope entry", async () => { const f = fixture(true, 1); for (const helper of [fetchWithResetRetry, fetchWithTransientRetry, fetchWithResetRetry]) { await helper(async () => new Response(), { attempts: 1, onSendsConsumed: f.reporter() }); @@ -64,7 +81,7 @@ test("delayed key A batch remains on A after key B reselection", () => { expect(f.ledger.snapshot("identity", "B")?.settled).toBe(7); }); -test("combo B adopts B seed on a shared parent observer", () => { +test("derived budget reporter adopts B seed on a shared parent observer", () => { const f = fixture(); const child = deriveRequestExecutionBudget(f.budget, CODEX_TEXT_GUARDED_BUDGET_POLICY); const report = createPhysicalSendReporter(child, () => ({ poolId: "B", identityId: "account-B" })); @@ -77,7 +94,7 @@ test("combo B adopts B seed on a shared parent observer", () => { expect(f.ledger.snapshot("identity", "account-B")?.settled).toBe(11); }); -test("all helper native adapter WS and compact executors enforce frozen L", async () => { +test("retry helpers enforce the frozen physical limit despite policy mutation", async () => { const f = fixture(); let wires = 0; for (let i = 0; i < 4; i++) await fetchWithResetRetry(async () => { wires++; return new Response(); }, { attempts: 1, onSendsConsumed: f.reporter() }); @@ -87,7 +104,7 @@ test("all helper native adapter WS and compact executors enforce frozen L", asyn expect(f.budget.physicalLimit).toBe(4); }); -test("native Messages without a budget shares the finite enforced ingress limit", () => { +test("physical reporter refuses a fifth count claim at the default limit", () => { const f = fixture(); const report = f.reporter(); for (let i = 0; i < 4; i++) expect(report.beforeSend?.()).toBe(true); @@ -96,7 +113,7 @@ test("native Messages without a budget shares the finite enforced ingress limit" expect(f.budget.physicalStarted).toBe(4); }); -test("async terminal cancel and shutdown drain reporters before forgetting", async () => { +test("terminal settlement and ledger drain wait for an open reporter", async () => { const f = fixture(); const report = f.reporter(); expect(report.beforeSend?.()).toBe(true); @@ -128,3 +145,291 @@ test("unconfigured helper keeps shipped no-refusal behavior", async () => { expect(wires).toBe(1); expect(f.tracker.refusals).toBe(0); }); + + +// These cases exercise production entrypoints; only the external wire is faked. + +const executorKinds = ["messages", "chat", "compact", "generic", "passthrough"] as const; +type ExecutorKind = typeof executorKinds[number]; +async function withExecutorHome(run: (ledger: ReturnType, home: string) => Promise, capacity = 8) { + const home = mkdtempSync(join(tmpdir(), "ocx-spend-executors-")); + const previousHome = process.env.OPENCODEX_HOME; + const originalFetch = globalThis.fetch; + process.env.OPENCODEX_HOME = home; + const release = acquireOwnedSpendHome(); + globalThis.fetch = (async () => { throw new Error("Unexpected external transport in executor fixture"); }) as typeof fetch; + try { + configureSharedSpendLedger({ ...DEFAULT_SPEND_RESERVATION_POLICY, maxTrackedSends: capacity, pool: { maxTokens: 100000 } }); + await run(sharedSpendLedger(), home); + } finally { + release(); globalThis.fetch = originalFetch; resetProviderRequestPacingForTest(); + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + removeTreeWithRetry(home); + } +} +const executorPool = (kind: ExecutorKind) => kind === "compact" ? "openai-apikey" : "P"; +function executorConfig(kind: ExecutorKind, wire: typeof fetch): OcxConfig { + return { port: 0, defaultProvider: executorPool(kind), providers: { [executorPool(kind)]: { + adapter: kind === "messages" ? "anthropic" : kind === "chat" || kind === "generic" ? "openai-chat" : "openai-responses", + authMode: "key", apiKey: "fixture-only", baseUrl: kind === "compact" ? "https://api.openai.com/v1" : "https://executor.example.test/v1", + transientRetryOn5xx: { attempts: 3 }, fetch: wire, + } } }; +} +function executorSuccess(kind: ExecutorKind): Response { + if (kind === "messages") return Response.json({ id: "msg_fixture", type: "message", role: "assistant", model: "model", + content: [{ type: "text", text: "ok" }], stop_reason: "end_turn", usage: { input_tokens: 2, output_tokens: 1 } }); + if (kind === "chat" || kind === "generic") return Response.json({ id: "chat_fixture", object: "chat.completion", model: "model", + choices: [{ index: 0, message: { role: "assistant", content: "ok" }, finish_reason: "stop" }], usage: { prompt_tokens: 2, completion_tokens: 1 } }); + + return Response.json({ id: "resp_fixture", object: "response", status: "completed", output: kind === "compact" ? [{ type: "compaction", encrypted_content: "fixture-opaque" }] : [], usage: { input_tokens: 2, output_tokens: 1 } }); +} +async function invokeExecutor(kind: ExecutorKind, config: OcxConfig, logCtx: RequestLogContext, sendBudget?: RequestExecutionBudget, signal?: AbortSignal): Promise { + const translatorBudget = createTranslatorBudget(); + const body = kind === "messages" || kind === "chat" + ? { model: "model", messages: [{ role: "user", content: "hello" }], max_tokens: 1, stream: false } + : { model: `${executorPool(kind)}/model`, input: "hello", max_output_tokens: 1, stream: false }; + const req = new Request(`http://localhost/v1/${kind === "messages" ? "messages" : kind === "chat" ? "chat/completions" : kind === "compact" ? "responses/compact" : "responses"}`, { + method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body), signal, + }); + try { + let response: Response; + if (kind === "messages") response = await handleNativeMessages({ req, config, logCtx, route: routeModel(config, "P/model"), body, requestedModel: "P/model", translatorBudget }); + else if (kind === "chat") { + const options = { req, config, logCtx, route: routeModel(config, "P/model"), chatBody: body, requestedModel: "P/model", requestedStream: false, translatorBudget }; + response = sendBudget ? await runNativeChatAttempt({ ...options, sendBudget, finishLog: () => {} }, beginInferenceAttempt(logCtx, { provider: "P", model: "model", adapter: "openai-chat" })) : await handleNativeChatCompletions(options); + } else if (kind === "compact") response = await handleResponsesCompact(req, config, logCtx, undefined, undefined, { sendBudget }); + else response = await handleResponses(req, config, logCtx, { sendBudget, translatorBudget, abortSignal: signal }); + // Consume while its translator owner is alive; preserve status/headers for assertions. + const text = await response.text(); + return new Response(text, { status: response.status, headers: response.headers }); + } finally { translatorBudget.dispose(); } +} + +for (const kind of executorKinds) { + test(`${kind} production executor refuses NORMAL capacity before any wire call`, async () => withExecutorHome(async ledger => { + expect(ledger.reserve({ sendId: "busy", scopes: { poolId: "P" }, inputTokens: 1, outputCeilingTokens: 0 }).reserved).toBe(true); + let wires = 0; + const wire = (async () => { wires++; return executorSuccess(kind); }) as typeof fetch; + globalThis.fetch = wire; + const ctx: RequestLogContext = { model: "model", provider: "P", spendPoolId: "P", usageLogInputTokens: 2, spendOutputCeilingTokens: 1 }; + const response = await invokeExecutor(kind, executorConfig(kind, wire), ctx); + expect(wires).toBe(0); + expect(response.status).toBe(429); + expect(ctx.errorCode).toBe(kind === "messages" || kind === "chat" ? "workflow_spend_exhausted" : "workflow_tracking_exhausted"); + }, 1)); + + test(`${kind} production executor books exactly one initial physical send`, async () => withExecutorHome(async (ledger, home) => { + let wires = 0; + const wire = (async (input) => { + wires++; + const path = new URL(input instanceof Request ? input.url : String(input)).pathname; + expect(path).toBe(kind === "messages" ? "/v1/messages" : kind === "chat" || kind === "generic" ? "/v1/chat/completions" : kind === "compact" ? "/v1/responses/compact" : "/v1/responses"); + expect(ledger.snapshot("pool", executorPool(kind))?.reserved).toBeGreaterThan(0); + return executorSuccess(kind); + }) as typeof fetch; + globalThis.fetch = wire; + const ctx: RequestLogContext = { model: "model", provider: "P", spendPoolId: "P", usageLogInputTokens: 2, spendOutputCeilingTokens: 1 }; + const response = await invokeExecutor(kind, executorConfig(kind, wire), ctx); + expect(response.status, await response.clone().text()).toBe(200); + expect(wires).toBe(1); + ctx.spendTracker?.settle({ inputTokens: 2, outputTokens: 1 }); + await ledger.waitForReporterDrain(); + const records = (await Bun.file(join(home, "spend-ledger.jsonl")).text()).trim().split("\n").map(line => JSON.parse(line)); + expect(records.filter(record => record.kind === "reserve")).toHaveLength(1); + expect(ledger.snapshot("pool", executorPool(kind))).toMatchObject({ reserved: 0, settled: 3 }); + })); +} + +for (const kind of ["chat", "compact", "generic", "passthrough"] as const) { + test(`${kind} production retries honor the inherited frozen physical limit`, async () => withExecutorHome(async ledger => { + const ctx: RequestLogContext = { model: "model", provider: executorPool(kind), spendPoolId: executorPool(kind), spendInputEstimateTokens: 2, spendOutputCeilingTokens: 1 }; + const tracker = createRequestSpendTracker(ctx, undefined, ledger); + ctx.spendTracker = tracker; + const budget = createRequestExecutionBudget({ ...CODEX_TEXT_GUARDED_BUDGET_POLICY, maxTotalModelSends: 1, baseSendAllowance: 1 }, undefined, tracker); + let wires = 0; + const wire = (async () => { wires++; Object.assign(budget.policy, { maxTotalModelSends: 100 }); return Response.json({ error: { message: "temporary outage" } }, { status: 503 }); }) as typeof fetch; + globalThis.fetch = wire; + const response = await invokeExecutor(kind, executorConfig(kind, wire), ctx, budget); + expect(response.ok).toBe(false); + expect(wires).toBe(1); + expect(budget.physicalStarted).toBe(1); + expect(budget.physicalLimit).toBe(1); + tracker.settle(undefined); + await ledger.waitForReporterDrain(); + expect(ledger.snapshot("pool", executorPool(kind))?.reserved).toBe(0); + })); +} + +test("native Messages without a supplied budget refuses the fifth real rate-limit retry", async () => withExecutorHome(async (ledger, home) => { + let wires = 0; + const wire = (async () => { wires++; return Response.json({ error: { type: "rate_limit_error", message: "busy" } }, { status: 429 }); }) as typeof fetch; + globalThis.fetch = wire; + const config = executorConfig("messages", wire); + config.providers.P!.retryOn429 = { attempts: 6, intervalMs: 1, maxIntervalMs: 1, respectRetryAfter: false }; + const ctx: RequestLogContext = { model: "model", provider: "P" }; + const response = await invokeExecutor("messages", config, ctx); + expect(response.status).toBe(429); + expect(wires).toBe(4); + ctx.spendTracker?.settle(undefined); + await ledger.waitForReporterDrain(); + const records = (await Bun.file(join(home, "spend-ledger.jsonl")).text()).trim().split("\n").map(line => JSON.parse(line)); + expect(records.filter(record => record.kind === "reserve")).toHaveLength(4); + expect(ledger.snapshot("pool", "P")?.reserved).toBe(0); +})); + +for (const kind of executorKinds) { + test(`${kind} terminal settlement waits for its actual pending transport`, async () => withExecutorHome(async ledger => { + const started = Promise.withResolvers(); + const completion = Promise.withResolvers(); + let wires = 0; + const wire = (async () => { wires++; started.resolve(); return completion.promise; }) as typeof fetch; + globalThis.fetch = wire; + const ctx: RequestLogContext = { model: "model", provider: executorPool(kind), spendPoolId: executorPool(kind), usageLogInputTokens: 2, spendOutputCeilingTokens: 1 }; + const pending = invokeExecutor(kind, executorConfig(kind, wire), ctx); + try { + await started.promise; + expect(wires).toBe(1); + ctx.spendTracker!.settle(undefined); + let drained = false; + const drain = ledger.waitForReporterDrain().then(() => { drained = true; }); + await Promise.resolve(); + expect(drained).toBe(false); + expect(ledger.snapshot("pool", executorPool(kind))?.reserved).toBeGreaterThan(0); + completion.resolve(executorSuccess(kind)); + expect((await pending).status).toBe(200); + await drain; + expect(ledger.snapshot("pool", executorPool(kind))?.reserved).toBe(0); + expect(ledger.snapshot("pool", executorPool(kind))?.unresolved).toBeGreaterThan(0); + } finally { completion.resolve(executorSuccess(kind)); await pending; } + })); + + test(`${kind} cancellation drains its actual transport before releasing the seed`, async () => withExecutorHome(async ledger => { + const started = Promise.withResolvers(); + const controller = new AbortController(); + let wires = 0; + const wire = (async (_input, init) => { + wires++; + return new Promise((_resolve, reject) => { + init!.signal!.addEventListener("abort", () => reject(init!.signal!.reason), { once: true }); + started.resolve(); + }); + }) as typeof fetch; + globalThis.fetch = wire; + const ctx: RequestLogContext = { model: "model", provider: executorPool(kind), spendPoolId: executorPool(kind), usageLogInputTokens: 2, spendOutputCeilingTokens: 1 }; + const pending = invokeExecutor(kind, executorConfig(kind, wire), ctx, undefined, controller.signal); + try { + await started.promise; + expect(ledger.snapshot("pool", executorPool(kind))?.reserved).toBeGreaterThan(0); + controller.abort(new DOMException("Fixture cancellation", "AbortError")); + const response = await pending; + expect(response.ok).toBe(false); + ctx.spendTracker!.settle(undefined); + await ledger.waitForReporterDrain(); + expect(wires).toBe(1); + expect(ledger.snapshot("pool", executorPool(kind))?.reserved).toBe(0); + expect(ledger.snapshot("pool", executorPool(kind))?.unresolved).toBeGreaterThan(0); + } finally { controller.abort(); await pending; } + })); +} + +type SocketListener = (event: unknown) => void; +class SpendFixtureSocket { + static instances: SpendFixtureSocket[] = []; + listeners = new Map(); + sent: string[] = []; + closed = false; + constructor(_url: string, _options?: unknown) { + SpendFixtureSocket.instances.push(this); + queueMicrotask(() => this.emit("open", {})); + } + addEventListener(type: string, listener: SocketListener) { + this.listeners.set(type, [...this.listeners.get(type) ?? [], listener]); + } + removeEventListener(type: string, listener: SocketListener) { + this.listeners.set(type, (this.listeners.get(type) ?? []).filter(value => value !== listener)); + } + emit(type: string, event: unknown) { for (const listener of this.listeners.get(type) ?? []) listener(event); } + send(data: string) { + this.sent.push(data); + queueMicrotask(() => this.emit("message", { data: JSON.stringify({ type: "response.completed", response: { + id: "resp_ws_fixture", status: "completed", output: [], usage: { input_tokens: 2, output_tokens: 1 }, + } }) })); + } + close() { if (!this.closed) { this.closed = true; this.emit("close", {}); } } +} +for (const mode of ["normal", "capacity", "exhausted-limit"] as const) { + test(`Responses WS executor ${mode} preserves seed and count admission before response.create`, async () => withExecutorHome(async ledger => { + const originalSocket = globalThis.WebSocket; + SpendFixtureSocket.instances = []; + globalThis.WebSocket = SpendFixtureSocket as unknown as typeof WebSocket; + const translatorBudget = createTranslatorBudget(); + const ctx: RequestLogContext = { model: "model", provider: "P", spendPoolId: "P", usageLogInputTokens: 2, spendOutputCeilingTokens: 1 }; + const tracker = createRequestSpendTracker(ctx, undefined, ledger); + ctx.spendTracker = tracker; + const budget = createRequestExecutionBudget({ ...CODEX_TEXT_GUARDED_BUDGET_POLICY, maxTotalModelSends: 1 }, undefined, tracker); + if (mode === "capacity") expect(ledger.reserve({ sendId: "busy", scopes: { poolId: "P" }, inputTokens: 1, outputCeilingTokens: 0 }).reserved).toBe(true); + if (mode === "exhausted-limit") { + const report = createPhysicalSendReporter(budget, () => ({ poolId: "P" })); + expect(report.beforeSend?.()).toBe(true); report(1); report.close?.(); + Object.assign(budget.policy, { maxTotalModelSends: 100 }); + } + const config: OcxConfig = { port: 0, defaultProvider: "P", providers: { P: { + adapter: "openai-responses", authMode: "key", apiKey: "fixture-only", baseUrl: "https://api.openai.com/v1", upstreamWebsocket: true, + } } }; + try { + const response = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "P/model", input: "hello", stream: true, max_output_tokens: 1 }), + }), config, ctx, { sendBudget: budget, translatorBudget, codexWsRuntimeIdentity: "1.4.0" }); + const text = await response.text(); + expect(SpendFixtureSocket.instances).toHaveLength(mode === "normal" ? 1 : 0); + if (mode === "normal") { + expect(response.status).toBe(200); + expect(text).toContain("response.completed"); + expect(SpendFixtureSocket.instances[0]!.sent).toHaveLength(1); + expect(JSON.parse(SpendFixtureSocket.instances[0]!.sent[0]!).type).toBe("response.create"); + expect(budget.physicalStarted).toBe(1); + } else expect(response.status).toBe(429); + tracker.settle(undefined); + await ledger.waitForReporterDrain(); + } finally { + for (const socket of SpendFixtureSocket.instances) socket.close(); + globalThis.WebSocket = originalSocket; + translatorBudget.dispose(); + } + }, mode === "capacity" ? 1 : 8)); +} + +test("server stop waits for a native Messages transport reporter after listener stop", async () => withExecutorHome(async (ledger, home) => { + const lifecycle = acquireSpendLedgerServerLifecycle(home); + const wireStarted = Promise.withResolvers(); + const wireCompletion = Promise.withResolvers(); + const listenerStopped = Promise.withResolvers(); + const wire = (async () => { wireStarted.resolve(); return wireCompletion.promise; }) as typeof fetch; + globalThis.fetch = wire; + const ctx: RequestLogContext = { model: "model", provider: "P", usageLogInputTokens: 2, spendOutputCeilingTokens: 1 }; + const pending = invokeExecutor("messages", executorConfig("messages", wire), ctx); + const listener = lifecycle.track({ async stop() { listenerStopped.resolve(); } }); + let stopping: Promise | undefined; + try { + await wireStarted.promise; + ctx.spendTracker!.settle(undefined); + let finished = false; + stopping = listener.stop().then(() => { finished = true; }); + await listenerStopped.promise; + await Promise.resolve(); + expect(finished).toBe(false); + expect(ledger.snapshot("pool", "P")?.reserved).toBeGreaterThan(0); + wireCompletion.resolve(executorSuccess("messages")); + expect((await pending).status).toBe(200); + await stopping; + expect(finished).toBe(true); + expect(ledger.snapshot("pool", "P")?.reserved).toBe(0); + } finally { + wireCompletion.resolve(executorSuccess("messages")); + await pending; + await stopping; + lifecycle.release(); + } +})); diff --git a/tests/server/inference-send-budget.test.ts b/tests/server/inference-send-budget.test.ts index bab0e384467..49012d184fb 100644 --- a/tests/server/inference-send-budget.test.ts +++ b/tests/server/inference-send-budget.test.ts @@ -208,3 +208,24 @@ describe("createInferenceSendBudget", () => { expect(started.policy).toEqual(CODEX_TEXT_GUARDED_BUDGET_POLICY); }); }); + +test("ingress reuses its tracker only for the same request even when log context is reused", () => { + const logCtx: RequestLogContext = { model: "m", provider: "p" }; + const first = new Request("http://localhost/v1/responses"); + createInferenceSendBudget(first, logCtx); + const initial = logCtx.spendTracker; + createInferenceSendBudget(first, logCtx); + expect(logCtx.spendTracker).toBe(initial); + createInferenceSendBudget(new Request("http://localhost/v1/responses"), logCtx); + expect(logCtx.spendTracker).not.toBe(initial); +}); + +test("checking an unconfigured ingress budget does not open its journal", async () => { + const { existsSync } = await import("node:fs"); + const { createPhysicalSendReporter } = await import("../../src/lib/request-execution-budget"); + const budget = createInferenceSendBudget(new Request("http://localhost/v1/responses"), { model: "m", provider: "p" }); + expect(budget.spendEnforced).toBe(false); + expect(createPhysicalSendReporter(budget, () => ({ poolId: "p" })).beforeSend).toBeUndefined(); + expect(existsSync(join(home, "spend-ledger.salt"))).toBe(false); + expect(existsSync(join(home, "spend-ledger.jsonl"))).toBe(false); +}); From 9617434e2a301ba537d8e77dbb111ac7ee040cd5 Mon Sep 17 00:00:00 2001 From: JUN Date: Thu, 8 Oct 2026 23:56:10 +0900 Subject: [PATCH 10/15] test(kiro): provide complete retry permit cleanup stub --- tests/providers/kiro/kiro-transport-parity.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/providers/kiro/kiro-transport-parity.test.ts b/tests/providers/kiro/kiro-transport-parity.test.ts index 7ea133c6861..a6c74190df9 100644 --- a/tests/providers/kiro/kiro-transport-parity.test.ts +++ b/tests/providers/kiro/kiro-transport-parity.test.ts @@ -103,7 +103,7 @@ test("budget refusal prevents alternate dispatch", async () => { }) as typeof fetch; const sendBudget = { reserveDispatch: () => { reservations++; - return reservations === 1 ? { allowed: true, permit: { use: () => true } } : { allowed: false, reason: "total-exhausted" }; + return reservations === 1 ? { allowed: true, permit: { use: () => true, release: () => undefined } } : { allowed: false, reason: "total-exhausted" }; } } as never; await expect(fetchKiroWithRetry(request, { executor, sendBudget })).rejects.toMatchObject({ name: "SendBudgetExhaustedError" }); expect(urls).toEqual([request.url]); From 756c933022eb4d2fc86c868fd92b0b950d314511 Mon Sep 17 00:00:00 2001 From: JUN Date: Fri, 9 Oct 2026 01:09:55 +0900 Subject: [PATCH 11/15] fix(spend): isolate provider pools and completed tracker cleanup Address the seven review threads with read-time provider self-binding, strict mapping destinations, owner-scoped reporter cleanup, deferred persistence recovery, reverse indexes, live roster refresh and compact refusal parity. Co-authored-by: Epinephrine --- .../docs/fr/reference/configuration/server.md | 30 +++++ .../docs/ja/reference/configuration/server.md | 26 +++-- .../docs/ko/reference/configuration/server.md | 29 +++-- .../docs/reference/configuration/server.md | 58 ++++----- .../docs/ru/reference/configuration/server.md | 36 +++--- .../docs/tr/reference/configuration/server.md | 30 +++++ .../zh-cn/reference/configuration/server.md | 21 ++-- .../zh-tw/reference/configuration/server.md | 30 +++++ src/config/diagnostics.ts | 2 + src/lib/spend-pool-alias-validation.ts | 8 +- src/lib/spend-pool-continuity.ts | 12 +- src/lib/spend-reservation-ledger.ts | 87 ++++++++++---- src/lib/state-store-registrations.ts | 5 + src/server/responses/compact.ts | 3 +- src/server/responses/request-spend.ts | 58 +++++---- structure/config.md | 2 +- structure/runtime.md | 2 +- structure/transports/responses-spend.md | 43 ++++--- tests/config/config-spend-ceilings.test.ts | 2 +- .../config-spend-pool-collision.test.ts | 13 +++ tests/lib/spend-pool-continuity.test.ts | 45 ++++--- tests/lib/spend-seed-overflow.test.ts | 110 +++++++++++++++++- ...esponses-compact-handoff-admission.test.ts | 29 +++++ .../responses-spend-contract-c.test.ts | 20 ++-- tests/server/spend-ledger-lifecycle.test.ts | 63 +++++++++- 25 files changed, 585 insertions(+), 179 deletions(-) diff --git a/docs-site/src/content/docs/fr/reference/configuration/server.md b/docs-site/src/content/docs/fr/reference/configuration/server.md index d82a29a23ca..debf3946fa7 100644 --- a/docs-site/src/content/docs/fr/reference/configuration/server.md +++ b/docs-site/src/content/docs/fr/reference/configuration/server.md @@ -286,3 +286,33 @@ This takes effect on the **next routed `ocx claude` launch**, injecting `CLAUDE_ Claude Code **2.1.257 or newer** is required for FORCE. Plugin and built-in agents (including Explore/Plan) and per-call model arguments are overridden. Forks and subagent skills with `model: inherit` keep the main conversation model. The main loop and Haiku/small-fast sidecars are unaffected. Existing roster files remain available. The dashboard warns about old or unknown CLI versions, unavailable targets, and either variable already present in `settings.json` → `env` (which overrides launch env). Detection is read-only and server-local: it cannot inspect another launch shell, another machine, or project-local settings. An unknown result is not proof of force support. + +## Continuité de l’historique des pools + +Le plafond d’un pool s’applique au fournisseur canonique choisi par le routage, indépendamment du libellé de compte dans les journaux. Pour chaque fournisseur `P` actuellement configuré, OpenCodex associe automatiquement `h(pool, P)`, son alias de pool salé propre à cette installation, à `P` lors de la lecture des soldes. Cela inclut les soldes historiques conservés sous ce même alias. Les plafonds indépendants ne nécessitent ni association manuelle du fournisseur à lui-même ni calcul avec le sel. + +Les autres libellés historiques, y compris ceux contenant un numéro de compte, restent sans propriétaire confirmé. Chaque solde d’origine est compté une seule fois dans son groupe associé ; tout solde positif encore non associé est ajouté, par prudence, une fois à chaque pool candidat. Les montants réglés, réservés et non résolus comptent tous. Un historique inconnu peut donc limiter un pool inutilisé. Les noms actuels et l’ordre des comptes ne prouvent pas à quel fournisseur cet historique appartient. Les plafonds de racine et d’identité restent indépendants. + +Pour un historique dont vous avez vérifié le propriétaire, ajoutez une entrée dans `spendPoolAliases`, au premier niveau de `config.json`, en dehors de `spend`. La clé est l’alias salé exact du pool dans le journal de cette installation, composé de 32 caractères hexadécimaux minuscules ; la valeur est l’ID exact d’un fournisseur vérifié et actuellement configuré, sans espaces au début ni à la fin. Gardez le journal, le sel et les éléments justificatifs privés. L’alias propre d’un fournisseur configuré ne peut pas être attribué à un autre fournisseur. La validation est répétée lorsque les fournisseurs changent et lit le sel existant sans en créer. + +Les associations automatiques et explicites s’appliquent uniquement à la lecture : elles ne déplacent pas les soldes et n’inscrivent aucun lien d’identité dans le journal. Supprimer une association explicite rend l’historique inconnu, sauf si son alias est celui d’un fournisseur actuellement configuré. Une association invalide est rejetée à l’écriture ; une modification manuelle invalide conserve les plafonds mais bloque l’admission du pool. + +Un historique inconnu et inactif expire seulement quand sa dernière activité est strictement antérieure à la limite de `spend.retentionDays`. Un manque de capacité ne raccourcit pas ce délai pour un solde positif. Les réservations actives et leurs cibles restent protégées, même à zéro jeton. La suppression doit être enregistrée durablement avant de réduire le total utilisé pour l’admission. + +### Réservations et limite d’envoi + +Lorsqu’un plafond de racine, d’identité ou de pool s’applique, le premier envoi pour chaque cible ou clé nécessite une réservation dans la capacité normale de suivi. Sans réservation, aucun envoi n’atteint le fournisseur. Les nouvelles tentatives stables réutilisent les mêmes périmètres ; une autre cible ou clé nécessite une nouvelle réservation. `L` est la limite des envois physiques pour l’ensemble de la requête, figée au premier envoi physique : quatre par défaut, ou jusqu’à dix-huit avec l’extension OAuth existante appliquée avant tout envoi. + +Le règlement final attend les rapports des envois démarrés. Un ID d’envoi n’est oublié qu’après enregistrement durable de sa comptabilité ; son solde demeure. La limite borne les envois, pas la facture : toute utilisation réelle supérieure à l’estimation reste comptabilisée. Sans plafond applicable, le comportement reste en observation seule, y compris l’omission des écritures lorsque la capacité de suivi est pleine. Aucun point de contrôle d’identité n’est ajouté. + +Claude CLI, CodeBuddy et Qoder comptent chaque invocation CLI comme un envoi et exigent une réservation normale avant le lancement. Les nouvelles tentatives et les tours d’outils internes au CLI ne consomment pas séparément la limite d’envois de la requête. Leur coût est réglé selon l’utilisation réelle rapportée, y compris la part dépassant l’estimation initiale. + +### Retour à 2.80.0 : contrat C + +Les nouveaux enregistrements utilisent le format v1 ordinaire et le domaine d’alias `pool`. La version 2.80.0 non modifiée les lit, compacte le journal et applique la rétention selon ses propres règles par libellé. Conservez le journal actuel et son sel : restaurer une ancienne copie perd les dépenses enregistrées depuis. Aucun correctif rétroporté, blocage du lanceur ou commande de rapprochement n’est nécessaire. + +Le retour à 2.80.0 ne garantit ni l’agrégation canonique ni le même solde disponible que si un trafic identique avait toujours été traité par 2.80.0. Lors d’une nouvelle mise à niveau, les associations automatiques des fournisseurs configurés et les correspondances explicites actuelles s’appliquent aux soldes d’origine encore conservés. Aucun nouveau point de contrôle ne dépend de métadonnées de liaison d’identité à préserver par l’ancien lecteur. + +Les journaux de versions expérimentales non publiées contenant `pool-current` ou `poolContinuity` sont exclus du contrat C, même après compactage. Leurs soldes d’origine restent un historique inconnu jusqu’à l’expiration normale, sans conversion ni suppression immédiate. + +Un enregistrement complet mais invalide, y compris un `null` final, bloque l’admission sous plafond. Le compactage peut conserver les données valides dans un point de contrôle propre, mais le refus reste actif jusqu’au redémarrage avec ce journal propre. Une dernière ligne JSON incomplète suit les règles de récupération existantes. Voir les limites et codes d’erreur dans la [référence anglaise](/reference/configuration/server/#historical-pool-continuity). diff --git a/docs-site/src/content/docs/ja/reference/configuration/server.md b/docs-site/src/content/docs/ja/reference/configuration/server.md index 135fdfe4cd4..dfe859a7d07 100644 --- a/docs-site/src/content/docs/ja/reference/configuration/server.md +++ b/docs-site/src/content/docs/ja/reference/configuration/server.md @@ -203,29 +203,37 @@ The dashboard warns about old or unknown CLI versions, unavailable targets, and ## 過去のプール使用量の継承 プロバイダープールの上限は正規のルーティング先に適用し、ログのアカウント表示名とは分けて扱います。 -`spendPoolAliases` は `config.json` のトップレベルに置きます。キーは同じインストールの過去のプール別名 -（32 文字の小文字 16 進数）、値は確認済みの正規プロバイダー ID です。現在の名前やアカウント一覧から推測しません。 - -対応関係は読み取り時だけ適用し、元の残高を移動したり、ジャーナルにリンクを保存したりしません。 -削除した対応関係の履歴は再び所有先不明になります。各元残高は確認済みグループか、全候補への保守的な加算の -どちらかで一度だけ計上します。現在の正規別名と一致していても、未登録の履歴を二重計上しません。 +現在設定されているプロバイダー `P` では、`h(pool, P)`（このインストールのソルト付きプール別名）を、 +残高の読み取り時に自動で `P` に対応付けます。同じ別名の過去残高も対象です。プロバイダー別の上限を使うために、 +自己対応の手動登録やソルトからの計算をする必要はありません。 + +異なる過去の表示名やアカウント番号付きラベルは、この自動対応の対象になりません。 +所有先を確認した履歴だけを `config.json` のトップレベルの `spendPoolAliases` に登録してください。 +キーは同じインストールの過去のプール別名（32 文字の小文字 16 進数）、値は現在設定されている、確認済みの正規プロバイダー ID です。 +ID の前後に空白を付けないでください。 +現在の名前やアカウントの並び順から所有先を推測しません。ジャーナルとソルトは非公開で保管してください。 + +自動対応と明示的な対応関係は読み取り時だけ適用し、元の残高を移動したり、ジャーナルに識別情報を保存したりしません。 +明示的な対応を削除した履歴は、現在設定されたプロバイダー自身の別名でない限り、再び所有先不明になります。 +各元残高は対応先のグループに一度だけ計上し、所有先不明の正の残高は全候補プールに一度ずつ保守的に加算します。 設定済みプロバイダー自身のソルト付き別名を別のプロバイダーに割り当てる設定は拒否します。 プロバイダー追加時にも再検証し、検証だけでソルトを作成することはありません。 休眠中の所有先不明の履歴は、最終活動時刻が `spend.retentionDays` の期限より古くなってから失効します。 容量不足を理由に正の履歴を早く削除しません。送信中の予約と seed はゼロトークンでも保持し、削除記録の保存後に残高を減らします。 適用される上限がある送信は、各送信先・キーの初回に通常容量の予約を必要とします。 -再試行は同じ範囲を使い、要求全体の有限の送信回数上限（通常 4、既存の OAuth 拡張では最大 18）を共有します。 +再試行は同じ範囲を使います。`L` は要求全体の物理送信回数上限で、最初の物理送信時に固定します。 +通常は 4 回で、既存の OAuth 拡張は送信開始前に最大 18 回まで拡大できます。すべての物理送信がこの有限の上限を共有します。 最終精算は報告完了を待ち、送信 ID の削除後も残高を保持します。上限未設定では従来の監視のみの動作を保ち、容量満杯時の記録省略も変わりません。 Claude CLI、CodeBuddy、Qoder は CLI 呼び出し 1 回を送信 1 回と数え、起動前に通常の予約を必要とします。 -CLI 内部の再試行・ツールターンは L に個別計上しませんが、初期見積もりを超えた分も実際に報告された使用量で精算します。 +CLI 内部の再試行・ツールターンは要求全体の送信回数上限に個別計上しませんが、初期見積もりを超えた分も実際に報告された使用量で精算します。 ### 2.80.0 へのロールバック（契約 C） 新しい記録は通常の v1 形式と `pool` 別名を使います。未改変の 2.80.0 は自身のラベル別規則で読み取り、圧縮・保持処理を行えます。 最新のジャーナルとソルトを維持してください。ダウングレード後の正規プール合算や、同じ通信を最初から 2.80.0 で処理した場合の残り枠は保証しません。 -再アップグレード時には、残存する元残高に現在の設定を適用します。起動フェンスや追加の照合コマンドは不要です。 +再アップグレード時には、残存する元残高に、現在設定されたプロバイダーの自動対応と明示的な対応関係を適用します。起動フェンスや追加の照合コマンドは不要です。 未公開の実験版の `pool-current` / `poolContinuity` 履歴は契約 C の対象外です。残高は所有先不明の履歴として保持し、通常の期限でのみ失効させます。 完全だが不正な記録（最終行の `null` も含む）は上限付きの受け入れを拒否します。圧縮は有効な残高を保持して続行し、 diff --git a/docs-site/src/content/docs/ko/reference/configuration/server.md b/docs-site/src/content/docs/ko/reference/configuration/server.md index cfde815855e..6a8fdd00217 100644 --- a/docs-site/src/content/docs/ko/reference/configuration/server.md +++ b/docs-site/src/content/docs/ko/reference/configuration/server.md @@ -260,31 +260,40 @@ The dashboard warns about old or unknown CLI versions, unavailable targets, and ## 과거 풀 사용량의 연속성 제공자 풀 한도는 실제로 선택된 제공자에 적용합니다. 요청 로그의 계정 표시 이름과는 구분합니다. -`config.json` 최상위의 `spendPoolAliases`에 검증된 매핑을 넣으세요. 키는 같은 설치의 과거 풀 별칭인 -32자리 소문자 16진수이고, 값은 확인된 제공자 ID입니다. 현재 이름이나 계정 목록만 보고 추측하지 마세요. - -매핑은 잔액을 읽을 때만 적용합니다. 원래 잔액을 옮기거나 저널에 연결 정보를 저장하지 않습니다. -매핑을 삭제하면 그 이력은 다시 소유자 미확인 상태가 됩니다. 각 잔액은 확인된 그룹에 넣거나 모든 후보 풀에 -보수적으로 더하며, 한 번만 계산합니다. 현재 제공자의 해시와 같은 미확인 잔액도 두 번 세지 않습니다. +현재 설정된 제공자 `P`는 잔액을 읽을 때 자신의 솔트 풀 별칭인 `h(pool, P)`와 자동으로 연결됩니다. +같은 별칭에 남아 있는 과거 잔액도 여기에 속합니다. 제공자마다 독립적인 한도를 쓰려고 자기 매핑을 직접 넣거나 +솔트로 해시를 계산할 필요는 없습니다. + +다른 과거 표시 이름이나 계정 순번이 붙은 라벨은 자동으로 연결하지 않습니다. +소유자를 확인한 과거 이력은 `config.json` 최상위의 `spendPoolAliases`에 매핑을 넣으세요. +키는 같은 설치의 과거 풀 별칭인 32자리 소문자 16진수이고, 값은 현재 설정에 있는 확인된 제공자의 정확한 ID입니다. +ID 앞뒤에 공백을 넣지 마세요. +현재 이름이나 계정 순서만 보고 소유자를 추측하지 마세요. 저널과 솔트는 비공개로 보관하세요. + +자동 연결과 명시적 매핑은 잔액을 읽을 때만 적용합니다. 원래 잔액을 옮기거나 저널에 식별 연결 정보를 저장하지 않습니다. +명시적 매핑을 삭제하면 그 이력은 다시 소유자 미확인 상태가 됩니다. 단, 현재 설정된 제공자 자신의 별칭이면 자동 연결이 남습니다. +각 잔액은 연결된 제공자 그룹에 한 번만 계산합니다. 연결되지 않은 양수 잔액은 모든 후보 풀에 한 번씩 보수적으로 더합니다. 설정된 제공자 자신의 솔트 별칭을 다른 제공자에 연결하면 거부합니다. 제공자를 추가할 때도 검사하며, 검증 과정에서 새 솔트를 만들지는 않습니다. 매핑이 잘못돼도 기존 한도는 유지합니다. 휴면 상태인 미확인 이력은 마지막 활동이 `spend.retentionDays`보다 오래됐을 때만 만료됩니다. 용량이 부족해도 양수 이력을 먼저 지우지 않습니다. 진행 중인 예약과 seed는 0토큰이어도 보존하며, 삭제 기록이 저장된 뒤에만 허용 여부를 판단하는 잔액을 줄입니다. -한도가 적용되는 요청은 대상·키별 첫 전송 전에 일반 추적 용량으로 예약해야 합니다. 재시도는 같은 범위를 쓰며, -요청 전체가 첫 전송 때 고정한 유한한 전송 횟수 한도(기본 4회, 기존 OAuth 확장은 최대 18회)를 공유합니다. +한도가 적용되는 요청은 대상·키별 첫 전송 전에 일반 추적 용량으로 예약해야 합니다. 재시도는 같은 범위를 씁니다. +`L`은 요청 전체가 공유하는 물리 전송 횟수 한도이며 첫 물리 전송 때 고정됩니다. 기본은 4회이고, +기존 OAuth 확장은 전송 시작 전에 최대 18회까지 늘릴 수 있습니다. 모든 물리 전송이 이 유한한 한도를 함께 씁니다. 최종 정산은 보고가 끝날 때까지 기다립니다. 전송 ID를 지워도 잔액은 남습니다. 적용되는 한도가 없으면 기존 관측 동작을 유지하며, 추적 용량이 찼을 때 기록을 생략하는 동작도 그대로입니다. Claude CLI·CodeBuddy·Qoder는 CLI 호출 한 번을 전송 한 번으로 셉니다. 실행 전 일반 예약이 필요하며, 예약하지 못하면 실행을 거부합니다. -CLI 내부 재시도·도구 턴은 L에 따로 세지 않지만 비용은 보고된 실제 사용량으로 정산하며, 최초 추정량을 넘은 비용도 기록합니다. +CLI 내부 재시도·도구 턴은 요청 전체 전송 횟수 한도에 따로 세지 않지만 비용은 보고된 실제 사용량으로 정산하며, 최초 추정량을 넘은 비용도 기록합니다. ### 2.80.0으로 롤백: 계약 C 새 기록은 기존 v1 형식과 `pool` 별칭을 씁니다. 수정하지 않은 2.80.0도 자신의 라벨별 규칙대로 읽고 압축·보존할 수 있습니다. 최신 저널과 솔트를 유지하세요. 롤백 후 제공자별 통합 집계가 유지되거나, 같은 요청을 처음부터 2.80.0에서 처리했을 때와 -남은 한도가 같다는 보장은 없습니다. 다시 업그레이드하면 남아 있는 원래 잔액에 현재 매핑을 적용합니다. +남은 한도가 같다는 보장은 없습니다. 다시 업그레이드하면 남아 있는 원래 잔액에 +현재 설정된 제공자의 자동 연결과 명시적 매핑을 적용합니다. 별도의 실행 차단 장치나 조정 명령은 필요하지 않습니다. 미공개 실험 버전의 `pool-current` / `poolContinuity` 이력은 계약 C에서 제외합니다. 원래 잔액을 미확인 이력으로 보존하고 일반 보존 기간이 지난 뒤 만료시키며, 즉시 변환하거나 삭제하지 않습니다. diff --git a/docs-site/src/content/docs/reference/configuration/server.md b/docs-site/src/content/docs/reference/configuration/server.md index bdb3b22a286..57626d78562 100644 --- a/docs-site/src/content/docs/reference/configuration/server.md +++ b/docs-site/src/content/docs/reference/configuration/server.md @@ -944,27 +944,32 @@ The dashboard warns about old or unknown CLI versions, unavailable targets, and ## Historical pool continuity -Provider-pool ceilings use the canonical routed provider. Request logs still use account-specific -display labels. Earlier journals may record spend under those labels; salted aliases alone cannot -prove which provider owned them. OpenCodex does not infer ownership from current accounts or names. - -With `spend.pool.maxTokens`, each original balance contributes once to its verified provider group -or, when unmapped and positive, to every candidate pool. This includes an unmapped bucket whose -hash matches the current canonical provider; it is never counted twice. Settled, reserved and -unresolved usage all count. Unknown usage is not a refund. This conservative view may restrict an -unused pool until its historical mappings are verified. Root and identity ceilings remain independent. - -Add verified mappings to the top-level `spendPoolAliases` object in `config.json`. Each key is an -exact 32-character lowercase hexadecimal pool alias from this installation's journal; each value -is its verified canonical provider ID. Keep evidence for the mapping and keep the journal and salt -private. A canonical-looking historical alias also needs an explicit mapping to establish ownership. - -Mappings apply when balances are read. They do not move original balances or persist links in the -journal: removing a mapping makes that history unbound again. Keep this object outside `spend`, -because older versions reject unknown fields inside that strict section. Invalid mappings are -rejected on writes; invalid hand edits retain ceilings and refuse pool admission. A configured -provider's own salted pool alias cannot map to another provider. This is checked again when the -provider set changes. Validation reads an existing salt without creating one. +Provider-pool ceilings use the canonical routed provider. For every currently configured provider +`P`, OpenCodex automatically binds its salted pool alias, `h(pool, P)`, to `P` when reading +balances. No manual self-mapping or salt calculation is needed for independent +provider ceilings. This rule also applies to retained historical balances with that exact alias. + +Request logs still use account-specific display labels. A different historical label, including an +account ordinal, does not establish provider ownership. With `spend.pool.maxTokens`, each original +balance contributes once to its bound provider group or, when still unbound and positive, to every +candidate pool. Settled, reserved and unresolved usage all count; unknown usage is not a refund. +Unbound history may therefore restrict an otherwise unused pool. Current account order and display +names do not resolve that history. Root and identity ceilings remain independent. + +For distinct historical labels whose ownership you have verified, add mappings to the top-level +`spendPoolAliases` object in `config.json`. Each key is an exact 32-character lowercase hexadecimal +pool alias from this installation's journal; each value is the exact ID of a verified, currently +configured canonical provider, with no surrounding whitespace. +Keep evidence for the mapping and keep the journal and salt private. + +Automatic self-bindings and explicit mappings apply only when balances are read. They do not move +original balances or persist identity links in the journal. Removing an explicit historical mapping +makes that history unbound again unless its alias is a currently configured provider's own alias. +Keep `spendPoolAliases` outside `spend`, because older versions reject unknown fields inside that +strict section. Invalid mappings are rejected on writes; invalid hand edits retain ceilings and +refuse pool admission. A configured provider's own salted pool alias cannot map to another provider. +This is checked again when the provider set changes. Validation reads an existing salt without +creating one. Dormant unbound history expires only after its last activity is strictly older than `spend.retentionDays`. Capacity pressure cannot shorten that interval for positive unknown history. @@ -977,8 +982,9 @@ active/exhausted protections. No automatic tool reconstructs unverifiable mappin When a root, identity or pool ceiling applies, the first physical send for each selected target/key requires a normal-capacity reservation. If that cannot be recorded, the send is refused before it reaches upstream. Stable retries reuse that reservation's scopes; a different target/key needs a -new one. Additional sends are counted against a finite request-wide limit fixed at the first start +new one. `L` is the request-wide physical-send limit, frozen at the first physical start (default four; the existing bounded OAuth profile can expand to eighteen before sending). +Every additional physical send shares this finite limit. Delayed reports retain already-started usage. Terminal settlement waits for reporters, and send IDs are forgotten only after their accounting is durable; forgetting an ID does not remove its balance. @@ -989,8 +995,8 @@ added for unconfigured traffic. Claude CLI, CodeBuddy and Qoder count each CLI invocation as one send. Their first invocation still needs a normal reservation and is refused if that reservation cannot be obtained. Internal -CLI retries and tool turns do not separately consume L; their cost settles from actual reported -usage, including amounts above the initial estimate. This exception preserves CLI use with configured +CLI retries and tool turns do not separately consume this request-wide send limit; their cost +settles from actual reported usage, including amounts above the initial estimate. This exception preserves CLI use with configured spend ceilings while limiting the number of child invocations. ### Rollback to 2.80.0: contract C @@ -1002,8 +1008,8 @@ an older copy loses the spend recorded since that copy. This does not preserve canonical aggregation after downgrade, nor promise the same remaining allowance that 2.80.0 would have calculated for identical traffic without the upgrade. It may book -new traffic under account labels. On re-upgrade, the current alias configuration applies to the -original balances still retained by 2.80.0. New checkpoints contain no identity-link metadata that +new traffic under account labels. On re-upgrade, current configured-provider self-bindings and explicit +alias mappings apply to the original balances still retained by 2.80.0. New checkpoints contain no identity-link metadata that an older reader must preserve. Journals written by unpublished experimental builds with `pool-current` aliases or `poolContinuity` diff --git a/docs-site/src/content/docs/ru/reference/configuration/server.md b/docs-site/src/content/docs/ru/reference/configuration/server.md index 378dc4a1350..81d6e635500 100644 --- a/docs-site/src/content/docs/ru/reference/configuration/server.md +++ b/docs-site/src/content/docs/ru/reference/configuration/server.md @@ -251,36 +251,44 @@ The dashboard warns about old or unknown CLI versions, unavailable targets, and ## Непрерывный учёт истории пулов Лимиты пула относятся к каноническому провайдеру маршрута, а не к отображаемой метке аккаунта. -Верхнеуровневый `spendPoolAliases` в `config.json` связывает точный исторический псевдоним пула -(32 строчных шестнадцатеричных символа из журнала этой установки) с проверенным ID провайдера. -Текущие имена и список аккаунтов не доказывают владельца истории. - -Соответствия применяются при чтении: исходные балансы не перемещаются, связи в журнал не записываются. -Удаление соответствия снова делает историю неопознанной. Каждый баланс учитывается один раз: -в подтверждённой группе либо консервативно для каждого возможного пула. Совпадение с текущим -каноническим хешем не приводит к двойному учёту. Псевдоним самого настроенного провайдера нельзя -назначить другому провайдеру. Проверка повторяется при добавлении провайдеров и не создаёт соль. +Для каждого настроенного сейчас провайдера `P` его солёный псевдоним пула `h(pool, P)` автоматически +связывается с `P` при чтении балансов. Это касается и сохранённой истории с точно таким же псевдонимом. +Независимые лимиты провайдеров не требуют ручного назначения собственных псевдонимов или вычислений с солью. + +Другие исторические метки, включая метки с порядковым номером аккаунта, автоматически не связываются. +Для истории с подтверждённым владельцем верхнеуровневый `spendPoolAliases` в `config.json` задаёт связь +точного псевдонима пула (32 строчных шестнадцатеричных символа из журнала этой установки) с точным ID +проверенного и настроенного сейчас провайдера, без пробелов по краям. +Текущие имена и порядок аккаунтов не доказывают владельца истории. Храните журнал и соль в секрете. + +Автоматические и явные связи применяются только при чтении: исходные балансы не перемещаются, +идентификационные связи в журнал не записываются. Удаление явной связи снова делает историю неопознанной, +если её псевдоним не принадлежит самому настроенному сейчас провайдеру. Каждый исходный баланс +учитывается один раз в связанной группе; неопознанный положительный баланс консервативно добавляется +один раз к каждому возможному пулу. Псевдоним самого настроенного провайдера нельзя назначить другому +провайдеру. Проверка повторяется при добавлении провайдеров и не создаёт соль. Некорректное соответствие сохраняет лимиты и блокирует допуск к пулу. Неактивная неопознанная история истекает только когда её последняя активность строго старше `spend.retentionDays`. Нехватка места не сокращает этот срок для положительных балансов. Активные резервирования и seed сохраняются даже при нулевой оценке; уменьшение баланса требует сначала записать удаление в журнал. При применимом лимите первая отправка для каждой цели/ключа -требует обычного резервирования. Повторы используют те же области и общий конечный лимит отправок, -фиксируемый при первом старте (обычно 4, существующее расширение OAuth — до 18). +требует обычного резервирования. Повторы используют те же области. `L` — общий лимит физических отправок +одного запроса, фиксируемый при первой физической отправке. Обычно он равен 4; существующее расширение +OAuth может увеличить его до 18 до начала отправок. Все физические отправки делят этот конечный лимит. Окончательный расчёт ждёт отчётов; удаление ID отправки не удаляет баланс. Без применимого лимита сохраняется режим наблюдения, включая пропуск записей при заполненной ёмкости отслеживания. Для Claude CLI, CodeBuddy и Qoder один вызов CLI считается одной отправкой и требует обычного резерва до запуска. -Внутренние повторы и инструментальные ходы CLI отдельно не расходуют L, но стоимость учитывается по фактически сообщённому использованию, включая превышение первоначальной оценки. +Внутренние повторы и инструментальные ходы CLI отдельно не расходуют общий лимит отправок запроса, но стоимость учитывается по фактически сообщённому использованию, включая превышение первоначальной оценки. ### Откат к 2.80.0: контракт C Новые записи используют обычный формат v1 и псевдонимы `pool`. Неизменённая 2.80.0 читает их, уплотняет журнал и применяет срок хранения по своим правилам отдельных меток. Сохраните текущие журнал и соль. После отката не гарантируется каноническая агрегация или тот же остаток лимита, -который 2.80.0 вычислила бы для того же трафика без обновления. При повторном обновлении текущая -конфигурация применяется к сохранившимся исходным балансам. Барьер запуска и команда сверки не требуются. +который 2.80.0 вычислила бы для того же трафика без обновления. При повторном обновлении автоматические +связи настроенных сейчас провайдеров и явные соответствия применяются к сохранившимся исходным балансам. Барьер запуска и команда сверки не требуются. История неопубликованных экспериментальных сборок с `pool-current` / `poolContinuity` исключена из контракта C. Исходные балансы сохраняются как неопознанная история до обычного истечения срока; немедленного преобразования или удаления нет. diff --git a/docs-site/src/content/docs/tr/reference/configuration/server.md b/docs-site/src/content/docs/tr/reference/configuration/server.md index 2fedcb91188..33734ad82f3 100644 --- a/docs-site/src/content/docs/tr/reference/configuration/server.md +++ b/docs-site/src/content/docs/tr/reference/configuration/server.md @@ -317,3 +317,33 @@ This takes effect on the **next routed `ocx claude` launch**, injecting `CLAUDE_ Claude Code **2.1.257 or newer** is required for FORCE. Plugin and built-in agents (including Explore/Plan) and per-call model arguments are overridden. Forks and subagent skills with `model: inherit` keep the main conversation model. The main loop and Haiku/small-fast sidecars are unaffected. Existing roster files remain available. The dashboard warns about old or unknown CLI versions, unavailable targets, and either variable already present in `settings.json` → `env` (which overrides launch env). Detection is read-only and server-local: it cannot inspect another launch shell, another machine, or project-local settings. An unknown result is not proof of force support. + +## Havuz harcama geçmişinin sürekliliği + +Havuz sınırı, istek günlüğündeki hesap etiketinden bağımsız olarak yönlendirmenin seçtiği kanonik sağlayıcıya uygulanır. Şu anda yapılandırılmış her `P` sağlayıcısı için bu kuruluma ait tuzlanmış havuz takma adı `h(pool, P)`, bakiyeler okunurken otomatik olarak `P` ile eşleştirilir. Aynı takma ad altında tutulan eski bakiyeler de buna dahildir. Bağımsız sağlayıcı sınırları için elle öz eşleme eklemek veya tuzu kullanarak karma hesaplamak gerekmez. + +Hesap sıra numarası içeren etiketler dahil diğer geçmiş etiketlerin sahibi otomatik olarak belirlenmez. Her özgün bakiye, eşleştiği sağlayıcı grubunda bir kez sayılır. Hâlâ eşleşmemiş pozitif bakiyeler, ihtiyatlı olmak için her aday havuza birer kez eklenir. Kesinleşmiş, rezerve edilmiş ve çözümlenmemiş kullanımın tümü hesaba katılır. Bu yüzden belirsiz geçmiş, kullanılmamış bir havuzu da sınırlayabilir. Güncel adlar ve hesap sırası geçmişin sahibini kanıtlamaz. Kök ve kimlik sınırları bağımsız kalır. + +Sahibini doğruladığınız geçmiş için `config.json` dosyasının en üst düzeyindeki `spendPoolAliases` alanına eşleme ekleyin; bu alan `spend` içinde olmamalıdır. Anahtar, bu kurulumun günlüğündeki tam 32 küçük harfli onaltılık karakterden oluşan tuzlanmış havuz takma adıdır. Değer, doğrulanmış ve şu anda yapılandırılmış sağlayıcının tam kimliğidir; başında veya sonunda boşluk bulunamaz. Günlüğü, tuzu ve eşleme kanıtını gizli tutun. Yapılandırılmış bir sağlayıcının kendi takma adı başka bir sağlayıcıya atanamaz. Sağlayıcı kümesi değiştiğinde doğrulama tekrarlanır; doğrulama mevcut tuzu okur, yenisini oluşturmaz. + +Otomatik ve açık eşlemeler yalnızca okuma sırasında uygulanır; özgün bakiyeleri taşımaz veya günlüğe kimlik bağlantısı yazmaz. Açık bir eşleme kaldırılınca geçmiş yeniden belirsiz olur; ancak takma ad halen yapılandırılmış bir sağlayıcının kendi takma adıysa otomatik eşleme sürer. Geçersiz eşlemeler yazılırken reddedilir. Elle yapılmış geçersiz bir değişiklik mevcut sınırları korur ve havuza yeni istek kabulünü engeller. + +Etkin olmayan belirsiz geçmiş, yalnızca son etkinliği `spend.retentionDays` eşiğinden kesin olarak daha eski olduğunda sona erer. Kapasite baskısı pozitif bakiyeler için bu süreyi kısaltmaz. Etkin rezervasyonlar ve hedefleri sıfır belirteçte bile korunur. Kabul kararında daha düşük toplam kullanılmadan önce silme kaydı kalıcı olarak yazılır. + +### Rezervasyonlar ve gönderim sınırı + +Kök, kimlik veya havuz sınırı geçerliyse her hedef ya da anahtarın ilk gönderimi normal izleme kapasitesinden bir rezervasyon gerektirir. Rezervasyon alınamazsa sağlayıcıya gönderim yapılmaz. Aynı hedefteki yeniden denemeler aynı kapsamları kullanır; farklı hedef veya anahtar yeni rezervasyon gerektirir. `L`, ilk fiziksel gönderimde sabitlenen, isteğin tamamı için fiziksel gönderim sınırıdır. Varsayılan değer dörttür; mevcut OAuth genişletmesi gönderimler başlamadan önce bunu en fazla on sekize çıkarabilir. + +Son hesaplaşma, başlamış gönderimlerin raporlarını bekler. Bir gönderim kimliği ancak muhasebesi kalıcılaştıktan sonra unutulur; bakiyesi silinmez. Sınır faturayı değil gönderim sayısını sınırlar: ilk tahmini aşan gerçek kullanımın tamamı kaydedilir. Geçerli bir harcama sınırı yoksa yalnızca gözlem davranışı sürer; izleme kapasitesi dolduğunda kayıtların atlanması da değişmez. Kimlik bağlantısı içeren yeni kontrol noktaları eklenmez. + +Claude CLI, CodeBuddy ve Qoder için her CLI çağrısı bir gönderim sayılır. Başlatmadan önce normal rezervasyon gerekir; alınamazsa çağrı reddedilir. CLI içindeki yeniden denemeler ve araç turları istek genelindeki gönderim sınırını ayrıca tüketmez. Maliyetleri, ilk tahmini aşan bölüm dahil, raporlanan gerçek kullanıma göre hesaplanır. + +### 2.80.0 sürümüne dönüş: C sözleşmesi + +Yeni kayıtlar olağan v1 biçimini ve `pool` takma ad alanını kullanır. Değiştirilmemiş 2.80.0 bunları kendi etiket bazlı kurallarıyla okur, günlüğü sıkıştırır ve saklama süresini uygular. Güncel günlüğü ve tuzu koruyun; eski bir kopyayı geri yüklemek, o kopyadan sonra kaydedilen harcamayı kaybettirir. Geriye dönük yama, başlatıcı engeli veya mutabakat komutu gerekmez. + +Sürüm düşürme, kanonik toplamların korunacağını veya aynı trafik baştan beri 2.80.0 üzerinde işlenseydi kalacak kotayla eşitliği garanti etmez. Yeniden yükseltmede, yapılandırılmış sağlayıcıların otomatik öz eşlemeleri ve güncel açık eşlemeler, 2.80.0 tarafından hâlâ tutulan özgün bakiyelere uygulanır. Yeni kontrol noktaları eski okuyucunun koruması gereken kimlik bağlantısı metaverisi içermez. + +Yayımlanmamış deneysel sürümlerin `pool-current` veya `poolContinuity` içeren günlükleri, daha sonra sıkıştırılsalar bile C sözleşmesinin dışındadır. Özgün bakiyeleri normal saklama süresi dolana kadar belirsiz geçmiş olarak tutulur; hemen dönüştürülmez veya silinmez. + +Tam ama geçersiz bir kayıt, son satırdaki `null` dahil, sınır uygulanan isteklerin kabulünü engeller. Sıkıştırma geçerli muhasebeyi temiz bir kontrol noktasında koruyabilir; çalışan süreç temiz günlükle yeniden başlatılana kadar ret durumu sürer. Yarım kalmış son JSON satırı mevcut kurtarma kurallarına tabidir. Ayrıntılar ve hata kodları için [İngilizce kaynağa](/reference/configuration/server/#historical-pool-continuity) bakın. diff --git a/docs-site/src/content/docs/zh-cn/reference/configuration/server.md b/docs-site/src/content/docs/zh-cn/reference/configuration/server.md index bb731ecb98c..bb039332a8d 100644 --- a/docs-site/src/content/docs/zh-cn/reference/configuration/server.md +++ b/docs-site/src/content/docs/zh-cn/reference/configuration/server.md @@ -217,12 +217,16 @@ The dashboard warns about old or unknown CLI versions, unavailable targets, and ## 历史池用量的连续性 提供方池额度按路由选定的规范提供方计算，请求日志仍保留账户显示标签。 -在 `config.json` 顶层配置 `spendPoolAliases`：键是本安装日志中的历史池加盐别名 -（32 位小写十六进制），值是已核实的提供方 ID。当前名称或账户列表不能证明历史归属。 +对于当前配置的每个提供方 `P`，读取余额时会自动将本安装的加盐池别名 `h(pool, P)` 绑定到 `P`， +相同别名下保留的历史余额也适用。独立的提供方额度不需要手动添加自映射，也不需要用盐值自行计算哈希。 -映射仅在读取时应用，不移动原始余额，也不把关联写入日志。删除映射会使该历史重新成为归属未知。 -每份原始余额只计算一次：计入已确认的组，或保守地计入每个候选池。 -即使未知别名与当前规范提供方的哈希相同，也不会重复计算。 +其他历史标签，包括带账户序号的标签，不会自动绑定。对于已核实归属的历史，在 `config.json` 顶层配置 +`spendPoolAliases`：键是本安装日志中的历史池加盐别名（32 位小写十六进制），值必须是已核实且当前已配置的提供方准确 ID，不得有首尾空格。 +当前名称或账户排列顺序不能证明历史归属。请妥善保密保存日志和盐值。 + +自动绑定和显式映射都仅在读取时应用，不移动原始余额，也不把身份关联写入日志。 +删除显式映射会使该历史重新成为归属未知；但如果别名属于当前配置的提供方自身，自动绑定仍然生效。 +每份原始余额只计入所属组一次；仍未绑定的正余额则保守地计入每个候选池一次。 已配置提供方自己的池别名不能映射到另一个提供方；新增提供方时会重新验证。验证只读取现有盐值，不创建盐值。 无效映射保留原有额度并拒绝池准入。 @@ -230,17 +234,18 @@ The dashboard warns about old or unknown CLI versions, unavailable targets, and 容量压力不会提前删除正余额的未知历史。活动预留和 seed 即使为零令牌也会保留； 只有删除记录持久化后，准入计算才使用减少后的余额。 有适用额度的请求必须在每个目标/密钥首次发送前取得普通容量的预留。 -重试沿用相同范围，并共享首次发送时固定的有限发送次数上限（通常 4，现有 OAuth 扩展最多 18）。 +重试沿用相同范围。`L` 是整个请求共享的物理发送次数上限，在首次物理发送时固定。 +默认是 4 次，现有 OAuth 扩展可在发送开始前增加到最多 18 次。所有物理发送共享这一有限上限。 最终结算等待报告完成；遗忘发送 ID 不会删除余额。无适用额度时保留原有仅观察行为，包括容量已满时省略记账。 Claude CLI、CodeBuddy 和 Qoder 将一次 CLI 调用计为一次发送，启动前仍必须取得普通预留；无法预留就拒绝启动。 -CLI 内部重试和工具轮次不单独计入 L，但费用按报告的实际使用量结算，包括超出初始估算的部分。 +CLI 内部重试和工具轮次不单独计入请求级发送次数上限，但费用按报告的实际使用量结算，包括超出初始估算的部分。 ### 回滚至 2.80.0：契约 C 新记录使用普通 v1 格式和 `pool` 别名。未经修改的 2.80.0 可以按自身的逐标签规则读取、压缩并执行保留策略。 请保留最新日志和盐值。降级后不保证规范提供方聚合，也不保证与从未升级、始终运行 2.80.0 的相同流量具有相同剩余额度。 -再次升级时，当前映射应用于仍然保留的原始余额。不需要启动屏障或对账命令。 +再次升级时，当前配置提供方的自动绑定及显式映射应用于仍然保留的原始余额。不需要启动屏障或对账命令。 未发布实验版本的 `pool-current` / `poolContinuity` 历史不属于契约 C；原始余额作为未知历史保留到正常过期， 不会立即转换或删除。 diff --git a/docs-site/src/content/docs/zh-tw/reference/configuration/server.md b/docs-site/src/content/docs/zh-tw/reference/configuration/server.md index 00fd7020f53..b6b2c0d3217 100644 --- a/docs-site/src/content/docs/zh-tw/reference/configuration/server.md +++ b/docs-site/src/content/docs/zh-tw/reference/configuration/server.md @@ -230,3 +230,33 @@ This takes effect on the **next routed `ocx claude` launch**, injecting `CLAUDE_ Claude Code **2.1.257 or newer** is required for FORCE. Plugin and built-in agents (including Explore/Plan) and per-call model arguments are overridden. Forks and subagent skills with `model: inherit` keep the main conversation model. The main loop and Haiku/small-fast sidecars are unaffected. Existing roster files remain available. The dashboard warns about old or unknown CLI versions, unavailable targets, and either variable already present in `settings.json` → `env` (which overrides launch env). Detection is read-only and server-local: it cannot inspect another launch shell, another machine, or project-local settings. An unknown result is not proof of force support. + +## 歷史集區用量的連續性 + +提供者集區額度依路由選定的標準提供者計算，請求日誌仍保留帳戶顯示標籤。對於目前設定的每個提供者 `P`，讀取餘額時會自動將本安裝環境的加鹽集區別名 `h(pool, P)` 綁定至 `P`，相同別名下保留的歷史餘額也適用。各提供者的獨立額度不需要手動加入自身對應，也不需要用鹽值自行計算雜湊。 + +其他歷史標籤，包括帶有帳戶序號的標籤，不會自動綁定。每份原始餘額只計入所屬群組一次；仍未綁定的正餘額則保守地計入每個候選集區一次。已結算、預留及尚未確認的用量都會計入，因此未知歷史可能限制尚未使用的集區。目前名稱或帳戶排列順序不能證明歷史歸屬。根請求及身分額度仍各自獨立。 + +對於已確認歸屬的歷史，請在 `config.json` 最上層設定 `spendPoolAliases`，不要放在 `spend` 內。索引鍵是本安裝環境日誌中的完整歷史集區加鹽別名，由 32 個小寫十六進位字元組成；值必須是已確認且目前已設定的提供者完整 ID，前後不得有空白。請保密保存日誌、鹽值及確認依據。已設定提供者自己的集區別名不能對應到另一個提供者；提供者集合變更時會重新驗證。驗證只讀取現有鹽值，不會建立新鹽值。 + +自動綁定及明確對應都只在讀取時套用，不搬移原始餘額，也不將身分關聯寫入日誌。刪除明確對應會讓該歷史重新成為歸屬未知；但如果別名屬於目前設定的提供者自身，自動綁定仍然有效。無效對應在寫入時會遭拒；手動編輯留下無效對應時，既有額度仍保留，但拒絕新的集區請求。 + +閒置的未知歷史只有在最後活動時間嚴格早於 `spend.retentionDays` 的期限後才會到期。容量不足不會提早刪除正餘額的未知歷史。活動中的預留及其目標即使為零權杖也會保留；只有刪除紀錄已持久化後，請求准入判斷才會使用減少後的餘額。 + +### 預留及傳送次數上限 + +若根請求、身分或集區額度適用，每個目標或金鑰的首次傳送都必須先取得一般追蹤容量的預留。無法預留時，不會傳送至提供者。同一目標的重試沿用原有範圍；不同目標或金鑰需要新的預留。`L` 是整個請求共用的實體傳送次數上限，在首次實體傳送時固定。預設為四次；既有 OAuth 擴充可在開始傳送前增加至最多十八次。 + +最終結算會等待已開始傳送的回報完成。只有帳務已持久化後才會遺忘傳送 ID，餘額仍會保留。上限限制的是傳送次數而非帳單，超過初始估算的實際用量仍全數計入。沒有適用額度時，維持原有的僅觀察行為，包括追蹤容量已滿時省略記帳；不會新增身分關聯檢查點。 + +Claude CLI、CodeBuddy 及 Qoder 將每次 CLI 呼叫計為一次傳送，啟動前仍須取得一般預留，否則拒絕啟動。CLI 內部重試及工具回合不會另外消耗整個請求的傳送次數上限。費用依回報的實際用量結算，包括超出初始估算的部分。 + +### 回復至 2.80.0：契約 C + +新紀錄使用一般 v1 格式及 `pool` 別名範圍。未修改的 2.80.0 可依其逐標籤規則讀取、壓縮日誌及套用保留期限。請保留最新日誌及鹽值；還原較舊副本會遺失該副本之後記錄的支出。不需要回補修正、啟動阻擋機制或對帳指令。 + +降版後不保證維持標準提供者的彙總，也不保證與相同流量從未升級、始終由 2.80.0 處理時具有相同的剩餘額度。再次升級時，目前設定提供者的自動綁定及明確對應會套用至 2.80.0 仍保留的原始餘額。新檢查點不含需要舊版讀取器保留的身分關聯中繼資料。 + +未發布實驗版本的日誌若含 `pool-current` 或 `poolContinuity`，即使後來經過壓縮，仍不屬於契約 C。其原始餘額會作為未知歷史保留至正常到期，不會立即轉換或刪除。 + +完整但無效的紀錄，包括最後一行的 `null`，會使有額度限制的請求遭拒。壓縮仍可將有效帳務保留在乾淨的檢查點；目前程序的拒絕狀態會持續到重新啟動並讀取乾淨日誌。寫入中斷的最後一行 JSON 沿用原有復原規則。詳細範圍及錯誤碼請見[英文原文](/reference/configuration/server/#historical-pool-continuity)。 diff --git a/src/config/diagnostics.ts b/src/config/diagnostics.ts index ab622d0b73a..912a52e1382 100644 --- a/src/config/diagnostics.ts +++ b/src/config/diagnostics.ts @@ -92,6 +92,8 @@ function rawPoolAliasesError(value: unknown): string | undefined { if (!mapping || Object.keys(mapping).length === 0) return undefined; const providers = rawConfigRecord(record.providers ?? {}); if (!providers) return "providers must be an object to validate spendPoolAliases"; + const targetError = spendPoolAliasesError(aliases, Object.keys(providers)); + if (targetError) return targetError; const salt = readExistingSpendSalt(getConfigDir()); if (!salt) return "spendPoolAliases requires a safely readable existing spend salt"; return validatePoolAliasOwners(aliases, salt, Object.keys(providers)); diff --git a/src/lib/spend-pool-alias-validation.ts b/src/lib/spend-pool-alias-validation.ts index e116754ab07..344ad179967 100644 --- a/src/lib/spend-pool-alias-validation.ts +++ b/src/lib/spend-pool-alias-validation.ts @@ -38,15 +38,17 @@ export function hashSpendAlias(salt: string, kind: string, id: string): string { const isAlias = (value: unknown): value is string => typeof value === "string" && /^[0-9a-f]{32}$/.test(value); -/** Shape check. Kept outside `spend`: older strict spend schemas must keep all configured ceilings. */ -export function spendPoolAliasesError(value: unknown): string | undefined { +/** Shape and optional roster check, outside `spend` so old schemas keep configured ceilings. */ +export function spendPoolAliasesError(value: unknown, providerIds?: Iterable): string | undefined { if (value === undefined) return undefined; if (!value || typeof value !== "object" || Array.isArray(value)) return "spendPoolAliases must be an object"; if (Object.keys(value).length > 4_096) return "spendPoolAliases has too many entries"; + const providers = providerIds === undefined ? undefined : new Set(providerIds); for (const [alias, provider] of Object.entries(value)) { - if (!isAlias(alias) || typeof provider !== "string" || !provider.trim() || provider.length > 256) { + if (!isAlias(alias) || typeof provider !== "string" || !provider.trim() || provider.trim() !== provider || provider.length > 256) { return "spendPoolAliases requires exact 32-character salted pool aliases and nonempty provider IDs"; } + if (providers && !providers.has(provider)) return "spendPoolAliases targets a provider that is not configured"; } return undefined; } diff --git a/src/lib/spend-pool-continuity.ts b/src/lib/spend-pool-continuity.ts index f05c8aa04ec..af2e60431b3 100644 --- a/src/lib/spend-pool-continuity.ts +++ b/src/lib/spend-pool-continuity.ts @@ -1,13 +1,19 @@ -import { hashSpendAlias, validatePoolAliasOwners } from "./spend-pool-alias-validation"; +import { hashSpendAlias, spendPoolAliasesError, validatePoolAliasOwners } from "./spend-pool-alias-validation"; export { spendPoolAliasesError } from "./spend-pool-alias-validation"; /** Current config supplies a read-time graph; original counters never move between buckets. */ -export function resolvePoolAliases(config: unknown, salt: string, providerIds: Iterable = []) { - const error = validatePoolAliasOwners(config, salt, providerIds); +export function resolvePoolAliases(config: unknown, salt: string, providerIds?: Iterable) { + const providers = providerIds === undefined ? undefined : [...providerIds]; + const error = spendPoolAliasesError(config, providers) ?? validatePoolAliasOwners(config, salt, providers ?? []); const bindings = new Map(); if (!error) for (const [alias, provider] of Object.entries(config ?? {})) { bindings.set(alias, hashSpendAlias(salt, "pool", provider as string)); } + // Current provider IDs own their ordinary v1 pool buckets; no identity record is written. + if (!error) for (const provider of providers ?? []) { + const alias = hashSpendAlias(salt, "pool", provider); + bindings.set(alias, alias); + } const resolve = (original: string): string => { let alias = original; const seen = new Set(); diff --git a/src/lib/spend-reservation-ledger.ts b/src/lib/spend-reservation-ledger.ts index 9ef81145556..90171becf83 100644 --- a/src/lib/spend-reservation-ledger.ts +++ b/src/lib/spend-reservation-ledger.ts @@ -624,11 +624,14 @@ export interface SpendSeed { readonly sendId: string } export type SpendSeedDecision = { reserved: true; seed: SpendSeed } | { reserved: false; denial: SpendDenial }; export interface SpendReservationLedger { - reserveSeed(request: SpendReservationRequest): SpendSeedDecision; + /** Tracker owners isolate cleanup barriers; omitted owners retain the installation barrier. */ + reserveSeed(request: SpendReservationRequest, owner?: object): SpendSeedDecision; reserveReportedFromSeed(seed: SpendSeed, request: { sendId: string; inputTokens: number; outputCeilingTokens: number }): boolean; - forgetResolved(sendIds: readonly string[]): boolean; - finishSeed(seed: SpendSeed): boolean; - registerReporter(): { close(): void }; + forgetResolved(sendIds: readonly string[], owner?: object): boolean; + finishSeed(seed: SpendSeed, owner?: object): boolean; + registerReporter(owner?: object): { close(): void }; + /** Retry completed tracker cleanup on later admission or maintenance after storage recovers. */ + deferCleanup(owner: object, cleanup: () => boolean): void; waitForReporterDrain(): Promise; reserve(request: SpendReservationRequest): SpendReservationDecision; /** Pre-dispatch guard, including transports which report their sends after dispatch. */ @@ -731,13 +734,26 @@ export function createSpendReservationLedger(options: { const scopes = new Map(); const reservations = new Map(); // Object identity is the capability: a copied sendId or another ledger's seed is not proof. - const seeds = new Map; abandoned: boolean }>(); - const seedForSend = (send: string) => [...seeds.values()].find(seed => seed.sends.has(send)); - const pinnedScope = (key: string): boolean => [...seeds.values()].some(seed => - seed.targets.some(ref => scopeKey(ref.scope, ref.alias) === key)); + type SeedState = { targets: readonly ScopeRef[]; sends: Set; abandoned: boolean; owner?: object }; + const seeds = new Map(); + const seedBySend = new Map(); + const pinnedKeys = new Map(); + const seedForSend = (send: string) => seedBySend.get(send); + const pinnedScope = (key: string): boolean => pinnedKeys.has(key); const pendingRecords: JournalRecord[] = []; let reporters = 0; + const ownerReporters = new WeakMap(); + const activeReporters = (owner?: object): number => owner === undefined ? reporters : ownerReporters.get(owner) ?? 0; const drainWaiters = new Set<() => void>(); + const deferredCleanups = new Map boolean>(); + let retryingCleanup = false; + const retryCleanup = (): void => { + if (retryingCleanup) return; + retryingCleanup = true; + try { + for (const [owner, cleanup] of deferredCleanups) if (cleanup()) deferredCleanups.delete(owner); + } finally { retryingCleanup = false; } + }; let poolContinuity = resolvePoolAliases(policy.poolAliases, salt, policy.canonicalProviderIds); let persistFailures = 0; let corruptRecords = 0; @@ -774,9 +790,8 @@ export function createSpendReservationLedger(options: { const member = key.slice(5); const unbound = !poolContinuity.known(member); if (unbound) { - // Ownership cannot be inferred because an old alias hashes to a current provider ID. - // Keep that original balance out of the proven group, then charge it once through the - // conservative overlay below for every candidate provider. + // Configured provider buckets bind at read time. Only unmapped historical labels + // absent from that roster remain unbound and overlay every candidate provider. if (state.settled + state.reserved + state.unresolved === 0) continue; includesUnboundHistory = true; } else if (poolContinuity.resolve(member) !== group) { @@ -1151,21 +1166,27 @@ export function createSpendReservationLedger(options: { }; const ledger: SpendReservationLedger = { - reserveSeed(request): SpendSeedDecision { + reserveSeed(request, owner): SpendSeedDecision { assertOwnedAccounting?.(); // Initial anchors never borrow already-sent capacity or bypass token/durability checks. const decision = ledger.reserve({ ...request, alreadySent: false }); if (!decision.reserved) return decision; const seed: SpendSeed = Object.freeze({ sendId: request.sendId }); const send = aliasFor("send", seed.sendId); - seeds.set(seed, { targets: reservations.get(send)!.targets, sends: new Set([send]), abandoned: false }); + const state: SeedState = { targets: reservations.get(send)!.targets, sends: new Set([send]), abandoned: false, owner }; + seeds.set(seed, state); + seedBySend.set(send, state); + for (const ref of state.targets) { + const key = scopeKey(ref.scope, ref.alias); + pinnedKeys.set(key, (pinnedKeys.get(key) ?? 0) + 1); + } return { reserved: true, seed }; }, reserveReportedFromSeed(seed, request): boolean { assertOwnedAccounting?.(); const owned = seeds.get(seed); const send = aliasFor("send", request.sendId); - if (!owned || owned.abandoned || reservations.has(send) + if (!owned || owned.abandoned || reservations.has(send) || seedBySend.has(send) || owned.targets.some(ref => !scopes.has(scopeKey(ref.scope, ref.alias)))) return false; const tokens = sanitizeTokens(request.inputTokens) + sanitizeTokens(request.outputCeilingTokens); const at = now(); @@ -1174,13 +1195,15 @@ export function createSpendReservationLedger(options: { if (pendingRecords.length > 0 || !append(record)) pendingRecords.push(record); applyReserve(send, owned.targets, tokens, at); owned.sends.add(send); + seedBySend.set(send, owned); compact(at); return true; }, - forgetResolved(sendIds): boolean { + forgetResolved(sendIds, owner): boolean { assertOwnedAccounting?.(); - if (reporters > 0 || !flushPending()) return false; + if (activeReporters(owner) > 0) return false; const sends = [...new Set(sendIds.map(id => aliasFor("send", id)))]; + if (sends.some(send => { const seed = seedForSend(send); return seed && seed.owner !== owner; }) || !flushPending()) return false; if (sends.some(send => { const entry = reservations.get(send); return entry && isLive(entry.status); })) return false; for (const send of sends) { if (!reservations.has(send)) continue; @@ -1189,10 +1212,10 @@ export function createSpendReservationLedger(options: { } return true; }, - finishSeed(seed): boolean { + finishSeed(seed, owner): boolean { assertOwnedAccounting?.(); const owned = seeds.get(seed); - if (!owned || !flushPending()) return false; + if (!owned || owned.owner !== owner || !flushPending()) return false; if (owned.abandoned) { // A never-sent provisional anchor cannot have delayed physical reports. Rebinding // during another live producer is safe only for this explicitly abandoned capability. @@ -1201,23 +1224,39 @@ export function createSpendReservationLedger(options: { if (!append({ v: 1, kind: "forget", send, at: now() })) return false; applyForget(send); } - } else if (reporters > 0) return false; + } else if (activeReporters(owner) > 0) return false; if ([...owned.sends].some(send => reservations.has(send))) return false; seeds.delete(seed); + for (const send of owned.sends) seedBySend.delete(send); + for (const ref of owned.targets) { + const key = scopeKey(ref.scope, ref.alias); + const count = pinnedKeys.get(key)! - 1; + if (count === 0) pinnedKeys.delete(key); else pinnedKeys.set(key, count); + } return true; }, - registerReporter() { + registerReporter(owner) { assertOwnedAccounting?.(); reporters += 1; + if (owner) ownerReporters.set(owner, activeReporters(owner) + 1); let closed = false; return { close() { if (closed) return; assertOwnedAccounting?.(); closed = true; reporters -= 1; + if (owner) { + const remaining = activeReporters(owner) - 1; + if (remaining === 0) ownerReporters.delete(owner); else ownerReporters.set(owner, remaining); + } + retryCleanup(); if (reporters === 0) { for (const resolve of drainWaiters) resolve(); drainWaiters.clear(); } } }; }, + deferCleanup(owner, cleanup) { + assertOwnedAccounting?.(); + deferredCleanups.set(owner, cleanup); + }, waitForReporterDrain() { assertOwnedAccounting?.(); return reporters === 0 ? Promise.resolve() : new Promise(resolve => { drainWaiters.add(resolve); }); @@ -1243,6 +1282,7 @@ export function createSpendReservationLedger(options: { reserve(request: SpendReservationRequest): SpendReservationDecision { assertOwnedAccounting?.(); + retryCleanup(); const tokens = sanitizeTokens(request.inputTokens) + sanitizeTokens(request.outputCeilingTokens); const at = request.at ?? now(); const send = aliasFor("send", request.sendId); @@ -1252,7 +1292,7 @@ export function createSpendReservationLedger(options: { // A send id this ledger already knows is REFUSED. Returning success while booking // nothing -- the old behaviour -- let one id authorise an unlimited number of physical // sends with the scope totals never moving. - if (reservations.has(send)) { + if (reservations.has(send) || seedBySend.has(send)) { return { reserved: false, denial: { reason: "duplicate-send-id", sendId: request.sendId } }; } // Replay could not prove these totals are complete, so a configured ceiling cannot be @@ -1370,7 +1410,8 @@ export function createSpendReservationLedger(options: { knows(sendId: string): boolean { assertOwnedAccounting?.(); - return reservations.has(aliasFor("send", sendId)); + const send = aliasFor("send", sendId); + return reservations.has(send) || seedBySend.has(send); }, snapshot(scope: SpendScope, scopeId: string): ScopeSpendSnapshot | undefined { @@ -1401,6 +1442,7 @@ export function createSpendReservationLedger(options: { prune(at: number = now()): void { assertOwnedAccounting?.(); + retryCleanup(); // Removal requires BOTH inactive and not exhausted inside the window. An // exhausted-but-idle scope that was dropped would be recreated fresh under the // same id -- the exact laundering the ceiling exists to stop. @@ -1410,6 +1452,7 @@ export function createSpendReservationLedger(options: { reconfigure(next: SpendReservationPolicy): void { assertOwnedAccounting?.(); + retryCleanup(); policy = next; poolContinuity = resolvePoolAliases(policy.poolAliases, salt, policy.canonicalProviderIds); }, diff --git a/src/lib/state-store-registrations.ts b/src/lib/state-store-registrations.ts index fec8bb42c9a..7a05ce9d636 100644 --- a/src/lib/state-store-registrations.ts +++ b/src/lib/state-store-registrations.ts @@ -41,6 +41,7 @@ import { sweepAbandonedResponseStateTemps, sweepExpiredResponseStates, sweepOrph import { sweepExpiredAntigravityReplay } from "../adapters/google-antigravity-replay"; import { reconcileProviderAccountQuotaRows } from "../providers/quota"; import { reconcileRouterWarningMemos } from "../router"; +import { configureSharedSpendLedger, spendPolicyFromConfig } from "./spend-reservation-ledger"; import type { OcxConfig } from "../types"; import { type GenerationContext, @@ -58,6 +59,10 @@ export function setLiveStateStoreConfig(config: OcxConfig): void { export function reconcileLiveStateStores() { if (!liveServerConfig) return { storesVisited: 0, rowsRemoved: 0 }; + // Only adopted live providers own canonical pools; detached disk snapshots may + // contain providers the routing instance deliberately has not activated yet. + configureSharedSpendLedger(spendPolicyFromConfig(liveServerConfig.spend, + liveServerConfig.spendPoolAliases, Object.keys(liveServerConfig.providers))); return reconcileStateGeneration(buildGenerationContext()); } diff --git a/src/server/responses/compact.ts b/src/server/responses/compact.ts index 83b04bb230e..90535b1d5e3 100644 --- a/src/server/responses/compact.ts +++ b/src/server/responses/compact.ts @@ -1,5 +1,6 @@ import { createPhysicalSendReporter } from "../../lib/request-execution-budget"; import { createInferenceSendBudget } from "../inference/context"; +import { unboundPoolSpendRefusalResponse } from "../workflow-refusal"; import { capturePoolQuotaWriter } from "../../codex/account-store"; import { previewXaiOauthWireModel } from "./core-normalize"; import { @@ -1092,7 +1093,7 @@ export async function handleResponsesCompact( const localDispatchRefusal = (error: unknown): Response | undefined => { const cause = unwrapUpstreamRetryEvidenceError(error); const response = cause instanceof SendBudgetExhaustedError - ? formatErrorResponse(429, "request_send_budget_exhausted", cause.message) + ? unboundPoolSpendRefusalResponse(logCtx) ?? formatErrorResponse(429, "request_send_budget_exhausted", cause.message) : mapCodexAuthContextErrorToResponse(cause, { now: Date.now(), accountSelector: route.codexAccountNamespace, }); diff --git a/src/server/responses/request-spend.ts b/src/server/responses/request-spend.ts index 81cf6fbc884..5ba7e80a98f 100644 --- a/src/server/responses/request-spend.ts +++ b/src/server/responses/request-spend.ts @@ -79,7 +79,6 @@ export function createRequestSpendTracker( let reporters = 0; const seeds = new Map(); const resolvedSendIds = new Set(); - let awaitingDrain = false; const completedSeeds = new Set(); const physical = new Map(); const selectedScopes = (target: SpendTargetIdentity = {}): SpendScopes => ({ @@ -96,49 +95,41 @@ export function createRequestSpendTracker( const estimates = () => ({ inputTokens: logCtx.spendInputEstimateTokens ?? logCtx.usageLogInputTokens ?? 0, outputCeilingTokens: logCtx.spendOutputCeilingTokens ?? 0 }); const seedState = (seed: SpendSeed) => [...seeds.values()].find(state => state.seed === seed); - const finalizeSeeds = (): void => { - if (!finalRequested || reporters !== 0 || resolved || !seededAccounting) return; + const tryFinalizeSeeds = (): boolean => { // All starts are booked before producers close. Ordinal, never callback order, owns usage. const entries = [...physical.entries()].sort((a, b) => a[0] - b[0]); const terminal = entries.at(-1)?.[1].sendId; const ids: string[] = []; for (const [, entry] of entries) { - if (!entry.sendId) return; + if (!entry.sendId) return false; const reported = entry.sendId === terminal && (typeof finalUsage?.inputTokens === "number" || typeof finalUsage?.outputTokens === "number"); if (!resolvedSendIds.has(entry.sendId)) { const ok = reported ? ledger().settle(entry.sendId, { inputTokens: finalUsage?.inputTokens ?? 0, outputTokens: finalUsage?.outputTokens ?? 0 }) : ledger().markLost(entry.sendId); - if (!ok) return; + if (!ok) return false; resolvedSendIds.add(entry.sendId); } ids.push(entry.sendId); } for (const state of seeds.values()) if (!state.started && !completedSeeds.has(state.seed)) { - if (!resolvedSendIds.has(state.seed.sendId) && !ledger().abandon(state.seed.sendId)) return; + if (!resolvedSendIds.has(state.seed.sendId) && !ledger().abandon(state.seed.sendId)) return false; resolvedSendIds.add(state.seed.sendId); ids.push(state.seed.sendId); } - if (!ledger().forgetResolved(ids)) { - if (!awaitingDrain) { - awaitingDrain = true; - void ledger().waitForReporterDrain().then(() => { - awaitingDrain = false; - // One retry after the installation producer barrier; storage failure keeps debt. - if (!ledger().forgetResolved(ids)) return; - for (const state of seeds.values()) if (!completedSeeds.has(state.seed)) { - if (!ledger().finishSeed(state.seed)) return; - completedSeeds.add(state.seed); - } - resolved = true; - }); - } - return; - } + if (!ledger().forgetResolved(ids, tracker)) return false; for (const state of seeds.values()) if (!completedSeeds.has(state.seed)) { - if (!ledger().finishSeed(state.seed)) return; - completedSeeds.add(state.seed); - } + if (!ledger().finishSeed(state.seed, tracker)) return false; + completedSeeds.add(state.seed); + } resolved = true; + return true; + }; + const finalizeSeeds = (): void => { + if (!finalRequested || reporters !== 0 || resolved || !seededAccounting) return; + if (!tryFinalizeSeeds()) { + // Failed durable cleanup remains owned by this ledger until later activity retries it. + ledger().deferCleanup(tracker, () => resolved || tryFinalizeSeeds()); + } }; /** * Confirm the sends this request has already moved past. @@ -214,10 +205,10 @@ export function createRequestSpendTracker( } // Rebind an unused provisional anchor before selection changes. Dispatched anchors stay live. for (const [oldKey, state] of seeds) if (!state.started) { - if (!ledger().abandon(state.seed.sendId)) return undefined; - // The ledger pins abandoned anchors until its reporter drain. Keep the capability - // for final cleanup if another producer is currently registered. - if (!ledger().finishSeed(state.seed)) continue; + if (!resolvedSendIds.has(state.seed.sendId) && !ledger().abandon(state.seed.sendId)) return undefined; + resolvedSendIds.add(state.seed.sendId); + // Keep a durably abandoned capability for final cleanup if forgetting fails. + if (!ledger().finishSeed(state.seed, tracker)) continue; seeds.delete(oldKey); const index = live.indexOf(state.seed.sendId); if (index >= 0) live.splice(index, 1); @@ -226,7 +217,7 @@ export function createRequestSpendTracker( // Exact prepaid proofs originate from this tracker and are found above; foreign receipts // cannot enroll arbitrary scopes. Normal reservation remains the only initial admission. if (prepaidProof && prepaidProof.ledger !== ledger()) return undefined; - const decision = ledger().reserveSeed({ sendId: randomUUID(), scopes, ...estimates() }); + const decision = ledger().reserveSeed({ sendId: randomUUID(), scopes, ...estimates() }, tracker); if (!decision.reserved) { // Use the established denial mapping without making another reservation. refusal(decision.denial); @@ -238,7 +229,7 @@ export function createRequestSpendTracker( }, beginReporter() { if (resolved || finalRequested && reporters === 0) throw new Error("Spend reporter registered after final closure"); - const lease = ledger().registerReporter(); + const lease = ledger().registerReporter(tracker); reporters++; let closed = false; let reported = 0; @@ -368,7 +359,10 @@ export function createRequestSpendTracker( refund(proof): void { if (proof && proof.ledger === ledgerRef && seededAccounting) { const match = [...seeds.entries()].find(([, state]) => state.seed.sendId === proof.sendId); - if (match && !match[1].started && ledger().abandon(proof.sendId) && ledger().finishSeed(match[1].seed)) seeds.delete(match[0]); + if (match && !match[1].started && (resolvedSendIds.has(proof.sendId) || ledger().abandon(proof.sendId))) { + resolvedSendIds.add(proof.sendId); + if (ledger().finishSeed(match[1].seed, tracker)) seeds.delete(match[0]); + } return; } // null is an exact budget reservation that obtained no durable booking. diff --git a/structure/config.md b/structure/config.md index c6af3364d9a..29f0c191cb0 100644 --- a/structure/config.md +++ b/structure/config.md @@ -101,7 +101,7 @@ A schema-invalid top-level JSON value is repairable only when it is a non-array | Compaction and memory routing | `compactionRouting.model`, optional `compactionRouting.reasoningEffort`, optional `compactionRouting.triggers`, optional `compactionRouting.sourceModels`; `memoryModels.extract`, `memoryModels.consolidation` | Explicit Codex compaction metadata whose `compaction.trigger` is one the block names activates a request-local override; `triggers` defaults to `["manual"]`. Optional source selectors restrict the override to exact incoming models or `provider/*`. See [Responses compaction](transports/responses-failover.md#compaction-routing-overrides). Invalid hand edits disable the block with a load warning without discarding providers; candidate writes reject invalid blocks. Each optional memory phase names a nonblank model and optional declared `reasoningEffort`; absence preserves the current route, candidate writes reject malformed values, and load degrades only the invalid phase with a warning. See [memory phase routing](transports/responses-failover.md#memory-phase-routing). | | Catalog | `disabledModels`, `customModels`, `modelCacheTtlMs`, `providerContextCaps`, `contextCapValue`, per-provider `modelDisplayNames`, `codexAccountNamespaces`, `codexAccountPickerEnabled` | Catalog state is derived; config only records intent. Exact provider model display names are durable display only overlays. The picker flag is an explicit visibility override, while selector mappings remain the durable exact-routing contract. | | Retained state | `appOwnedMemoryBudgetMb` | Process-wide eviction target for app-owned logs, caches, blobs, and continuation payloads. Default 256 MiB, valid 64..4096; pinned state may temporarily exceed the target, but every pin-capable store has a finite local cap and their documented aggregate stays below `APP_OWNED_WORST_CASE_PINNED_BYTES` (512 MiB). Neither value caps RSS or native runtime memory. | -| Spend | `spend.root`, `spend.identity`, `spend.pool`, `spend.retentionDays`, top-level `spendPoolAliases` | Durable token ceilings for the spend-reservation ledger. Absent is the default and means observe-only accounting: spend is still journaled and nothing is refused, so observe-only and enforced servers take the same state-directory writer lease. One live process may write one directory; explicit sibling instances need separate `OPENCODEX_HOME` directories. There is no default figure for any scope — the ledger is on by default, so a shipped ceiling would refuse real traffic on upgrade against a number nobody chose. Strictly validated and positive-integer only, because 0 would read as a budget and refuse everything; a malformed section degrades to no ceiling, which is why the write path rejects it and load diagnostics report it. Top-level `spendPoolAliases` is preserved verbatim on load; malformed mappings leave ceilings intact and refuse pool admission, while candidate writes reject them. Mappings apply at read time and are not persisted as journal links. `src/lib/spend-pool-alias-validation.ts` checks the existing home salt without creating it and rejects a configured provider’s own pool alias mapped to a different provider, including when providers are added later. It stays outside the legacy strict `spend` object. Resolution and application live in `src/lib/spend-reservation-ledger.ts`; see [historical continuity and rollback](transports/responses-spend.md#historical-pool-continuity-and-rollback). | +| Spend | `spend.root`, `spend.identity`, `spend.pool`, `spend.retentionDays`, top-level `spendPoolAliases` | Durable token ceilings for the spend-reservation ledger. Absent is the default and means observe-only accounting: spend is still journaled and nothing is refused, so observe-only and enforced servers take the same state-directory writer lease. One live process may write one directory; explicit sibling instances need separate `OPENCODEX_HOME` directories. There is no default figure for any scope — the ledger is on by default, so a shipped ceiling would refuse real traffic on upgrade against a number nobody chose. Strictly validated and positive-integer only, because 0 would read as a budget and refuse everything; a malformed section degrades to no ceiling, which is why the write path rejects it and load diagnostics report it. Top-level `spendPoolAliases` is preserved verbatim on load; malformed mappings leave ceilings intact and refuse pool admission, while candidate writes reject them. Current provider IDs automatically own their ordinary salted pool buckets at read time; other historical labels need explicit mappings or remain conservatively unbound. Neither self-bindings nor explicit mappings persist as journal links. Padded destinations and destinations absent from the provider roster are rejected on write and warn/refuse pool admission on load. `src/lib/spend-pool-alias-validation.ts` checks the existing home salt without creating it and rejects a configured provider’s own pool alias mapped to a different provider, including when providers are added later. Live provider and full-config adoption call `reconcileLiveStateStores` in `src/lib/state-store-registrations.ts` to refresh this policy from the active roster; detached config reads do not change it. It stays outside the legacy strict `spend` object. Resolution and application live in `src/lib/spend-reservation-ledger.ts`; see [historical continuity and rollback](transports/responses-spend.md#historical-pool-continuity-and-rollback). | | Transport | stream mode, timeouts, proxy settings, `websockets`, `emptyCompletionRetry` | `streamMode` persists in config.json; Windows services need a persisted input, and macOS uses it for explicit eager-relay opt-in. Empty-completion replay is an explicit top-level opt-in because its second upstream request may be billable. | | Canonical ChatGPT upstream transport | `providers.openai.upstreamWebsocket` | Omitted uses upstream WebSocket when eligible; explicit `false` selects HTTP/SSE without changing the canonical provider identity. `true` is rejected on the canonical row. This is independent of the client-facing `websockets` setting. | | Provider egress | `providers..proxy`, `providers..noProxy` | An absent `proxy` inherits global egress; `"direct"` or `null` forces direct egress; HTTP(S) and SOCKS5(H) URLs select a provider-owned proxy. `noProxy` uses NO_PROXY syntax and sends a matching destination direct across either a provider-owned or inherited global proxy. `src/lib/provider-egress.ts` owns parsing and request-local resolution. `forwardClientHeaders` opts into `originator`, `x-client-request-id`, `x-codex-app-version`, or `user-agent` on Responses inference only; provider headers win and all other names are rejected on load/write and filtered at runtime. | diff --git a/structure/runtime.md b/structure/runtime.md index 0dc41e446b9..70f240411ce 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -218,7 +218,7 @@ and build a ledger by replaying that directory's own journal. A second process o directory is refused even for observe-only spend configuration, while a separate directory is independent. Ordinary stop drains the spend reporters after listener teardown before releasing the final reference. `src/server/index/spend-ledger-lifecycle.ts` also holds the lease through failed-start listener rollback. Enforced seeds remain live until reporters close and settlement plus send-ID forgetting are durable; scope totals survive forgetting. SQLite and the OS release a crashed owner; no PID, timestamp, -TTL or lock-file deletion participates in recovery. Startup passes top-level pool-alias evidence with the spend policy. HTTP admission checks [historical continuity](transports/responses-spend.md#historical-pool-continuity-and-rollback) even without a root; failed accounting reads release the active-turn lease. Rooted continuity denials record exactly one workflow refusal event before response formatting. +TTL or lock-file deletion participates in recovery. Startup passes the provider roster and top-level pool aliases with the spend policy; live state reconciliation refreshes both without opening an unused ledger. HTTP admission checks [historical continuity](transports/responses-spend.md#historical-pool-continuity-and-rollback) even without a root; failed accounting reads release the active-turn lease. Rooted continuity denials record exactly one workflow refusal event before response formatting. An explicit Codex integration OFF skips startup cache invalidation before the user-scoped catalog serialization lock is resolved. Explicit `sync` and `sync-cache` retain their catalog-only override. diff --git a/structure/transports/responses-spend.md b/structure/transports/responses-spend.md index 2faf0c19316..db48d04fe4f 100644 --- a/structure/transports/responses-spend.md +++ b/structure/transports/responses-spend.md @@ -131,21 +131,22 @@ target provider; child account labels and the logical `combo` label do not creat ### Historical pool continuity and rollback -`src/lib/spend-pool-continuity.ts` applies the current top-level `spendPoolAliases` mapping -at read time. Keys are exact salted historical pool aliases; values are verified canonical -provider IDs. Original scope counters and send targets stay unchanged. Every original bucket -contributes once: either to its mapped group or, when unmapped and positive, to every candidate -pool as conservative unbound history. A candidate's own unmapped canonical bucket participates -in that overlay once, never again as a group member. Account rosters, label prefixes and a -matching canonical hash do not establish historical ownership, including after restart. - -Mappings are configuration, not journal metadata. Removing or changing a mapping changes the -next read-time view; it does not rewrite original counters or persist an identity link. -`src/lib/spend-pool-alias-validation.ts` owns the salted hash and validates that a configured -provider P's own pool alias cannot map to another provider Q. Configuration reads and writes, -provider-set changes, live reconfiguration and selected-provider admission enforce this rule. -Validation reads an existing home salt without creating one. A malformed or conflicting mapping -keeps existing ceilings and refuses applicable pool admission with `workflow_pool_history_unresolved`. +`src/lib/spend-pool-continuity.ts` applies the current provider roster and top-level +`spendPoolAliases` mapping at read time. Each configured provider P automatically owns its +ordinary salted `pool` alias, so its canonical bucket counts only toward P's group. Explicit +mapping keys are exact salted historical aliases and values are current provider IDs. Other +historical labels, including account-ordinal labels, remain unbound until explicitly mapped. +Each positive unbound bucket overlays every candidate once; original counters and send targets +never move or double-count. Restart reconstructs the same view from the current roster and mapping. + +Mappings and automatic self-bindings are read-time configuration, not journal metadata. Removing +a provider or mapping changes the next view without rewriting counters or persisting identity links. +`src/lib/spend-pool-alias-validation.ts` owns the salted hash and validates that P's own alias cannot +map to Q. Padded mapping values and destinations absent from the current roster are rejected at the +write boundary; hand-edited invalid mappings warn on load and refuse applicable pool admission with +`workflow_pool_history_unresolved`, keeping existing ceilings. Configuration writes, provider-set +changes, live reconfiguration and selected-provider admission enforce owner conflicts. Validation +reads an existing home salt without creating one. Dormant unbound history expires only when its last activity is strictly before the configured retention cutoff. Capacity pressure cannot expire positive unknown history early. Live reservations @@ -159,7 +160,7 @@ domain. Contract C is compatibility with the shipped 2.80.0 reader's own record 2.80.0 reads these counters, compacts them and expires them using its existing per-label rules. Downgrade does not preserve canonical cross-alias aggregation or promise the allowance 2.80.0 would have computed for the same traffic without an upgrade. Keep the current journal and salt; -restoring an older copy omits subsequent spend. Re-upgrade applies current configured mappings +restoring an older copy omits subsequent spend. Re-upgrade applies current provider self-bindings and explicit mappings to the original balances that survived ordinary old-version retention. There is no launch fence, wrapper enrollment or reconciliation command. `tests/lib/spend-contract-c.test.ts` uses the complete frozen 2.80.0 reader in `tests/fixtures/spend-ledger-2-80-0.ts.txt` for this contract. @@ -193,10 +194,14 @@ closes its producer after reconciliation. Adapter permits claim once at their ph rebindable until OAuth selection reaches the wire, then reports and closes it. Zed completion requests and token-refresh replays use `src/adapters/physical-send.ts`; authentication exchanges remain outside the inference count. Kiro retains this span through its executor-side rebuild. -Telemetry and raw numeric budget updates cannot create extra enforced send records. +Telemetry and raw numeric budget updates cannot create extra enforced send records. Native compact +uses the same unbound-history refusal explanation and workflow header as other Responses executors. `request-spend.ts` waits for producers before terminal settlement, marks unknown usage lost, then -persists ordinary `forget` records before releasing seeds. Forgotten send IDs do not subtract -scope totals. Persistence failure retains the accounting debt. Shutdown drains reporters before +persists ordinary `forget` records before releasing seeds. Each tracker waits only for its own +reporters, so overlapping traffic cannot pin a completed request. Storage-failed cleanup stays +queued on the ledger and retries on later admission, pruning, reconfiguration or reporter closure. +Send-to-seed and reference-counted scope-pin indexes avoid scanning all seeds on admission. +Forgotten send IDs do not subtract scope totals. Shutdown drains all reporters before releasing ledger ownership; restart resolves orphan reservations conservatively without reviving request capabilities. diff --git a/tests/config/config-spend-ceilings.test.ts b/tests/config/config-spend-ceilings.test.ts index 3226000baf9..40ecca2e8be 100644 --- a/tests/config/config-spend-ceilings.test.ts +++ b/tests/config/config-spend-ceilings.test.ts @@ -131,7 +131,7 @@ test("top-level pool aliases preserve all ceilings and malformed hand edits fail writeFileSync(join(home, "spend-ledger.salt"), "7".repeat(64) + "\n", { mode: 0o600 }); const alias = "a".repeat(32); const spend = { root: { maxTokens: 200 }, identity: { maxTokens: 150 }, pool: { maxTokens: 100 } }; - const valid = { ...candidate(spend), spendPoolAliases: { [alias]: "fixture-provider" } }; + const valid = { ...candidate(spend), spendPoolAliases: { [alias]: "xai" } }; expect(validateConfigCandidate(valid).ok).toBe(true); for (const aliases of [null, [], { wrong: "fixture-provider" }, { [alias]: 1 }]) { const raw = { ...valid, spendPoolAliases: aliases }; diff --git a/tests/config/config-spend-pool-collision.test.ts b/tests/config/config-spend-pool-collision.test.ts index 54d20a1e24e..e2009ce9f8f 100644 --- a/tests/config/config-spend-pool-collision.test.ts +++ b/tests/config/config-spend-pool-collision.test.ts @@ -104,3 +104,16 @@ test("unreadable malformed and linked existing salts cannot validate a mapping", } } finally { process.env.OPENCODEX_HOME = home; removeTreeWithRetry(other); } }); + +for (const target of [" P", "P ", "missing-provider"]) { + test(`invalid alias target ${JSON.stringify(target)} rejects writes and warns on load without hiding spend`, () => { + installSalt(); + const raw = candidate({ [alias("historical-account")]: target }); + expect(validateConfigCandidate(raw).ok).toBe(false); + const loaded = configDiagnosticsFromRaw(JSON.stringify(raw)); + expect(loaded.config.spend).toEqual(raw.spend); + expect(loaded.warnings?.join("\n")).toContain("spendPoolAliases invalid"); + const ledger = createSpendReservationLedger({ salt, policy: policy(raw.spendPoolAliases, ["P", "Q"]) }); + expect(reserve(ledger)).toMatchObject({ reserved: false, denial: { reason: "pool-history-unresolved" } }); + }); +} diff --git a/tests/lib/spend-pool-continuity.test.ts b/tests/lib/spend-pool-continuity.test.ts index 52e0d4631bb..92faf9387f2 100644 --- a/tests/lib/spend-pool-continuity.test.ts +++ b/tests/lib/spend-pool-continuity.test.ts @@ -119,26 +119,37 @@ describe("historical pool identity continuity", () => { expect(ledger.hasUnboundPositivePoolHistory()).toBe(true); }); - test("same-spelling unknown history stays unbound and is not counted twice", () => { + test("configured provider history self-binds without manual aliases and survives restart", () => { const disk = journal([checkpoint([["provider", 40, 0]])]); - const ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(), now: () => 2 }); - - // Matching the current provider's salted alias is not identity evidence by itself. - expect(ledger.checkPoolContinuity()).toBeUndefined(); - expect(ledger.hasUnboundPositivePoolHistory()).toBe(true); - expect(reserve(ledger, "same-spelling-exact", "provider", 60).reserved).toBe(true); + const current = policy(undefined, { canonicalProviderIds: ["provider", "unrelated-provider"] }); + let ledger = createSpendReservationLedger({ journal: disk, salt, policy: current, now: () => 2 }); + expect(ledger.hasUnboundPositivePoolHistory()).toBe(false); + expect(reserve(ledger, "exact", "provider", 60).reserved).toBe(true); expect(ledger.snapshot("pool", "provider")).toEqual({ settled: 40, reserved: 60, unresolved: 0, exhausted: true }); - // New bookings use the ordinary pool domain; the unbound original counts once. expect(JSON.parse(disk.lines.at(-1)!).targets).toEqual([{ scope: "pool", alias: pool("provider") }]); - expect(reserve(ledger, "unrelated-under-overlay", "unrelated-provider", 1)).toMatchObject({ - reserved: false, denial: { reason: "spend-limit-exceeded", projected: 101 }, - }); - const mapped = createSpendReservationLedger({ journal: disk, salt, - policy: policy({ [pool("provider")]: "provider" }), now: () => 3 }); - expect(mapped.checkPoolContinuity()).toBeUndefined(); - expect(mapped.hasUnboundPositivePoolHistory()).toBe(false); - expect(mapped.snapshot("pool", "provider")).toMatchObject({ settled: 40, unresolved: 60 }); - expect(reserve(mapped, "unrelated-after-mapping", "unrelated-provider", 100).reserved).toBe(true); + expect(reserve(ledger, "unrelated", "unrelated-provider", 100).reserved).toBe(true); + ledger = createSpendReservationLedger({ journal: disk, salt, policy: current, now: () => 3 }); + expect(ledger.snapshot("pool", "provider")).toMatchObject({ settled: 40, unresolved: 60 }); + expect(ledger.snapshot("pool", "unrelated-provider")).toMatchObject({ settled: 0, unresolved: 100 }); + expect(ledger.hasUnboundPositivePoolHistory()).toBe(false); + }); + + test("five configured provider ceilings stay independent while historical labels overlay each once", () => { + const ids = ["P", "Q", "R", "S", "T"]; + for (const historical of [0, 20]) { + const disk = journal(historical ? [checkpoint([["P-account-2", historical, 0]])] : []); + const current = policy(undefined, { canonicalProviderIds: ids }); + let ledger = createSpendReservationLedger({ journal: disk, salt, policy: current, now: () => 2 }); + for (const id of ids) { + expect(reserve(ledger, `fill-${id}`, id, 100 - historical).reserved).toBe(true); + expect(ledger.settle(`fill-${id}`, { inputTokens: 100 - historical, outputTokens: 0 })).toBe(true); + expect(reserve(ledger, `over-${id}`, id, 1)).toMatchObject({ reserved: false, denial: { projected: 101 } }); + } + ledger = createSpendReservationLedger({ journal: disk, salt, policy: current, now: () => 3 }); + for (const id of ids) expect(ledger.snapshot("pool", id)).toMatchObject({ settled: 100 }); + expect(ledger.hasUnboundPositivePoolHistory()).toBe(historical > 0); + expect(disk.lines.some(line => JSON.parse(line).poolContinuity !== undefined)).toBe(false); + } }); test("unbound history is overlaid when a pool ceiling is enabled later", () => { diff --git a/tests/lib/spend-seed-overflow.test.ts b/tests/lib/spend-seed-overflow.test.ts index ff524cf4cf8..450326ed09d 100644 --- a/tests/lib/spend-seed-overflow.test.ts +++ b/tests/lib/spend-seed-overflow.test.ts @@ -134,7 +134,7 @@ for (const sends of [1, 2]) test(`terminal usage survives append failure without reporter.close(); disk.failAppend = true; tracker.settle({ inputTokens: 70, outputTokens: 0 }); - // Let the tracker's one drain callback run while storage is still unavailable. + // A terminal write failure must survive without another caller settlement. await Promise.resolve(); expect(ledger.snapshot("pool", "P")).toMatchObject({ settled: 70, reserved: 0, unresolved: sends - 1 }); expect(ledger.knows(anchor.sendId)).toBe(true); @@ -143,6 +143,7 @@ for (const sends of [1, 2]) test(`terminal usage survives append failure without expect(ledger.reserve(request("next", { poolId: "P" }, 1))).toMatchObject({ reserved: false, denial: { reason: "spend-limit-exceeded", projected: 70 + sends }, }); + expect(ledger.knows(anchor.sendId)).toBe(false); const old = createShippedSpendLedger({ journal: disk, salt, now: () => 2 }); expect(old.snapshot("pool", "P")).toMatchObject({ settled: 70, unresolved: sends - 1 }); expect(old.corruptRecords).toBe(0); @@ -182,3 +183,110 @@ test("partial pending flush preserves reserve dispatch and terminal order withou expect(createShippedSpendLedger({ journal: disk, salt, now: () => 2 }).snapshot("pool", "P")) .toMatchObject({ settled: 70, unresolved: 1 }); }); + +test("completed trackers reclaim seeds while another tracker still reports", async () => { + const disk = spendTestJournal(); + const ledger = factory(disk, 1, 2); + const busy = createRequestSpendTracker({ provider: "P" }, undefined, ledger); + const busySeed = busy.ensureSeed({}); + if (!busySeed) throw new Error("busy seed denied"); + const busyReporter = busy.beginReporter(); + busyReporter.start(busySeed, 1); + let drained = false; + const drain = ledger.waitForReporterDrain().then(() => { drained = true; }); + for (let index = 0; index < 3; index++) { + const tracker = createRequestSpendTracker({ provider: "P" }, undefined, ledger); + const anchor = tracker.ensureSeed({}); + expect(anchor).toBeDefined(); + if (!anchor) throw new Error("overlapping seed denied"); + const reporter = tracker.beginReporter(); + reporter.start(anchor, 1); + tracker.settle({ inputTokens: 1, outputTokens: 0 }); + expect(ledger.knows(anchor.sendId)).toBe(true); + reporter.close(); + expect(ledger.knows(anchor.sendId)).toBe(false); + } + await Promise.resolve(); + expect(drained).toBe(false); + busyReporter.close(); + busy.settle({ inputTokens: 1, outputTokens: 0 }); + await drain; + expect(drained).toBe(true); + expect(ledger.knows(busySeed.sendId)).toBe(false); + expect(createShippedSpendLedger({ journal: disk, salt, now: () => 2 }).snapshot("pool", "P")?.settled).toBe(4); +}); + +test("later admission retries tracker cleanup after partial durable forgetting fails", async () => { + const disk = spendTestJournal(); + const append = disk.append.bind(disk); + let forgetsBeforeFailure = 1; + disk.append = line => { + if (JSON.parse(line).kind === "forget" && forgetsBeforeFailure-- <= 0) throw new Error("forget unavailable"); + append(line); + }; + const ledger = factory(disk); + const tracker = createRequestSpendTracker({ provider: "P", spendInputEstimateTokens: 1 }, undefined, ledger); + const anchor = tracker.ensureSeed({}); + if (!anchor) throw new Error("seed denied"); + const reporter = tracker.beginReporter(); + reporter.start(anchor, 1); + reporter.start(anchor, 2); + reporter.close(); + tracker.settle({ inputTokens: 70, outputTokens: 0 }); + await Promise.resolve(); + expect(disk.lines.filter(line => JSON.parse(line).kind === "forget")).toHaveLength(1); + expect(ledger.reserveSeed(request("still-full"))).toMatchObject({ reserved: false, denial: { reason: "tracking-capacity-exhausted" } }); + forgetsBeforeFailure = Number.POSITIVE_INFINITY; + // No second settle call: ordinary admission must release the durable completed seed. + expect(ledger.reserveSeed(request("recovered"))).toMatchObject({ reserved: true }); + expect(disk.lines.filter(line => JSON.parse(line).kind === "forget")).toHaveLength(2); + expect(createShippedSpendLedger({ journal: disk, salt, now: () => 2 }).snapshot("pool", "P")) + .toMatchObject({ settled: 70, unresolved: 1 }); +}); + +test("seed reverse indexes release all sends and retain shared scope pins until the last seed closes", () => { + const ledger = factory(spendTestJournal(), 1, 2); + ledger.reconfigure(spendTestPolicy({ maxTrackedScopes: 1, maxTrackedSends: 2, poolAliases: { [spendAlias("pool", "P")]: "P" } })); + const first = seed(ledger, "first"); + const last = seed(ledger, "last"); + expect(reported(ledger, first, "overflow")).toBe(true); + for (const sendId of ["first", "overflow"]) ledger.settle(sendId, { inputTokens: 0, outputTokens: 0 }); + expect(ledger.forgetResolved(["first", "overflow"])).toBe(true); + expect(ledger.finishSeed(first)).toBe(true); + ledger.prune(1e12); + expect(ledger.snapshot("pool", "P")).toBeDefined(); + expect(reported(ledger, last, "overflow")).toBe(true); + for (const sendId of ["last", "overflow"]) ledger.settle(sendId, { inputTokens: 0, outputTokens: 0 }); + expect(ledger.forgetResolved(["last", "overflow"])).toBe(true); + expect(ledger.finishSeed(last)).toBe(true); + ledger.prune(1e12); + expect(ledger.snapshot("pool", "P")).toBeUndefined(); + expect(ledger.reserveSeed(request("first", { poolId: "Q" }))).toMatchObject({ reserved: true }); + expect(reported(ledger, first, "late")).toBe(false); +}); + +test("seed cleanup checks the owning reporter count and rejects another owner", () => { + const ledger = factory(); + const owner = {}; + const foreign = {}; + const decision = ledger.reserveSeed(request("owned"), owner); + if (!decision.reserved) throw new Error("seed denied"); + const first = ledger.registerReporter(owner); + const second = ledger.registerReporter(owner); + const unrelated = ledger.registerReporter(foreign); + ledger.settle("owned", { inputTokens: 1, outputTokens: 0 }); + first.close(); + first.close(); + expect(ledger.forgetResolved(["owned"], owner)).toBe(false); + expect(ledger.finishSeed(decision.seed, owner)).toBe(false); + second.close(); + expect(ledger.forgetResolved(["owned"], {})).toBe(false); + expect(ledger.finishSeed(decision.seed, {})).toBe(false); + expect(ledger.forgetResolved(["owned"], owner)).toBe(true); + // A forgotten alias stays pinned until its seed closes; it cannot be rebound underneath it. + expect(ledger.knows("owned")).toBe(true); + expect(ledger.reserve(request("owned"))).toMatchObject({ reserved: false, denial: { reason: "duplicate-send-id" } }); + expect(ledger.finishSeed(decision.seed, owner)).toBe(true); + unrelated.close(); + expect(ledger.reserveSeed(request("owned"), foreign)).toMatchObject({ reserved: true }); +}); diff --git a/tests/responses/responses-compact-handoff-admission.test.ts b/tests/responses/responses-compact-handoff-admission.test.ts index 3ebe8c78706..df3f383cc45 100644 --- a/tests/responses/responses-compact-handoff-admission.test.ts +++ b/tests/responses/responses-compact-handoff-admission.test.ts @@ -10,6 +10,7 @@ import { handleResponsesCompact } from "../../src/server/responses"; import type { OcxConfig } from "../../src/types"; import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { configureSharedSpendLedger, sharedSpendLedger, spendPolicyFromConfig } from "../../src/lib/spend-reservation-ledger"; const originalFetch = globalThis.fetch; @@ -105,6 +106,34 @@ describe("compact handoff route admission namespacing", () => { }; } + for (const historicalPool of ["retired-display-label", "openai"]) test(`native compact explains spend refusal with history from ${historicalPool}`, async () => { + await withPoolEnv(async config => { + configureSharedSpendLedger(spendPolicyFromConfig({ pool: { maxTokens: 100 } }, undefined, ["openai"])); + const ledger = sharedSpendLedger(); + expect(ledger.reserve({ sendId: "historical", scopes: { poolId: historicalPool }, + inputTokens: 90, outputCeilingTokens: 0 }).reserved).toBe(true); + ledger.markDispatched("historical"); + ledger.settle("historical", { inputTokens: 90, outputTokens: 0 }); + let calls = 0; + globalThis.fetch = (async () => { calls++; throw new Error("compact must refuse before wire"); }) as typeof fetch; + const response = await handleResponsesCompact(compactionRequest(compactionBody("gpt-5.6-sol"), {}), + config, { model: "", provider: "", spendInputEstimateTokens: 20 }); + expect(response.status).toBe(429); + const body = await response.text(); + if (historicalPool === "retired-display-label") { + expect(response.headers.get("x-opencodex-local-refusal")).toBe("workflow_spend_exhausted"); + expect(body).toContain("unassigned historical provider-pool balances"); + } else { + expect(response.headers.get("x-opencodex-local-refusal")).toBeNull(); + expect(body).toContain("request_send_budget_exhausted"); + expect(body).not.toContain("unassigned historical provider-pool balances"); + } + expect(body).not.toContain("retired-display-label"); + expect(calls).toBe(0); + expect(ledger.snapshot("pool", "openai")?.settled).toBe(90); + }); + }); + test("a remembered route is claimed only by the principal that stored it", async () => { await withPoolEnv(async config => { config.providers.deepseek = { diff --git a/tests/responses/responses-spend-contract-c.test.ts b/tests/responses/responses-spend-contract-c.test.ts index d0db6bb7680..fe69f74636b 100644 --- a/tests/responses/responses-spend-contract-c.test.ts +++ b/tests/responses/responses-spend-contract-c.test.ts @@ -22,7 +22,7 @@ import { removeTreeWithRetry } from "../helpers/remove-tree"; const salt = "6".repeat(64); const pool = (id: string) => createHash("sha256").update(salt).update("\0pool\0").update(id).digest("hex").slice(0, 32); const policy = (poolAliases?: unknown, limit = 100): SpendReservationPolicy => ({ - ...DEFAULT_SPEND_RESERVATION_POLICY, pool: { maxTokens: limit }, poolAliases, + ...DEFAULT_SPEND_RESERVATION_POLICY, pool: { maxTokens: limit }, poolAliases, canonicalProviderIds: ["P", "Q"], }); const journal = (records: unknown[] = []): SpendJournal & { lines: string[] } => { const lines = records.map(record => JSON.stringify(record)); @@ -95,7 +95,7 @@ test("Responses A B and Messages share canonical ceiling", async () => { } }); -test("own unbound canonical bucket is counted once for absent partial complete maps", () => { +test("self-bound canonical bucket is counted once for absent partial complete maps", () => { for (const aliases of [undefined, { [pool("old-A")]: "P" }, { [pool("old-A")]: "P", [pool("old-B")]: "P", [pool("P")]: "P" }]) { const disk = journal([checkpoint([["P", 20, 0], ["old-A", 30, 0], ["old-B", 0, 10]])]); @@ -109,20 +109,22 @@ test("own unbound canonical bucket is counted once for absent partial complete m } }); -test("restart does not self-bind ambiguous canonical history", () => { +test("restart keeps configured canonical history self-bound without persisted metadata", () => { const disk = journal(); let ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(), now: () => 2 }); expect(reserve(ledger, "initial", "P", 40).reserved).toBe(true); expect(ledger.settle("initial", { inputTokens: 40, outputTokens: 0 })).toBe(true); ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(), now: () => 3 }); - expect(ledger.hasUnboundPositivePoolHistory()).toBe(true); - expect(ledger.snapshot("pool", "Q")?.settled).toBe(40); - expect(reserve(ledger, "other-over", "Q", 61)).toMatchObject({ reserved: false, denial: { projected: 101 } }); - ledger.reconfigure(policy({ [pool("P")]: "P" })); expect(ledger.hasUnboundPositivePoolHistory()).toBe(false); + expect(ledger.snapshot("pool", "P")?.settled).toBe(40); expect(ledger.snapshot("pool", "Q")).toBeUndefined(); - ledger.reconfigure(policy()); + expect(reserve(ledger, "other-independent", "Q", 100).reserved).toBe(true); + ledger.abandon("other-independent"); + ledger.reconfigure({ ...policy(), canonicalProviderIds: ["Q"] }); + expect(ledger.hasUnboundPositivePoolHistory()).toBe(true); expect(ledger.snapshot("pool", "Q")?.settled).toBe(40); + ledger.reconfigure(policy()); + expect(ledger.snapshot("pool", "Q")?.settled ?? 0).toBe(0); expect(disk.lines.some(line => JSON.parse(line).poolContinuity !== undefined)).toBe(false); }); @@ -133,7 +135,7 @@ test("combo A to B preserves exact current permit exclusion", async () => { }, combos: { spend: { strategy: "failover", targets: [{ provider: "A", model: "m" }, { provider: "B", model: "m" }] } } }; const disk = journal(); const ledger = createSpendReservationLedger({ journal: disk, salt, - policy: policy({ [pool("A")]: "A", [pool("B")]: "B" }, 10) }); + policy: { ...policy({ [pool("A")]: "A", [pool("B")]: "B" }, 10), canonicalProviderIds: ["A", "B"] } }); const ctx = log("stale"); const tracker = createRequestSpendTracker(ctx, undefined, ledger); const budget = createRequestExecutionBudget(undefined, "combo-contract-c", tracker); diff --git a/tests/server/spend-ledger-lifecycle.test.ts b/tests/server/spend-ledger-lifecycle.test.ts index 85607f05376..bff73703675 100644 --- a/tests/server/spend-ledger-lifecycle.test.ts +++ b/tests/server/spend-ledger-lifecycle.test.ts @@ -4,12 +4,14 @@ * ownership exists to prevent, and it cannot be observed from a passing startup. */ import { afterEach, beforeEach, expect, test } from "bun:test"; -import { mkdtempSync } from "node:fs"; +import { existsSync, mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { acquireSpendLedgerServerLifecycle } from "../../src/server/index/spend-ledger-lifecycle"; import { spendLedgerOwnerSnapshot } from "../../src/lib/spend-ledger-owner"; -import { resetSharedSpendLedgerForTest } from "../../src/lib/spend-reservation-ledger"; +import { resetSharedSpendLedgerForTest, sharedSpendLedger } from "../../src/lib/spend-reservation-ledger"; +import { getDefaultConfig, reconcileLiveConfigFromDisk, saveConfig } from "../../src/config"; +import { reconcileLiveStateStores, setLiveStateStoreConfig } from "../../src/lib/state-store-registrations"; import { removeTreeWithRetry } from "../helpers/remove-tree"; let root = ""; @@ -103,3 +105,60 @@ test("a listener whose stop rejects still stops the rest and still returns the d lifecycle.release(); expect(spendLedgerOwnerSnapshot().ownership).toBe("unheld"); }); + +test("live provider mutations and disk adoption refresh spend policy from the active roster", () => { + const lifecycle = acquireSpendLedgerServerLifecycle(home); + const config = { ...getDefaultConfig(), defaultProvider: "stable", spend: { pool: { maxTokens: 100 } }, + providers: { stable: { adapter: "openai-chat" as const, baseUrl: "https://example.test/v1", apiKey: "fixture-key" } } }; + setLiveStateStoreConfig(config); + lifecycle.configure(config.spend, undefined, Object.keys(config.providers)); + try { + const ledger = sharedSpendLedger(); + expect(ledger.reserve({ sendId: "retained", scopes: { poolId: "stable" }, inputTokens: 20, outputCeilingTokens: 0 }).reserved).toBe(true); + ledger.markDispatched("retained"); + ledger.settle("retained", { inputTokens: 20, outputTokens: 0 }); + expect(ledger.reserve({ sendId: "unassigned", scopes: { poolId: "added" }, inputTokens: 10, outputCeilingTokens: 0 }).reserved).toBe(true); + ledger.markDispatched("unassigned"); + ledger.settle("unassigned", { inputTokens: 10, outputTokens: 0 }); + expect(ledger.snapshot("pool", "stable")?.settled).toBe(30); + const added = { ...config.providers.stable }; + Object.assign(config.providers, { added }); + config.spend.pool.maxTokens = 120; + reconcileLiveStateStores(); + expect(ledger.policy.canonicalProviderIds).toEqual(["stable", "added"]); + expect(ledger.policy.pool.maxTokens).toBe(120); + expect(ledger.snapshot("pool", "stable")?.settled).toBe(20); + + Reflect.deleteProperty(config.providers, "added"); + reconcileLiveStateStores(); + expect(ledger.policy.canonicalProviderIds).toEqual(["stable"]); + expect(ledger.snapshot("pool", "stable")?.settled).toBe(30); + saveConfig(config); + const baseline = structuredClone(config); + saveConfig({ ...baseline, providers: { ...baseline.providers, fromdisk: added }, spend: { pool: { maxTokens: 140 } } }); + // Detached reads/writes do not make disk-only providers live. + expect(ledger.policy.canonicalProviderIds).toEqual(["stable"]); + reconcileLiveConfigFromDisk(config, baseline); + reconcileLiveStateStores(); + expect(ledger.policy.canonicalProviderIds).toEqual(["stable", "fromdisk"]); + expect(ledger.policy.pool.maxTokens).toBe(140); + expect(ledger.snapshot("pool", "stable")?.settled).toBe(30); + } finally { + lifecycle.release(); + } +}); + +test("live refresh with no spend ceiling creates neither journal nor salt", () => { + const lifecycle = acquireSpendLedgerServerLifecycle(home); + const config = getDefaultConfig(); + delete config.spend; + delete config.spendPoolAliases; + setLiveStateStoreConfig(config); + try { + reconcileLiveStateStores(); + expect(existsSync(join(home, "spend-ledger.jsonl"))).toBe(false); + expect(existsSync(join(home, "spend-ledger.salt"))).toBe(false); + } finally { + lifecycle.release(); + } +}); From ed8cc785e5f02f2c8c79c46ba0cd5231cbdf3801 Mon Sep 17 00:00:00 2001 From: JUN Date: Fri, 9 Oct 2026 01:38:07 +0900 Subject: [PATCH 12/15] fix(spend): retain accounting through policy activation and cleanup errors Co-authored-by: Epinephrine --- src/lib/request-execution-budget.ts | 49 +++-- src/lib/spend-reservation-ledger.ts | 23 +-- src/server/responses/request-spend.ts | 163 ++++++++++++---- structure/transports/responses-spend.md | 18 +- tests/lib/execution-budget-permits.test.ts | 206 +++++++++++++++++++++ tests/lib/spend-seed-overflow.test.ts | 130 ++++++++++++- tests/server/inference-send-budget.test.ts | 2 +- 7 files changed, 526 insertions(+), 65 deletions(-) diff --git a/src/lib/request-execution-budget.ts b/src/lib/request-execution-budget.ts index 795801ea0bb..39fcaf64c09 100644 --- a/src/lib/request-execution-budget.ts +++ b/src/lib/request-execution-budget.ts @@ -153,7 +153,13 @@ export interface RequestSendObserver { /** Confirm the exact reservation once its dispatch is known. */ readonly enforced?: boolean; ensureSeed?(target: SpendScopes & { targetKey?: string }, proof?: SpendReservationProof): SpendSeed | undefined; - beginReporter?(): { start(seed: SpendSeed, ordinal: number): void; rebindTarget(target: SpendScopes & { targetKey?: string }, ordinal: number): boolean; report(sends: number): void; close(): void }; + beginReporter?(): { + start(seed: SpendSeed, ordinal: number): void; + startObserved?(target: SpendScopes & { targetKey?: string }, ordinal: number, proof?: SpendReservationProof): void; + rebindTarget(target: SpendScopes & { targetKey?: string }, ordinal: number): boolean; + report(sends: number): void; + close(): void; + }; dispatch?(proof: SpendReservationProof): void; /** Give back a booking whose send never happened. */ refund(proof?: SpendReservationProof | null): void; @@ -292,7 +298,7 @@ export type PhysicalSendReporter = ((sends: number) => void) & { const activePhysicalReporters = new AsyncLocalStorage(); /** Selection owners rebind this helper's current start before invoking the executor. */ export function rebindPhysicalSend(budget: TransientSendBudget | undefined, target: SpendScopes & { targetKey?: string }): boolean { - if (!(budget as RequestExecutionBudget | undefined)?.spendEnforced) return true; + if (!budget) return true; return activePhysicalReporters.getStore()?.bind?.(target) ?? true; } export function createPhysicalSendReporter( @@ -319,24 +325,27 @@ export function createPhysicalSendReporter( if (counter) counter.spent += count - prepaid; telemetry?.(count); }; - if (observer?.enforced === true) report.beforeSend = () => { + if (observer?.beginReporter) report.beforeSend = () => { if (closed) return false; - if (observer?.enforced !== true) return true; producer ??= observer.beginReporter?.(); const receipt = permit && dispatchSpendProofs.get(permit); const proof = receipt?.owner === counter ? receipt?.claim() : undefined; - const seed = observer.ensureSeed?.(target(), proof); - if (!seed) return false; + const selected = target(); + const enforced = observer.enforced === true; + const seed = enforced ? observer.ensureSeed?.(selected, proof) : undefined; + if (enforced && !seed) return false; const ordinal = (budget as RequestExecutionBudget).claimPhysicalSend?.(); if (ordinal === undefined) return false; - producer?.start(seed, ordinal); + if (seed) producer?.start(seed, ordinal); + else producer?.startObserved?.(selected, ordinal, proof); currentOrdinal = ordinal; started++; return true; }; report.execute = run => activePhysicalReporters.run(report, run); report.bind = (selected: SpendScopes & { targetKey?: string }) => { - if (!producer || currentOrdinal === undefined || observer?.enforced !== true) return true; + if (!producer || currentOrdinal === undefined) return true; + if (observer?.enforced && (counter?.physicalLimit === undefined || currentOrdinal > counter.physicalLimit)) return false; return producer.rebindTarget(selected, currentOrdinal); }; report.close = () => { @@ -397,12 +406,11 @@ function createRequestExecutionBudgetWithLedger( get spendEnforced(): boolean { return observer?.enforced === true; }, beginSpendProducer() { return observer?.enforced ? observer.beginReporter?.() : undefined; }, claimPhysicalSend(): number | undefined { - if (observer?.enforced !== true) return undefined; if (counter.physicalLimit === undefined) { - if (!Number.isSafeInteger(policy.maxTotalModelSends) || policy.maxTotalModelSends <= 0) return undefined; - counter.physicalLimit = policy.maxTotalModelSends; + if (Number.isSafeInteger(policy.maxTotalModelSends) && policy.maxTotalModelSends > 0) counter.physicalLimit = policy.maxTotalModelSends; + else if (observer?.enforced === true) return undefined; } - if (counter.physicalStarted >= counter.physicalLimit) return undefined; + if (observer?.enforced === true && (counter.physicalLimit === undefined || counter.physicalStarted >= counter.physicalLimit)) return undefined; return ++counter.physicalStarted; }, get used(): number { return counter.spent; }, @@ -487,13 +495,15 @@ function createRequestExecutionBudgetWithLedger( let executing = false; let physicalOrdinal: number | undefined; const startPhysical = (): boolean => { - if (observer?.enforced !== true) return true; - const seed = observer.ensureSeed?.({ targetKey: intent.targetKey }, spendProof); - if (!seed) return false; + if (!observer?.beginReporter) return true; + const enforced = observer.enforced === true; + const seed = enforced ? observer.ensureSeed?.({ targetKey: intent.targetKey }, spendProof) : undefined; + if (enforced && !seed) return false; const ordinal = budget.claimPhysicalSend?.(); if (ordinal === undefined) return false; producer = observer.beginReporter?.(); - producer?.start(seed, ordinal); + if (seed) producer?.start(seed, ordinal); + else producer?.startObserved?.({ targetKey: intent.targetKey }, ordinal, spendProof); physicalOrdinal = ordinal; if (!executing) { producer?.report(1); producer?.close(); } return true; @@ -505,8 +515,11 @@ function createRequestExecutionBudgetWithLedger( if (executing) throw new Error("Dispatch permit executor already active"); executing = true; const binding: PhysicalSendReporter = () => {}; - binding.bind = (target: SpendScopes & { targetKey?: string }) => physicalOrdinal === undefined || !producer - ? true : producer.rebindTarget(target, physicalOrdinal); + binding.bind = (target: SpendScopes & { targetKey?: string }) => { + if (physicalOrdinal === undefined || !producer) return true; + if (observer?.enforced && (counter.physicalLimit === undefined || physicalOrdinal > counter.physicalLimit)) return false; + return producer.rebindTarget(target, physicalOrdinal); + }; try { return await activePhysicalReporters.run(binding, run); } finally { executing = false; diff --git a/src/lib/spend-reservation-ledger.ts b/src/lib/spend-reservation-ledger.ts index 90171becf83..b1fede93f76 100644 --- a/src/lib/spend-reservation-ledger.ts +++ b/src/lib/spend-reservation-ledger.ts @@ -654,14 +654,15 @@ export interface SpendReservationLedger { /** * Settle with real usage. Returns false when the send is unknown or already resolved -- * double settlement is as wrong as none, so a repeat call changes nothing. - * Seeded terminal debt applies immediately; failed writes queue before durable forgetting. + * Seeded or explicitly retained terminal debt applies immediately; failed writes queue + * for ordered retry. This intent grants no reservation capacity or seed capability. */ - settle(sendId: string, usage: SpendUsage): boolean; + settle(sendId: string, usage: SpendUsage, options?: { retainOnFailure: boolean }): boolean; /** * Usage never arrived. The reservation moves to unresolved spend -- it may have been * billed -- rather than being released. Idempotent on the same key as settle. */ - markLost(sendId: string): boolean; + markLost(sendId: string, options?: { retainOnFailure: boolean }): boolean; snapshot(scope: SpendScope, scopeId: string): ScopeSpendSnapshot | undefined; /** Exclude only a proven current dispatch's still-open reservation in this scope. */ exhausted(scope: SpendScope, scopeId: string, excludingSendId?: string): boolean; @@ -1377,7 +1378,7 @@ export function createSpendReservationLedger(options: { return true; }, - settle(sendId: string, usage: SpendUsage): boolean { + settle(sendId: string, usage: SpendUsage, options?: { retainOnFailure: boolean }): boolean { assertOwnedAccounting?.(); const send = aliasFor("send", sendId); const reservation = reservations.get(send); @@ -1385,26 +1386,26 @@ export function createSpendReservationLedger(options: { const tokens = sanitizeTokens(usage.inputTokens) + sanitizeTokens(usage.outputTokens); const at = now(); const record: JournalRecord = { v: 1, kind: "settle", send, tokens, at }; - const seeded = seedForSend(send); + const retained = seedForSend(send) || options?.retainOnFailure; // Terminal usage cannot be refused: the tracker may never report it again, and // the estimate can be smaller than the actual bill. Preserve ordered retryable debt. - if (seeded && (pendingRecords.length > 0 || !append(record))) pendingRecords.push(record); + if (retained && (pendingRecords.length > 0 || !append(record))) pendingRecords.push(record); applyResolve(send, "settled", tokens, at); - if (!seeded) append(record); + if (!retained) append(record); return true; }, - markLost(sendId: string): boolean { + markLost(sendId: string, options?: { retainOnFailure: boolean }): boolean { assertOwnedAccounting?.(); const send = aliasFor("send", sendId); const reservation = reservations.get(send); if (!reservation || !isLive(reservation.status)) return false; const at = now(); const record: JournalRecord = { v: 1, kind: "lost", send, at }; - const seeded = seedForSend(send); - if (seeded && (pendingRecords.length > 0 || !append(record))) pendingRecords.push(record); + const retained = seedForSend(send) || options?.retainOnFailure; + if (retained && (pendingRecords.length > 0 || !append(record))) pendingRecords.push(record); applyResolve(send, "lost", 0, at); - if (!seeded) append(record); + if (!retained) append(record); return true; }, diff --git a/src/server/responses/request-spend.ts b/src/server/responses/request-spend.ts index 5ba7e80a98f..1e9168f05fe 100644 --- a/src/server/responses/request-spend.ts +++ b/src/server/responses/request-spend.ts @@ -19,6 +19,7 @@ export interface RequestSpendSettlement { export interface SpendTargetIdentity extends SpendScopes { readonly targetKey?: string } export interface RequestSpendReporter { start(seed: SpendSeed, ordinal: number): void; + startObserved(target: SpendTargetIdentity, ordinal: number, proof?: SpendReservationProof): void; rebindTarget(target: SpendTargetIdentity, ordinal: number): boolean; report(sends: number): void; close(): void; @@ -70,17 +71,25 @@ export function createRequestSpendTracker( // Outstanding entries; exact dispatch reports move their reservation to the end. const live: string[] = []; const pendingDispatch = new Set(); + const legacyOrder = new Map(); + let dispatchOrder = 0; let refusals = 0; let resolved = false; let terminalProcessed = false; let seededAccounting = false; + let enforcementObserved = false; let finalRequested = false; let finalUsage: TerminalSpendUsage | undefined; let reporters = 0; const seeds = new Map(); + // Abandoned capabilities retain only a cleanup obligation, never an admission lookup. + const retiredSeeds = new Set(); const resolvedSendIds = new Set(); const completedSeeds = new Set(); - const physical = new Map(); + type PhysicalEntry = { order: number; seed?: SpendSeed; sendId?: string; observed?: { + scopes: SpendScopes; inputTokens: number; outputCeilingTokens: number; + }; reported?: boolean }; + const physical = new Map(); const selectedScopes = (target: SpendTargetIdentity = {}): SpendScopes => ({ ...(rootId !== undefined ? { rootId } : {}), ...(target.identityId ?? logCtx.accountLogLabel ? { identityId: target.identityId ?? logCtx.accountLogLabel } : {}), @@ -88,35 +97,68 @@ export function createRequestSpendTracker( }); const applicable = (scopes: SpendScopes): boolean => { const policy = ledgerRef?.policy ?? sharedSpendPolicy(); - return seededAccounting || (scopes.rootId !== undefined && policy.root.maxTokens !== undefined) + const enforced = seededAccounting || (scopes.rootId !== undefined && policy.root.maxTokens !== undefined) || (scopes.identityId !== undefined && policy.identity.maxTokens !== undefined) || (scopes.poolId !== undefined && policy.pool.maxTokens !== undefined); + if (enforced) enforcementObserved = true; + return enforced; }; const estimates = () => ({ inputTokens: logCtx.spendInputEstimateTokens ?? logCtx.usageLogInputTokens ?? 0, outputCeilingTokens: logCtx.spendOutputCeilingTokens ?? 0 }); const seedState = (seed: SpendSeed) => [...seeds.values()].find(state => state.seed === seed); + // Observe-only starts use ordinary records, never overflow capabilities. A full normal + // table may omit the booking. If policy activates, retain that debt until normal room is available. + const bookObserved = (entry: PhysicalEntry): boolean => { + if (entry.sendId) return true; + if (!entry.observed) return false; + const sendId = randomUUID(); + const decision = ledger().reserve({ sendId, ...entry.observed, alreadySent: true }); + if (!decision.reserved) return false; + entry.sendId = sendId; + if (entry.reported) ledger().markDispatched(sendId); + return true; + }; + const cleanupRetiredSeeds = (): boolean => { + for (const seed of retiredSeeds) if (ledger().finishSeed(seed, tracker)) retiredSeeds.delete(seed); + return retiredSeeds.size === 0; + }; const tryFinalizeSeeds = (): boolean => { + applicable(selectedScopes()); + const durability = { retainOnFailure: enforcementObserved }; // All starts are booked before producers close. Ordinal, never callback order, owns usage. const entries = [...physical.entries()].sort((a, b) => a[0] - b[0]); - const terminal = entries.at(-1)?.[1].sendId; + const terminalOrder = Math.max(0, ...entries.map(([, entry]) => entry.order), ...live.map(id => legacyOrder.get(id) ?? 0)); const ids: string[] = []; for (const [, entry] of entries) { + if (!entry.sendId && entry.observed && !enforcementObserved) continue; + if (!entry.sendId && !bookObserved(entry)) return false; if (!entry.sendId) return false; - const reported = entry.sendId === terminal && (typeof finalUsage?.inputTokens === "number" || typeof finalUsage?.outputTokens === "number"); + const reported = entry.order === terminalOrder && (typeof finalUsage?.inputTokens === "number" || typeof finalUsage?.outputTokens === "number"); if (!resolvedSendIds.has(entry.sendId)) { - const ok = reported ? ledger().settle(entry.sendId, { inputTokens: finalUsage?.inputTokens ?? 0, outputTokens: finalUsage?.outputTokens ?? 0 }) - : ledger().markLost(entry.sendId); + const ok = reported ? ledger().settle(entry.sendId, { inputTokens: finalUsage?.inputTokens ?? 0, outputTokens: finalUsage?.outputTokens ?? 0 }, durability) + : ledger().markLost(entry.sendId, durability); if (!ok) return false; resolvedSendIds.add(entry.sendId); } - ids.push(entry.sendId); + if (entry.seed) ids.push(entry.sendId); + } + for (const sendId of live) { + if (!resolvedSendIds.has(sendId)) { + const reported = legacyOrder.get(sendId) === terminalOrder + && (typeof finalUsage?.inputTokens === "number" || typeof finalUsage?.outputTokens === "number"); + const ok = reported ? ledger().settle(sendId, { inputTokens: finalUsage?.inputTokens ?? 0, outputTokens: finalUsage?.outputTokens ?? 0 }, durability) + : ledger().markLost(sendId, durability); + if (!ok) return false; + resolvedSendIds.add(sendId); + } } for (const state of seeds.values()) if (!state.started && !completedSeeds.has(state.seed)) { if (!resolvedSendIds.has(state.seed.sendId) && !ledger().abandon(state.seed.sendId)) return false; resolvedSendIds.add(state.seed.sendId); ids.push(state.seed.sendId); } - if (!ledger().forgetResolved(ids, tracker)) return false; + if (!cleanupRetiredSeeds()) return false; + if ((ids.length > 0 || enforcementObserved) && !ledger().forgetResolved(ids, tracker)) return false; for (const state of seeds.values()) if (!completedSeeds.has(state.seed)) { if (!ledger().finishSeed(state.seed, tracker)) return false; completedSeeds.add(state.seed); @@ -124,13 +166,30 @@ export function createRequestSpendTracker( resolved = true; return true; }; + const deferCleanup = (): void => ledger().deferCleanup(tracker, () => resolved + || (finalRequested && reporters === 0 ? tryFinalizeSeeds() : cleanupRetiredSeeds())); const finalizeSeeds = (): void => { - if (!finalRequested || reporters !== 0 || resolved || !seededAccounting) return; + if (!finalRequested || reporters !== 0 || resolved || (!seededAccounting && physical.size === 0)) return; + // Own terminal retry before an append can throw; propagation must not orphan the debt. + deferCleanup(); if (!tryFinalizeSeeds()) { // Failed durable cleanup remains owned by this ledger until later activity retries it. - ledger().deferCleanup(tracker, () => resolved || tryFinalizeSeeds()); + deferCleanup(); } }; + const retireSeed = (key: string, seed: SpendSeed): boolean => { + if (!resolvedSendIds.has(seed.sendId) && !ledger().abandon(seed.sendId)) return false; + resolvedSendIds.add(seed.sendId); + seeds.delete(key); + const index = live.indexOf(seed.sendId); + if (index >= 0) live.splice(index, 1); + pendingDispatch.delete(seed.sendId); + // Keep cleanup owned even if finishSeed throws after durable abandonment. + retiredSeeds.add(seed); + deferCleanup(); + if (ledger().finishSeed(seed, tracker)) retiredSeeds.delete(seed); + return true; + }; /** * Confirm the sends this request has already moved past. * @@ -183,6 +242,20 @@ export function createRequestSpendTracker( if (resolved || finalRequested && reporters === 0) return undefined; const scopes = selectedScopes(target); if (!applicable(scopes)) return undefined; + // Activation cannot authorize a new send while an earlier observe-only liability + // still lacks normal tracking capacity. It never grants that liability a seed. + for (const entry of physical.values()) if (entry.observed && !bookObserved(entry)) { + refusal({ reason: "tracking-capacity-exhausted" }); + return undefined; + } + if (prepaidProof && prepaidProof.ledger === ledgerRef && pendingDispatch.has(prepaidProof.sendId)) { + // A reservation made before activation is not an enforced seed. Release its + // undispatched booking before asking NORMAL admission under the new policy. + if (!ledger().abandon(prepaidProof.sendId)) return undefined; + pendingDispatch.delete(prepaidProof.sendId); + const index = live.indexOf(prepaidProof.sendId); + if (index >= 0) live.splice(index, 1); + } const key = JSON.stringify([scopes.rootId, scopes.identityId, scopes.poolId, target.targetKey]); const existing = seeds.get(key); if (existing) return existing.seed; @@ -205,14 +278,7 @@ export function createRequestSpendTracker( } // Rebind an unused provisional anchor before selection changes. Dispatched anchors stay live. for (const [oldKey, state] of seeds) if (!state.started) { - if (!resolvedSendIds.has(state.seed.sendId) && !ledger().abandon(state.seed.sendId)) return undefined; - resolvedSendIds.add(state.seed.sendId); - // Keep a durably abandoned capability for final cleanup if forgetting fails. - if (!ledger().finishSeed(state.seed, tracker)) continue; - seeds.delete(oldKey); - const index = live.indexOf(state.seed.sendId); - if (index >= 0) live.splice(index, 1); - pendingDispatch.delete(state.seed.sendId); + if (!retireSeed(oldKey, state.seed)) return undefined; } // Exact prepaid proofs originate from this tracker and are found above; foreign receipts // cannot enroll arbitrary scopes. Normal reservation remains the only initial admission. @@ -233,28 +299,56 @@ export function createRequestSpendTracker( reporters++; let closed = false; let reported = 0; - const starts: Array<{ seed: SpendSeed; ordinal: number }> = []; + const starts: Array<{ seed?: SpendSeed; ordinal: number }> = []; return { start(seed, ordinal) { if (closed || !seedState(seed) || physical.has(ordinal)) throw new Error("Invalid spend reporter start"); const state = seedState(seed)!; state.started = true; starts.push({ seed, ordinal }); - physical.set(ordinal, { seed }); - + physical.set(ordinal, { seed, order: ++dispatchOrder }); + }, + startObserved(target, ordinal, proof) { + if (closed || physical.has(ordinal)) throw new Error("Invalid observed reporter start"); + const entry: PhysicalEntry = { order: ++dispatchOrder, observed: { scopes: selectedScopes(target), ...estimates() } }; + if (proof && proof.ledger === ledgerRef && pendingDispatch.has(proof.sendId)) { + entry.sendId = proof.sendId; + pendingDispatch.delete(proof.sendId); + const index = live.indexOf(proof.sendId); + if (index >= 0) live.splice(index, 1); + } + physical.set(ordinal, entry); + starts.push({ ordinal }); + bookObserved(entry); }, rebindTarget(target, ordinal) { const entry = physical.get(ordinal); const owned = starts.find(start => start.ordinal === ordinal); - if (closed || !entry || !owned || entry.sendId) throw new Error("Invalid spend reporter rebind"); - const old = seedState(entry.seed)!; + if (closed || !entry || !owned || entry.reported || entry.seed && entry.sendId) throw new Error("Invalid spend reporter rebind"); + if (entry.observed) { + const scopes = selectedScopes(target); + const enforced = applicable(scopes); + if (!enforced && JSON.stringify(scopes) === JSON.stringify(entry.observed.scopes)) return true; + if (entry.sendId && !ledger().abandon(entry.sendId)) return false; + physical.delete(ordinal); + if (enforced) { + const seed = tracker.ensureSeed(target); + if (!seed) { starts.splice(starts.indexOf(owned), 1); return false; } + physical.set(ordinal, { seed, order: entry.order }); owned.seed = seed; seedState(seed)!.started = true; + } else { + const rebound: PhysicalEntry = { order: entry.order, observed: { scopes, ...estimates() } }; + physical.set(ordinal, rebound); bookObserved(rebound); + } + return true; + } + const old = seedState(entry.seed!)!; // A helper start is provisional until selection reaches the executor. If this is // the first start on its seed, abandon/rebind at NORMAL capacity before the wire. physical.delete(ordinal); old.started = [...physical.values()].some(start => start.seed === old.seed); const seed = tracker.ensureSeed(target); if (!seed) { starts.splice(starts.indexOf(owned), 1); return false; } - physical.set(ordinal, { seed }); + physical.set(ordinal, { seed, order: entry.order }); owned.seed = seed; seedState(seed)!.started = true; return true; @@ -264,7 +358,12 @@ export function createRequestSpendTracker( const count = Number.isFinite(sends) ? Math.max(0, Math.trunc(sends)) : 0; for (let i = 0; i < count && reported < starts.length; i++) { const entry = starts[reported++]!; - tracker.reportFromSeed(entry.seed, entry.ordinal); + if (entry.seed) tracker.reportFromSeed(entry.seed, entry.ordinal); + else { + const observed = physical.get(entry.ordinal)!; + observed.reported = true; + if (bookObserved(observed) && observed.sendId) ledger().markDispatched(observed.sendId); + } } }, close() { @@ -341,17 +440,18 @@ export function createRequestSpendTracker( if (!alreadySent) options?.onReserved?.({ ledger: bookedLedger, sendId }); live.push(sendId); if (options?.deferDispatch) pendingDispatch.add(sendId); - else confirmOlderSends(); + else { legacyOrder.set(sendId, ++dispatchOrder); confirmOlderSends(); } // It has already left, so the reservation cannot be handed back for free: from here only // a settlement or unresolved spend is honest about it. if (alreadySent) ledger().markDispatched(sendId); return true; }, dispatch(proof): void { - if (seededAccounting && proof.ledger === ledgerRef) return; + if (proof.ledger === ledgerRef && [...seeds.values()].some(state => state.seed.sendId === proof.sendId)) return; if (proof.ledger !== ledgerRef || !pendingDispatch.has(proof.sendId)) return; ledger().markDispatched(proof.sendId); pendingDispatch.delete(proof.sendId); + legacyOrder.set(proof.sendId, ++dispatchOrder); // Terminal usage follows dispatch/report order, not reservation order. const index = live.indexOf(proof.sendId); if (index >= 0) live.push(...live.splice(index, 1)); @@ -359,11 +459,10 @@ export function createRequestSpendTracker( refund(proof): void { if (proof && proof.ledger === ledgerRef && seededAccounting) { const match = [...seeds.entries()].find(([, state]) => state.seed.sendId === proof.sendId); - if (match && !match[1].started && (resolvedSendIds.has(proof.sendId) || ledger().abandon(proof.sendId))) { - resolvedSendIds.add(proof.sendId); - if (ledger().finishSeed(match[1].seed, tracker)) seeds.delete(match[0]); + if (match) { + if (!match[1].started) retireSeed(match[0], match[1].seed); + return; } - return; } // null is an exact budget reservation that obtained no durable booking. if (proof === null || (proof && proof.ledger !== ledgerRef)) return; @@ -378,7 +477,7 @@ export function createRequestSpendTracker( }, settle(usage: TerminalSpendUsage | undefined): void { if (resolved) return; - if (seededAccounting) { tracker.requestFinalSettlement(usage); return; } + if (seededAccounting || physical.size > 0 || reporters > 0) { tracker.requestFinalSettlement(usage); return; } try { if (!terminalProcessed && live.length > 0) { const terminal = live[live.length - 1] as string; diff --git a/structure/transports/responses-spend.md b/structure/transports/responses-spend.md index db48d04fe4f..fb575d18f1e 100644 --- a/structure/transports/responses-spend.md +++ b/structure/transports/responses-spend.md @@ -116,10 +116,22 @@ caller is `request-spend.ts`, installed on the execution budget at genuine ingre and parked on the log context so `addFinalRequestLog` can settle it. Native Chat installs the same tracker before its independent physical-send ladder and charges it immediately before each dispatch, so that fast path cannot bypass root, identity or provider-pool ceilings. Unconfigured requests preserve the legacy counter observer: reservations increment the send -counter, refunds decrement it, and named reports reconcile prepaid sends. Enforced requests use -the seed and physical-start protocol below across Responses, native Messages and native Chat. +counter, refunds decrement it, and named reports reconcile prepaid sends. Physical helpers retain +their start order even while observing, without creating seed capabilities or imposing a new +observe-only send limit. Enforced requests use the seed protocol below across Responses, native Messages and native Chat. Each transport claims immediately before inference I/O and retains a producer through completion. +Enabling a ceiling during an in-flight call preserves that call's accounting and the request's +existing physical-start count. Subsequent starts require normal seed admission under the current +policy; a prepaid reservation from before activation is not an enforced admission. Selection-time +rebinding checks the current policy even when the target stays the same. Terminal usage follows +the last actual start across both modes. Once enforcement is observed, an earlier omitted booking +remains pending debt and must obtain normal capacity before another enforced start is admitted; +it never becomes an overflow capability. Requests that remain observe-only retain capacity omission. +After activation, ordinary observed terminal records also retain failed writes in the ledger's +ordered retry queue. This persistence intent grants no admission capacity. Cleanup is registered +before terminal writes can throw, so later ledger activity retries without another settlement call. + Provider-pool accounting uses the canonical routed provider identity, not the mutable display label that may identify an OAuth account in request logs. Responses final-route normalization captures that identity before credential selection labels the account, and replaces it when a fallback @@ -200,6 +212,8 @@ uses the same unbound-history refusal explanation and workflow header as other R persists ordinary `forget` records before releasing seeds. Each tracker waits only for its own reporters, so overlapping traffic cannot pin a completed request. Storage-failed cleanup stays queued on the ledger and retries on later admission, pruning, reconfiguration or reporter closure. +An abandoned seed leaves every admission lookup immediately. Its remaining cleanup obligation +stays separate, including when durable forgetting throws an ownership or unsafe-file error. Send-to-seed and reference-counted scope-pin indexes avoid scanning all seeds on admission. Forgotten send IDs do not subtract scope totals. Shutdown drains all reporters before releasing ledger ownership; restart resolves orphan reservations conservatively without reviving diff --git a/tests/lib/execution-budget-permits.test.ts b/tests/lib/execution-budget-permits.test.ts index edce9f1f980..18e14fcb861 100644 --- a/tests/lib/execution-budget-permits.test.ts +++ b/tests/lib/execution-budget-permits.test.ts @@ -2,10 +2,17 @@ import { describe, expect, test } from "bun:test"; import { CODEX_TEXT_GUARDED_BUDGET_POLICY, createRequestExecutionBudget, + createPhysicalSendReporter, deriveRequestExecutionBudget, reportDispatchSends, + rebindPhysicalSend, type RequestExecutionBudgetPolicy, } from "../../src/lib/request-execution-budget"; +import { createSpendReservationLedger, DEFAULT_SPEND_RESERVATION_POLICY } from "../../src/lib/spend-reservation-ledger"; +import { createRequestSpendTracker } from "../../src/server/responses/request-spend"; +import { createShippedSpendLedger, spendTestJournal, spendTestPolicy, testSpendSalt } from "../helpers/shipped-spend-ledger"; +import { SpendLedgerOwnerError } from "../../src/lib/spend-ledger-owner"; +import { fetchWithTransientRetry, SendBudgetExhaustedError } from "../../src/lib/upstream-retry"; /** * The permit is the charge (#4546). @@ -26,6 +33,205 @@ const ONE_SEND_LEFT: RequestExecutionBudgetPolicy = { maxTargetTransitions: 1, }; +function policyActivationFixture(limit = 4, maxTrackedSends = 100) { + const lines: string[] = []; + const policy = { ...DEFAULT_SPEND_RESERVATION_POLICY, pool: {}, canonicalProviderIds: ["P"], maxTrackedSends }; + const ledger = createSpendReservationLedger({ policy, salt: "activation-fixture", journal: { + read: () => [...lines], append: line => { lines.push(line); }, rewrite: next => { lines.splice(0, lines.length, ...next); }, + } }); + const tracker = createRequestSpendTracker({ provider: "P", accountLogLabel: "A", spendInputEstimateTokens: 10 }, undefined, ledger); + const budget = createRequestExecutionBudget({ ...ONE_SEND_LEFT, maxTotalModelSends: limit, baseSendAllowance: limit }, undefined, tracker); + const enable = (maxTokens = 50) => ledger.reconfigure({ ...policy, pool: { maxTokens } }); + return { ledger, tracker, budget, enable, lines }; +} + +test("a ceiling enabled inside an observe-only physical fetch retains its 70 actual tokens", async () => { + const { ledger, tracker, budget, enable, lines } = policyActivationFixture(); + let wires = 0; + const telemetry: number[] = []; + await fetchWithTransientRetry(async () => { wires++; enable(); return new Response(); }, { + attempts: 1, onSendsConsumed: createPhysicalSendReporter(budget, () => ({ poolId: "P" }), undefined, count => telemetry.push(count)), + }); + tracker.settle({ inputTokens: 70, outputTokens: 0 }); + expect(wires).toBe(1); + expect(budget.used).toBe(1); + expect(budget.physicalStarted).toBe(1); + expect(telemetry.reduce((total, count) => total + count, 0)).toBe(1); + expect(ledger.snapshot("pool", "P")).toMatchObject({ settled: 70, reserved: 0, unresolved: 0 }); + expect(lines.length).toBeGreaterThan(0); +}); + +for (const ceiling of [15, 50]) test(`activation applies NORMAL admission to the next start at ceiling ${ceiling}`, async () => { + const { ledger, tracker, budget, enable } = policyActivationFixture(); + let wires = 0; + const result = fetchWithTransientRetry(async () => { + if (++wires === 1) { enable(ceiling); return new Response(null, { status: 503 }); } + return new Response(); + }, { attempts: 2, onSendsConsumed: createPhysicalSendReporter(budget, () => ({ poolId: "P" })) }); + if (ceiling === 15) await expect(result).rejects.toThrow(SendBudgetExhaustedError); + else await result; + tracker.settle({ inputTokens: 70, outputTokens: 0 }); + expect(wires).toBe(ceiling === 15 ? 1 : 2); + expect(budget.used).toBe(wires); + expect(budget.physicalStarted).toBe(wires); + expect(ledger.snapshot("pool", "P")).toMatchObject({ settled: 70, reserved: 0, unresolved: ceiling === 15 ? 0 : 10 }); +}); + +test("the physical L includes starts made before policy activation", async () => { + const { tracker, budget, enable } = policyActivationFixture(1); + let wires = 0; + await expect(fetchWithTransientRetry(async () => { + wires++; enable(); return new Response(null, { status: 503 }); + }, { attempts: 2, onSendsConsumed: createPhysicalSendReporter(budget, () => ({ poolId: "P" })) })).rejects.toThrow(SendBudgetExhaustedError); + tracker.settle({ inputTokens: 70 }); + expect(wires).toBe(1); + expect(budget.physicalStarted).toBe(1); +}); + +test("an off-mode prepaid reservation converts through NORMAL admission without duplicate liability", async () => { + const { ledger, tracker, budget, enable } = policyActivationFixture(); + const decision = budget.reserveDispatch({ sendClass: "initial", targetKey: "P" }); + if (!decision.allowed) throw new Error("fixture reservation denied"); + enable(); + expect(decision.permit.use()).toBe(true); + tracker.settle({ inputTokens: 7 }); + expect(budget.used).toBe(1); + expect(ledger.snapshot("pool", "P")).toMatchObject({ settled: 7, reserved: 0, unresolved: 0 }); +}); + +test("capacity-full observe-only actual usage remains debt when enforcement activates", async () => { + const { ledger, tracker, budget, enable } = policyActivationFixture(4, 1); + expect(ledger.reserve({ sendId: "occupant", scopes: { poolId: "P" }, inputTokens: 1, outputCeilingTokens: 0 }).reserved).toBe(true); + await fetchWithTransientRetry(async () => { enable(); return new Response(); }, { + attempts: 1, onSendsConsumed: createPhysicalSendReporter(budget, () => ({ poolId: "P" })), + }); + tracker.settle({ inputTokens: 70 }); + expect(budget.used).toBe(1); + expect(ledger.abandon("occupant")).toBe(true); + ledger.prune(Date.now()); + expect(ledger.snapshot("pool", "P")).toMatchObject({ settled: 70, reserved: 0, unresolved: 0 }); +}); + +test("an entirely observe-only capacity omission keeps terminal usage off the previous send", async () => { + const { ledger, tracker, budget, lines } = policyActivationFixture(1, 1); + const report = createPhysicalSendReporter(budget, () => ({ poolId: "P" })); + let wires = 0; + await fetchWithTransientRetry(async () => ++wires === 1 ? new Response(null, { status: 503 }) : new Response(), { + attempts: 2, onSendsConsumed: report, + }); + tracker.settle({ inputTokens: 70 }); + expect(lines.some(line => JSON.parse(line).kind === "forget")).toBe(false); + expect(wires).toBe(2); // L is not newly enforced for a wholly observe-only request. + expect(budget.used).toBe(2); + expect(ledger.snapshot("pool", "P")).toMatchObject({ settled: 0, reserved: 0, unresolved: 10 }); +}); + +test("activation before same-target selection rebind requires NORMAL admission before wire", async () => { + const { ledger, tracker, budget, enable } = policyActivationFixture(); + expect(ledger.reserve({ sendId: "prior", scopes: { poolId: "P" }, inputTokens: 50, outputCeilingTokens: 0 }).reserved).toBe(true); + ledger.settle("prior", { inputTokens: 50, outputTokens: 0 }); + let wires = 0; + await expect(fetchWithTransientRetry(async () => { + enable(); + if (!rebindPhysicalSend(budget, { poolId: "P" })) throw new SendBudgetExhaustedError(); + wires++; return new Response(); + }, { attempts: 1, onSendsConsumed: createPhysicalSendReporter(budget, () => ({ poolId: "P" })) })).rejects.toThrow(SendBudgetExhaustedError); + tracker.settle(undefined); + expect(wires).toBe(0); + expect(ledger.snapshot("pool", "P")).toMatchObject({ settled: 50, reserved: 0, unresolved: 0 }); +}); + +test("an adapter permit started observe-only retains liability before an enforced second send", async () => { + const { ledger, tracker, budget, enable } = policyActivationFixture(); + for (const sendClass of ["initial", "transient"] as const) { + const decision = budget.reserveDispatch({ sendClass, targetKey: "P" }); + if (!decision.allowed) throw new Error("fixture reservation denied"); + expect(decision.permit.use()).toBe(true); + enable(); + } + tracker.settle({ inputTokens: 70 }); + expect(budget.used).toBe(2); + expect(budget.physicalStarted).toBe(2); + expect(ledger.snapshot("pool", "P")).toMatchObject({ settled: 70, reserved: 0, unresolved: 10 }); +}); + +test("an older off-mode pending proof stays refundable after another helper enables seeds", async () => { + const { ledger, tracker, budget, enable } = policyActivationFixture(); + const reserved = budget.reserveDispatch({ sendClass: "initial", targetKey: "P", countedExternally: true }); + if (!reserved.allowed) throw new Error("fixture reservation denied"); + enable(); + await fetchWithTransientRetry(async () => new Response(), { + attempts: 1, onSendsConsumed: createPhysicalSendReporter(budget, () => ({ poolId: "P" })), + }); + reserved.permit.release(); + tracker.settle({ inputTokens: 70 }); + expect(ledger.snapshot("pool", "P")).toMatchObject({ settled: 70, reserved: 0, unresolved: 0 }); +}); + +for (const legacyLast of [false, true]) test(`mixed direct legacy charge and physical send settle in dispatch order (${legacyLast})`, async () => { + const { ledger, tracker, budget, enable } = policyActivationFixture(); + if (!legacyLast) tracker.charge({ alreadySent: true }); + enable(); + await fetchWithTransientRetry(async () => new Response(), { + attempts: 1, onSendsConsumed: createPhysicalSendReporter(budget, () => ({ poolId: "P", identityId: "B" })), + }); + if (legacyLast) tracker.charge({ alreadySent: true }); + tracker.settle({ inputTokens: 70 }); + expect(ledger.snapshot("pool", "P")).toMatchObject({ settled: 70, reserved: 0, unresolved: 10 }); + expect(ledger.snapshot("identity", "A")).toMatchObject({ settled: legacyLast ? 70 : 0, unresolved: legacyLast ? 0 : 10 }); + expect(ledger.snapshot("identity", "B")).toMatchObject({ settled: legacyLast ? 0 : 70, unresolved: legacyLast ? 10 : 0 }); +}); + +for (const dispatched of [false, true]) test(`a mixed old pending proof ${dispatched ? "dispatches" : "remains conservative"} after seeding`, async () => { + const { ledger, tracker, budget, enable } = policyActivationFixture(); + let proof: Parameters>[0] | undefined; + tracker.charge({ deferDispatch: true, onReserved: value => { proof = value; } }); + enable(); + await fetchWithTransientRetry(async () => new Response(), { + attempts: 1, onSendsConsumed: createPhysicalSendReporter(budget, () => ({ poolId: "P", identityId: "B" })), + }); + if (dispatched) tracker.dispatch!(proof!); + tracker.settle({ inputTokens: 70 }); + expect(ledger.snapshot("pool", "P")).toMatchObject({ settled: 70, reserved: 0, unresolved: 10 }); + expect(ledger.snapshot("identity", "A")).toMatchObject({ settled: dispatched ? 70 : 0, unresolved: dispatched ? 0 : 10 }); + expect(ledger.snapshot("identity", "B")).toMatchObject({ settled: dispatched ? 0 : 70, unresolved: dispatched ? 10 : 0 }); +}); + +for (const mixed of [false, true]) for (const ownerError of [false, true]) test(`activated observed terminal debt survives ${ownerError ? "owner throw" : "append failure"} (${mixed ? "mixed" : "ordinary"})`, async () => { + const journal = spendTestJournal(); + const append = journal.append.bind(journal); + let throwOwner = false; + journal.append = line => { + if (throwOwner) throw new SpendLedgerOwnerError("SPEND_LEDGER_OWNER_NOT_HELD", "fixture"); + append(line); + }; + const policy = spendTestPolicy({ pool: {}, canonicalProviderIds: ["P"] }); + const ledger = createSpendReservationLedger({ journal, policy, salt: testSpendSalt, now: () => 2 }); + const tracker = createRequestSpendTracker({ provider: "P", spendInputEstimateTokens: 10 }, undefined, ledger); + const budget = createRequestExecutionBudget(undefined, undefined, tracker); + await fetchWithTransientRetry(async () => { + ledger.reconfigure({ ...policy, pool: { maxTokens: 50 } }); return new Response(); + }, { attempts: 1, onSendsConsumed: createPhysicalSendReporter(budget, () => ({ poolId: "P" })) }); + if (mixed) await fetchWithTransientRetry(async () => new Response(), { + attempts: 1, onSendsConsumed: createPhysicalSendReporter(budget, () => ({ poolId: "P" })), + }); + if (ownerError) throwOwner = true; + else journal.failAppend = true; + if (ownerError) expect(() => tracker.settle({ inputTokens: 70 })).toThrow(SpendLedgerOwnerError); + else tracker.settle({ inputTokens: 70 }); + throwOwner = false; journal.failAppend = false; + ledger.prune(); + const expected = { settled: 70, reserved: 0, unresolved: mixed ? 10 : 0 }; + expect(ledger.snapshot("pool", "P")).toMatchObject(expected); + for (const create of [createShippedSpendLedger, createSpendReservationLedger]) { + expect(create({ journal: spendTestJournal(journal.lines), policy, salt: testSpendSalt, now: () => 2 }).snapshot("pool", "P")).toMatchObject(expected); + } + const kinds = journal.lines.map(line => JSON.parse(line).kind); + expect(kinds.filter(kind => kind === "settle")).toHaveLength(1); + expect(kinds.filter(kind => kind === "lost")).toHaveLength(mixed ? 1 : 0); + expect(kinds.filter(kind => kind === "reserve")).toHaveLength(mixed ? 2 : 1); +}); + describe("atomic dispatch permits", () => { test("two interleaved reserves for one remaining send produce exactly one permit", () => { const budget = createRequestExecutionBudget(ONE_SEND_LEFT); diff --git a/tests/lib/spend-seed-overflow.test.ts b/tests/lib/spend-seed-overflow.test.ts index 450326ed09d..cfaa84f6319 100644 --- a/tests/lib/spend-seed-overflow.test.ts +++ b/tests/lib/spend-seed-overflow.test.ts @@ -5,9 +5,11 @@ import { mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { acquireSpendLedgerServerLifecycle } from "../../src/server/index/spend-ledger-lifecycle"; -import { spendLedgerOwnerSnapshot } from "../../src/lib/spend-ledger-owner"; +import { SpendLedgerOwnerError, spendLedgerOwnerSnapshot } from "../../src/lib/spend-ledger-owner"; import { removeTreeWithRetry } from "../helpers/remove-tree"; import { createRequestSpendTracker } from "../../src/server/responses/request-spend"; +import { createRequestExecutionBudget, createPhysicalSendReporter } from "../../src/lib/request-execution-budget"; +import { fetchWithTransientRetry } from "../../src/lib/upstream-retry"; const request = (sendId: string, scopes = {poolId:"P"}, tokens=0) => ({sendId,scopes,inputTokens:tokens,outputCeilingTokens:0}); const seed = (ledger: SpendReservationLedger, id="seed", scopes={poolId:"P"}, tokens=0): SpendSeed => { const decision=ledger.reserveSeed(request(id,scopes,tokens)); expect(decision.reserved).toBe(true); @@ -290,3 +292,129 @@ test("seed cleanup checks the owning reporter count and rejects another owner", unrelated.close(); expect(ledger.reserveSeed(request("owned"), foreign)).toMatchObject({ reserved: true }); }); + +for (const lookup of ["exact", "prepaid", "scope"] as const) test(`abandoned seeds cannot be reused through ${lookup} lookup after failed forgetting`, () => { + const disk = spendTestJournal(); + const append = disk.append.bind(disk); + let failForget = true; + disk.append = line => { + if (failForget && JSON.parse(line).kind === "forget") throw new Error("forget unavailable"); + append(line); + }; + const ledger = createSpendReservationLedger({ journal: disk, salt, now: () => 2, + policy: spendTestPolicy({ canonicalProviderIds: ["P", "Q"], maxTrackedSends: 2 }) }); + const tracker = createRequestSpendTracker({ provider: "P", spendInputEstimateTokens: 10 }, undefined, ledger); + const original = tracker.ensureSeed({ poolId: "P", targetKey: "original" }); + if (!original) throw new Error("seed denied"); + expect(tracker.ensureSeed({ poolId: "Q", targetKey: "other" })).toBeDefined(); + expect(ledger.snapshot("pool", "P")?.reserved).toBe(0); + const target = { poolId: "P", ...(lookup === "exact" ? { targetKey: "original" } : lookup === "prepaid" ? { targetKey: "next" } : {}) }; + const proof = lookup === "prepaid" ? { ledger, sendId: original.sendId } : undefined; + // Both retired anchors remain pinned while forgetting fails: no new NORMAL admission fits. + expect(tracker.ensureSeed(target, proof)).toBeUndefined(); + const reporter = tracker.beginReporter(); + expect(() => reporter.start(original, 1)).toThrow("Invalid spend reporter start"); + failForget = false; + const fresh = tracker.ensureSeed(target, proof); + expect(fresh).toBeDefined(); + expect(fresh).not.toBe(original); + if (!fresh) throw new Error("recovered seed denied"); + expect(ledger.knows(original.sendId)).toBe(false); + expect(ledger.snapshot("pool", "P")?.reserved).toBe(10); + reporter.start(fresh, 1); + reporter.report(1); + reporter.close(); + tracker.settle({ inputTokens: 70, outputTokens: 0 }); + expect(ledger.snapshot("pool", "P")).toMatchObject({ settled: 70, reserved: 0, unresolved: 0 }); + expect(createShippedSpendLedger({ journal: disk, salt, now: () => 2 }).snapshot("pool", "P")?.settled).toBe(70); +}); + +test("refunded seed with failed forgetting refuses the retry helper before fetch and recovers NORMAL admission", async () => { + const disk = spendTestJournal(); + const append = disk.append.bind(disk); + let failForget = true; + disk.append = line => { + if (failForget && JSON.parse(line).kind === "forget") throw new Error("forget unavailable"); + append(line); + }; + const ledger = factory(disk); + const tracker = createRequestSpendTracker({ provider: "P", spendInputEstimateTokens: 10 }, undefined, ledger); + const anchor = tracker.ensureSeed({ poolId: "P" }); + if (!anchor) throw new Error("seed denied"); + tracker.refund({ ledger, sendId: anchor.sendId }); + const budget = createRequestExecutionBudget(undefined, undefined, tracker); + let wires = 0; + const send = () => fetchWithTransientRetry(async () => { wires++; return new Response(); }, { + attempts: 1, onSendsConsumed: createPhysicalSendReporter(budget, () => ({ poolId: "P" })), + }); + await expect(send()).rejects.toThrow(); + expect(wires).toBe(0); + expect(ledger.knows(anchor.sendId)).toBe(true); + failForget = false; + await send(); + expect(wires).toBe(1); + expect(ledger.knows(anchor.sendId)).toBe(false); + tracker.settle({ inputTokens: 70, outputTokens: 0 }); + expect(createShippedSpendLedger({ journal: disk, salt, now: () => 2 }).snapshot("pool", "P")?.settled).toBe(70); +}); + +test("retired seed cleanup failure cannot postpone active terminal liability", () => { + const disk = spendTestJournal(); + const append = disk.append.bind(disk); + let failForget = true; + disk.append = line => { + if (failForget && JSON.parse(line).kind === "forget") throw new Error("forget unavailable"); + append(line); + }; + const ledger = createSpendReservationLedger({ journal: disk, salt, now: () => 2, + policy: spendTestPolicy({ canonicalProviderIds: ["P", "Q"], maxTrackedSends: 2 }) }); + const tracker = createRequestSpendTracker({ provider: "P", spendInputEstimateTokens: 1 }, undefined, ledger); + const retired = tracker.ensureSeed({ poolId: "P" }); + const active = tracker.ensureSeed({ poolId: "Q" }); + if (!retired || !active) throw new Error("seed denied"); + const reporter = tracker.beginReporter(); + reporter.start(active, 1); + reporter.close(); + tracker.settle({ inputTokens: 70, outputTokens: 0 }); + expect(ledger.snapshot("pool", "Q")).toMatchObject({ settled: 70, reserved: 0, unresolved: 0 }); + expect(ledger.knows(retired.sendId)).toBe(true); + expect(ledger.knows(active.sendId)).toBe(true); + failForget = false; + expect(ledger.reserveSeed(request("after-recovery", { poolId: "Q" }, 1))).toMatchObject({ reserved: true }); + expect(ledger.knows(retired.sendId)).toBe(false); + expect(ledger.knows(active.sendId)).toBe(false); + expect(createShippedSpendLedger({ journal: disk, salt, now: () => 2 }).snapshot("pool", "Q")?.settled).toBe(70); +}); + +for (const retirement of ["rebind", "refund"] as const) { + for (const recovery of ["settle", "prune"] as const) { + test(`${retirement} retains seed cleanup after owner error until ${recovery} recovery`, () => { + const disk = spendTestJournal(); + const append = disk.append.bind(disk); + const ownerError = new SpendLedgerOwnerError("SPEND_LEDGER_OWNER_UNAVAILABLE", "injected unsafe journal"); + let failForget = true; + disk.append = line => { + if (failForget && JSON.parse(line).kind === "forget") throw ownerError; + append(line); + }; + const ledger = createSpendReservationLedger({ journal: disk, salt, now: () => 2, + policy: spendTestPolicy({ canonicalProviderIds: ["P", "Q"], maxTrackedSends: 1 }) }); + const tracker = createRequestSpendTracker({ provider: "P", spendInputEstimateTokens: 1 }, undefined, ledger); + const anchor = tracker.ensureSeed({ poolId: "P" }); + if (!anchor) throw new Error("seed denied"); + expect(() => retirement === "rebind" ? tracker.ensureSeed({ poolId: "Q" }) + : tracker.refund({ ledger, sendId: anchor.sendId })).toThrow(ownerError); + expect(disk.lines.map(line => JSON.parse(line).kind)).toEqual(["reserve", "abandon"]); + expect(ledger.knows(anchor.sendId)).toBe(true); + expect(ledger.snapshot("pool", "P")).toMatchObject({ settled: 0, reserved: 0, unresolved: 0 }); + failForget = false; + if (recovery === "settle") tracker.settle(undefined); + ledger.prune(); + expect(ledger.knows(anchor.sendId)).toBe(false); + expect(ledger.reserveSeed(request("next", { poolId: "Q" }, 1))).toMatchObject({ reserved: true }); + expect(disk.lines.filter(line => JSON.parse(line).kind === "forget")).toHaveLength(1); + expect(createShippedSpendLedger({ journal: disk, salt, now: () => 2 }).snapshot("pool", "P")) + .toMatchObject({ settled: 0, reserved: 0, unresolved: 0 }); + }); + } +} diff --git a/tests/server/inference-send-budget.test.ts b/tests/server/inference-send-budget.test.ts index 49012d184fb..91bb2232b03 100644 --- a/tests/server/inference-send-budget.test.ts +++ b/tests/server/inference-send-budget.test.ts @@ -225,7 +225,7 @@ test("checking an unconfigured ingress budget does not open its journal", async const { createPhysicalSendReporter } = await import("../../src/lib/request-execution-budget"); const budget = createInferenceSendBudget(new Request("http://localhost/v1/responses"), { model: "m", provider: "p" }); expect(budget.spendEnforced).toBe(false); - expect(createPhysicalSendReporter(budget, () => ({ poolId: "p" })).beforeSend).toBeUndefined(); + expect(createPhysicalSendReporter(budget, () => ({ poolId: "p" })).beforeSend).toBeFunction(); expect(existsSync(join(home, "spend-ledger.salt"))).toBe(false); expect(existsSync(join(home, "spend-ledger.jsonl"))).toBe(false); }); From 46f7418a0af1bf18181697f06ab339fceac3da64 Mon Sep 17 00:00:00 2001 From: JUN Date: Fri, 9 Oct 2026 02:03:14 +0900 Subject: [PATCH 13/15] fix(spend): freeze request policy at first admission Co-authored-by: Epinephrine --- .../docs/fr/reference/configuration/server.md | 4 +- .../docs/ja/reference/configuration/server.md | 9 +- .../docs/ko/reference/configuration/server.md | 9 +- .../docs/reference/configuration/server.md | 12 +- .../docs/ru/reference/configuration/server.md | 10 +- .../docs/tr/reference/configuration/server.md | 4 +- .../zh-cn/reference/configuration/server.md | 8 +- .../zh-tw/reference/configuration/server.md | 4 +- src/lib/request-execution-budget.ts | 71 ++-- src/lib/spend-reservation-ledger.ts | 71 ++-- src/lib/workflow-budget.ts | 18 +- src/server/chat-native.ts | 4 +- src/server/messages-native.ts | 12 +- src/server/responses/adapter-continuation.ts | 3 + src/server/responses/adapter-dispatch.ts | 8 + src/server/responses/request-send-budget.ts | 6 +- src/server/responses/request-spend.ts | 152 +++------ structure/runtime.md | 2 +- structure/transports/responses-spend.md | 35 +- .../messages-native.test.ts | 47 ++- tests/lib/execution-budget-permits.test.ts | 311 ++++++++---------- tests/responses/chat-native-spend.test.ts | 69 +++- .../responses/responses-spend-anchor.test.ts | 120 +++++++ .../responses-spend-capacity-guard.test.ts | 21 +- 24 files changed, 614 insertions(+), 396 deletions(-) diff --git a/docs-site/src/content/docs/fr/reference/configuration/server.md b/docs-site/src/content/docs/fr/reference/configuration/server.md index debf3946fa7..14bbc651b04 100644 --- a/docs-site/src/content/docs/fr/reference/configuration/server.md +++ b/docs-site/src/content/docs/fr/reference/configuration/server.md @@ -301,7 +301,9 @@ Un historique inconnu et inactif expire seulement quand sa dernière activité e ### Réservations et limite d’envoi -Lorsqu’un plafond de racine, d’identité ou de pool s’applique, le premier envoi pour chaque cible ou clé nécessite une réservation dans la capacité normale de suivi. Sans réservation, aucun envoi n’atteint le fournisseur. Les nouvelles tentatives stables réutilisent les mêmes périmètres ; une autre cible ou clé nécessite une nouvelle réservation. `L` est la limite des envois physiques pour l’ensemble de la requête, figée au premier envoi physique : quatre par défaut, ou jusqu’à dix-huit avec l’extension OAuth existante appliquée avant tout envoi. +Lorsqu’un plafond de racine, d’identité ou de pool s’applique, le premier envoi pour chaque cible ou clé nécessite une réservation dans la capacité normale de suivi. Sans réservation, aucun envoi n’atteint le fournisseur. Les nouvelles tentatives stables réutilisent les mêmes périmètres ; une autre cible ou clé nécessite une nouvelle réservation. `L` est la limite des envois physiques pour l’ensemble de la requête, figée au démarrage de la requête : quatre par défaut, ou jusqu’à dix-huit avec le profil de requête OAuth existant. + +L’activation des limites, les plafonds de jetons applicables à la racine, à l’identité et au pool, ainsi que `L`, sont déterminés au démarrage de chaque requête. Les changements de configuration s’appliquent uniquement aux requêtes démarrées après le changement. Une requête déjà en cours garde sa politique initiale pour toutes les nouvelles tentatives et continuations : activer ou abaisser un plafond ne la restreint pas davantage ; le relever ou le supprimer ne l’assouplit pas. Une requête commencée en observation seule reste dans ce mode jusqu’à sa fin. Le règlement final attend les rapports des envois démarrés. Un ID d’envoi n’est oublié qu’après enregistrement durable de sa comptabilité ; son solde demeure. La limite borne les envois, pas la facture : toute utilisation réelle supérieure à l’estimation reste comptabilisée. Sans plafond applicable, le comportement reste en observation seule, y compris l’omission des écritures lorsque la capacité de suivi est pleine. Aucun point de contrôle d’identité n’est ajouté. diff --git a/docs-site/src/content/docs/ja/reference/configuration/server.md b/docs-site/src/content/docs/ja/reference/configuration/server.md index dfe859a7d07..ba9807287fb 100644 --- a/docs-site/src/content/docs/ja/reference/configuration/server.md +++ b/docs-site/src/content/docs/ja/reference/configuration/server.md @@ -222,8 +222,13 @@ ID の前後に空白を付けないでください。 休眠中の所有先不明の履歴は、最終活動時刻が `spend.retentionDays` の期限より古くなってから失効します。 容量不足を理由に正の履歴を早く削除しません。送信中の予約と seed はゼロトークンでも保持し、削除記録の保存後に残高を減らします。 適用される上限がある送信は、各送信先・キーの初回に通常容量の予約を必要とします。 -再試行は同じ範囲を使います。`L` は要求全体の物理送信回数上限で、最初の物理送信時に固定します。 -通常は 4 回で、既存の OAuth 拡張は送信開始前に最大 18 回まで拡大できます。すべての物理送信がこの有限の上限を共有します。 +再試行は同じ範囲を使います。`L` は要求全体の物理送信回数上限で、要求開始時に固定します。 +通常は 4 回で、既存の OAuth 要求プロファイルでは最大 18 回です。すべての物理送信がこの有限の上限を共有します。 + +制限を適用するかどうか、適用対象のルート・識別子・プールのトークン上限、および `L` は要求開始時に確定します。 +設定変更は、変更後に開始する要求だけに適用します。実行中の要求は再試行・継続も含め、開始時のポリシーを維持します。 +上限の有効化・引き下げで実行中の要求を厳しくしたり、引き上げ・削除で緩和したりしません。 +監視のみで始まった要求は、完了まで監視のみのままです。 最終精算は報告完了を待ち、送信 ID の削除後も残高を保持します。上限未設定では従来の監視のみの動作を保ち、容量満杯時の記録省略も変わりません。 Claude CLI、CodeBuddy、Qoder は CLI 呼び出し 1 回を送信 1 回と数え、起動前に通常の予約を必要とします。 diff --git a/docs-site/src/content/docs/ko/reference/configuration/server.md b/docs-site/src/content/docs/ko/reference/configuration/server.md index 6a8fdd00217..eeabdb41dab 100644 --- a/docs-site/src/content/docs/ko/reference/configuration/server.md +++ b/docs-site/src/content/docs/ko/reference/configuration/server.md @@ -280,8 +280,13 @@ ID 앞뒤에 공백을 넣지 마세요. 용량이 부족해도 양수 이력을 먼저 지우지 않습니다. 진행 중인 예약과 seed는 0토큰이어도 보존하며, 삭제 기록이 저장된 뒤에만 허용 여부를 판단하는 잔액을 줄입니다. 한도가 적용되는 요청은 대상·키별 첫 전송 전에 일반 추적 용량으로 예약해야 합니다. 재시도는 같은 범위를 씁니다. -`L`은 요청 전체가 공유하는 물리 전송 횟수 한도이며 첫 물리 전송 때 고정됩니다. 기본은 4회이고, -기존 OAuth 확장은 전송 시작 전에 최대 18회까지 늘릴 수 있습니다. 모든 물리 전송이 이 유한한 한도를 함께 씁니다. +`L`은 요청 전체가 공유하는 물리 전송 횟수 한도이며 요청이 시작할 때 고정됩니다. 기본은 4회이고, +기존 OAuth 요청 프로필은 최대 18회까지 허용합니다. 모든 물리 전송이 이 유한한 한도를 함께 씁니다. + +한도 적용 여부, 적용할 루트·계정·제공자 풀의 토큰 한도, `L`은 요청이 시작할 때 정합니다. +설정을 바꾸면 변경 후 시작하는 요청에만 적용됩니다. 이미 실행 중인 요청은 재시도와 후속 전송에서도 시작할 때의 정책을 유지합니다. +한도를 켜거나 낮춰도 실행 중인 요청을 더 제한하지 않고, 높이거나 없애도 더 허용하지 않습니다. +관측 모드로 시작한 요청은 끝날 때까지 관측 모드를 유지합니다. 최종 정산은 보고가 끝날 때까지 기다립니다. 전송 ID를 지워도 잔액은 남습니다. 적용되는 한도가 없으면 기존 관측 동작을 유지하며, 추적 용량이 찼을 때 기록을 생략하는 동작도 그대로입니다. diff --git a/docs-site/src/content/docs/reference/configuration/server.md b/docs-site/src/content/docs/reference/configuration/server.md index 57626d78562..b9d367a2a32 100644 --- a/docs-site/src/content/docs/reference/configuration/server.md +++ b/docs-site/src/content/docs/reference/configuration/server.md @@ -29,7 +29,7 @@ runs helper features around provider requests. | `usageLedgerMaxBytes?` | `number` | unset | Opt-in ceiling in bytes for `usage.jsonl`. Absent means the request history grows without limit, which stays the default. See [usage history size](#usage-history-size). | | `appOwnedMemoryBudgetMb?` | `number` | `256` | Cap in MiB for evictable app-owned logs, caches, blobs, and continuation payloads. Range 64–4096; not an RSS cap. | | `metricsExport.enabled?` | `boolean` | `false` | Enable process-local aggregate request metrics at authenticated `GET /api/metrics`. Restart required; disabled mode returns 404 and starts no exporter activity. | -| `spend?` | `{ root?: { maxTokens?: number }; identity?: { maxTokens?: number }; pool?: { maxTokens?: number }; retentionDays?: number }` | unset | Durable token ceilings, off unless you write one. Each scope bounds settled spend plus in-flight reservations plus unresolved spend: `root` is one task including its whole fan-out, `identity` is one account across every task it serves, and `pool` is one provider pool. They intersect, so a request is admitted only when all three have room — which is what holds a ceiling against a client that mints a new task id per request. A reservation is the request's whole input plus its enforceable output ceiling, counted as if every cached prefix misses. Observe-only mode still journals, so every server owns the state directory's single-writer lease; an explicit sibling must use a separate `OPENCODEX_HOME`. Spend survives an ordinary process restart when its writes reached the filesystem, but the journal does not promise survival across host power loss because each append is not fsynced. Raising or removing the value is what grants more. `maxTokens` must be a positive integer (0 would refuse everything), `retentionDays` is 1–365 and defaults to 7, and an unknown key in this section is rejected rather than ignored. A refusal is a local HTTP 429 carrying `x-opencodex-local-refusal: workflow_spend_exhausted`, and its message names the scope and the ceiling; no provider is contacted. With an applicable ceiling, dispatch is also refused if its token reservation cannot be booked, including full tracking capacity. Requests without an applicable ceiling remain observe-only. | +| `spend?` | `{ root?: { maxTokens?: number }; identity?: { maxTokens?: number }; pool?: { maxTokens?: number }; retentionDays?: number }` | unset | Durable token ceilings, off unless you write one. Each scope bounds settled spend plus in-flight reservations plus unresolved spend: `root` is one task including its whole fan-out, `identity` is one account across every task it serves, and `pool` is one provider pool. They intersect, so a request is admitted only when all three have room — which is what holds a ceiling against a client that mints a new task id per request. A reservation is the request's whole input plus its enforceable output ceiling, counted as if every cached prefix misses. Observe-only mode still journals, so every server owns the state directory's single-writer lease; an explicit sibling must use a separate `OPENCODEX_HOME`. Spend survives an ordinary process restart when its writes reached the filesystem, but the journal does not promise survival across host power loss because each append is not fsynced. Raising or removing a ceiling changes the allowance for requests started afterward; requests already running keep their starting policy. `maxTokens` must be a positive integer (0 would refuse everything), `retentionDays` is 1–365 and defaults to 7, and an unknown key in this section is rejected rather than ignored. A refusal is a local HTTP 429 carrying `x-opencodex-local-refusal: workflow_spend_exhausted`, and its message names the scope and the ceiling; no provider is contacted. With an applicable ceiling, dispatch is also refused if its token reservation cannot be booked, including full tracking capacity. Requests without an applicable ceiling remain observe-only. | | `codexAutoStart?` | `boolean` | `true` | Let the Codex shim run `ocx ensure` before launching Codex. False makes ensure a no-op. | | `codexShimAutoRestore?` | `boolean` | `true` | Restore an installed shim after a completed external Codex update replaces it. Environment opt-out: `OPENCODEX_CODEX_SHIM_AUTO_RESTORE=0`. | @@ -982,10 +982,16 @@ active/exhausted protections. No automatic tool reconstructs unverifiable mappin When a root, identity or pool ceiling applies, the first physical send for each selected target/key requires a normal-capacity reservation. If that cannot be recorded, the send is refused before it reaches upstream. Stable retries reuse that reservation's scopes; a different target/key needs a -new one. `L` is the request-wide physical-send limit, frozen at the first physical start -(default four; the existing bounded OAuth profile can expand to eighteen before sending). +new one. `L` is the request-wide physical-send limit, frozen when the request starts +(default four; the existing bounded OAuth request profile allows up to eighteen). Every additional physical send shares this finite limit. +The enforcement state, applicable root/identity/pool token ceilings, and `L` are captured when each +request starts. Configuration changes apply only to requests started after the change. Requests +already running keep their starting policy for all retries and continuations: enabling or lowering +a ceiling does not tighten them, and raising or removing a ceiling does not relax them. A request +that started in observe-only mode remains observe-only until it finishes. + Delayed reports retain already-started usage. Terminal settlement waits for reporters, and send IDs are forgotten only after their accounting is durable; forgetting an ID does not remove its balance. The limit bounds retries, not the final bill: actual usage or already-started retries can exceed a diff --git a/docs-site/src/content/docs/ru/reference/configuration/server.md b/docs-site/src/content/docs/ru/reference/configuration/server.md index 81d6e635500..9795c728847 100644 --- a/docs-site/src/content/docs/ru/reference/configuration/server.md +++ b/docs-site/src/content/docs/ru/reference/configuration/server.md @@ -274,8 +274,14 @@ The dashboard warns about old or unknown CLI versions, unavailable targets, and Активные резервирования и seed сохраняются даже при нулевой оценке; уменьшение баланса требует сначала записать удаление в журнал. При применимом лимите первая отправка для каждой цели/ключа требует обычного резервирования. Повторы используют те же области. `L` — общий лимит физических отправок -одного запроса, фиксируемый при первой физической отправке. Обычно он равен 4; существующее расширение -OAuth может увеличить его до 18 до начала отправок. Все физические отправки делят этот конечный лимит. +одного запроса, фиксируемый при запуске запроса. Обычно он равен 4; существующий профиль запроса OAuth +допускает до 18. Все физические отправки делят этот конечный лимит. + +Режим применения ограничений, действующие токенные лимиты корня, идентичности и пула, а также `L` +фиксируются при запуске запроса. Изменение конфигурации применяется только к запросам, начатым после него. +Уже выполняющийся запрос сохраняет исходную политику для всех повторов и продолжений: включение или +снижение лимита не ужесточает её, а повышение или удаление не ослабляет. Запрос, начатый в режиме +наблюдения, остаётся в этом режиме до завершения. Окончательный расчёт ждёт отчётов; удаление ID отправки не удаляет баланс. Без применимого лимита сохраняется режим наблюдения, включая пропуск записей при заполненной ёмкости отслеживания. diff --git a/docs-site/src/content/docs/tr/reference/configuration/server.md b/docs-site/src/content/docs/tr/reference/configuration/server.md index 33734ad82f3..29b7666b5c7 100644 --- a/docs-site/src/content/docs/tr/reference/configuration/server.md +++ b/docs-site/src/content/docs/tr/reference/configuration/server.md @@ -332,7 +332,9 @@ Etkin olmayan belirsiz geçmiş, yalnızca son etkinliği `spend.retentionDays` ### Rezervasyonlar ve gönderim sınırı -Kök, kimlik veya havuz sınırı geçerliyse her hedef ya da anahtarın ilk gönderimi normal izleme kapasitesinden bir rezervasyon gerektirir. Rezervasyon alınamazsa sağlayıcıya gönderim yapılmaz. Aynı hedefteki yeniden denemeler aynı kapsamları kullanır; farklı hedef veya anahtar yeni rezervasyon gerektirir. `L`, ilk fiziksel gönderimde sabitlenen, isteğin tamamı için fiziksel gönderim sınırıdır. Varsayılan değer dörttür; mevcut OAuth genişletmesi gönderimler başlamadan önce bunu en fazla on sekize çıkarabilir. +Kök, kimlik veya havuz sınırı geçerliyse her hedef ya da anahtarın ilk gönderimi normal izleme kapasitesinden bir rezervasyon gerektirir. Rezervasyon alınamazsa sağlayıcıya gönderim yapılmaz. Aynı hedefteki yeniden denemeler aynı kapsamları kullanır; farklı hedef veya anahtar yeni rezervasyon gerektirir. `L`, istek başladığında sabitlenen, isteğin tamamı için fiziksel gönderim sınırıdır. Varsayılan değer dörttür; mevcut OAuth istek profili en fazla on sekiz gönderime izin verir. + +Sınırların uygulanıp uygulanmayacağı, geçerli kök, kimlik ve havuz belirteç sınırları ile `L`, her isteğin başlangıcında belirlenir. Yapılandırma değişiklikleri yalnızca değişiklikten sonra başlayan istekleri etkiler. Devam eden istekler, tüm yeniden denemeler ve devam çağrıları için başlangıç politikalarını korur: bir sınırı etkinleştirmek veya düşürmek onları daha fazla kısıtlamaz; yükseltmek veya kaldırmak da daha fazla izin vermez. Yalnızca gözlem modunda başlayan bir istek, tamamlanana kadar bu modda kalır. Son hesaplaşma, başlamış gönderimlerin raporlarını bekler. Bir gönderim kimliği ancak muhasebesi kalıcılaştıktan sonra unutulur; bakiyesi silinmez. Sınır faturayı değil gönderim sayısını sınırlar: ilk tahmini aşan gerçek kullanımın tamamı kaydedilir. Geçerli bir harcama sınırı yoksa yalnızca gözlem davranışı sürer; izleme kapasitesi dolduğunda kayıtların atlanması da değişmez. Kimlik bağlantısı içeren yeni kontrol noktaları eklenmez. diff --git a/docs-site/src/content/docs/zh-cn/reference/configuration/server.md b/docs-site/src/content/docs/zh-cn/reference/configuration/server.md index bb039332a8d..1027288da9f 100644 --- a/docs-site/src/content/docs/zh-cn/reference/configuration/server.md +++ b/docs-site/src/content/docs/zh-cn/reference/configuration/server.md @@ -234,8 +234,12 @@ The dashboard warns about old or unknown CLI versions, unavailable targets, and 容量压力不会提前删除正余额的未知历史。活动预留和 seed 即使为零令牌也会保留； 只有删除记录持久化后，准入计算才使用减少后的余额。 有适用额度的请求必须在每个目标/密钥首次发送前取得普通容量的预留。 -重试沿用相同范围。`L` 是整个请求共享的物理发送次数上限，在首次物理发送时固定。 -默认是 4 次，现有 OAuth 扩展可在发送开始前增加到最多 18 次。所有物理发送共享这一有限上限。 +重试沿用相同范围。`L` 是整个请求共享的物理发送次数上限，在请求开始时固定。 +默认是 4 次，现有 OAuth 请求配置最多允许 18 次。所有物理发送共享这一有限上限。 + +是否执行额度限制、适用的根请求／身份／池令牌额度，以及 `L`，都在请求开始时确定。 +配置变更仅影响变更后开始的请求。已经运行的请求在所有重试和后续发送中保留开始时的策略： +启用或降低额度不会收紧它，提高或删除额度也不会放宽它。以仅观察模式开始的请求会保持该模式直到结束。 最终结算等待报告完成；遗忘发送 ID 不会删除余额。无适用额度时保留原有仅观察行为，包括容量已满时省略记账。 Claude CLI、CodeBuddy 和 Qoder 将一次 CLI 调用计为一次发送，启动前仍必须取得普通预留；无法预留就拒绝启动。 diff --git a/docs-site/src/content/docs/zh-tw/reference/configuration/server.md b/docs-site/src/content/docs/zh-tw/reference/configuration/server.md index b6b2c0d3217..59b3555f3ed 100644 --- a/docs-site/src/content/docs/zh-tw/reference/configuration/server.md +++ b/docs-site/src/content/docs/zh-tw/reference/configuration/server.md @@ -245,7 +245,9 @@ The dashboard warns about old or unknown CLI versions, unavailable targets, and ### 預留及傳送次數上限 -若根請求、身分或集區額度適用，每個目標或金鑰的首次傳送都必須先取得一般追蹤容量的預留。無法預留時，不會傳送至提供者。同一目標的重試沿用原有範圍；不同目標或金鑰需要新的預留。`L` 是整個請求共用的實體傳送次數上限，在首次實體傳送時固定。預設為四次；既有 OAuth 擴充可在開始傳送前增加至最多十八次。 +若根請求、身分或集區額度適用，每個目標或金鑰的首次傳送都必須先取得一般追蹤容量的預留。無法預留時，不會傳送至提供者。同一目標的重試沿用原有範圍；不同目標或金鑰需要新的預留。`L` 是整個請求共用的實體傳送次數上限，在請求開始時固定。預設為四次；既有 OAuth 請求設定最多允許十八次。 + +是否執行額度限制、適用的根請求／身分／集區權杖額度，以及 `L`，都在請求開始時確定。設定變更只影響變更後開始的請求。已經執行中的請求在所有重試及後續傳送中保留開始時的策略：啟用或降低額度不會收緊它，提高或刪除額度也不會放寬它。以僅觀察模式開始的請求會維持該模式直到結束。 最終結算會等待已開始傳送的回報完成。只有帳務已持久化後才會遺忘傳送 ID，餘額仍會保留。上限限制的是傳送次數而非帳單，超過初始估算的實際用量仍全數計入。沒有適用額度時，維持原有的僅觀察行為，包括追蹤容量已滿時省略記帳；不會新增身分關聯檢查點。 diff --git a/src/lib/request-execution-budget.ts b/src/lib/request-execution-budget.ts index 39fcaf64c09..f481a0ee96a 100644 --- a/src/lib/request-execution-budget.ts +++ b/src/lib/request-execution-budget.ts @@ -15,7 +15,7 @@ import { AsyncLocalStorage } from "node:async_hooks"; * sends plus one alternate -- by funding the alternate from a reserve that a validated * sanitized repair can spend instead, but never both. */ -import type { SpendSeed, SpendScopes, SpendReservationProof } from "./spend-reservation-ledger"; +import type { SpendSeed, SpendScopes, SpendReservationProof, SpendAdmissionPolicy } from "./spend-reservation-ledger"; import type { TransientSendBudget } from "./upstream-retry"; export type SendClass = @@ -152,14 +152,12 @@ export interface RequestSendObserver { charge(options?: { alreadySent?: boolean; deferDispatch?: boolean; onReserved?: (proof: SpendReservationProof) => void; targetKey?: string }): boolean; /** Confirm the exact reservation once its dispatch is known. */ readonly enforced?: boolean; + readonly policyStarted?: boolean; + readonly spendAdmissionPolicy?: SpendAdmissionPolicy; + /** Capture request policy at its first admission or actual physical start. */ + startRequest?(target?: SpendScopes & { targetKey?: string }): void; ensureSeed?(target: SpendScopes & { targetKey?: string }, proof?: SpendReservationProof): SpendSeed | undefined; - beginReporter?(): { - start(seed: SpendSeed, ordinal: number): void; - startObserved?(target: SpendScopes & { targetKey?: string }, ordinal: number, proof?: SpendReservationProof): void; - rebindTarget(target: SpendScopes & { targetKey?: string }, ordinal: number): boolean; - report(sends: number): void; - close(): void; - }; + beginReporter?(): { start(seed: SpendSeed, ordinal: number): void; rebindTarget(target: SpendScopes & { targetKey?: string }, ordinal: number): boolean; report(sends: number): void; close(): void }; dispatch?(proof: SpendReservationProof): void; /** Give back a booking whose send never happened. */ refund(proof?: SpendReservationProof | null): void; @@ -174,6 +172,9 @@ export interface RequestExecutionBudget extends TransientSendBudget { readonly physicalStarted?: number; readonly physicalLimit?: number; readonly spendEnforced?: boolean; + readonly spendPolicyStarted?: boolean; + readonly spendAdmissionPolicy?: SpendAdmissionPolicy; + startRequest?(target?: SpendScopes & { targetKey?: string }): void; claimPhysicalSend?(): number | undefined; beginSpendProducer?(): { close(): void } | undefined; readonly logicalRequestId: string; @@ -298,7 +299,7 @@ export type PhysicalSendReporter = ((sends: number) => void) & { const activePhysicalReporters = new AsyncLocalStorage(); /** Selection owners rebind this helper's current start before invoking the executor. */ export function rebindPhysicalSend(budget: TransientSendBudget | undefined, target: SpendScopes & { targetKey?: string }): boolean { - if (!budget) return true; + if (!(budget as RequestExecutionBudget | undefined)?.spendEnforced) return true; return activePhysicalReporters.getStore()?.bind?.(target) ?? true; } export function createPhysicalSendReporter( @@ -325,27 +326,27 @@ export function createPhysicalSendReporter( if (counter) counter.spent += count - prepaid; telemetry?.(count); }; - if (observer?.beginReporter) report.beforeSend = () => { + if (observer?.startRequest || observer?.enforced === true) report.beforeSend = () => { if (closed) return false; + const selected = target(); + if ((budget as RequestExecutionBudget).startRequest) (budget as RequestExecutionBudget).startRequest!(selected); + else observer?.startRequest?.(selected); + if (observer?.enforced !== true) return true; producer ??= observer.beginReporter?.(); const receipt = permit && dispatchSpendProofs.get(permit); const proof = receipt?.owner === counter ? receipt?.claim() : undefined; - const selected = target(); - const enforced = observer.enforced === true; - const seed = enforced ? observer.ensureSeed?.(selected, proof) : undefined; - if (enforced && !seed) return false; + const seed = observer.ensureSeed?.(selected, proof); + if (!seed) return false; const ordinal = (budget as RequestExecutionBudget).claimPhysicalSend?.(); if (ordinal === undefined) return false; - if (seed) producer?.start(seed, ordinal); - else producer?.startObserved?.(selected, ordinal, proof); + producer?.start(seed, ordinal); currentOrdinal = ordinal; started++; return true; }; report.execute = run => activePhysicalReporters.run(report, run); report.bind = (selected: SpendScopes & { targetKey?: string }) => { - if (!producer || currentOrdinal === undefined) return true; - if (observer?.enforced && (counter?.physicalLimit === undefined || currentOrdinal > counter.physicalLimit)) return false; + if (!producer || currentOrdinal === undefined || observer?.enforced !== true) return true; return producer.rebindTarget(selected, currentOrdinal); }; report.close = () => { @@ -404,13 +405,19 @@ function createRequestExecutionBudgetWithLedger( get physicalStarted(): number { return counter.physicalStarted; }, get physicalLimit(): number | undefined { return counter.physicalLimit; }, get spendEnforced(): boolean { return observer?.enforced === true; }, + get spendPolicyStarted(): boolean { return observer?.policyStarted === true; }, + get spendAdmissionPolicy(): SpendAdmissionPolicy | undefined { return observer?.spendAdmissionPolicy; }, + startRequest(target) { + observer?.startRequest?.(target); + if (observer?.enforced && counter.physicalLimit === undefined) { + counter.physicalLimit = Number.isSafeInteger(policy.maxTotalModelSends) && policy.maxTotalModelSends > 0 ? policy.maxTotalModelSends : 0; + } + }, beginSpendProducer() { return observer?.enforced ? observer.beginReporter?.() : undefined; }, claimPhysicalSend(): number | undefined { - if (counter.physicalLimit === undefined) { - if (Number.isSafeInteger(policy.maxTotalModelSends) && policy.maxTotalModelSends > 0) counter.physicalLimit = policy.maxTotalModelSends; - else if (observer?.enforced === true) return undefined; - } - if (observer?.enforced === true && (counter.physicalLimit === undefined || counter.physicalStarted >= counter.physicalLimit)) return undefined; + budget.startRequest?.(); + if (observer?.enforced !== true || counter.physicalLimit === undefined) return undefined; + if (counter.physicalStarted >= counter.physicalLimit) return undefined; return ++counter.physicalStarted; }, get used(): number { return counter.spent; }, @@ -473,6 +480,7 @@ function createRequestExecutionBudgetWithLedger( // Consulted last, because it is the only bound here that WRITES. A ledger entry booked // for a dispatch a cheaper check above would have refused is spend this request never // makes, and it would hold those tokens against the scope until retention expired. + budget.startRequest?.({ targetKey: intent.targetKey }); let spendProof: SpendReservationProof | undefined; if (observer && !observer.charge({ targetKey: intent.targetKey, deferDispatch: true, onReserved: proof => { spendProof = proof; } })) { return { allowed: false, reason: "spend-exhausted" }; @@ -495,15 +503,13 @@ function createRequestExecutionBudgetWithLedger( let executing = false; let physicalOrdinal: number | undefined; const startPhysical = (): boolean => { - if (!observer?.beginReporter) return true; - const enforced = observer.enforced === true; - const seed = enforced ? observer.ensureSeed?.({ targetKey: intent.targetKey }, spendProof) : undefined; - if (enforced && !seed) return false; + if (observer?.enforced !== true) return true; + const seed = observer.ensureSeed?.({ targetKey: intent.targetKey }, spendProof); + if (!seed) return false; const ordinal = budget.claimPhysicalSend?.(); if (ordinal === undefined) return false; producer = observer.beginReporter?.(); - if (seed) producer?.start(seed, ordinal); - else producer?.startObserved?.({ targetKey: intent.targetKey }, ordinal, spendProof); + producer?.start(seed, ordinal); physicalOrdinal = ordinal; if (!executing) { producer?.report(1); producer?.close(); } return true; @@ -515,11 +521,8 @@ function createRequestExecutionBudgetWithLedger( if (executing) throw new Error("Dispatch permit executor already active"); executing = true; const binding: PhysicalSendReporter = () => {}; - binding.bind = (target: SpendScopes & { targetKey?: string }) => { - if (physicalOrdinal === undefined || !producer) return true; - if (observer?.enforced && (counter.physicalLimit === undefined || physicalOrdinal > counter.physicalLimit)) return false; - return producer.rebindTarget(target, physicalOrdinal); - }; + binding.bind = (target: SpendScopes & { targetKey?: string }) => physicalOrdinal === undefined || !producer + ? true : producer.rebindTarget(target, physicalOrdinal); try { return await activePhysicalReporters.run(binding, run); } finally { executing = false; diff --git a/src/lib/spend-reservation-ledger.ts b/src/lib/spend-reservation-ledger.ts index b1fede93f76..81c46733ee9 100644 --- a/src/lib/spend-reservation-ledger.ts +++ b/src/lib/spend-reservation-ledger.ts @@ -152,7 +152,11 @@ export interface SpendUsage { readonly outputTokens: number; } +/** Immutable numeric ceilings captured by one request; totals and capacity remain shared. */ +export type SpendAdmissionPolicy = Readonly>; + export interface SpendReservationRequest { + readonly admissionPolicy?: SpendAdmissionPolicy; /** Stable id of the physical send. Settlement is idempotent on this key. */ readonly sendId: string; readonly scopes: SpendScopes; @@ -654,18 +658,17 @@ export interface SpendReservationLedger { /** * Settle with real usage. Returns false when the send is unknown or already resolved -- * double settlement is as wrong as none, so a repeat call changes nothing. - * Seeded or explicitly retained terminal debt applies immediately; failed writes queue - * for ordered retry. This intent grants no reservation capacity or seed capability. + * Seeded terminal debt applies immediately; failed writes queue before durable forgetting. */ - settle(sendId: string, usage: SpendUsage, options?: { retainOnFailure: boolean }): boolean; + settle(sendId: string, usage: SpendUsage): boolean; /** * Usage never arrived. The reservation moves to unresolved spend -- it may have been * billed -- rather than being released. Idempotent on the same key as settle. */ - markLost(sendId: string, options?: { retainOnFailure: boolean }): boolean; + markLost(sendId: string): boolean; snapshot(scope: SpendScope, scopeId: string): ScopeSpendSnapshot | undefined; /** Exclude only a proven current dispatch's still-open reservation in this scope. */ - exhausted(scope: SpendScope, scopeId: string, excludingSendId?: string): boolean; + exhausted(scope: SpendScope, scopeId: string, excludingSendId?: string, admissionPolicy?: SpendAdmissionPolicy): boolean; /** * Drop dormant scopes per the retention rule in SpendReservationPolicy. Cleanup also runs * automatically on every reservation, so nothing depends on a caller remembering this. @@ -742,6 +745,9 @@ export function createSpendReservationLedger(options: { const seedForSend = (send: string) => seedBySend.get(send); const pinnedScope = (key: string): boolean => pinnedKeys.has(key); const pendingRecords: JournalRecord[] = []; + // Bounded by live ordinary reservations; never persisted as new metadata. + const nondurableReserves = new Map(); + const retainedOrdinary = new Set(); let reporters = 0; const ownerReporters = new WeakMap(); const activeReporters = (owner?: object): number => owner === undefined ? reporters : ownerReporters.get(owner) ?? 0; @@ -852,6 +858,15 @@ export function createSpendReservationLedger(options: { return true; }; + const repairReservePrefix = (send: string, reservation: Reservation): void => { + const reserve = nondurableReserves.get(send); + if (!reserve) return; + pendingRecords.push(reserve); + if (reservation.status === "dispatched") pendingRecords.push({ v: 1, kind: "dispatch", send, at: reservation.resolvedAt }); + nondurableReserves.delete(send); + retainedOrdinary.add(send); + }; + const applyReserve = (send: string, targets: readonly ScopeRef[], tokens: number, at: number): void => { if (reservations.has(send)) return; reservations.set(send, { targets, tokens, status: "open", at, resolvedAt: at }); @@ -872,6 +887,8 @@ export function createSpendReservationLedger(options: { const applyResolve = (send: string, outcome: "settled" | "lost" | "abandoned", tokens: number, at: number): void => { const reservation = reservations.get(send); if (!reservation || !isLive(reservation.status)) return; + nondurableReserves.delete(send); + retainedOrdinary.delete(send); reservation.status = outcome; reservation.resolvedAt = at; for (const ref of reservation.targets) { @@ -895,6 +912,8 @@ export function createSpendReservationLedger(options: { const reservation = reservations.get(send); if (!reservation || isLive(reservation.status)) return; reservations.delete(send); + nondurableReserves.delete(send); + retainedOrdinary.delete(send); }; const applyDrop = (scope: SpendScope, alias: string): void => { @@ -1093,6 +1112,8 @@ export function createSpendReservationLedger(options: { for (const candidate of candidates) { if (!flushPending() || !append({ v: 1, kind: "forget", send: candidate.send, at })) continue; reservations.delete(candidate.send); + nondurableReserves.delete(candidate.send); + retainedOrdinary.delete(candidate.send); removed += 1; } return removed; @@ -1288,7 +1309,8 @@ export function createSpendReservationLedger(options: { const at = request.at ?? now(); const send = aliasFor("send", request.sendId); const refs = refsFor(request.scopes); - const enforced = refs.some((ref) => limitFor(ref.scope) !== undefined); + const admissionLimit = (scope: SpendScope) => (request.admissionPolicy ?? policy)[scope].maxTokens; + const enforced = refs.some((ref) => admissionLimit(ref.scope) !== undefined); // A send id this ledger already knows is REFUSED. Returning success while booking // nothing -- the old behaviour -- let one id authorise an unlimited number of physical @@ -1317,7 +1339,7 @@ export function createSpendReservationLedger(options: { for (const ref of refs) { // A recorded send has no limit to fail: it already happened, and the point of booking // it is to let the total go OVER the ceiling so the next request can be refused. - const limit = request.alreadySent === true ? undefined : limitFor(ref.scope); + const limit = request.alreadySent === true ? undefined : admissionLimit(ref.scope); if (limit === undefined) continue; const poolView = ref.scope === "pool" ? poolState(ref.alias) : undefined; const state = ref.scope === "pool" ? poolView?.totals : scopes.get(scopeKey(ref.scope, ref.alias)); @@ -1339,11 +1361,13 @@ export function createSpendReservationLedger(options: { // Durability BEFORE admission. The record goes to disk first, and under a configured // limit a failed write refuses the request rather than admitting one that a restart // would forget -- which is exactly the disk-full and permission case durability is for. - const durable = append({ v: 1, kind: "reserve", send, targets: refs, tokens, at }); + const record: JournalRecord = { v: 1, kind: "reserve", send, targets: refs, tokens, at }; + const durable = append(record); if (!durable && enforced && request.alreadySent !== true) { return { reserved: false, denial: { reason: "reserve-not-durable", sendId: request.sendId } }; } applyReserve(send, refs, tokens, at); + if (!durable) nondurableReserves.set(send, record); compact(at); return { reserved: true, sendId: request.sendId, tokens, durable }; }, @@ -1353,11 +1377,13 @@ export function createSpendReservationLedger(options: { const send = aliasFor("send", sendId); const reservation = reservations.get(send); if (!reservation || reservation.status !== "open") return false; + repairReservePrefix(send, reservation); const at = now(); applyDispatch(send, at); const record: JournalRecord = { v: 1, kind: "dispatch", send, at }; - if (seedForSend(send) && pendingRecords.length > 0) pendingRecords.push(record); - else if (!append(record) && seedForSend(send)) pendingRecords.push(record); + const retained = seedForSend(send) || retainedOrdinary.has(send); + if (retained && pendingRecords.length > 0) pendingRecords.push(record); + else if (!append(record) && retained) pendingRecords.push(record); return true; }, @@ -1378,34 +1404,36 @@ export function createSpendReservationLedger(options: { return true; }, - settle(sendId: string, usage: SpendUsage, options?: { retainOnFailure: boolean }): boolean { + settle(sendId: string, usage: SpendUsage): boolean { assertOwnedAccounting?.(); const send = aliasFor("send", sendId); const reservation = reservations.get(send); if (!reservation || !isLive(reservation.status)) return false; + repairReservePrefix(send, reservation); const tokens = sanitizeTokens(usage.inputTokens) + sanitizeTokens(usage.outputTokens); const at = now(); const record: JournalRecord = { v: 1, kind: "settle", send, tokens, at }; - const retained = seedForSend(send) || options?.retainOnFailure; + const seeded = seedForSend(send) || retainedOrdinary.has(send); // Terminal usage cannot be refused: the tracker may never report it again, and // the estimate can be smaller than the actual bill. Preserve ordered retryable debt. - if (retained && (pendingRecords.length > 0 || !append(record))) pendingRecords.push(record); + if (seeded && (pendingRecords.length > 0 || !append(record))) pendingRecords.push(record); applyResolve(send, "settled", tokens, at); - if (!retained) append(record); + if (!seeded) append(record); return true; }, - markLost(sendId: string, options?: { retainOnFailure: boolean }): boolean { + markLost(sendId: string): boolean { assertOwnedAccounting?.(); const send = aliasFor("send", sendId); const reservation = reservations.get(send); if (!reservation || !isLive(reservation.status)) return false; + repairReservePrefix(send, reservation); const at = now(); const record: JournalRecord = { v: 1, kind: "lost", send, at }; - const retained = seedForSend(send) || options?.retainOnFailure; - if (retained && (pendingRecords.length > 0 || !append(record))) pendingRecords.push(record); + const seeded = seedForSend(send) || retainedOrdinary.has(send); + if (seeded && (pendingRecords.length > 0 || !append(record))) pendingRecords.push(record); applyResolve(send, "lost", 0, at); - if (!retained) append(record); + if (!seeded) append(record); return true; }, @@ -1430,7 +1458,7 @@ export function createSpendReservationLedger(options: { }; }, - exhausted(scope: SpendScope, scopeId: string, excludingSendId?: string): boolean { + exhausted(scope: SpendScope, scopeId: string, excludingSendId?: string, admissionPolicy?: SpendAdmissionPolicy): boolean { assertOwnedAccounting?.(); const alias = scope === "pool" ? poolAliasFor(scopeId) : aliasFor(scope, scopeId); const state = stateFor(scope, alias); @@ -1438,11 +1466,14 @@ export function createSpendReservationLedger(options: { const own = excludingSendId === undefined ? undefined : reservations.get(aliasFor("send", excludingSendId)); const matches = own?.status === "open" && own.targets.some(target => target.scope === scope && (scope === "pool" ? (!poolContinuity.known(target.alias) || poolContinuity.resolve(target.alias) === poolContinuity.resolve(alias)) : target.alias === alias)); - return isExhausted(scope, matches ? { ...state, reserved: Math.max(0, state.reserved - own.tokens) } : state); + const totals = matches ? { ...state, reserved: Math.max(0, state.reserved - own.tokens) } : state; + const limit = (admissionPolicy ?? policy)[scope].maxTokens; + return limit !== undefined && totals.settled + totals.reserved + totals.unresolved >= limit; }, prune(at: number = now()): void { assertOwnedAccounting?.(); + flushPending(); retryCleanup(); // Removal requires BOTH inactive and not exhausted inside the window. An // exhausted-but-idle scope that was dropped would be recreated fresh under the diff --git a/src/lib/workflow-budget.ts b/src/lib/workflow-budget.ts index 4950b3183a3..90abeaf9ab5 100644 --- a/src/lib/workflow-budget.ts +++ b/src/lib/workflow-budget.ts @@ -22,6 +22,7 @@ import { sharedSpendLedger, spendCeilingsConfigured, + type SpendAdmissionPolicy, type SpendReservationLedger, type SpendReservationProof, type SpendScope, @@ -733,10 +734,11 @@ function spentRootCeiling( rootId: string, ledger: SpendReservationLedger, excludingSendId?: string, + admissionPolicy?: SpendAdmissionPolicy, ): WorkflowSpendDenialDetail | undefined { - const limit = ledger.policy.root.maxTokens; + const limit = (admissionPolicy ?? ledger.policy).root.maxTokens; if (limit === undefined) return undefined; - return ledger.exhausted("root", rootId, excludingSendId) ? { scope: "root", limit } : undefined; + return ledger.exhausted("root", rootId, excludingSendId, admissionPolicy) ? { scope: "root", limit } : undefined; } /** @@ -755,16 +757,20 @@ export function workflowSpendCeilingReached( spendLedger?: SpendReservationLedger, poolId?: string, reservation?: SpendReservationProof, + admissionPolicy?: SpendAdmissionPolicy, ): WorkflowSpendDenialDetail | undefined { if (!rootId && !poolId) return undefined; - const ledger = spendLedger ?? (spendCeilingsConfigured() ? sharedSpendLedger() : undefined); + // An explicit empty snapshot belongs to an observe-only request, even if current config + // enables a ceiling. Conversely, old enforced requests still read totals after disable. + if (admissionPolicy && admissionPolicy.root.maxTokens === undefined && admissionPolicy.pool.maxTokens === undefined) return undefined; + const ledger = spendLedger ?? (admissionPolicy || spendCeilingsConfigured() ? sharedSpendLedger() : undefined); if (!ledger) return undefined; // This reads the current alias partition; it never publishes continuity metadata. const excludingSendId = reservation?.ledger === ledger ? reservation.sendId : undefined; - const root = rootId ? spentRootCeiling(rootId, ledger, excludingSendId) : undefined; + const root = rootId ? spentRootCeiling(rootId, ledger, excludingSendId, admissionPolicy) : undefined; if (root) return root; - const limit = ledger.policy.pool.maxTokens; - return poolId && limit !== undefined && ledger.exhausted("pool", poolId, excludingSendId) + const limit = (admissionPolicy ?? ledger.policy).pool.maxTokens; + return poolId && limit !== undefined && ledger.exhausted("pool", poolId, excludingSendId, admissionPolicy) ? { scope: "pool", limit, diff --git a/src/server/chat-native.ts b/src/server/chat-native.ts index 6a4dfa90119..6b7302205b8 100644 --- a/src/server/chat-native.ts +++ b/src/server/chat-native.ts @@ -458,7 +458,9 @@ export async function runNativeChatAttempt( const spendReport = createPhysicalSendReporter(physicalBudget, () => ({ poolId: logCtx.spendPoolId ?? route.providerName, identityId: logCtx.accountLogLabel, }), execution.comboDispatchPermit); - if (physicalBudget.spendEnforced && !spendReport.beforeSend?.()) { + // Capture this request's spend policy at its first physical boundary, including + // observe-only starts; later configuration changes belong to later requests. + if (spendReport.beforeSend && !spendReport.beforeSend()) { spendReport.close?.(); if ((logCtx.spendTracker as { refusals?: number } | undefined)?.refusals) throw new NativeChatSpendRefusal(); throw new SendBudgetExhaustedError(); diff --git a/src/server/messages-native.ts b/src/server/messages-native.ts index 1dddf17e838..dc6d3e6d711 100644 --- a/src/server/messages-native.ts +++ b/src/server/messages-native.ts @@ -536,12 +536,12 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions) })); try { if (init.signal?.aborted) throw init.signal.reason; - if (physicalBudget.spendEnforced) { - if (!spendReport.beforeSend?.()) { - if (spendTracker.refusals) throw new NativeMessagesSpendRefusal(); - throw new SendBudgetExhaustedError(); - } - } else if (!spendTracker.charge()) throw new NativeMessagesSpendRefusal(); + // Capture before either accounting path so this request retains its start policy. + if (spendReport.beforeSend && !spendReport.beforeSend()) { + if (spendTracker.refusals) throw new NativeMessagesSpendRefusal(); + throw new SendBudgetExhaustedError(); + } + if (!physicalBudget.spendEnforced && !spendTracker.charge()) throw new NativeMessagesSpendRefusal(); noteProviderAttemptSend(logCtx, route.providerName, activeProvider, logCtx.usageLogInputTokens, transportRecovery ?? recovery); dispatched = await sendWithConnectionPolicy( (activeProvider as OcxProviderTransport).fetch ?? execute, diff --git a/src/server/responses/adapter-continuation.ts b/src/server/responses/adapter-continuation.ts index 24f644ebfd3..7e4a14f28dc 100644 --- a/src/server/responses/adapter-continuation.ts +++ b/src/server/responses/adapter-continuation.ts @@ -241,6 +241,9 @@ export function createAdapterContinuations( : fetchWithResetRetry; return await fetchContinuationWithRetryPolicy( recovery => { + sendBudgetState.adapterSendBudget?.startRequest?.({ + poolId: logCtx.spendPoolId ?? route.providerName, identityId: logCtx.accountLogLabel, + }); transportState.noteRoutedAttemptSend(continuationEstimate, recovery ?? replayKind); return fetchWithHeaderTimeout( builtContinuationRequest.url, diff --git a/src/server/responses/adapter-dispatch.ts b/src/server/responses/adapter-dispatch.ts index 2e1276598de..7a20e8e95f1 100644 --- a/src/server/responses/adapter-dispatch.ts +++ b/src/server/responses/adapter-dispatch.ts @@ -382,6 +382,11 @@ export async function prepareAdapterExchange( : fetchWithResetRetry; upstreamResponse = await fetchWithRetryPolicy( recovery => { + // Unconfigured generic sends may omit the counting reporter. Capture policy + // independently so a later recovery still belongs to this request's start. + sendBudgetState.adapterSendBudget?.startRequest?.({ + poolId: logCtx.spendPoolId ?? route.providerName, identityId: logCtx.accountLogLabel, + }); if (compactPrepaid && !compactPrepaidUsed) { if (!compactPrepaid.use()) throw new SendBudgetExhaustedError(safeHostLabel(builtInitialRequest.url)); compactPrepaidUsed = true; @@ -617,6 +622,9 @@ export async function prepareAdapterExchange( try { return await refetchWithPolicy( recoveryKind => { + sendBudgetState.adapterSendBudget?.startRequest?.({ + poolId: logCtx.spendPoolId ?? route.providerName, identityId: logCtx.accountLogLabel, + }); if (refetchAllowance?.permit && !refetchAllowance.permit.use()) { throw new SendBudgetExhaustedError(safeHostLabel(retryRequest.url)); } diff --git a/src/server/responses/request-send-budget.ts b/src/server/responses/request-send-budget.ts index 9cca3ec2386..da331ff3291 100644 --- a/src/server/responses/request-send-budget.ts +++ b/src/server/responses/request-send-budget.ts @@ -98,7 +98,8 @@ export function createResponsesSendBudget( } const prepaid = isRequestExecutionBudget(sendBudget) ? claimDispatchSpendProof(sendBudget, options.compactionRecoveryPermit ?? options.comboDispatchPermit) : undefined; - const spentCeiling = workflowSpendCeilingReached(workflowRootId, undefined, logCtx.spendPoolId ?? logCtx.provider, prepaid); + const admissionPolicy = isRequestExecutionBudget(sendBudget) ? sendBudget.spendAdmissionPolicy : undefined; + const spentCeiling = workflowSpendCeilingReached(workflowRootId, undefined, logCtx.spendPoolId ?? logCtx.provider, prepaid, admissionPolicy); if (spentCeiling) { return workflowRefusalResponse( "workflow-spend-exhausted", @@ -341,6 +342,9 @@ function adapterDispatchBudgetView( get physicalStarted() { return budget.physicalStarted; }, get physicalLimit() { return budget.physicalLimit; }, get spendEnforced() { return budget.spendEnforced; }, + get spendPolicyStarted() { return budget.spendPolicyStarted; }, + get spendAdmissionPolicy() { return budget.spendAdmissionPolicy; }, + startRequest: target => budget.startRequest?.(target), claimPhysicalSend: () => budget.claimPhysicalSend?.(), beginSpendProducer: () => budget.beginSpendProducer?.(), get used(): number { return budget.used; }, diff --git a/src/server/responses/request-spend.ts b/src/server/responses/request-spend.ts index 1e9168f05fe..7a2bc6d6e4f 100644 --- a/src/server/responses/request-spend.ts +++ b/src/server/responses/request-spend.ts @@ -1,6 +1,6 @@ import { randomUUID } from "node:crypto"; import type { RequestSendObserver } from "../../lib/request-execution-budget"; -import { sharedSpendLedger, sharedSpendPolicy, type SpendReservationLedger, type SpendScopes, type SpendSeed, type SpendReservationProof } from "../../lib/spend-reservation-ledger"; +import { sharedSpendLedger, sharedSpendPolicy, type SpendReservationLedger, type SpendScopes, type SpendSeed, type SpendReservationProof, type SpendAdmissionPolicy } from "../../lib/spend-reservation-ledger"; import { SpendLedgerOwnerError } from "../../lib/spend-ledger-owner"; import { markLocalRequestLogRefusal, type RequestLogContext } from "../request-log"; import { recordWorkflowRefusalEvent, workflowDenialSummary, type WorkflowDenial } from "../../lib/workflow-budget"; @@ -19,7 +19,6 @@ export interface RequestSpendSettlement { export interface SpendTargetIdentity extends SpendScopes { readonly targetKey?: string } export interface RequestSpendReporter { start(seed: SpendSeed, ordinal: number): void; - startObserved(target: SpendTargetIdentity, ordinal: number, proof?: SpendReservationProof): void; rebindTarget(target: SpendTargetIdentity, ordinal: number): boolean; report(sends: number): void; close(): void; @@ -68,89 +67,58 @@ export function createRequestSpendTracker( // one. It also means the home in effect at dispatch is the one that gets written. let ledgerRef: SpendReservationLedger | undefined = injected; const ledger = (): SpendReservationLedger => (ledgerRef ??= sharedSpendLedger()); + let admissionPolicy: SpendAdmissionPolicy | undefined; + let initialEnforced: boolean | undefined; // Outstanding entries; exact dispatch reports move their reservation to the end. const live: string[] = []; const pendingDispatch = new Set(); - const legacyOrder = new Map(); - let dispatchOrder = 0; let refusals = 0; let resolved = false; let terminalProcessed = false; let seededAccounting = false; - let enforcementObserved = false; let finalRequested = false; let finalUsage: TerminalSpendUsage | undefined; let reporters = 0; const seeds = new Map(); - // Abandoned capabilities retain only a cleanup obligation, never an admission lookup. + // Abandoned capabilities retain cleanup ownership, never admission lookup. const retiredSeeds = new Set(); const resolvedSendIds = new Set(); const completedSeeds = new Set(); - type PhysicalEntry = { order: number; seed?: SpendSeed; sendId?: string; observed?: { - scopes: SpendScopes; inputTokens: number; outputCeilingTokens: number; - }; reported?: boolean }; - const physical = new Map(); + const physical = new Map(); const selectedScopes = (target: SpendTargetIdentity = {}): SpendScopes => ({ ...(rootId !== undefined ? { rootId } : {}), ...(target.identityId ?? logCtx.accountLogLabel ? { identityId: target.identityId ?? logCtx.accountLogLabel } : {}), ...(target.poolId ?? logCtx.spendPoolId ?? logCtx.provider ? { poolId: target.poolId ?? logCtx.spendPoolId ?? logCtx.provider } : {}), }); const applicable = (scopes: SpendScopes): boolean => { - const policy = ledgerRef?.policy ?? sharedSpendPolicy(); - const enforced = seededAccounting || (scopes.rootId !== undefined && policy.root.maxTokens !== undefined) + if (initialEnforced !== undefined) return initialEnforced; + const policy = admissionPolicy ?? ledgerRef?.policy ?? sharedSpendPolicy(); + return (scopes.rootId !== undefined && policy.root.maxTokens !== undefined) || (scopes.identityId !== undefined && policy.identity.maxTokens !== undefined) || (scopes.poolId !== undefined && policy.pool.maxTokens !== undefined); - if (enforced) enforcementObserved = true; - return enforced; }; const estimates = () => ({ inputTokens: logCtx.spendInputEstimateTokens ?? logCtx.usageLogInputTokens ?? 0, outputCeilingTokens: logCtx.spendOutputCeilingTokens ?? 0 }); const seedState = (seed: SpendSeed) => [...seeds.values()].find(state => state.seed === seed); - // Observe-only starts use ordinary records, never overflow capabilities. A full normal - // table may omit the booking. If policy activates, retain that debt until normal room is available. - const bookObserved = (entry: PhysicalEntry): boolean => { - if (entry.sendId) return true; - if (!entry.observed) return false; - const sendId = randomUUID(); - const decision = ledger().reserve({ sendId, ...entry.observed, alreadySent: true }); - if (!decision.reserved) return false; - entry.sendId = sendId; - if (entry.reported) ledger().markDispatched(sendId); - return true; - }; const cleanupRetiredSeeds = (): boolean => { for (const seed of retiredSeeds) if (ledger().finishSeed(seed, tracker)) retiredSeeds.delete(seed); return retiredSeeds.size === 0; }; const tryFinalizeSeeds = (): boolean => { - applicable(selectedScopes()); - const durability = { retainOnFailure: enforcementObserved }; // All starts are booked before producers close. Ordinal, never callback order, owns usage. const entries = [...physical.entries()].sort((a, b) => a[0] - b[0]); - const terminalOrder = Math.max(0, ...entries.map(([, entry]) => entry.order), ...live.map(id => legacyOrder.get(id) ?? 0)); + const terminal = entries.at(-1)?.[1].sendId; const ids: string[] = []; for (const [, entry] of entries) { - if (!entry.sendId && entry.observed && !enforcementObserved) continue; - if (!entry.sendId && !bookObserved(entry)) return false; if (!entry.sendId) return false; - const reported = entry.order === terminalOrder && (typeof finalUsage?.inputTokens === "number" || typeof finalUsage?.outputTokens === "number"); + const reported = entry.sendId === terminal && (typeof finalUsage?.inputTokens === "number" || typeof finalUsage?.outputTokens === "number"); if (!resolvedSendIds.has(entry.sendId)) { - const ok = reported ? ledger().settle(entry.sendId, { inputTokens: finalUsage?.inputTokens ?? 0, outputTokens: finalUsage?.outputTokens ?? 0 }, durability) - : ledger().markLost(entry.sendId, durability); + const ok = reported ? ledger().settle(entry.sendId, { inputTokens: finalUsage?.inputTokens ?? 0, outputTokens: finalUsage?.outputTokens ?? 0 }) + : ledger().markLost(entry.sendId); if (!ok) return false; resolvedSendIds.add(entry.sendId); } - if (entry.seed) ids.push(entry.sendId); - } - for (const sendId of live) { - if (!resolvedSendIds.has(sendId)) { - const reported = legacyOrder.get(sendId) === terminalOrder - && (typeof finalUsage?.inputTokens === "number" || typeof finalUsage?.outputTokens === "number"); - const ok = reported ? ledger().settle(sendId, { inputTokens: finalUsage?.inputTokens ?? 0, outputTokens: finalUsage?.outputTokens ?? 0 }, durability) - : ledger().markLost(sendId, durability); - if (!ok) return false; - resolvedSendIds.add(sendId); - } + ids.push(entry.sendId); } for (const state of seeds.values()) if (!state.started && !completedSeeds.has(state.seed)) { if (!resolvedSendIds.has(state.seed.sendId) && !ledger().abandon(state.seed.sendId)) return false; @@ -158,7 +126,7 @@ export function createRequestSpendTracker( ids.push(state.seed.sendId); } if (!cleanupRetiredSeeds()) return false; - if ((ids.length > 0 || enforcementObserved) && !ledger().forgetResolved(ids, tracker)) return false; + if (!ledger().forgetResolved(ids, tracker)) return false; for (const state of seeds.values()) if (!completedSeeds.has(state.seed)) { if (!ledger().finishSeed(state.seed, tracker)) return false; completedSeeds.add(state.seed); @@ -169,7 +137,7 @@ export function createRequestSpendTracker( const deferCleanup = (): void => ledger().deferCleanup(tracker, () => resolved || (finalRequested && reporters === 0 ? tryFinalizeSeeds() : cleanupRetiredSeeds())); const finalizeSeeds = (): void => { - if (!finalRequested || reporters !== 0 || resolved || (!seededAccounting && physical.size === 0)) return; + if (!finalRequested || reporters !== 0 || resolved || !seededAccounting) return; // Own terminal retry before an append can throw; propagation must not orphan the debt. deferCleanup(); if (!tryFinalizeSeeds()) { @@ -237,25 +205,21 @@ export function createRequestSpendTracker( return; }; const tracker: RequestSpendTracker = { + startRequest(target) { + if (admissionPolicy) return; + const policy = ledgerRef?.policy ?? sharedSpendPolicy(); + admissionPolicy = Object.freeze({ root: Object.freeze({ ...policy.root }), identity: Object.freeze({ ...policy.identity }), pool: Object.freeze({ ...policy.pool }) }); + initialEnforced = applicable(selectedScopes(target)); + if (!initialEnforced) admissionPolicy = Object.freeze({ root: Object.freeze({}), identity: Object.freeze({}), pool: Object.freeze({}) }); + }, get enforced(): boolean { return applicable(selectedScopes()); }, + get policyStarted(): boolean { return admissionPolicy !== undefined; }, + get spendAdmissionPolicy(): SpendAdmissionPolicy | undefined { return admissionPolicy; }, ensureSeed(target, prepaidProof) { + tracker.startRequest?.(target); if (resolved || finalRequested && reporters === 0) return undefined; const scopes = selectedScopes(target); if (!applicable(scopes)) return undefined; - // Activation cannot authorize a new send while an earlier observe-only liability - // still lacks normal tracking capacity. It never grants that liability a seed. - for (const entry of physical.values()) if (entry.observed && !bookObserved(entry)) { - refusal({ reason: "tracking-capacity-exhausted" }); - return undefined; - } - if (prepaidProof && prepaidProof.ledger === ledgerRef && pendingDispatch.has(prepaidProof.sendId)) { - // A reservation made before activation is not an enforced seed. Release its - // undispatched booking before asking NORMAL admission under the new policy. - if (!ledger().abandon(prepaidProof.sendId)) return undefined; - pendingDispatch.delete(prepaidProof.sendId); - const index = live.indexOf(prepaidProof.sendId); - if (index >= 0) live.splice(index, 1); - } const key = JSON.stringify([scopes.rootId, scopes.identityId, scopes.poolId, target.targetKey]); const existing = seeds.get(key); if (existing) return existing.seed; @@ -283,7 +247,7 @@ export function createRequestSpendTracker( // Exact prepaid proofs originate from this tracker and are found above; foreign receipts // cannot enroll arbitrary scopes. Normal reservation remains the only initial admission. if (prepaidProof && prepaidProof.ledger !== ledger()) return undefined; - const decision = ledger().reserveSeed({ sendId: randomUUID(), scopes, ...estimates() }, tracker); + const decision = ledger().reserveSeed({ sendId: randomUUID(), scopes, ...estimates(), admissionPolicy }, tracker); if (!decision.reserved) { // Use the established denial mapping without making another reservation. refusal(decision.denial); @@ -299,56 +263,28 @@ export function createRequestSpendTracker( reporters++; let closed = false; let reported = 0; - const starts: Array<{ seed?: SpendSeed; ordinal: number }> = []; + const starts: Array<{ seed: SpendSeed; ordinal: number }> = []; return { start(seed, ordinal) { if (closed || !seedState(seed) || physical.has(ordinal)) throw new Error("Invalid spend reporter start"); const state = seedState(seed)!; state.started = true; starts.push({ seed, ordinal }); - physical.set(ordinal, { seed, order: ++dispatchOrder }); - }, - startObserved(target, ordinal, proof) { - if (closed || physical.has(ordinal)) throw new Error("Invalid observed reporter start"); - const entry: PhysicalEntry = { order: ++dispatchOrder, observed: { scopes: selectedScopes(target), ...estimates() } }; - if (proof && proof.ledger === ledgerRef && pendingDispatch.has(proof.sendId)) { - entry.sendId = proof.sendId; - pendingDispatch.delete(proof.sendId); - const index = live.indexOf(proof.sendId); - if (index >= 0) live.splice(index, 1); - } - physical.set(ordinal, entry); - starts.push({ ordinal }); - bookObserved(entry); + physical.set(ordinal, { seed }); + }, rebindTarget(target, ordinal) { const entry = physical.get(ordinal); const owned = starts.find(start => start.ordinal === ordinal); - if (closed || !entry || !owned || entry.reported || entry.seed && entry.sendId) throw new Error("Invalid spend reporter rebind"); - if (entry.observed) { - const scopes = selectedScopes(target); - const enforced = applicable(scopes); - if (!enforced && JSON.stringify(scopes) === JSON.stringify(entry.observed.scopes)) return true; - if (entry.sendId && !ledger().abandon(entry.sendId)) return false; - physical.delete(ordinal); - if (enforced) { - const seed = tracker.ensureSeed(target); - if (!seed) { starts.splice(starts.indexOf(owned), 1); return false; } - physical.set(ordinal, { seed, order: entry.order }); owned.seed = seed; seedState(seed)!.started = true; - } else { - const rebound: PhysicalEntry = { order: entry.order, observed: { scopes, ...estimates() } }; - physical.set(ordinal, rebound); bookObserved(rebound); - } - return true; - } - const old = seedState(entry.seed!)!; + if (closed || !entry || !owned || entry.sendId) throw new Error("Invalid spend reporter rebind"); + const old = seedState(entry.seed)!; // A helper start is provisional until selection reaches the executor. If this is // the first start on its seed, abandon/rebind at NORMAL capacity before the wire. physical.delete(ordinal); old.started = [...physical.values()].some(start => start.seed === old.seed); const seed = tracker.ensureSeed(target); if (!seed) { starts.splice(starts.indexOf(owned), 1); return false; } - physical.set(ordinal, { seed, order: entry.order }); + physical.set(ordinal, { seed }); owned.seed = seed; seedState(seed)!.started = true; return true; @@ -358,12 +294,7 @@ export function createRequestSpendTracker( const count = Number.isFinite(sends) ? Math.max(0, Math.trunc(sends)) : 0; for (let i = 0; i < count && reported < starts.length; i++) { const entry = starts[reported++]!; - if (entry.seed) tracker.reportFromSeed(entry.seed, entry.ordinal); - else { - const observed = physical.get(entry.ordinal)!; - observed.reported = true; - if (bookObserved(observed) && observed.sendId) ledger().markDispatched(observed.sendId); - } + tracker.reportFromSeed(entry.seed, entry.ordinal); } }, close() { @@ -395,6 +326,7 @@ export function createRequestSpendTracker( finalizeSeeds(); }, charge(options?: Parameters[0]): boolean { + tracker.startRequest?.(); // A send that has already left is RECORDED, never refused: the tokens are spent, and a // booking the ledger drops is a booking the ceiling can never see. This is the reporting // transports' path -- the passthrough ladder reports through `onSendsConsumed` after the @@ -418,13 +350,14 @@ export function createRequestSpendTracker( ? { poolId: logCtx.spendPoolId ?? logCtx.provider } : {}), }; - const policy = bookedLedger.policy; + const policy = admissionPolicy!; const enforced = (scopes.rootId !== undefined && policy.root.maxTokens !== undefined) || (scopes.identityId !== undefined && policy.identity.maxTokens !== undefined) || (scopes.poolId !== undefined && policy.pool.maxTokens !== undefined); const decision = bookedLedger.reserve({ sendId, scopes, + admissionPolicy, inputTokens: logCtx.spendInputEstimateTokens ?? logCtx.usageLogInputTokens ?? 0, outputCeilingTokens: logCtx.spendOutputCeilingTokens ?? 0, ...(alreadySent ? { alreadySent: true } : {}), @@ -440,18 +373,17 @@ export function createRequestSpendTracker( if (!alreadySent) options?.onReserved?.({ ledger: bookedLedger, sendId }); live.push(sendId); if (options?.deferDispatch) pendingDispatch.add(sendId); - else { legacyOrder.set(sendId, ++dispatchOrder); confirmOlderSends(); } + else confirmOlderSends(); // It has already left, so the reservation cannot be handed back for free: from here only // a settlement or unresolved spend is honest about it. if (alreadySent) ledger().markDispatched(sendId); return true; }, dispatch(proof): void { - if (proof.ledger === ledgerRef && [...seeds.values()].some(state => state.seed.sendId === proof.sendId)) return; + if (seededAccounting && proof.ledger === ledgerRef) return; if (proof.ledger !== ledgerRef || !pendingDispatch.has(proof.sendId)) return; ledger().markDispatched(proof.sendId); pendingDispatch.delete(proof.sendId); - legacyOrder.set(proof.sendId, ++dispatchOrder); // Terminal usage follows dispatch/report order, not reservation order. const index = live.indexOf(proof.sendId); if (index >= 0) live.push(...live.splice(index, 1)); @@ -459,10 +391,8 @@ export function createRequestSpendTracker( refund(proof): void { if (proof && proof.ledger === ledgerRef && seededAccounting) { const match = [...seeds.entries()].find(([, state]) => state.seed.sendId === proof.sendId); - if (match) { - if (!match[1].started) retireSeed(match[0], match[1].seed); - return; - } + if (match && !match[1].started) retireSeed(match[0], match[1].seed); + return; } // null is an exact budget reservation that obtained no durable booking. if (proof === null || (proof && proof.ledger !== ledgerRef)) return; @@ -477,7 +407,7 @@ export function createRequestSpendTracker( }, settle(usage: TerminalSpendUsage | undefined): void { if (resolved) return; - if (seededAccounting || physical.size > 0 || reporters > 0) { tracker.requestFinalSettlement(usage); return; } + if (seededAccounting) { tracker.requestFinalSettlement(usage); return; } try { if (!terminalProcessed && live.length > 0) { const terminal = live[live.length - 1] as string; diff --git a/structure/runtime.md b/structure/runtime.md index 70f240411ce..fadb058d119 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -218,7 +218,7 @@ and build a ledger by replaying that directory's own journal. A second process o directory is refused even for observe-only spend configuration, while a separate directory is independent. Ordinary stop drains the spend reporters after listener teardown before releasing the final reference. `src/server/index/spend-ledger-lifecycle.ts` also holds the lease through failed-start listener rollback. Enforced seeds remain live until reporters close and settlement plus send-ID forgetting are durable; scope totals survive forgetting. SQLite and the OS release a crashed owner; no PID, timestamp, -TTL or lock-file deletion participates in recovery. Startup passes the provider roster and top-level pool aliases with the spend policy; live state reconciliation refreshes both without opening an unused ledger. HTTP admission checks [historical continuity](transports/responses-spend.md#historical-pool-continuity-and-rollback) even without a root; failed accounting reads release the active-turn lease. Rooted continuity denials record exactly one workflow refusal event before response formatting. +TTL or lock-file deletion participates in recovery. Startup passes the provider roster and top-level pool aliases with the spend policy; live state reconciliation refreshes both without opening an unused ledger. Under the [request-start policy](transports/responses-spend.md#durable-spend-reservations), existing requests retain their initial enforcement mode, numeric ceilings and L; changed limits apply to later requests. HTTP admission checks [historical continuity](transports/responses-spend.md#historical-pool-continuity-and-rollback) even without a root; failed accounting reads release the active-turn lease. Rooted continuity denials record exactly one workflow refusal event before response formatting. An explicit Codex integration OFF skips startup cache invalidation before the user-scoped catalog serialization lock is resolved. Explicit `sync` and `sync-cache` retain their catalog-only override. diff --git a/structure/transports/responses-spend.md b/structure/transports/responses-spend.md index fb575d18f1e..7ffa726b9a5 100644 --- a/structure/transports/responses-spend.md +++ b/structure/transports/responses-spend.md @@ -116,21 +116,20 @@ caller is `request-spend.ts`, installed on the execution budget at genuine ingre and parked on the log context so `addFinalRequestLog` can settle it. Native Chat installs the same tracker before its independent physical-send ladder and charges it immediately before each dispatch, so that fast path cannot bypass root, identity or provider-pool ceilings. Unconfigured requests preserve the legacy counter observer: reservations increment the send -counter, refunds decrement it, and named reports reconcile prepaid sends. Physical helpers retain -their start order even while observing, without creating seed capabilities or imposing a new -observe-only send limit. Enforced requests use the seed protocol below across Responses, native Messages and native Chat. +counter, refunds decrement it, and named reports reconcile prepaid sends. Enforced requests use +the seed and physical-start protocol below across Responses, native Messages and native Chat. Each transport claims immediately before inference I/O and retains a producer through completion. -Enabling a ceiling during an in-flight call preserves that call's accounting and the request's -existing physical-start count. Subsequent starts require normal seed admission under the current -policy; a prepaid reservation from before activation is not an enforced admission. Selection-time -rebinding checks the current policy even when the target stays the same. Terminal usage follows -the last actual start across both modes. Once enforcement is observed, an earlier omitted booking -remains pending debt and must obtain normal capacity before another enforced start is admitted; -it never becomes an overflow capability. Requests that remain observe-only retain capacity omission. -After activation, ordinary observed terminal records also retain failed writes in the ledger's -ordered retry queue. This persistence intent grants no admission capacity. Cleanup is registered -before terminal writes can throw, so later ledger activity retries without another settlement call. +A request fixes its enforcement mode, token ceilings and physical-send limit L when it actually +starts. Configuration changes apply to requests that start afterward. An in-flight request keeps +its starting policy until completion, including when an operator enables, disables, raises or +lowers a ceiling. Changing policy never resets that request's physical-send allowance. A request +that starts observe-only keeps the existing observe-only path throughout its lifetime. +The shared tracker captures once at the first actual spend admission or start; constructing a +budget or reading its enforcement preview does not capture policy. Generic Responses paths that +omit retry reporters still invoke that same capture before the wire. Re-entry's token-ceiling +preflight and exact seed admission use the same frozen numeric policy and prepaid proof; +pool continuity validation and shared tracking-capacity safeguards remain active. Provider-pool accounting uses the canonical routed provider identity, not the mutable display label that may identify an OAuth account in request logs. Responses final-route normalization captures @@ -190,6 +189,12 @@ such as `null` are corruption. An unparseable torn final line retains the existi replay rule. Storage/corruption refusals use `workflow_spend_undurable`; unsafe-file and ownership errors remain storage failures. `tests/lib/spend-corruption-compat.test.ts` checks these boundaries. +An ordinary observe-only reservation whose first write failed keeps bounded in-memory repair +metadata until it resolves or is evicted. Before reporting dispatch or terminal usage, the ledger +queues the missing ordinary reserve prefix ahead of those records. Later pruning or admission +flushes that queue, so a recovered writer cannot leave terminal usage without its reservation. +This grants no seed or overflow capacity and adds no journal record type or identity metadata. + For requests with an applicable root, identity or pool ceiling, each selected target/key first obtains a normal-capacity pre-send seed through `reserveSeed`. A failed initial seed refuses the wire call, including rootless passthrough and generic adapter ingress. Stable retries reuse that @@ -199,8 +204,8 @@ Reported additional sends reuse the seed's exact scope references through `reser and allocate no new scope keys. Already-started sends remain booked even during persistence failure. `src/lib/request-execution-budget.ts` shares a physical-start counter across derived budgets and -freezes the finite positive limit L at the first claim. The default guarded limit is four; bounded -OAuth expansion happens before the first start and allows at most eighteen for that profile. +freezes the finite positive limit L when the request starts. The default guarded limit is four; +the initial bounded OAuth request profile allows at most eighteen. `createPhysicalSendReporter` owns selected-seed start receipts, reports only claimed sends and closes its producer after reconciliation. Adapter permits claim once at their physical boundary; their executor span keeps the start rebindable until OAuth selection reaches the wire, then reports and closes it. Zed completion diff --git a/tests/claude-integration/messages-native.test.ts b/tests/claude-integration/messages-native.test.ts index 389c2376b04..9e58f16711e 100644 --- a/tests/claude-integration/messages-native.test.ts +++ b/tests/claude-integration/messages-native.test.ts @@ -6,7 +6,7 @@ * or service is reached. */ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; -import { mkdtempSync } from "node:fs"; +import { mkdtempSync, readFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { saveConfig } from "../../src/config"; @@ -18,6 +18,7 @@ import type { AdmissionLease } from "../../src/lib/admission"; import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; import { startTruncatedSseUpstream } from "../helpers/truncated-sse-upstream"; +import { configureSharedSpendLedger, spendPolicyFromConfig } from "../../src/lib/spend-reservation-ledger"; interface Seen { path: string; @@ -191,6 +192,50 @@ async function settledRowFor(requestId: string) { } describe("managed native Messages", () => { + for (const initiallyEnforced of [false, true]) test(`freezes ${initiallyEnforced ? "enforced" : "observe-only"} policy across retries`, async () => { + const config = fixtureConfig(startUpstream((_entry, index) => { + if (index === 0) configureSharedSpendLedger(spendPolicyFromConfig( + initiallyEnforced ? {} : { pool: { maxTokens: 1 } }, undefined, ["anth"])); + return Response.json({ type: "error", error: { type: "rate_limit_error", message: "temporary" } }, { status: 429 }); + })); + config.providers.anth!.retryOn429 = { attempts: 5, intervalMs: 1, maxIntervalMs: 1, respectRetryAfter: false }; + configureSharedSpendLedger(spendPolicyFromConfig(initiallyEnforced ? { pool: { maxTokens: 100_000 } } : {}, undefined, ["anth"])); + const body = { model: "anth/claude-x", messages: [{ role: "user", content: "hello" }], max_tokens: 1 }; + const { response, requestId } = await send(config, body); + expect(rowFor(requestId).protocolTrace?.mode).toBe("native"); + expect(seen).toHaveLength(initiallyEnforced ? 4 : 6); + expect(response.status).toBe(429); + const records = readFileSync(join(testDir, "spend-ledger.jsonl"), "utf8").trim().split("\n").map(line => JSON.parse(line)); + expect(records.filter(record => record.kind === "reserve")).toHaveLength(seen.length); + if (!initiallyEnforced) { + const next = await send(config, body); + expect(next.response.status).toBe(429); + expect(next.response.headers.get("x-opencodex-local-refusal")).toBe("workflow_spend_exhausted"); + expect(seen).toHaveLength(6); + } + }, 20_000); + + for (const change of ["disable", "raise"] as const) test(`retains its initial ceiling when ${change} occurs before key failover`, async () => { + const body = { model: "anth/claude-x", messages: [{ role: "user", content: "hello" }], max_tokens: 1 }; + const config = fixtureConfig(startUpstream((entry, index) => { + if (index === 0) { + configureSharedSpendLedger(spendPolicyFromConfig(change === "disable" ? {} : { pool: { maxTokens: 100_000 } }, undefined, ["anth"])); + return Response.json({ type: "error", error: { type: "rate_limit_error", message: "key rate limited" } }, + { status: 429, headers: { "retry-after": "30" } }); + } + return ok(entry); + }), { pool: true }); + configureSharedSpendLedger(spendPolicyFromConfig({ pool: { maxTokens: estimateClaudeRequestTokens(body, body.model) + 1 } }, undefined, ["anth"])); + const { response, requestId } = await send(config, body); + expect(rowFor(requestId).protocolTrace?.mode).toBe("native"); + expect(response.status).toBe(429); + expect(response.headers.get("x-opencodex-local-refusal")).toBe("workflow_spend_exhausted"); + expect(seen).toHaveLength(1); + const next = await send(config, body); + expect(next.response.status).toBe(200); + expect(seen).toHaveLength(2); + }); + test("sends exactly the allowlisted source body with the provider key and no caller credential", async () => { const config = fixtureConfig(startUpstream()); const { requestId, response, text } = await send(config, { ...SOURCE_BODY, stream: false }, { diff --git a/tests/lib/execution-budget-permits.test.ts b/tests/lib/execution-budget-permits.test.ts index 18e14fcb861..0acb8b62716 100644 --- a/tests/lib/execution-budget-permits.test.ts +++ b/tests/lib/execution-budget-permits.test.ts @@ -5,14 +5,13 @@ import { createPhysicalSendReporter, deriveRequestExecutionBudget, reportDispatchSends, - rebindPhysicalSend, type RequestExecutionBudgetPolicy, } from "../../src/lib/request-execution-budget"; import { createSpendReservationLedger, DEFAULT_SPEND_RESERVATION_POLICY } from "../../src/lib/spend-reservation-ledger"; import { createRequestSpendTracker } from "../../src/server/responses/request-spend"; -import { createShippedSpendLedger, spendTestJournal, spendTestPolicy, testSpendSalt } from "../helpers/shipped-spend-ledger"; -import { SpendLedgerOwnerError } from "../../src/lib/spend-ledger-owner"; import { fetchWithTransientRetry, SendBudgetExhaustedError } from "../../src/lib/upstream-retry"; +import { createShippedSpendLedger, spendTestJournal, testSpendSalt } from "../helpers/shipped-spend-ledger"; +import { SpendLedgerOwnerError } from "../../src/lib/spend-ledger-owner"; /** * The permit is the charge (#4546). @@ -33,203 +32,163 @@ const ONE_SEND_LEFT: RequestExecutionBudgetPolicy = { maxTargetTransitions: 1, }; -function policyActivationFixture(limit = 4, maxTrackedSends = 100) { - const lines: string[] = []; - const policy = { ...DEFAULT_SPEND_RESERVATION_POLICY, pool: {}, canonicalProviderIds: ["P"], maxTrackedSends }; - const ledger = createSpendReservationLedger({ policy, salt: "activation-fixture", journal: { - read: () => [...lines], append: line => { lines.push(line); }, rewrite: next => { lines.splice(0, lines.length, ...next); }, - } }); - const tracker = createRequestSpendTracker({ provider: "P", accountLogLabel: "A", spendInputEstimateTokens: 10 }, undefined, ledger); - const budget = createRequestExecutionBudget({ ...ONE_SEND_LEFT, maxTotalModelSends: limit, baseSendAllowance: limit }, undefined, tracker); - const enable = (maxTokens = 50) => ledger.reconfigure({ ...policy, pool: { maxTokens } }); - return { ledger, tracker, budget, enable, lines }; +function policyEpochFixture(maxTokens?: number, limit = 4) { + const journal = spendTestJournal(); + const policy = { ...DEFAULT_SPEND_RESERVATION_POLICY, pool: { maxTokens }, canonicalProviderIds: ["P"] }; + const ledger = createSpendReservationLedger({ policy, journal, salt: testSpendSalt, now: () => 2 }); + const request = () => { + const tracker = createRequestSpendTracker({ provider: "P", spendInputEstimateTokens: 10 }, undefined, ledger); + const budget = createRequestExecutionBudget({ ...ONE_SEND_LEFT, maxTotalModelSends: limit, baseSendAllowance: limit }, undefined, tracker); + const send = (identityId = "A", run = async () => new Response()) => fetchWithTransientRetry(run, { + attempts: 1, onSendsConsumed: createPhysicalSendReporter(budget, () => ({ poolId: "P", identityId })), + }); + return { tracker, budget, send }; + }; + return { ledger, journal, policy, request, configure: (maxTokens?: number) => ledger.reconfigure({ ...policy, pool: { maxTokens } }) }; } -test("a ceiling enabled inside an observe-only physical fetch retains its 70 actual tokens", async () => { - const { ledger, tracker, budget, enable, lines } = policyActivationFixture(); - let wires = 0; - const telemetry: number[] = []; - await fetchWithTransientRetry(async () => { wires++; enable(); return new Response(); }, { - attempts: 1, onSendsConsumed: createPhysicalSendReporter(budget, () => ({ poolId: "P" }), undefined, count => telemetry.push(count)), - }); - tracker.settle({ inputTokens: 70, outputTokens: 0 }); - expect(wires).toBe(1); - expect(budget.used).toBe(1); - expect(budget.physicalStarted).toBe(1); - expect(telemetry.reduce((total, count) => total + count, 0)).toBe(1); - expect(ledger.snapshot("pool", "P")).toMatchObject({ settled: 70, reserved: 0, unresolved: 0 }); - expect(lines.length).toBeGreaterThan(0); +test("request construction and read-only policy checks do not capture before actual start", async () => { + const { request, configure, journal } = policyEpochFixture(); + const { tracker, budget, send } = request(); + expect(budget.spendEnforced).toBe(false); + expect(budget.spendPolicyStarted).toBe(false); + expect(journal.lines).toHaveLength(0); + configure(5); + await expect(send()).rejects.toThrow(SendBudgetExhaustedError); + expect(tracker.enforced).toBe(true); + expect(budget.spendPolicyStarted).toBe(true); + expect(budget.physicalStarted).toBe(0); }); -for (const ceiling of [15, 50]) test(`activation applies NORMAL admission to the next start at ceiling ${ceiling}`, async () => { - const { ledger, tracker, budget, enable } = policyActivationFixture(); - let wires = 0; - const result = fetchWithTransientRetry(async () => { - if (++wires === 1) { enable(ceiling); return new Response(null, { status: 503 }); } - return new Response(); - }, { attempts: 2, onSendsConsumed: createPhysicalSendReporter(budget, () => ({ poolId: "P" })) }); - if (ceiling === 15) await expect(result).rejects.toThrow(SendBudgetExhaustedError); - else await result; - tracker.settle({ inputTokens: 70, outputTokens: 0 }); - expect(wires).toBe(ceiling === 15 ? 1 : 2); - expect(budget.used).toBe(wires); - expect(budget.physicalStarted).toBe(wires); - expect(ledger.snapshot("pool", "P")).toMatchObject({ settled: 70, reserved: 0, unresolved: ceiling === 15 ? 0 : 10 }); +test("an off-start request stays legacy while the next request applies the new ceiling", async () => { + const { request, configure } = policyEpochFixture(undefined, 1); + const old = request(); + await old.send("A", async () => { configure(5); return new Response(); }); + await old.send("B"); + expect(old.budget.spendEnforced).toBe(false); + expect(old.budget.physicalStarted).toBe(0); + expect(old.budget.used).toBe(2); + old.tracker.settle({ inputTokens: 70 }); + await expect(request().send()).rejects.toThrow(SendBudgetExhaustedError); }); -test("the physical L includes starts made before policy activation", async () => { - const { tracker, budget, enable } = policyActivationFixture(1); - let wires = 0; - await expect(fetchWithTransientRetry(async () => { - wires++; enable(); return new Response(null, { status: 503 }); - }, { attempts: 2, onSendsConsumed: createPhysicalSendReporter(budget, () => ({ poolId: "P" })) })).rejects.toThrow(SendBudgetExhaustedError); - tracker.settle({ inputTokens: 70 }); - expect(wires).toBe(1); - expect(budget.physicalStarted).toBe(1); +for (const next of [undefined, 100, 5]) test(`an enforced request retains its original ceiling after reconfigure to ${next}`, async () => { + const { request, configure } = policyEpochFixture(15); + const old = request(); + await old.send(); + configure(next); + await expect(old.send("B")).rejects.toThrow(SendBudgetExhaustedError); + old.tracker.settle({ inputTokens: 10 }); + const fresh = request(); + if (next === 5) await expect(fresh.send()).rejects.toThrow(SendBudgetExhaustedError); + else { await fresh.send(); fresh.tracker.settle({ inputTokens: 10 }); } }); -test("an off-mode prepaid reservation converts through NORMAL admission without duplicate liability", async () => { - const { ledger, tracker, budget, enable } = policyActivationFixture(); - const decision = budget.reserveDispatch({ sendClass: "initial", targetKey: "P" }); - if (!decision.allowed) throw new Error("fixture reservation denied"); - enable(); - expect(decision.permit.use()).toBe(true); - tracker.settle({ inputTokens: 7 }); - expect(budget.used).toBe(1); - expect(ledger.snapshot("pool", "P")).toMatchObject({ settled: 7, reserved: 0, unresolved: 0 }); +test("overlapping requests use different immutable ceilings against the same totals", async () => { + const { request, configure, ledger } = policyEpochFixture(50); + const older = request(); + await older.send(); + configure(15); + const newer = request(); + await expect(newer.send()).rejects.toThrow(SendBudgetExhaustedError); + await older.send("B"); + older.tracker.settle({ inputTokens: 30 }); + expect(ledger.snapshot("pool", "P")).toMatchObject({ settled: 30, unresolved: 10, reserved: 0 }); }); -test("capacity-full observe-only actual usage remains debt when enforcement activates", async () => { - const { ledger, tracker, budget, enable } = policyActivationFixture(4, 1); - expect(ledger.reserve({ sendId: "occupant", scopes: { poolId: "P" }, inputTokens: 1, outputCeilingTokens: 0 }).reserved).toBe(true); - await fetchWithTransientRetry(async () => { enable(); return new Response(); }, { - attempts: 1, onSendsConsumed: createPhysicalSendReporter(budget, () => ({ poolId: "P" })), - }); - tracker.settle({ inputTokens: 70 }); - expect(budget.used).toBe(1); - expect(ledger.abandon("occupant")).toBe(true); - ledger.prune(Date.now()); - expect(ledger.snapshot("pool", "P")).toMatchObject({ settled: 70, reserved: 0, unresolved: 0 }); +test("an enforced physical L remains fixed after a ceiling is disabled", async () => { + const { request, configure } = policyEpochFixture(100, 1); + const old = request(); + await old.send(); configure(); + await expect(old.send()).rejects.toThrow(SendBudgetExhaustedError); + expect(old.budget.physicalStarted).toBe(1); + old.tracker.settle({ inputTokens: 10 }); }); -test("an entirely observe-only capacity omission keeps terminal usage off the previous send", async () => { - const { ledger, tracker, budget, lines } = policyActivationFixture(1, 1); - const report = createPhysicalSendReporter(budget, () => ({ poolId: "P" })); - let wires = 0; - await fetchWithTransientRetry(async () => ++wires === 1 ? new Response(null, { status: 503 }) : new Response(), { - attempts: 2, onSendsConsumed: report, - }); - tracker.settle({ inputTokens: 70 }); - expect(lines.some(line => JSON.parse(line).kind === "forget")).toBe(false); - expect(wires).toBe(2); // L is not newly enforced for a wholly observe-only request. - expect(budget.used).toBe(2); - expect(ledger.snapshot("pool", "P")).toMatchObject({ settled: 0, reserved: 0, unresolved: 10 }); +for (const ownerError of [false, true]) test(`fixed enforced terminal debt retries after ${ownerError ? "owner error" : "append failure"}`, async () => { + const { request, journal, ledger, policy } = policyEpochFixture(50); + const old = request(); await old.send(); + const append = journal.append.bind(journal); + journal.append = line => { if (ownerError) throw new SpendLedgerOwnerError("SPEND_LEDGER_OWNER_NOT_HELD", "fixture"); append(line); }; + journal.failAppend = !ownerError; + if (ownerError) expect(() => old.tracker.settle({ inputTokens: 70 })).toThrow(SpendLedgerOwnerError); + else old.tracker.settle({ inputTokens: 70 }); + journal.append = append; journal.failAppend = false; ledger.prune(); + expect(createShippedSpendLedger({ policy, journal: spendTestJournal(journal.lines), salt: testSpendSalt, now: () => 2 }).snapshot("pool", "P")).toMatchObject({ settled: 70, unresolved: 0 }); }); -test("activation before same-target selection rebind requires NORMAL admission before wire", async () => { - const { ledger, tracker, budget, enable } = policyActivationFixture(); - expect(ledger.reserve({ sendId: "prior", scopes: { poolId: "P" }, inputTokens: 50, outputCeilingTokens: 0 }).reserved).toBe(true); - ledger.settle("prior", { inputTokens: 50, outputTokens: 0 }); - let wires = 0; - await expect(fetchWithTransientRetry(async () => { - enable(); - if (!rebindPhysicalSend(budget, { poolId: "P" })) throw new SendBudgetExhaustedError(); - wires++; return new Response(); - }, { attempts: 1, onSendsConsumed: createPhysicalSendReporter(budget, () => ({ poolId: "P" })) })).rejects.toThrow(SendBudgetExhaustedError); - tracker.settle(undefined); - expect(wires).toBe(0); - expect(ledger.snapshot("pool", "P")).toMatchObject({ settled: 50, reserved: 0, unresolved: 0 }); +test("the actual first target determines the frozen enforcement mode", async () => { + const journal = spendTestJournal(); + const ledger = createSpendReservationLedger({ journal, salt: testSpendSalt, policy: { ...DEFAULT_SPEND_RESERVATION_POLICY, identity: { maxTokens: 5 } } }); + const tracker = createRequestSpendTracker({ spendInputEstimateTokens: 10 }, undefined, ledger); + const budget = createRequestExecutionBudget(undefined, undefined, tracker); + expect(budget.spendEnforced).toBe(false); + await expect(fetchWithTransientRetry(async () => new Response(), { + attempts: 1, onSendsConsumed: createPhysicalSendReporter(budget, () => ({ identityId: "selected" })), + })).rejects.toThrow(SendBudgetExhaustedError); + expect(budget.spendEnforced).toBe(true); }); -test("an adapter permit started observe-only retains liability before an enforced second send", async () => { - const { ledger, tracker, budget, enable } = policyActivationFixture(); - for (const sendClass of ["initial", "transient"] as const) { - const decision = budget.reserveDispatch({ sendClass, targetKey: "P" }); - if (!decision.allowed) throw new Error("fixture reservation denied"); - expect(decision.permit.use()).toBe(true); - enable(); +for (const usage of [70, undefined]) test(`ordinary failed reserve repairs its prefix before terminal ${usage}`, async () => { + const { request, journal, ledger, policy } = policyEpochFixture(); + journal.failAppend = true; + const old = request(); await old.send(); + old.tracker.settle(usage === undefined ? undefined : { inputTokens: usage }); + journal.failAppend = false; ledger.prune(); + const expected = { settled: usage ?? 0, unresolved: usage === undefined ? 10 : 0, reserved: 0 }; + expect(ledger.snapshot("pool", "P")).toMatchObject(expected); + for (const create of [createSpendReservationLedger, createShippedSpendLedger]) { + expect(create({ policy, journal: spendTestJournal(journal.lines), salt: testSpendSalt, now: () => 2 }).snapshot("pool", "P")).toMatchObject(expected); } - tracker.settle({ inputTokens: 70 }); - expect(budget.used).toBe(2); - expect(budget.physicalStarted).toBe(2); - expect(ledger.snapshot("pool", "P")).toMatchObject({ settled: 70, reserved: 0, unresolved: 10 }); -}); - -test("an older off-mode pending proof stays refundable after another helper enables seeds", async () => { - const { ledger, tracker, budget, enable } = policyActivationFixture(); - const reserved = budget.reserveDispatch({ sendClass: "initial", targetKey: "P", countedExternally: true }); - if (!reserved.allowed) throw new Error("fixture reservation denied"); - enable(); - await fetchWithTransientRetry(async () => new Response(), { - attempts: 1, onSendsConsumed: createPhysicalSendReporter(budget, () => ({ poolId: "P" })), - }); - reserved.permit.release(); - tracker.settle({ inputTokens: 70 }); - expect(ledger.snapshot("pool", "P")).toMatchObject({ settled: 70, reserved: 0, unresolved: 0 }); + expect(journal.lines.map(line => JSON.parse(line).kind)).toEqual(["reserve", "dispatch", usage === undefined ? "lost" : "settle"]); }); -for (const legacyLast of [false, true]) test(`mixed direct legacy charge and physical send settle in dispatch order (${legacyLast})`, async () => { - const { ledger, tracker, budget, enable } = policyActivationFixture(); - if (!legacyLast) tracker.charge({ alreadySent: true }); - enable(); - await fetchWithTransientRetry(async () => new Response(), { - attempts: 1, onSendsConsumed: createPhysicalSendReporter(budget, () => ({ poolId: "P", identityId: "B" })), - }); - if (legacyLast) tracker.charge({ alreadySent: true }); - tracker.settle({ inputTokens: 70 }); - expect(ledger.snapshot("pool", "P")).toMatchObject({ settled: 70, reserved: 0, unresolved: 10 }); - expect(ledger.snapshot("identity", "A")).toMatchObject({ settled: legacyLast ? 70 : 0, unresolved: legacyLast ? 0 : 10 }); - expect(ledger.snapshot("identity", "B")).toMatchObject({ settled: legacyLast ? 0 : 70, unresolved: legacyLast ? 10 : 0 }); +test("captured ceiling preflight preserves exact prepaid exclusion across policy changes", () => { + const { request, configure, ledger } = policyEpochFixture(10); + const old = request(); + expect(old.budget.spendAdmissionPolicy).toBeUndefined(); + const seed = old.tracker.ensureSeed({ poolId: "P" }); + expect(seed).toBeDefined(); + const captured = old.budget.spendAdmissionPolicy!; + expect(Object.isFrozen(captured)).toBe(true); + expect(Object.isFrozen(captured.pool)).toBe(true); + configure(); + expect(ledger.exhausted("pool", "P")).toBe(false); + expect(ledger.exhausted("pool", "P", undefined, captured)).toBe(true); + expect(ledger.exhausted("pool", "P", seed!.sendId, captured)).toBe(false); + expect(ledger.exhausted("pool", "P", "foreign", captured)).toBe(true); + ledger.markDispatched(seed!.sendId); + expect(ledger.exhausted("pool", "P", seed!.sendId, captured)).toBe(true); }); -for (const dispatched of [false, true]) test(`a mixed old pending proof ${dispatched ? "dispatches" : "remains conservative"} after seeding`, async () => { - const { ledger, tracker, budget, enable } = policyActivationFixture(); - let proof: Parameters>[0] | undefined; - tracker.charge({ deferDispatch: true, onReserved: value => { proof = value; } }); - enable(); - await fetchWithTransientRetry(async () => new Response(), { - attempts: 1, onSendsConsumed: createPhysicalSendReporter(budget, () => ({ poolId: "P", identityId: "B" })), - }); - if (dispatched) tracker.dispatch!(proof!); - tracker.settle({ inputTokens: 70 }); - expect(ledger.snapshot("pool", "P")).toMatchObject({ settled: 70, reserved: 0, unresolved: 10 }); - expect(ledger.snapshot("identity", "A")).toMatchObject({ settled: dispatched ? 70 : 0, unresolved: dispatched ? 0 : 10 }); - expect(ledger.snapshot("identity", "B")).toMatchObject({ settled: dispatched ? 0 : 70, unresolved: dispatched ? 10 : 0 }); +test("started-off preflight retains an explicit empty ceiling view", () => { + const { request, configure, ledger } = policyEpochFixture(); + const old = request(); + old.budget.startRequest?.({ poolId: "P" }); + configure(1); + ledger.reserve({ sendId: "prior", scopes: { poolId: "P" }, inputTokens: 2, outputCeilingTokens: 0, alreadySent: true }); + expect(old.budget.spendAdmissionPolicy).toEqual({ root: {}, identity: {}, pool: {} }); + expect(ledger.exhausted("pool", "P")).toBe(true); + expect(ledger.exhausted("pool", "P", undefined, old.budget.spendAdmissionPolicy)).toBe(false); }); -for (const mixed of [false, true]) for (const ownerError of [false, true]) test(`activated observed terminal debt survives ${ownerError ? "owner throw" : "append failure"} (${mixed ? "mixed" : "ordinary"})`, async () => { - const journal = spendTestJournal(); - const append = journal.append.bind(journal); - let throwOwner = false; - journal.append = line => { - if (throwOwner) throw new SpendLedgerOwnerError("SPEND_LEDGER_OWNER_NOT_HELD", "fixture"); - append(line); - }; - const policy = spendTestPolicy({ pool: {}, canonicalProviderIds: ["P"] }); - const ledger = createSpendReservationLedger({ journal, policy, salt: testSpendSalt, now: () => 2 }); - const tracker = createRequestSpendTracker({ provider: "P", spendInputEstimateTokens: 10 }, undefined, ledger); - const budget = createRequestExecutionBudget(undefined, undefined, tracker); - await fetchWithTransientRetry(async () => { - ledger.reconfigure({ ...policy, pool: { maxTokens: 50 } }); return new Response(); - }, { attempts: 1, onSendsConsumed: createPhysicalSendReporter(budget, () => ({ poolId: "P" })) }); - if (mixed) await fetchWithTransientRetry(async () => new Response(), { - attempts: 1, onSendsConsumed: createPhysicalSendReporter(budget, () => ({ poolId: "P" })), +test("a prepaid reservation freezes L before a wider derived scope claims its first wire", async () => { + const { request } = policyEpochFixture(100, 1); + const { tracker, budget } = request(); + const decision = budget.reserveDispatch({ sendClass: "initial", targetKey: "P", countedExternally: true }); + if (!decision.allowed) throw new Error("fixture admission denied"); + const child = deriveRequestExecutionBudget(budget, { ...ONE_SEND_LEFT, maxTotalModelSends: 4, baseSendAllowance: 4 }); + expect(child.physicalLimit).toBe(1); + let wires = 0; + await fetchWithTransientRetry(async () => { wires++; return new Response(); }, { + attempts: 1, onSendsConsumed: createPhysicalSendReporter(child, () => ({ poolId: "P" }), decision.permit), }); - if (ownerError) throwOwner = true; - else journal.failAppend = true; - if (ownerError) expect(() => tracker.settle({ inputTokens: 70 })).toThrow(SpendLedgerOwnerError); - else tracker.settle({ inputTokens: 70 }); - throwOwner = false; journal.failAppend = false; - ledger.prune(); - const expected = { settled: 70, reserved: 0, unresolved: mixed ? 10 : 0 }; - expect(ledger.snapshot("pool", "P")).toMatchObject(expected); - for (const create of [createShippedSpendLedger, createSpendReservationLedger]) { - expect(create({ journal: spendTestJournal(journal.lines), policy, salt: testSpendSalt, now: () => 2 }).snapshot("pool", "P")).toMatchObject(expected); - } - const kinds = journal.lines.map(line => JSON.parse(line).kind); - expect(kinds.filter(kind => kind === "settle")).toHaveLength(1); - expect(kinds.filter(kind => kind === "lost")).toHaveLength(mixed ? 1 : 0); - expect(kinds.filter(kind => kind === "reserve")).toHaveLength(mixed ? 2 : 1); + await expect(fetchWithTransientRetry(async () => { wires++; return new Response(); }, { + attempts: 1, onSendsConsumed: createPhysicalSendReporter(child, () => ({ poolId: "P" })), + })).rejects.toThrow(SendBudgetExhaustedError); + expect(wires).toBe(1); + expect(budget.physicalStarted).toBe(1); + tracker.settle({ inputTokens: 10 }); }); describe("atomic dispatch permits", () => { diff --git a/tests/responses/chat-native-spend.test.ts b/tests/responses/chat-native-spend.test.ts index 2a4690feee6..1d2bd3516ef 100644 --- a/tests/responses/chat-native-spend.test.ts +++ b/tests/responses/chat-native-spend.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, expect, spyOn, test } from "bun:test"; -import { mkdtempSync } from "node:fs"; +import { mkdtempSync, readFileSync } from "node:fs"; import { createServer } from "node:net"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -17,6 +17,8 @@ import { estimateTokens } from "../../src/lib/token-estimate"; import { getRequestLogEntries } from "../../src/server/request-log"; import * as stateStores from "../../src/lib/state-store-registrations"; import { translatorAggregateCurrentBytesForTests } from "../../src/lib/translator-budget"; +import { configureSharedSpendLedger, spendPolicyFromConfig } from "../../src/lib/spend-reservation-ledger"; +import { clearKeyCooldowns } from "../../src/providers/key-failover"; let previousHome: string | undefined; let testDir = ""; @@ -39,6 +41,7 @@ function stopFixtureServers(): Promise { })(); } beforeEach(() => { + clearKeyCooldowns(); activeServer = undefined; activeUpstream = undefined; activeRawUpstream = undefined; @@ -58,6 +61,7 @@ afterEach(async () => { if (isolatedCodexHome) await flushWindowsSecretAclReapsBeforeRemoval(isolatedCodexHome.path); expect(spendLedgerOwnerSnapshot().ownership).toBe("unheld"); resetProviderRequestPacingForTest(); + clearKeyCooldowns(); if (previousHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousHome; isolatedCodexHome?.restore(); @@ -156,6 +160,69 @@ test("native Chat reuses its normal seed on a stable transient retry and retains }); }); +for (const initiallyEnforced of [false, true]) test(`native Chat freezes ${initiallyEnforced ? "enforced" : "observe-only"} policy across retries`, async () => { + let wires = 0; + activeUpstream = Bun.serve({ port: 0, fetch() { + if (++wires === 1) configureSharedSpendLedger(spendPolicyFromConfig( + initiallyEnforced ? {} : { pool: { maxTokens: 1 } }, undefined, ["mock"])); + return Response.json({ error: { message: "temporary" } }, { status: 503 }); + } }); + const config = mockConfig(new URL("/v1", activeUpstream.url).href, { transientRetryOn5xx: { attempts: 6 } }); + if (initiallyEnforced) config.spend = { pool: { maxTokens: 100_000 } }; + saveConfig(config); + activeServer = startServer(0); + const post = () => fetch(new URL("/v1/chat/completions", activeServer!.url), { + method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "mock/test-model", messages: [{ role: "user", content: "hello" }], max_tokens: 1 }), + }); + const response = await post(); + await response.text(); + expect(wires).toBe(initiallyEnforced ? 4 : 6); + expect(response.status).toBe(initiallyEnforced ? 429 : 503); + const records = readFileSync(join(testDir, "spend-ledger.jsonl"), "utf8").trim().split("\n").map(line => JSON.parse(line)); + expect(records.filter(record => record.kind === "reserve")).toHaveLength(wires); + if (!initiallyEnforced) { + const next = await post(); + await next.text(); + expect(next.status).toBe(429); + expect(next.headers.get("x-opencodex-local-refusal")).toBe("workflow_spend_exhausted"); + expect(wires).toBe(6); + } +}, 20_000); + +for (const change of ["disable", "raise"] as const) test(`native Chat retains its initial ceiling when ${change} occurs before key failover`, async () => { + const messages = [{ role: "user", content: "hello" }]; + let wires = 0; + activeUpstream = Bun.serve({ port: 0, fetch() { + if (++wires === 1) { + configureSharedSpendLedger(spendPolicyFromConfig(change === "disable" ? {} : { pool: { maxTokens: 100_000 } }, undefined, ["mock"])); + return Response.json({ error: { message: "key rate limited" } }, { status: 429, headers: { "retry-after": "30" } }); + } + return Response.json({ id: "chat_fixture", object: "chat.completion", model: "test-model", + choices: [{ index: 0, message: { role: "assistant", content: "ok" }, finish_reason: "stop" }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } }); + } }); + const config = mockConfig(new URL("/v1", activeUpstream.url).href, { apiKeyPool: [ + { id: "first", key: "k", addedAt: 1 }, { id: "second", key: "k2", addedAt: 2 }, + ] }); + config.spend = { pool: { maxTokens: estimateTokens(JSON.stringify(messages), "mock/test-model") + 1 } }; + saveConfig(config); + activeServer = startServer(0); + const post = () => fetch(new URL("/v1/chat/completions", activeServer!.url), { + method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "mock/test-model", messages, max_tokens: 1 }), + }); + const response = await post(); + await response.text(); + expect(response.status).toBe(429); + expect(response.headers.get("x-opencodex-local-refusal")).toBe("workflow_spend_exhausted"); + expect(wires).toBe(1); + const next = await post(); + await next.text(); + expect(next.status).toBe(200); + expect(wires).toBe(2); +}); + test("native Chat includes tool definitions in its pre-dispatch spend reservation", async () => { const upstream = mockChatUpstreamCapturing(); const config = mockConfig(`${upstream.server.url.toString().replace(/\/$/, "")}/v1`); diff --git a/tests/responses/responses-spend-anchor.test.ts b/tests/responses/responses-spend-anchor.test.ts index e9acb5ec3c7..2c0320f08c3 100644 --- a/tests/responses/responses-spend-anchor.test.ts +++ b/tests/responses/responses-spend-anchor.test.ts @@ -20,6 +20,7 @@ import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; import { resetProviderRequestPacingForTest } from "../../src/providers/request-pacing"; import { acquireSpendLedgerServerLifecycle } from "../../src/server/index/spend-ledger-lifecycle"; +import { createResponsesSendBudget } from "../../src/server/responses/request-send-budget"; function fixture(enforced = true, capacity = 8) { const lines: string[] = []; @@ -206,6 +207,125 @@ async function invokeExecutor(kind: ExecutorKind, config: OcxConfig, logCtx: Req } finally { translatorBudget.dispose(); } } +for (const recovery of [false, true]) test(`generic observe-only executor captures before ${recovery ? "same-key recovery" : "an uncounted send"}`, async () => withExecutorHome(async ledger => { + configureSharedSpendLedger({ ...DEFAULT_SPEND_RESERVATION_POLICY, canonicalProviderIds: ["P"] }); + const ctx: RequestLogContext = { model: "model", provider: "P", spendPoolId: "P", spendInputEstimateTokens: 2, spendOutputCeilingTokens: 1 }; + const tracker = createRequestSpendTracker(ctx, undefined, ledger); + ctx.spendTracker = tracker; + const budget = createRequestExecutionBudget(undefined, undefined, tracker); + let wires = 0; + const wire = (async () => { + wires++; + expect(budget.spendPolicyStarted).toBe(true); + if (wires === 1) { + configureSharedSpendLedger({ ...DEFAULT_SPEND_RESERVATION_POLICY, canonicalProviderIds: ["P"], pool: { maxTokens: 1 } }); + if (recovery) return Response.json({ error: { message: "rate limited" } }, { status: 429 }); + } + expect(budget.spendEnforced).toBe(false); + return executorSuccess("generic"); + }) as typeof fetch; + globalThis.fetch = wire; + const config = executorConfig("generic", wire); + delete config.providers.P!.transientRetryOn5xx; + if (recovery) config.providers.P!.retryOn429 = { attempts: 1, intervalMs: 1, maxIntervalMs: 1, respectRetryAfter: false }; + const response = await invokeExecutor("generic", config, ctx, budget); + expect(response.status, await response.text()).toBe(200); + expect(wires).toBe(recovery ? 2 : 1); + expect(budget.physicalStarted).toBe(0); + expect(budget.used).toBe(0); +})); + +test("generic terminal continuation retains the policy captured by its uncounted initial send", async () => withExecutorHome(async ledger => { + configureSharedSpendLedger({ ...DEFAULT_SPEND_RESERVATION_POLICY, canonicalProviderIds: ["P"] }); + const ctx: RequestLogContext = { model: "model", provider: "P", spendPoolId: "P" }; + const tracker = createRequestSpendTracker(ctx, undefined, ledger); + ctx.spendTracker = tracker; + const budget = createRequestExecutionBudget(undefined, undefined, tracker); + let wires = 0; + const wire = (async () => { + wires++; + expect(budget.spendPolicyStarted).toBe(true); + if (wires === 1) configureSharedSpendLedger({ ...DEFAULT_SPEND_RESERVATION_POLICY, canonicalProviderIds: ["P"], pool: { maxTokens: 1 } }); + expect(budget.spendEnforced).toBe(false); + const delta = wires === 1 ? { content: "我接下来会修改相关文件。" } + : { tool_calls: [{ index: 0, id: "call_fixture", type: "function", function: { name: "exec_command", arguments: "{}" } }] }; + return new Response(`data: ${JSON.stringify({ choices: [{ index: 0, delta }] })}\n\ndata: ${JSON.stringify({ choices: [{ index: 0, delta: {}, finish_reason: wires === 1 ? "stop" : "tool_calls" }] })}\n\ndata: [DONE]\n\n`, { headers: { "content-type": "text/event-stream" } }); + }) as typeof fetch; + globalThis.fetch = wire; + const config = executorConfig("generic", wire); + delete config.providers.P!.transientRetryOn5xx; + config.providers.P!.terminalContinuationGuard = true; + const req = new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "P/model", input: "Edit the file", stream: true, tools: [{ type: "function", name: "exec_command", parameters: { type: "object", properties: {} } }] }) }); + const response = await handleResponses(req, config, ctx, { sendBudget: budget }); + const text = await response.text(); + expect(response.status, text).toBe(200); + expect(text).toContain("exec_command"); + expect(wires).toBe(2); + expect(budget.physicalStarted).toBe(0); + expect(budget.used).toBe(0); +})); + +for (const originallyEnforced of [false, true]) test(`Responses re-entry retains ${originallyEnforced ? "higher" : "disabled"} starting ceiling after a new lower ceiling`, async () => withExecutorHome(async ledger => { + configureSharedSpendLedger({ ...DEFAULT_SPEND_RESERVATION_POLICY, canonicalProviderIds: ["P"], pool: originallyEnforced ? { maxTokens: 100 } : {} }); + expect(ledger.reserve({ sendId: "history", scopes: { poolId: "P" }, inputTokens: 10, outputCeilingTokens: 0 }).reserved).toBe(true); + ledger.settle("history", { inputTokens: 10, outputTokens: 0 }); + const ctx: RequestLogContext = { model: "model", provider: "P", spendPoolId: "P", spendInputEstimateTokens: 1 }; + const tracker = createRequestSpendTracker(ctx, undefined, ledger); + const budget = createRequestExecutionBudget(undefined, undefined, tracker); + expect(budget.spendPolicyStarted).toBe(false); + budget.startRequest?.({ poolId: "P" }); + configureSharedSpendLedger({ ...DEFAULT_SPEND_RESERVATION_POLICY, canonicalProviderIds: ["P"], pool: { maxTokens: 1 } }); + const req = new Request("http://localhost/v1/responses"); + const child = createResponsesSendBudget({ req, logCtx: ctx, options: { sendBudget: budget } }); + expect(child).not.toBeInstanceOf(Response); + if (child instanceof Response) throw new Error("frozen request refused by current policy"); + const admission = child.adapterDispatchBudget!.reserveDispatch({ sendClass: "initial", targetKey: "P/model" }); + expect(admission.allowed).toBe(true); + if (admission.allowed) admission.permit.release(); + expect(createResponsesSendBudget({ req, logCtx: { ...ctx }, options: {} })).toBeInstanceOf(Response); + tracker.settle(undefined); +})); + +for (const change of ["disable", "raise"] as const) test(`Responses re-entry still applies the original seed ceiling after ${change}`, async () => withExecutorHome(async ledger => { + configureSharedSpendLedger({ ...DEFAULT_SPEND_RESERVATION_POLICY, canonicalProviderIds: ["P"], pool: { maxTokens: 1 } }); + const ctx: RequestLogContext = { model: "model", provider: "P", spendPoolId: "P", spendInputEstimateTokens: 2 }; + const tracker = createRequestSpendTracker(ctx, undefined, ledger); + const budget = createRequestExecutionBudget(undefined, undefined, tracker); + budget.startRequest?.({ poolId: "P" }); + configureSharedSpendLedger({ ...DEFAULT_SPEND_RESERVATION_POLICY, canonicalProviderIds: ["P"], pool: change === "disable" ? {} : { maxTokens: 100 } }); + const req = new Request("http://localhost/v1/responses"); + const child = createResponsesSendBudget({ req, logCtx: ctx, options: { sendBudget: budget } }); + expect(child).not.toBeInstanceOf(Response); + if (child instanceof Response) throw new Error("unexpected preflight refusal"); + expect(child.adapterDispatchBudget!.reserveDispatch({ sendClass: "initial", targetKey: "P/model" })) + .toMatchObject({ allowed: false, reason: "spend-exhausted" }); +})); + +for (const change of ["disable", "raise", "lower"] as const) { + for (const rootId of [undefined, "snapshot-root"]) test(`Responses frozen ${rootId ? "root" : "pool"} preflight keeps one-use prepaid exclusion after ${change}`, async () => withExecutorHome(async ledger => { + configureSharedSpendLedger({ ...DEFAULT_SPEND_RESERVATION_POLICY, canonicalProviderIds: ["P"], root: { maxTokens: 10 }, pool: { maxTokens: 10 } }); + const ctx: RequestLogContext = { model: "model", provider: "P", spendPoolId: "P", spendInputEstimateTokens: 10 }; + const tracker = createRequestSpendTracker(ctx, rootId, ledger); + const budget = createRequestExecutionBudget(undefined, undefined, tracker); + const prepaid = budget.reserveDispatch({ sendClass: "initial", targetKey: "P/model", countedExternally: true }); + if (!prepaid.allowed) throw new Error("prepaid admission refused"); + const scopePolicy = change === "disable" ? {} : { maxTokens: change === "raise" ? 100 : 1 }; + configureSharedSpendLedger({ ...DEFAULT_SPEND_RESERVATION_POLICY, canonicalProviderIds: ["P"], root: scopePolicy, pool: scopePolicy }); + const req = new Request("http://localhost/v1/responses", { headers: rootId ? { "x-codex-parent-thread-id": rootId } : {} }); + const options = { sendBudget: budget, compactionRecoveryPermit: prepaid.permit }; + expect(createResponsesSendBudget({ req, logCtx: ctx, options })).not.toBeInstanceOf(Response); + const repeated = createResponsesSendBudget({ req, logCtx: ctx, options }); + expect(repeated).toBeInstanceOf(Response); + if (repeated instanceof Response) { + expect(repeated.headers.get("x-opencodex-local-refusal")).toBe("workflow_spend_exhausted"); + expect(await repeated.text()).toContain("10"); + } + prepaid.permit.release(); + tracker.settle(undefined); + })); +} + for (const kind of executorKinds) { test(`${kind} production executor refuses NORMAL capacity before any wire call`, async () => withExecutorHome(async ledger => { expect(ledger.reserve({ sendId: "busy", scopes: { poolId: "P" }, inputTokens: 1, outputCeilingTokens: 0 }).reserved).toBe(true); diff --git a/tests/responses/responses-spend-capacity-guard.test.ts b/tests/responses/responses-spend-capacity-guard.test.ts index 2207678f0ea..32638a9be00 100644 --- a/tests/responses/responses-spend-capacity-guard.test.ts +++ b/tests/responses/responses-spend-capacity-guard.test.ts @@ -70,24 +70,27 @@ describe("configured spend cannot dispatch without a capacity booking", () => { expect(f.ledger.snapshot("pool", "fixture-pool")?.reserved).toBe(1); }); - test("enabling a ceiling affects the existing tracker on its next charge", () => { + test("enabling a ceiling applies to new requests while the started tracker remains observe-only", () => { const f = fixture({ maxTrackedScopes: 1 }); expect(f.attempt().allowed).toBe(true); f.ledger.reconfigure({ ...f.ledger.policy, pool: { maxTokens: 1000 } }); - expect(f.attempt().allowed).toBe(false); - expect(f.dispatched()).toBe(1); - expect(f.budget.used).toBe(1); - expect(f.tracker.refusals).toBe(1); + expect(f.attempt().allowed).toBe(true); + expect(f.dispatched()).toBe(2); + expect(f.tracker.refusals).toBe(0); + const next = createRequestSpendTracker(f.context, "fixture-root", f.ledger); + expect(next.charge()).toBe(false); }); - test("newly resolved identity applies on the next charge without reconstructing the tracker", () => { + test("a newly resolved identity does not change a request that already started observe-only", () => { const f = fixture({ identity: { maxTokens: 1000 }, maxTrackedScopes: 1 }); delete f.context.accountLogLabel; expect(f.attempt().allowed).toBe(true); f.context.accountLogLabel = "newly-resolved-account"; - expect(f.attempt().allowed).toBe(false); - expect(f.dispatched()).toBe(1); - expect(f.budget.used).toBe(1); + expect(f.attempt().allowed).toBe(true); + expect(f.dispatched()).toBe(2); + expect(f.budget.used).toBe(2); + const next = createRequestSpendTracker(f.context, "fixture-root", f.ledger); + expect(next.charge()).toBe(false); }); }); From fe0ffa6ed2aa4b008f0555df909ef6e04f9f023c Mon Sep 17 00:00:00 2001 From: JUN Date: Fri, 9 Oct 2026 02:20:06 +0900 Subject: [PATCH 14/15] fix(spend): retain reported sends through ownership errors Co-authored-by: Epinephrine --- src/adapters/coding-agent/turn.ts | 13 +- src/lib/request-execution-budget.ts | 24 ++-- src/lib/spend-reservation-ledger.ts | 29 ++-- src/server/chat-native.ts | 67 ++++----- src/server/messages-native.ts | 7 +- src/server/responses/compact.ts | 13 +- src/server/responses/passthrough-dispatch.ts | 5 +- src/server/responses/request-spend.ts | 41 ++++-- structure/transports/responses-spend.md | 4 + tests/lib/spend-seed-overflow.test.ts | 128 ++++++++++++++++++ .../codebuddy-tool-bridge-turn.test.ts | 54 +++++++- .../responses/responses-spend-anchor.test.ts | 48 ++++++- 12 files changed, 354 insertions(+), 79 deletions(-) diff --git a/src/adapters/coding-agent/turn.ts b/src/adapters/coding-agent/turn.ts index 82255bbae18..3a2596bb1b8 100644 --- a/src/adapters/coding-agent/turn.ts +++ b/src/adapters/coding-agent/turn.ts @@ -291,9 +291,10 @@ export async function runCodingAgentTurn(input: CodingAgentTurnInput): Promise undefined); + try { permit?.release(); } + finally { + try { producer?.close(); } + finally { + if (toolBridgeDir) await rm(toolBridgeDir, { recursive: true, force: true }).catch(() => undefined); + } + } } } diff --git a/src/lib/request-execution-budget.ts b/src/lib/request-execution-budget.ts index f481a0ee96a..33d0418bf23 100644 --- a/src/lib/request-execution-budget.ts +++ b/src/lib/request-execution-budget.ts @@ -319,12 +319,14 @@ export function createPhysicalSendReporter( if (closed) return; if (!producer) { if (observer?.enforced !== true) { reportDispatchSends(budget, sends, permit); telemetry?.(sends); } return; } const count = Math.min(Math.max(0, Math.trunc(sends)), started - reported); - producer.report(count); - reported += count; - const receipt = permit && dispatchSpendProofs.get(permit); - const prepaid = count > 0 && receipt && receipt.owner === counter && receipt.report() ? 1 : 0; - if (counter) counter.spent += count - prepaid; - telemetry?.(count); + try { producer.report(count); } + finally { + reported += count; + const receipt = permit && dispatchSpendProofs.get(permit); + const prepaid = count > 0 && receipt && receipt.owner === counter && receipt.report() ? 1 : 0; + if (counter) counter.spent += count - prepaid; + telemetry?.(count); + } }; if (observer?.startRequest || observer?.enforced === true) report.beforeSend = () => { if (closed) return false; @@ -351,9 +353,8 @@ export function createPhysicalSendReporter( }; report.close = () => { if (closed) return; - if (producer) report(started - reported); - closed = true; - producer?.close(); + try { if (producer) report(started - reported); } + finally { closed = true; producer?.close(); } }; return report; } @@ -511,7 +512,7 @@ function createRequestExecutionBudgetWithLedger( producer = observer.beginReporter?.(); producer?.start(seed, ordinal); physicalOrdinal = ordinal; - if (!executing) { producer?.report(1); producer?.close(); } + if (!executing) { try { producer?.report(1); } finally { producer?.close(); } } return true; }; let settled: "open" | "used" | "released" = "open"; @@ -526,8 +527,7 @@ function createRequestExecutionBudgetWithLedger( try { return await activePhysicalReporters.run(binding, run); } finally { executing = false; - producer?.report(1); - producer?.close(); + try { producer?.report(1); } finally { producer?.close(); } } }, use(): boolean { diff --git a/src/lib/spend-reservation-ledger.ts b/src/lib/spend-reservation-ledger.ts index 81c46733ee9..0f22c2170ee 100644 --- a/src/lib/spend-reservation-ledger.ts +++ b/src/lib/spend-reservation-ledger.ts @@ -850,6 +850,12 @@ export function createSpendReservationLedger(options: { } }; + // Already-sent debt owns its record before I/O, including an owner error that propagates. + const appendReported = (record: JournalRecord): void => { + pendingRecords.push(record); + if (pendingRecords.length === 1 && append(record)) pendingRecords.shift(); + }; + const flushPending = (): boolean => { while (pendingRecords.length > 0) { if (!append(pendingRecords[0]!)) return false; @@ -1208,16 +1214,19 @@ export function createSpendReservationLedger(options: { assertOwnedAccounting?.(); const owned = seeds.get(seed); const send = aliasFor("send", request.sendId); - if (!owned || owned.abandoned || reservations.has(send) || seedBySend.has(send) - || owned.targets.some(ref => !scopes.has(scopeKey(ref.scope, ref.alias)))) return false; + if (!owned || owned.abandoned || owned.targets.some(ref => !scopes.has(scopeKey(ref.scope, ref.alias)))) return false; const tokens = sanitizeTokens(request.inputTokens) + sanitizeTokens(request.outputCeilingTokens); + const existing = reservations.get(send); + if (existing || seedBySend.has(send)) { + return seedBySend.get(send) === owned && existing !== undefined && isLive(existing.status) && existing.tokens === tokens; + } const at = now(); const record: JournalRecord = { v: 1, kind: "reserve", send, targets: [...owned.targets], tokens, at }; - // The physical send already happened. Keep its debt even if storage is temporarily unavailable. - if (pendingRecords.length > 0 || !append(record)) pendingRecords.push(record); + // Enrollment is stable before an owner error can escape the journal write. applyReserve(send, owned.targets, tokens, at); owned.sends.add(send); seedBySend.set(send, owned); + appendReported(record); compact(at); return true; }, @@ -1264,15 +1273,16 @@ export function createSpendReservationLedger(options: { let closed = false; return { close() { if (closed) return; - assertOwnedAccounting?.(); closed = true; reporters -= 1; if (owner) { const remaining = activeReporters(owner) - 1; if (remaining === 0) ownerReporters.delete(owner); else ownerReporters.set(owner, remaining); } - retryCleanup(); - if (reporters === 0) { for (const resolve of drainWaiters) resolve(); drainWaiters.clear(); } + try { assertOwnedAccounting?.(); retryCleanup(); } + finally { + if (reporters === 0) { for (const resolve of drainWaiters) resolve(); drainWaiters.clear(); } + } } }; }, deferCleanup(owner, cleanup) { @@ -1376,14 +1386,15 @@ export function createSpendReservationLedger(options: { assertOwnedAccounting?.(); const send = aliasFor("send", sendId); const reservation = reservations.get(send); + if (reservation?.status === "dispatched" && seedForSend(send)) return true; if (!reservation || reservation.status !== "open") return false; repairReservePrefix(send, reservation); const at = now(); applyDispatch(send, at); const record: JournalRecord = { v: 1, kind: "dispatch", send, at }; const retained = seedForSend(send) || retainedOrdinary.has(send); - if (retained && pendingRecords.length > 0) pendingRecords.push(record); - else if (!append(record) && retained) pendingRecords.push(record); + if (retained) appendReported(record); + else append(record); return true; }, diff --git a/src/server/chat-native.ts b/src/server/chat-native.ts index 6b7302205b8..c6aa3d045a4 100644 --- a/src/server/chat-native.ts +++ b/src/server/chat-native.ts @@ -458,39 +458,42 @@ export async function runNativeChatAttempt( const spendReport = createPhysicalSendReporter(physicalBudget, () => ({ poolId: logCtx.spendPoolId ?? route.providerName, identityId: logCtx.accountLogLabel, }), execution.comboDispatchPermit); - // Capture this request's spend policy at its first physical boundary, including - // observe-only starts; later configuration changes belong to later requests. - if (spendReport.beforeSend && !spendReport.beforeSend()) { - spendReport.close?.(); - if ((logCtx.spendTracker as { refusals?: number } | undefined)?.refusals) throw new NativeChatSpendRefusal(); - throw new SendBudgetExhaustedError(); - } - if (!physicalBudget.spendEnforced && sendBudget) { - // Backstop for sends the helper cannot see coming (a reset replay). The first - // report settles the combo's booking; each later one is charged and booked. - if (physicalSends > 0 && sendBudget.remainingBaseSends(sharedSendCap) <= 0) { - throw new SendBudgetExhaustedError(safeHostLabel(request.url)); - } - physicalSends += 1; - reportDispatchSends(sendBudget, 1, execution.comboDispatchPermit); - } else if (!physicalBudget.spendEnforced && !spendTracker?.charge()) throw new NativeChatSpendRefusal(); - noteProviderAttemptSend(logCtx, route.providerName, activeProvider, logCtx.usageLogInputTokens, transportRecovery ?? recovery); - // A reselected provider transport is still a physical send: the connection policy - // and manual-redirect ownership wrap the selected implementation (#4992). let dispatched: Response; - try { dispatched = await sendWithConnectionPolicy( - (activeProvider as OcxProviderTransport).fetch ?? execute, - request.url, - applyUpstreamRecoveryInit({ - ...init, method: request.method, headers, body: request.body, - }, transportRecovery), - // Reselection can replace the provider transport and the wire shape, so the - // egress route is bound to the provider this send actually uses. Omitting it - // here would let a provider transport bypass its configured route entirely, - // because that transport wins over the executor that carries the binding. - { providerName: route.providerName, provider: activeProvider }, - ); - } finally { if (physicalBudget.spendEnforced) spendReport(1); spendReport.close?.(); } + try { + // Capture this request's spend policy at its first physical boundary, including + // observe-only starts; later configuration changes belong to later requests. + if (spendReport.beforeSend && !spendReport.beforeSend()) { + if ((logCtx.spendTracker as { refusals?: number } | undefined)?.refusals) throw new NativeChatSpendRefusal(); + throw new SendBudgetExhaustedError(); + } + if (!physicalBudget.spendEnforced && sendBudget) { + // Backstop for sends the helper cannot see coming (a reset replay). The first + // report settles the combo's booking; each later one is charged and booked. + if (physicalSends > 0 && sendBudget.remainingBaseSends(sharedSendCap) <= 0) { + throw new SendBudgetExhaustedError(safeHostLabel(request.url)); + } + physicalSends += 1; + reportDispatchSends(sendBudget, 1, execution.comboDispatchPermit); + } else if (!physicalBudget.spendEnforced && !spendTracker?.charge()) throw new NativeChatSpendRefusal(); + noteProviderAttemptSend(logCtx, route.providerName, activeProvider, logCtx.usageLogInputTokens, transportRecovery ?? recovery); + // A reselected provider transport is still a physical send: the connection policy + // and manual-redirect ownership wrap the selected implementation (#4992). + dispatched = await sendWithConnectionPolicy( + (activeProvider as OcxProviderTransport).fetch ?? execute, + request.url, + applyUpstreamRecoveryInit({ + ...init, method: request.method, headers, body: request.body, + }, transportRecovery), + // Reselection can replace the provider transport and the wire shape, so the + // egress route is bound to the provider this send actually uses. Omitting it + // here would let a provider transport bypass its configured route entirely, + // because that transport wins over the executor that carries the binding. + { providerName: route.providerName, provider: activeProvider }, + ); + } finally { + try { if (physicalBudget.spendEnforced) spendReport(1); } + finally { spendReport.close?.(); } + } if (!dispatched.ok) await recordKeyAttemptFailure(logCtx, dispatched, init.signal ?? upstream.signal); return dispatched; }, diff --git a/src/server/messages-native.ts b/src/server/messages-native.ts index dc6d3e6d711..39682da8e97 100644 --- a/src/server/messages-native.ts +++ b/src/server/messages-native.ts @@ -550,8 +550,11 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions) { providerName: route.providerName, provider: activeProvider }, ); } finally { - if (physicalBudget.spendEnforced) spendReport(1); - spendReport.close?.(); releaseFamily(); + try { if (physicalBudget.spendEnforced) spendReport(1); } + finally { + try { spendReport.close?.(); } + finally { releaseFamily(); } + } } if (ownsBearer && snapshot) { try { diff --git a/src/server/responses/compact.ts b/src/server/responses/compact.ts index 90535b1d5e3..bce7005946a 100644 --- a/src/server/responses/compact.ts +++ b/src/server/responses/compact.ts @@ -1026,14 +1026,21 @@ export async function handleResponsesCompact( // `kind === "pool"` and a main-pool credential left it false -- so 401 then 429 really did // reach five. The single sends spend the base allowance first and then the one shared // final-recovery reserve, which is the same rule the Responses path follows. - const sendSingleCompactAttempt = ( + const sendSingleCompactAttempt = async ( doFetch: () => Promise, ): Promise => { if (sendBudget.remainingBaseSends(TRANSIENT_RETRY_MAX_ATTEMPTS) > 0) { const report = createPhysicalSendReporter(sendBudget, () => ({ poolId: logCtx.spendPoolId ?? route.providerName, identityId: logCtx.accountLogLabel })); - if (report.beforeSend?.() === false) { report.close?.(); return Promise.reject(new SendBudgetExhaustedError(safeHostLabel(compactUrl))); } - return doFetch().finally(() => { report(1); report.close?.(); }); + let started = false; + try { + if (report.beforeSend?.() === false) throw new SendBudgetExhaustedError(safeHostLabel(compactUrl)); + started = true; + return await doFetch(); + } finally { + try { if (started) report(1); } + finally { report.close?.(); } + } } const decision = sendBudget.reserveDispatch({ sendClass: "auth-recovery", diff --git a/src/server/responses/passthrough-dispatch.ts b/src/server/responses/passthrough-dispatch.ts index df04ac4282b..e61ea985550 100644 --- a/src/server/responses/passthrough-dispatch.ts +++ b/src/server/responses/passthrough-dispatch.ts @@ -841,7 +841,10 @@ export async function preparePassthroughExchange( }), route.provider.authMode === "forward", ); - return (report.execute ? report.execute(run) : run()).finally(() => { if (started) report(1); report.close?.(); }); + return (report.execute ? report.execute(run) : run()).finally(() => { + try { if (started) report(1); } + finally { report.close?.(); } + }); }; /** * Refuse a built body that exceeds the operator's configured ceiling, before it is sent. diff --git a/src/server/responses/request-spend.ts b/src/server/responses/request-spend.ts index 7a2bc6d6e4f..ac217e6899e 100644 --- a/src/server/responses/request-spend.ts +++ b/src/server/responses/request-spend.ts @@ -84,7 +84,7 @@ export function createRequestSpendTracker( const retiredSeeds = new Set(); const resolvedSendIds = new Set(); const completedSeeds = new Set(); - const physical = new Map(); + const physical = new Map(); const selectedScopes = (target: SpendTargetIdentity = {}): SpendScopes => ({ ...(rootId !== undefined ? { rootId } : {}), ...(target.identityId ?? logCtx.accountLogLabel ? { identityId: target.identityId ?? logCtx.accountLogLabel } : {}), @@ -105,6 +105,7 @@ export function createRequestSpendTracker( return retiredSeeds.size === 0; }; const tryFinalizeSeeds = (): boolean => { + retryPhysicalReports(); // All starts are booked before producers close. Ordinal, never callback order, owns usage. const entries = [...physical.entries()].sort((a, b) => a[0] - b[0]); const terminal = entries.at(-1)?.[1].sendId; @@ -134,8 +135,15 @@ export function createRequestSpendTracker( resolved = true; return true; }; - const deferCleanup = (): void => ledger().deferCleanup(tracker, () => resolved - || (finalRequested && reporters === 0 ? tryFinalizeSeeds() : cleanupRetiredSeeds())); + const retryPhysicalReports = (): void => { + for (const [ordinal, entry] of physical) if (entry.reportPending && !entry.reportComplete) tracker.reportFromSeed(entry.seed, ordinal); + }; + const deferCleanup = (): void => ledger().deferCleanup(tracker, () => { + if (resolved) return true; + retryPhysicalReports(); + if (finalRequested && reporters === 0) return tryFinalizeSeeds(); + return cleanupRetiredSeeds() && reporters === 0; + }); const finalizeSeeds = (): void => { if (!finalRequested || reporters !== 0 || resolved || !seededAccounting) return; // Own terminal retry before an append can throw; propagation must not orphan the debt. @@ -259,6 +267,7 @@ export function createRequestSpendTracker( }, beginReporter() { if (resolved || finalRequested && reporters === 0) throw new Error("Spend reporter registered after final closure"); + deferCleanup(); const lease = ledger().registerReporter(tracker); reporters++; let closed = false; @@ -292,34 +301,40 @@ export function createRequestSpendTracker( report(sends) { if (closed) return; const count = Number.isFinite(sends) ? Math.max(0, Math.trunc(sends)) : 0; + // Own the whole reported batch before the first append can interrupt its cursor. + for (const start of starts.slice(reported, reported + count)) physical.get(start.ordinal)!.reportPending = true; for (let i = 0; i < count && reported < starts.length; i++) { - const entry = starts[reported++]!; + const entry = starts[reported]!; tracker.reportFromSeed(entry.seed, entry.ordinal); + reported++; } }, close() { if (closed) return; // Cancellation still books every claimed start conservatively. - this.report(starts.length - reported); - closed = true; - reporters--; - lease.close(); - finalizeSeeds(); + try { this.report(starts.length - reported); } + finally { + closed = true; + reporters--; + try { lease.close(); } finally { finalizeSeeds(); } + } }, }; }, reportFromSeed(seed, ordinal) { const entry = physical.get(ordinal); const state = seedState(seed); - if (!entry || entry.seed !== seed || entry.sendId || !state) return; + if (!entry || entry.seed !== seed || entry.reportComplete || !state) return; + entry.reportPending = true; const firstOrdinal = [...physical.entries()].filter(([, send]) => send.seed === seed).map(([order]) => order).sort((a, b) => a - b)[0]; - const sendId = ordinal === firstOrdinal ? seed.sendId : randomUUID(); - if (sendId !== seed.sendId && !ledger().reserveReportedFromSeed(seed, { sendId, ...estimates() })) { + const sendId = entry.sendId ??= ordinal === firstOrdinal ? seed.sendId : randomUUID(); + const estimate = entry.estimate ??= estimates(); + if (sendId !== seed.sendId && !ledger().reserveReportedFromSeed(seed, { sendId, ...estimate })) { throw new Error("Already-sent spend could not be persisted"); } state.reported = true; if (!ledger().markDispatched(sendId)) throw new Error("Already-sent spend dispatch could not be persisted"); - entry.sendId = sendId; + entry.reportComplete = true; }, requestFinalSettlement(usage) { if (!finalRequested) { finalRequested = true; finalUsage = usage; } diff --git a/structure/transports/responses-spend.md b/structure/transports/responses-spend.md index 7ffa726b9a5..90d32f14c12 100644 --- a/structure/transports/responses-spend.md +++ b/structure/transports/responses-spend.md @@ -217,6 +217,10 @@ uses the same unbound-history refusal explanation and workflow header as other R persists ordinary `forget` records before releasing seeds. Each tracker waits only for its own reporters, so overlapping traffic cannot pin a completed request. Storage-failed cleanup stays queued on the ledger and retries on later admission, pruning, reconfiguration or reporter closure. +Already-started reports retain a stable send ID and estimate through partial writes. Reporting a +batch owns every pending start before its first append; same-seed retries reconcile those records +without admitting another send. Owner errors propagate while preserving queued liability, and +caller finalizers still close reporter leases so shutdown can drain after a failed report. An abandoned seed leaves every admission lookup immediately. Its remaining cleanup obligation stays separate, including when durable forgetting throws an ownership or unsafe-file error. Send-to-seed and reference-counted scope-pin indexes avoid scanning all seeds on admission. diff --git a/tests/lib/spend-seed-overflow.test.ts b/tests/lib/spend-seed-overflow.test.ts index cfaa84f6319..ce3494c2139 100644 --- a/tests/lib/spend-seed-overflow.test.ts +++ b/tests/lib/spend-seed-overflow.test.ts @@ -418,3 +418,131 @@ for (const retirement of ["rebind", "refund"] as const) { }); } } + +for (const phase of ["first-dispatch", "second-reserve"] as const) { + for (const failure of ["append", "owner"] as const) { + test(`physical report recovers ${phase} ${failure} failure without another settlement`, async () => { + const disk = spendTestJournal(); + const append = disk.append.bind(disk); + let failing = false; + const ownerError = new SpendLedgerOwnerError("SPEND_LEDGER_OWNER_UNAVAILABLE", "injected report failure"); + disk.append = line => { + const kind = JSON.parse(line).kind; + if (failing && kind === (phase === "first-dispatch" ? "dispatch" : "reserve")) { + throw failure === "owner" ? ownerError : new Error("append unavailable"); + } + append(line); + }; + const policy = spendTestPolicy({ canonicalProviderIds: ["P"], pool: { maxTokens: 50 }, maxTrackedSends: 1 }); + const ledger = createSpendReservationLedger({ journal: disk, salt, policy, now: () => 2 }); + const tracker = createRequestSpendTracker({ provider: "P", spendInputEstimateTokens: 10 }, undefined, ledger); + const budget = createRequestExecutionBudget(undefined, undefined, tracker); + let wires = 0; + const sends = phase === "first-dispatch" ? 1 : 2; + const running = fetchWithTransientRetry(async () => { + wires++; + if (wires === sends) failing = true; + return new Response(null, { status: wires < sends ? 503 : 200 }); + }, { attempts: sends, onSendsConsumed: createPhysicalSendReporter(budget, () => ({ poolId: "P" })) }); + if (failure === "owner") await expect(running).rejects.toThrow(ownerError); + else await running; + expect(wires).toBe(sends); + expect(budget.used).toBe(sends); + expect(await Promise.race([ledger.waitForReporterDrain().then(() => true), Bun.sleep(50).then(() => false)])).toBe(true); + if (failure === "owner") expect(() => tracker.settle({ inputTokens: 70 })).toThrow(ownerError); + else tracker.settle({ inputTokens: 70 }); + failing = false; + ledger.prune(); + const expected = { settled: 70, reserved: 0, unresolved: sends === 2 ? 10 : 0 }; + expect(ledger.snapshot("pool", "P")).toMatchObject(expected); + expect(ledger.reserve(request("must-refuse", { poolId: "P" }, 1)).reserved).toBe(false); + for (const create of [createSpendReservationLedger, createShippedSpendLedger]) { + expect(create({ journal: spendTestJournal(disk.lines), salt, policy, now: () => 2 }).snapshot("pool", "P")).toMatchObject(expected); + } + const records = disk.lines.map(line => JSON.parse(line)); + expect(records.filter(record => record.kind === "reserve")).toHaveLength(sends); + expect(records.filter(record => record.kind === "dispatch")).toHaveLength(sends); + expect(records.filter(record => record.kind === "settle")).toHaveLength(1); + expect(records.filter(record => record.kind === "lost")).toHaveLength(sends - 1); + expect(new Set(records.filter(record => record.kind === "reserve").map(record => record.send)).size).toBe(sends); + }); + } +} + + +test("a failed first report retains every start in the same batch and releases its lease", async () => { + const disk = spendTestJournal(); + const append = disk.append.bind(disk); + let fail = false; + const ownerError = new SpendLedgerOwnerError("SPEND_LEDGER_OWNER_UNAVAILABLE", "batch report failure"); + disk.append = line => { if (fail && JSON.parse(line).kind === "dispatch") throw ownerError; append(line); }; + const ledger = factory(disk); + const tracker = createRequestSpendTracker({ provider: "P", spendInputEstimateTokens: 10 }, undefined, ledger); + const anchor = tracker.ensureSeed({ poolId: "P" }); + if (!anchor) throw new Error("fixture seed denied"); + const report = tracker.beginReporter(); + report.start(anchor, 1); report.start(anchor, 2); report.start(anchor, 3); + fail = true; + expect(() => report.report(3)).toThrow(ownerError); + report.close(); + await ledger.waitForReporterDrain(); + expect(() => tracker.settle({ inputTokens: 70 })).toThrow(ownerError); + fail = false; + ledger.prune(); + const expected = { settled: 70, reserved: 0, unresolved: 20 }; + expect(ledger.snapshot("pool", "P")).toMatchObject(expected); + expect(createShippedSpendLedger({ journal: spendTestJournal(disk.lines), salt, now: () => 2 }).snapshot("pool", "P")).toMatchObject(expected); + expect(disk.lines.filter(line => JSON.parse(line).kind === "reserve")).toHaveLength(3); + expect(disk.lines.filter(line => JSON.parse(line).kind === "dispatch")).toHaveLength(3); +}); + + +test("reporter acquisition failure and ownership loss during close cannot retain a lease", async () => { + let checks = 0; + let failAt = Infinity; + const ownerError = new SpendLedgerOwnerError("SPEND_LEDGER_OWNER_UNAVAILABLE", "ownership changed"); + const ledger = createSpendReservationLedger({ journal: spendTestJournal(), salt, policy: spendTestPolicy(), + assertOwnedAccounting: () => { if (++checks === failAt) throw ownerError; } }); + const tracker = createRequestSpendTracker({ provider: "P" }, undefined, ledger); + checks = 0; failAt = 2; + expect(() => tracker.beginReporter()).toThrow(ownerError); + failAt = Infinity; + expect(await Promise.race([ledger.waitForReporterDrain().then(() => true), Bun.sleep(50).then(() => false)])).toBe(true); + const lease = ledger.registerReporter(); + const drained = ledger.waitForReporterDrain(); + checks = 0; failAt = 1; + expect(() => lease.close()).toThrow(ownerError); + failAt = Infinity; + expect(await Promise.race([drained.then(() => true), Bun.sleep(50).then(() => false)])).toBe(true); +}); + + +test("a partially enrolled ordinal retains its estimate across owner loss and changed log context", () => { + const disk = spendTestJournal(); + const append = disk.append.bind(disk); + let failReserve = false; + let ownerLost = false; + const ownerError = new SpendLedgerOwnerError("SPEND_LEDGER_OWNER_UNAVAILABLE", "report owner lost"); + disk.append = line => { + if (failReserve && JSON.parse(line).kind === "reserve") { ownerLost = true; throw ownerError; } + append(line); + }; + const ledger = createSpendReservationLedger({ journal: disk, salt, policy: spendTestPolicy(), now: () => 2, + assertOwnedAccounting: () => { if (ownerLost) throw ownerError; } }); + const context = { provider: "P", spendInputEstimateTokens: 10 }; + const tracker = createRequestSpendTracker(context, undefined, ledger); + const anchor = tracker.ensureSeed({ poolId: "P" }); + if (!anchor) throw new Error("fixture seed denied"); + const report = tracker.beginReporter(); + report.start(anchor, 1); report.start(anchor, 2); report.report(1); + failReserve = true; + expect(() => report.report(1)).toThrow(ownerError); + expect(() => report.close()).toThrow(ownerError); + context.spendInputEstimateTokens = 100; + ownerLost = false; failReserve = false; + tracker.settle({ inputTokens: 70 }); ledger.prune(); + const expected = { settled: 70, reserved: 0, unresolved: 10 }; + expect(ledger.snapshot("pool", "P")).toMatchObject(expected); + expect(createShippedSpendLedger({ journal: spendTestJournal(disk.lines), salt, now: () => 2 }).snapshot("pool", "P")).toMatchObject(expected); + expect(disk.lines.map(line => JSON.parse(line)).filter(record => record.kind === "reserve").map(record => record.tokens)).toEqual([10, 10]); +}); diff --git a/tests/providers/codebuddy-tool-bridge-turn.test.ts b/tests/providers/codebuddy-tool-bridge-turn.test.ts index 0f01cf53720..e921b5eb550 100644 --- a/tests/providers/codebuddy-tool-bridge-turn.test.ts +++ b/tests/providers/codebuddy-tool-bridge-turn.test.ts @@ -1,7 +1,7 @@ -import { beforeEach, describe, expect, test } from "bun:test"; +import { beforeEach, describe, expect, spyOn, test } from "bun:test"; import { EventEmitter } from "node:events"; import { existsSync, readdirSync } from "node:fs"; -import { writeFile } from "node:fs/promises"; +import { rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { basename, dirname, join } from "node:path"; import { Readable, Writable } from "node:stream"; @@ -12,6 +12,12 @@ import { CODEBUDDY_GLOBAL_PROFILE, clearCodeBuddyBinaryCache } from "../../src/a import type { AdapterEvent, OcxParsedRequest, OcxProviderConfig, OcxTool } from "../../src/types"; import { createTestTranslatorBudget } from "../helpers/translator-budget"; +import { createRequestExecutionBudget, CODEX_TEXT_GUARDED_BUDGET_POLICY } from "../../src/lib/request-execution-budget"; +import { createSpendReservationLedger } from "../../src/lib/spend-reservation-ledger"; +import { createRequestSpendTracker } from "../../src/server/responses/request-spend"; +import { SpendLedgerOwnerError } from "../../src/lib/spend-ledger-owner"; +import { spendTestJournal, spendTestPolicy, testSpendSalt } from "../helpers/shipped-spend-ledger"; + const enc = new TextEncoder(); beforeEach(() => clearCodeBuddyBinaryCache()); @@ -93,6 +99,50 @@ const BLOCK_STOP = { type: "stream_event", event: { type: "content_block_stop" } const MESSAGE_STOP = { type: "stream_event", event: { type: "message_stop" } }; describe("CodeBuddy capture-only tool bridge turn", () => { + test.each(["acquire", "release", "close"])("owner error during %s still closes the producer and removes the tool bridge", async phase => { + const ledger = createSpendReservationLedger({ journal: spendTestJournal(), salt: testSpendSalt, + policy: spendTestPolicy({ pool: { maxTokens: 50 } }) }); + const tracker = createRequestSpendTracker({ provider: "codebuddy", spendInputEstimateTokens: 1 }, undefined, ledger); + const budget = createRequestExecutionBudget({ ...CODEX_TEXT_GUARDED_BUDGET_POLICY }, undefined, tracker); + const error = new SpendLedgerOwnerError("SPEND_LEDGER_OWNER_UNAVAILABLE", "injected cleanup owner refusal"); + const begin = budget.beginSpendProducer!.bind(budget); + let close: (() => void) | undefined; + let closed = false; + spyOn(budget, "beginSpendProducer").mockImplementation(() => { + if (phase === "acquire") throw error; + const producer = begin()!; + close = producer.close.bind(producer); + return { close() { close!(); closed = true; if (phase === "close") throw error; } }; + }); + const reserve = budget.reserveDispatch.bind(budget); + spyOn(budget, "reserveDispatch").mockImplementation(intent => { + const decision = reserve(intent); + if (decision.allowed && phase === "release") { + const release = decision.permit.release.bind(decision.permit); + decision.permit.release = () => { release(); throw error; }; + } + return decision; + }); + let bridgeDir: string | undefined; + let spawns = 0; + const adapter = createCodeBuddyAdapter(provider(), { + which: () => "/usr/bin/codebuddy", + spawn: () => { spawns++; return fakeChild(frameLines([INIT_OK, { type: "result", subtype: "success", is_error: false }])) as unknown as ChildProcess; }, + writeToolBridgeFile: async (path, data, options) => { bridgeDir = dirname(String(path)); await writeFile(path, data, options); }, + }); + try { + await expect(adapter.runTurn!(parsed([tool("exec")]), { ...incoming(), sendBudget: budget }, () => {})).rejects.toBe(error); + expect(spawns).toBe(phase === "acquire" ? 0 : 1); + expect(closed).toBe(phase !== "acquire"); + expect(bridgeDir).toBeDefined(); + expect(existsSync(bridgeDir!)).toBe(false); + await ledger.waitForReporterDrain(); + } finally { + close?.(); + if (bridgeDir) await rm(bridgeDir, { recursive: true, force: true }); + } + }); + test("active bridge continuation keeps prior tool calls under the bare request name", async () => { const p = parsed([tool("probe_echo")]); const callId = "cb_prior_call"; diff --git a/tests/responses/responses-spend-anchor.test.ts b/tests/responses/responses-spend-anchor.test.ts index 2c0320f08c3..a28f6c95211 100644 --- a/tests/responses/responses-spend-anchor.test.ts +++ b/tests/responses/responses-spend-anchor.test.ts @@ -1,4 +1,4 @@ -import { expect, test } from "bun:test"; +import { expect, spyOn, test } from "bun:test"; import { createSpendReservationLedger, DEFAULT_SPEND_RESERVATION_POLICY } from "../../src/lib/spend-reservation-ledger"; import { createRequestExecutionBudget, deriveRequestExecutionBudget, CODEX_TEXT_GUARDED_BUDGET_POLICY, createPhysicalSendReporter } from "../../src/lib/request-execution-budget"; import { createRequestSpendTracker } from "../../src/server/responses/request-spend"; @@ -21,6 +21,7 @@ import { removeTreeWithRetry } from "../helpers/remove-tree"; import { resetProviderRequestPacingForTest } from "../../src/providers/request-pacing"; import { acquireSpendLedgerServerLifecycle } from "../../src/server/index/spend-ledger-lifecycle"; import { createResponsesSendBudget } from "../../src/server/responses/request-send-budget"; +import { SpendLedgerOwnerError } from "../../src/lib/spend-ledger-owner"; function fixture(enforced = true, capacity = 8) { const lines: string[] = []; @@ -361,6 +362,51 @@ for (const kind of executorKinds) { })); } +for (const kind of ["chat", "messages"] as const) { + for (const phase of ["admission", "report"] as const) test(`${kind} closes its shutdown reporter after an owner error during ${phase}`, async () => withExecutorHome(async (ledger, home) => { + const leases: Array<{ close(): void }> = []; + const register = ledger.registerReporter.bind(ledger); + const registrations = spyOn(ledger, "registerReporter").mockImplementation(owner => { + const lease = register(owner); + leases.push(lease); + return lease; + }); + const error = new SpendLedgerOwnerError("SPEND_LEDGER_OWNER_UNAVAILABLE", "injected owner refusal"); + const fault = phase === "admission" + ? spyOn(ledger, "reserveSeed").mockImplementation(() => { throw error; }) + : spyOn(ledger, "markDispatched").mockImplementation(() => { throw error; }); + const lifecycle = acquireSpendLedgerServerLifecycle(home); + const listener = lifecycle.track({ stop: async () => {} }); + let stopping: Promise | undefined; + let timeout: ReturnType | undefined; + try { + let wires = 0; + const wire = (async () => { wires++; return executorSuccess(kind); }) as typeof fetch; + globalThis.fetch = wire; + const ctx: RequestLogContext = { model: "model", provider: "P", spendPoolId: "P", spendInputEstimateTokens: 2, spendOutputCeilingTokens: 1 }; + const outcome = await Promise.allSettled([invokeExecutor(kind, executorConfig(kind, wire), ctx)]); + if (outcome[0]!.status === "fulfilled") expect(outcome[0]!.value.status).toBeGreaterThanOrEqual(400); + expect(wires).toBe(phase === "admission" ? 0 : 1); + expect(leases).toHaveLength(1); + fault.mockRestore(); + ledger.prune(); + let stopped = false; + stopping = Promise.resolve(listener.stop()).then(() => { stopped = true; }); + await Promise.race([stopping, new Promise(resolve => { timeout = setTimeout(resolve, 100); })]); + expect(stopped).toBe(true); + } finally { + if (timeout) clearTimeout(timeout); + fault.mockRestore(); + // Keep a failing regression from retaining the fixture's ownership. This happens only + // after the assertion; production closure must be what lets shutdown complete above. + for (const lease of leases) lease.close(); + await stopping; + registrations.mockRestore(); + lifecycle.release(); + } + })); +} + for (const kind of ["chat", "compact", "generic", "passthrough"] as const) { test(`${kind} production retries honor the inherited frozen physical limit`, async () => withExecutorHome(async ledger => { const ctx: RequestLogContext = { model: "model", provider: executorPool(kind), spendPoolId: executorPool(kind), spendInputEstimateTokens: 2, spendOutputCeilingTokens: 1 }; From 6e4b899ba0075dcf330a8ce15301ec4a52e75735 Mon Sep 17 00:00:00 2001 From: JUN Date: Fri, 9 Oct 2026 02:36:45 +0900 Subject: [PATCH 15/15] fix(spend): retry repair journal on observe-only admission Flush pending ordinary repair records at each admission and preserve enforced durability refusal. Cover recovery through admission for settled and unknown usage with current and frozen readers. Co-authored-by: Epinephrine --- src/lib/spend-reservation-ledger.ts | 3 ++- structure/transports/responses-spend.md | 4 ++-- tests/lib/execution-budget-permits.test.ts | 8 +++++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/lib/spend-reservation-ledger.ts b/src/lib/spend-reservation-ledger.ts index 0f22c2170ee..514f3e06af5 100644 --- a/src/lib/spend-reservation-ledger.ts +++ b/src/lib/spend-reservation-ledger.ts @@ -1315,6 +1315,7 @@ export function createSpendReservationLedger(options: { reserve(request: SpendReservationRequest): SpendReservationDecision { assertOwnedAccounting?.(); retryCleanup(); + const pendingDurable = flushPending(); // Observe-only admissions also retry ordinary repair debt. const tokens = sanitizeTokens(request.inputTokens) + sanitizeTokens(request.outputCeilingTokens); const at = request.at ?? now(); const send = aliasFor("send", request.sendId); @@ -1333,7 +1334,7 @@ export function createSpendReservationLedger(options: { if (enforced && corruptRecords > 0) { return { reserved: false, denial: { reason: "journal-corrupt", corruptRecords } }; } - if (enforced && request.alreadySent !== true && !flushPending()) { + if (enforced && request.alreadySent !== true && !pendingDurable) { return { reserved: false, denial: { reason: "reserve-not-durable", sendId: request.sendId } }; } const continuityDenial = preparePoolContinuity(request.scopes.poolId); diff --git a/structure/transports/responses-spend.md b/structure/transports/responses-spend.md index 90d32f14c12..66f526f4067 100644 --- a/structure/transports/responses-spend.md +++ b/structure/transports/responses-spend.md @@ -191,8 +191,8 @@ errors remain storage failures. `tests/lib/spend-corruption-compat.test.ts` chec An ordinary observe-only reservation whose first write failed keeps bounded in-memory repair metadata until it resolves or is evicted. Before reporting dispatch or terminal usage, the ledger -queues the missing ordinary reserve prefix ahead of those records. Later pruning or admission -flushes that queue, so a recovered writer cannot leave terminal usage without its reservation. +queues the missing ordinary reserve prefix ahead of those records. Later pruning or any admission, +including observe-only admission, retries that queue so recovered storage retains the reservation. This grants no seed or overflow capacity and adds no journal record type or identity metadata. For requests with an applicable root, identity or pool ceiling, each selected target/key first diff --git a/tests/lib/execution-budget-permits.test.ts b/tests/lib/execution-budget-permits.test.ts index 0acb8b62716..28b42b7f649 100644 --- a/tests/lib/execution-budget-permits.test.ts +++ b/tests/lib/execution-budget-permits.test.ts @@ -129,18 +129,20 @@ test("the actual first target determines the frozen enforcement mode", async () expect(budget.spendEnforced).toBe(true); }); -for (const usage of [70, undefined]) test(`ordinary failed reserve repairs its prefix before terminal ${usage}`, async () => { +for (const recovery of ["prune", "admission"] as const) for (const usage of [70, undefined]) test(`ordinary failed reserve repairs its prefix before terminal ${usage} via ${recovery}`, async () => { const { request, journal, ledger, policy } = policyEpochFixture(); journal.failAppend = true; const old = request(); await old.send(); old.tracker.settle(usage === undefined ? undefined : { inputTokens: usage }); - journal.failAppend = false; ledger.prune(); + journal.failAppend = false; + if (recovery === "prune") ledger.prune(); + else expect(ledger.reserve({ sendId: "next-admission", scopes: {}, inputTokens: 0, outputCeilingTokens: 0 }).reserved).toBe(true); const expected = { settled: usage ?? 0, unresolved: usage === undefined ? 10 : 0, reserved: 0 }; expect(ledger.snapshot("pool", "P")).toMatchObject(expected); for (const create of [createSpendReservationLedger, createShippedSpendLedger]) { expect(create({ policy, journal: spendTestJournal(journal.lines), salt: testSpendSalt, now: () => 2 }).snapshot("pool", "P")).toMatchObject(expected); } - expect(journal.lines.map(line => JSON.parse(line).kind)).toEqual(["reserve", "dispatch", usage === undefined ? "lost" : "settle"]); + expect(journal.lines.map(line => JSON.parse(line).kind)).toEqual(["reserve", "dispatch", usage === undefined ? "lost" : "settle", ...(recovery === "admission" ? ["reserve"] : [])]); }); test("captured ceiling preflight preserves exact prepaid exclusion across policy changes", () => {