fix(proxy): preserve terminal service-tier provenance

Dockerfile

@@ -44,7 +44,7 @@ WORKDIR /app
 RUN apt-get update \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends --only-upgrade \
         bsdutils libblkid1 libc-bin libc6 libcap2 libmount1 libsmartcols1 libssl3t64 \
-        libsystemd0 libudev1 libuuid1 openssl sed util-linux \
+        libsystemd0 libudev1 libuuid1 openssl perl-base sed util-linux \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
         openssl-provider-legacy \
     && rm -rf /var/lib/apt/lists/*

app/core/clients/codex_continuation.py

@@ -731,6 +731,10 @@ def _reconstruct_terminal(
 ) -> dict[str, Any]:
     terminal_response = _response_payload(terminal or {})
     response = dict(base_response or terminal_response)
+    if "service_tier" in terminal_response:
+        response["service_tier"] = terminal_response["service_tier"]
+    else:
+        response.pop("service_tier", None)
     response["output"] = output_items
     response["usage"] = usage
     response["status"] = terminal_response.get("status", "completed")
@@ -754,6 +758,7 @@ def _synthetic_incomplete(
     billed_usage: dict[str, Any] | None,
 ) -> dict[str, Any]:
     response = dict(base_response or {})
+    response.pop("service_tier", None)
     response["output"] = output_items
     response["usage"] = usage
     response["status"] = "incomplete"

app/core/request_locality.py

@@ -36,6 +36,7 @@ def _combined_chain_header(headers: Mapping[str, str], header_name: str) -> str
         return ",".join(values) if values else None
     return headers.get(header_name)
 
+
 def _has_non_empty_header_value(
     headers: Mapping[str, str],
     header_name: str,

app/modules/proxy/_service/http_bridge/upstream_events.py

@@ -837,10 +837,11 @@ async def _process_http_bridge_upstream_text(
                     matched_request_state.latency_response_created_ms = int(
                         max(0.0, now - matched_request_state.started_at) * 1000
                     )
-                actual_service_tier = _service_tier_from_event_payload(payload)
-                if actual_service_tier is not None:
-                    matched_request_state.actual_service_tier = actual_service_tier
-                    matched_request_state.service_tier = actual_service_tier
+                # Preserve any-event tier observations for billing. The shared
+                # terminal finalizer alone assigns actual-tier evidence.
+                observed_service_tier = _service_tier_from_event_payload(payload)
+                if observed_service_tier is not None:
+                    matched_request_state.service_tier = observed_service_tier
                 _record_http_bridge_tool_call_lifecycle(
                     matched_request_state,
                     event_type=event_type,

app/modules/proxy/_service/observability.py

@@ -152,6 +152,7 @@ def _maybe_log_proxy_service_tier_trace(
         kind,
         requested_service_tier=requested_service_tier,
         actual_service_tier=actual_service_tier,
+        status=status,
     )
     settings = _service_get_settings()
     if not settings.log_proxy_service_tier_trace:
@@ -176,7 +177,10 @@ def _record_service_tier_mismatch(
     *,
     requested_service_tier: str | None,
     actual_service_tier: str | None,
+    status: str | None,
 ) -> None:
+    if status != "success":
+        return
     requested_tier = _service_tier_metric_label(requested_service_tier)
     actual_tier = _service_tier_metric_label(actual_service_tier)
     if requested_tier == "none" or actual_tier == "none" or requested_tier == actual_tier:

app/modules/proxy/_service/streaming/mixin.py

@@ -598,8 +598,9 @@ async def _record_or_defer_owner_recovery_health(error: UpstreamError, code: str
                 )
             event_service_tier = _facade()._service_tier_from_event_payload(first_payload)
             if event_service_tier is not None:
-                actual_service_tier = event_service_tier
                 service_tier = event_service_tier
+            if event_type == "response.completed":
+                actual_service_tier = event_service_tier
             if event and event.response and event.response.id:
                 response_id = event.response.id
                 settlement.response_id = response_id
@@ -765,8 +766,9 @@ async def _record_or_defer_owner_recovery_health(error: UpstreamError, code: str
                     )
                 event_service_tier = _facade()._service_tier_from_event_payload(event_payload)
                 if event_service_tier is not None:
-                    actual_service_tier = event_service_tier
                     service_tier = event_service_tier
+                if event_type == "response.completed":
+                    actual_service_tier = event_service_tier
                 line, event_payload, event, event_type = rewrite_parallel_tool_call_sse_line(line, event_payload)
                 if event_type in _facade()._TEXT_DELTA_EVENT_TYPES:
                     saw_text_delta = True
@@ -986,6 +988,8 @@ async def _record_or_defer_owner_recovery_health(error: UpstreamError, code: str
             output_tokens = usage_accounting.output_tokens
             cached_input_tokens = usage_accounting.cached_input_tokens
             reasoning_tokens = usage_accounting.reasoning_tokens
+            if status != "success":
+                actual_service_tier = None
             settlement.status = status
             settlement.model = model
             settlement.service_tier = service_tier

app/modules/proxy/_service/websocket/mixin.py

@@ -3784,10 +3784,11 @@ async def _process_upstream_websocket_text(
                     )
                 if event_type not in {"response.completed", "response.failed", "response.incomplete", "error"}:
                     _record_response_event(request_state, event_type)
-                actual_service_tier = _facade()._service_tier_from_event_payload(payload)
-                if actual_service_tier is not None:
-                    request_state.actual_service_tier = actual_service_tier
-                    request_state.service_tier = actual_service_tier
+                # Any upstream event may refine billing, but only the final
+                # response.completed event supplies actual-tier evidence.
+                observed_service_tier = _facade()._service_tier_from_event_payload(payload)
+                if observed_service_tier is not None:
+                    request_state.service_tier = observed_service_tier
                 completed_tool_call = _facade()._response_output_item_done_tool_call(payload)
                 if completed_tool_call is not None:
                     completed_call_id, completed_call_type = completed_tool_call
@@ -4007,6 +4008,8 @@ async def _process_upstream_websocket_text(
                 # to this pending request (and is then folded/suppressed).
                 fold_request_state.awaiting_response_created = True
                 fold_request_state.response_id = None
+                # A swallowed round cannot provide final actual-tier evidence.
+                fold_request_state.actual_service_tier = None
                 upstream_control.continuation_resend_body = fold_outcome.continuation_request
                 upstream_control.suppress_downstream_event = True
                 return text
@@ -4580,6 +4583,8 @@ async def _finalize_websocket_request_state(
         error_payload: UpstreamError | None = None
         response_id = request_state.response_id or request_state.request_id
         response_service_tier = request_state.service_tier
+        # Clear earlier attempts and in-flight events before every terminal path.
+        request_state.actual_service_tier = None
 
         if request_state.draining_until_terminal:
             await _release_websocket_response_create_gate(request_state, response_create_gate)
@@ -4622,10 +4627,11 @@ async def _finalize_websocket_request_state(
         billed_usage_payload = _proxy_billed_usage_from_event_payload(payload)
         usage_accounting = _stream_usage_accounting(usage, billed_usage_payload)
 
-        actual_service_tier = _facade()._service_tier_from_event_payload(payload)
-        if actual_service_tier is not None:
-            request_state.actual_service_tier = actual_service_tier
-            response_service_tier = actual_service_tier
+        observed_service_tier = _facade()._service_tier_from_event_payload(payload)
+        if observed_service_tier is not None:
+            response_service_tier = observed_service_tier
+        if event_type == "response.completed":
+            request_state.actual_service_tier = observed_service_tier
 
         settlement = _StreamSettlement(
             status=status,
@@ -4765,6 +4771,7 @@ async def _write_websocket_connect_failure(
     ) -> None:
         proxy = cast(_WebSocketServiceProtocol, self)
         _ = proxy
+        request_state.actual_service_tier = None
         if request_state.skip_request_log:
             return
         await proxy._write_request_log(
@@ -4950,6 +4957,7 @@ async def _fail_pending_websocket_requests(
 
         last_index = len(remaining) - 1
         for index, request_state in enumerate(remaining):
+            request_state.actual_service_tier = None
             proxy._cancel_request_state_api_key_reservation_heartbeat(request_state)
             request_error_code = request_state.error_code_override or error_code
             request_error_message = request_state.error_message_override or error_message

openspec/changes/fix-terminal-service-tier-provenance/.openspec.yaml

@@ -0,0 +1,2 @@
+schema: spec-driven
+created: 2026-09-14

openspec/changes/fix-terminal-service-tier-provenance/design.md

@@ -0,0 +1,19 @@
+## Context
+
+The initial created response is useful for stable IDs and output reconstruction, but its service tier is not a final processing report. Continuation currently copies that initial field into the terminal response. Direct collection and WebSocket finalization then treat the copied value as actual. Separately, any-event collection retains created-only tiers when completion omits the field, and a finally-block counter can count unsuccessful attempts.
+
+## Decisions
+
+1. The final upstream terminal owns its service-tier field. Reconstruction copies that field when present and removes any inherited field when absent. Synthetic incomplete output removes created tier claims.
+2. Streaming actual-tier evidence requires an explicit nonblank tier on `response.completed`. Failed, incomplete, cancelled and interrupted attempts do not gain evidence from earlier events. Hidden-round completions do not supply the final logical request's actual tier.
+3. Billable `service_tier` remains a separate fallback. Existing observed-event/request fallback is retained; explicit final terminal data takes precedence. This repair does not redesign multi-round pricing or historical accounting.
+4. The existing mismatch counter is gated on successful completion and actual terminal evidence. Compact success is terminal by definition. Counter names, labels and transport coverage stay unchanged.
+5. An explicit terminal `auto` is faithfully recorded but remains inconclusive about upstream scheduling. Neither a zero counter nor an absent tier proves Fast delivery.
+
+## Verification and rollout
+
+Use synthetic created, terminal, hidden-round, omitted-tier and interrupted-stream fixtures through folding and service/route settlement. Keep useful existing accounting assertions. Run all checks only after worker edits settle. Publish to ericjuta/codex-lb main through a reviewed PR, then use the existing direct deployment script with unchanged configuration. Verify image/source identity, readiness and safe error counts. Inspect grouped post-deployment metadata from ordinary traffic only, without prompts, response bodies, account identities or model benchmarks.
+
+## Risks and rollback
+
+New logs will have null actual tiers for created-only observations. Existing rows are not rewritten and must be separated from post-deployment observations by time. The deployment restarts the local proxy and interrupts in-flight connections. Keep the previous image available for rollback; do not change continuation or transport settings to make the evidence look better.

openspec/changes/fix-terminal-service-tier-provenance/proposal.md

@@ -0,0 +1,22 @@
+## Why
+
+Continuation reconstruction can replace a terminal response's explicit service tier with the tier from the first `response.created`. Request logs and mismatch counters can therefore report `auto` even when the terminal reports `priority` or `default`. Created-only tiers and failed attempts must not be presented as successful terminal evidence.
+
+## What Changes
+
+- Preserve the final upstream terminal's service-tier field through continuation reconstruction, including its absence.
+- Record streaming `actual_service_tier` only from an explicit successful completion, across direct streaming, native WebSocket, and the HTTP bridge.
+- Keep the existing accounting-tier fallback separate from terminal evidence.
+- Count service-tier mismatches only for successful completions with an explicit reported tier. Keep existing counter coverage and bounded labels.
+- Verify the repaired response and request-log behavior with synthetic upstream events, then deploy and inspect aggregate normal-traffic metadata.
+
+## Capabilities
+
+### Modified Capabilities
+
+- `responses-api-compat`: terminal tier preservation and actual-versus-accounting request-log semantics.
+- `proxy-runtime-observability`: successful-terminal mismatch evidence and missing/auto interpretation.
+
+## Impact
+
+The change affects continuation reconstruction and proxy tier reporting. It adds no database columns or runtime settings, does not rewrite historical request logs, and does not alter request tiers, reasoning effort, transport, account selection, or continuation limits. Repairing reporting does not itself prove Fast delivery or improve model latency.

openspec/changes/fix-terminal-service-tier-provenance/specs/proxy-runtime-observability/spec.md

@@ -0,0 +1,22 @@
+## ADDED Requirements
+
+### Requirement: Service-tier mismatches use successful terminal evidence
+The service-tier mismatch counter MUST count only successful requests with an explicit nonblank terminal-reported tier that differs from the normalized requested tier. A created or in-progress tier, requested-tier fallback, missing terminal tier, or unsuccessful attempt MUST NOT increment the counter. The change MUST preserve existing counter coverage and bounded labels rather than adding identities or payload data.
+
+An explicitly returned `auto` MUST remain observable as the reported tier. Operator guidance MUST describe it as inconclusive for Fast delivery rather than proof of a downgrade. Counter equality or absence MUST NOT be described as evidence of upstream scheduling.
+
+#### Scenario: Explicit terminal default differs from requested priority
+- **WHEN** a successful counted request asks for `priority` and completes with explicit `default`
+- **THEN** one mismatch is counted with requested `priority` and actual `default`
+
+#### Scenario: Created auto followed by terminal priority is not a mismatch
+- **WHEN** a successful counted request asks for `priority`, starts with created `auto`, and completes with explicit `priority`
+- **THEN** the created tier does not increment the mismatch counter
+
+#### Scenario: Missing terminal tier does not count a created fallback
+- **WHEN** a counted request starts with a tier different from the requested tier but completes without a tier
+- **THEN** the request does not increment the mismatch counter
+
+#### Scenario: Failure does not count a tier mismatch
+- **WHEN** a counted attempt fails, is cancelled, disconnects, or finishes incomplete
+- **THEN** it does not increment the mismatch counter even if an earlier event included a different tier

openspec/changes/fix-terminal-service-tier-provenance/specs/responses-api-compat/spec.md

@@ -0,0 +1,38 @@
+## MODIFIED Requirements
+
+### Requirement: Request logs persist requested, actual, and billable service tiers separately
+For Responses proxy traffic, the system MUST persist the operator-requested tier, the upstream-reported actual tier when available, and the effective billable tier used for pricing as separate request-log fields.
+
+The legacy `fast` alias MUST be normalized to `priority` before forwarding and before it is stored as the requested tier. For streaming requests, `actual_service_tier` MUST come only from a nonblank service tier explicitly present in `response.completed.response`. Created, in-progress, failed, incomplete, cancelled, and disconnected streams MUST NOT supply actual-tier evidence. A completed response that omits the tier MUST leave actual-tier evidence absent even if earlier events supplied one. Non-streaming successful compact responses MAY supply actual-tier evidence directly.
+
+The system MUST preserve the existing billable-tier fallback independently of actual-tier evidence. An explicit terminal tier MUST take precedence for billing; otherwise the existing observed-event/request fallback remains available without being copied into `actual_service_tier`.
+
+Continuation reconstruction MUST preserve the final upstream terminal response's `service_tier`, including its absence. It MUST NOT inherit a tier from the initial created response when the terminal omits the field. Proxy-synthesized incomplete responses MUST NOT claim an upstream terminal tier from the created response.
+
+#### Scenario: Upstream reports a downgraded actual tier
+- **WHEN** a client sends a Responses request with `service_tier: "priority"`
+- **AND** the upstream completion explicitly reports `service_tier: "default"`
+- **THEN** the persisted request log records requested tier `priority`, actual tier `default`, and billable tier `default`
+
+#### Scenario: Fast alias is logged as a priority request
+- **WHEN** a client sends `service_tier: "fast"`
+- **AND** the upstream completion explicitly reports `service_tier: "default"`
+- **THEN** the persisted request log records requested tier `priority`, actual tier `default`, and billable tier `default`
+
+#### Scenario: Upstream omits the actual tier
+- **WHEN** a client requests `priority` and no upstream event reports a tier
+- **THEN** actual tier is null and billable tier falls back to requested `priority`
+
+#### Scenario: Initial auto does not replace terminal priority
+- **WHEN** a continued response starts with created tier `auto` and finishes with explicit terminal tier `priority`
+- **THEN** the folded completion and persisted actual tier are `priority`
+- **AND** every upstream continuation request retains the requested tier
+
+#### Scenario: Created-only tier remains accounting fallback
+- **WHEN** a created response reports `default` and the completion omits `service_tier`
+- **THEN** the completion does not inherit `service_tier`
+- **AND** actual tier is null while the existing billable-tier fallback remains `default`
+
+#### Scenario: Failed or interrupted stream does not prove a terminal tier
+- **WHEN** a stream reports a created tier but fails, is cancelled, disconnects, or finishes incomplete
+- **THEN** actual-tier evidence remains absent

openspec/changes/fix-terminal-service-tier-provenance/tasks.md

@@ -0,0 +1,17 @@
+## 1. Implementation
+
+- [x] 1.1 Preserve terminal service-tier values and omission through continuation folding.
+- [x] 1.2 Separate direct-stream terminal evidence from accounting fallback and gate mismatch counters on successful completion.
+- [x] 1.3 Apply the same actual-tier contract to native WebSocket and HTTP bridge settlement.
+
+## 2. Verification
+
+- [x] 2.1 Prove conflicting, missing, and failed-terminal cases through response and request-log regressions.
+- [x] 2.2 Run focused tests, required repository validation, and independent review on the settled tree.
+- [ ] 2.3 Synchronize the normative specs and operational context, then validate and archive this change.
+
+## 3. Release
+
+- [ ] 3.1 Publish a PR targeting ericjuta/codex-lb main and satisfy current-head CI and review gates.
+- [ ] 3.2 Merge and deploy the verified revision without changing runtime settings.
+- [ ] 3.3 Verify deployment identity and readiness, then inspect aggregate normal-traffic terminal-tier evidence without model benchmarks or payload capture.