From 97ffafcce351dcadd1863d1301987687e3e442a0 Mon Sep 17 00:00:00 2001 From: Factory Agent Date: Fri, 16 Oct 2026 09:48:46 +0700 Subject: [PATCH 1/5] Expose identified request helpers to Codex workspaces Make the bounded JSONL reader and request-usage helpers crate-visible so the Codex workspace turn scanner can reuse them. No behavior change. --- rust/src/codex_costs.rs | 2 +- rust/src/codex_costs/identified_requests.rs | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/rust/src/codex_costs.rs b/rust/src/codex_costs.rs index 65d227f115..544db175c6 100644 --- a/rust/src/codex_costs.rs +++ b/rust/src/codex_costs.rs @@ -4,7 +4,7 @@ //! local-scan aggregation and pricing logic. mod host_costs; -mod identified_requests; +pub(crate) mod identified_requests; mod quota_windows; mod summary_contract; diff --git a/rust/src/codex_costs/identified_requests.rs b/rust/src/codex_costs/identified_requests.rs index e9d7b09b94..bd5c28eed0 100644 --- a/rust/src/codex_costs/identified_requests.rs +++ b/rust/src/codex_costs/identified_requests.rs @@ -490,9 +490,9 @@ fn collect_jsonl_files(root: &Path, files: &mut Vec) -> std::io::Result Ok(complete) } -type Usage = (i64, i64, i64, i64); +pub(crate) type Usage = (i64, i64, i64, i64); -fn non_empty_str(value: Option<&Value>) -> Option<&str> { +pub(crate) fn non_empty_str(value: Option<&Value>) -> Option<&str> { value .and_then(Value::as_str) .map(str::trim) @@ -525,7 +525,7 @@ fn token_int( } /// `(input, cached, output, reasoning)` from one request-usage object. -fn request_usage(value: Option<&Value>) -> Option { +pub(crate) fn request_usage(value: Option<&Value>) -> Option { let usage = value?.as_object()?; let input = token_int(usage, "input_tokens", None)?; let output = token_int(usage, "output_tokens", None)?; @@ -535,11 +535,11 @@ fn request_usage(value: Option<&Value>) -> Option { .then_some((input, cached, output, reasoning)) } -fn totals_at_least(total: Usage, usage: Usage) -> bool { +pub(crate) fn totals_at_least(total: Usage, usage: Usage) -> bool { total.0 >= usage.0 && total.1 >= usage.1 && total.2 >= usage.2 && total.3 >= usage.3 } -fn turn_id(payload: &Value) -> Option { +pub(crate) fn turn_id(payload: &Value) -> Option { [Some(payload), payload.get("info")] .into_iter() .flatten() @@ -591,7 +591,7 @@ fn request_usage_record(object: &Value) -> Option { }) } -fn timestamp_unix_ms(timestamp: &str) -> Option { +pub(crate) fn timestamp_unix_ms(timestamp: &str) -> Option { let parsed = chrono::DateTime::parse_from_rfc3339(timestamp.trim()).ok()?; let round_up = parsed.timestamp_subsec_nanos() % 1_000_000 >= 500_000; Some(parsed.timestamp_millis() + i64::from(round_up)) @@ -603,13 +603,16 @@ fn file_fingerprint(path: &Path) -> Option<(u64, Option)> { .map(|metadata| (metadata.len(), metadata.modified().ok())) } -enum JsonlLine { +pub(crate) enum JsonlLine { Complete(Vec), /// Over [`MAX_LINE_BYTES`]; only the retained prefix is kept. Oversized(Vec), } -fn read_line(reader: &mut R, consumed: &mut u64) -> std::io::Result> { +pub(crate) fn read_line( + reader: &mut R, + consumed: &mut u64, +) -> std::io::Result> { let mut bytes = Vec::new(); let mut length = 0_usize; let mut saw_any = false; From 4b387264e66fee00cdf0919aeb36c024bdcb51fa Mon Sep 17 00:00:00 2001 From: Factory Agent Date: Fri, 16 Oct 2026 09:48:56 +0700 Subject: [PATCH 2/5] Add per-session Codex turn performance scanner Scan completed Codex turns from local rollout logs and summarize first-token latency, turn duration, and output throughput per session. A turn counts only with a successful task_complete, owned usage records that sum to the reported total, and no contradictory copies. Scans reuse the bounded JSONL reader and the indexer merges results by turn id. The sidecar payload version is bumped so stale snapshots rebuild. --- rust/src/codex_workspaces/indexer.rs | 23 +- rust/src/codex_workspaces/mod.rs | 1 + .../codex_workspaces/project_names/tests.rs | 1 + .../src/codex_workspaces/projectless/tests.rs | 1 + rust/src/codex_workspaces/sidecar.rs | 2 +- rust/src/codex_workspaces/turn_performance.rs | 682 ++++++++++++++++++ .../turn_performance/tests.rs | 477 ++++++++++++ rust/src/codex_workspaces/types.rs | 6 + rust/src/codex_workspaces/types/tests.rs | 1 + 9 files changed, 1192 insertions(+), 2 deletions(-) create mode 100644 rust/src/codex_workspaces/turn_performance.rs create mode 100644 rust/src/codex_workspaces/turn_performance/tests.rs diff --git a/rust/src/codex_workspaces/indexer.rs b/rust/src/codex_workspaces/indexer.rs index 5250f1e4e4..36890b4012 100644 --- a/rust/src/codex_workspaces/indexer.rs +++ b/rust/src/codex_workspaces/indexer.rs @@ -1,6 +1,6 @@ //! Scan Codex rollout JSONL + read-only catalog → project usage snapshot. -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::fs; use std::path::{Path, PathBuf}; @@ -16,6 +16,7 @@ use super::project_names::apply_saved_project_names; use super::projectless::apply_projectless_chats; use super::sidecar::{SidecarError, WorkspaceUsageSidecar}; use super::thread_names::apply_session_names_and_ranking; +use super::turn_performance::{CodexTurnPerformanceSummary, TurnSample, scan_session_turns}; use super::types::{ CodexLocalProjectUsageSnapshot, CostEstimate, DailyPoint, Progress, ProgressPhase, ProjectUsage, SessionUsage, SourceStatus, UsageTotals, untitled_session_label, @@ -220,6 +221,9 @@ impl CodexWorkspacesIndex { progress(Progress::phase(ProgressPhase::IndexingProjects)); let mut session_buckets: HashMap = HashMap::new(); + // Completed turns keyed by turn id per session, so a turn copied into + // two rollout files counts once. + let mut session_turns: HashMap> = HashMap::new(); let mut daily_acc: HashMap = HashMap::new(); let mut indexed = 0u32; let mut skipped = 0u32; @@ -228,6 +232,11 @@ impl CodexWorkspacesIndex { match index_one_file(path, &range) { Some(parsed) => { merge_daily(&mut daily_acc, &parsed.day_models); + let turns = scan_session_turns(path, &parsed.session_id, &range); + session_turns + .entry(parsed.session_id.clone()) + .or_default() + .extend(turns.into_iter().map(|turn| (turn.turn_id.clone(), turn))); let entry = session_buckets .entry(parsed.session_id.clone()) .or_insert_with(|| SessionBucket::new(&parsed)); @@ -254,6 +263,13 @@ impl CodexWorkspacesIndex { } } + for (session_id, turns) in session_turns { + if let Some(bucket) = session_buckets.get_mut(&session_id) { + let samples = turns.into_values().collect::>(); + bucket.turn_performance = CodexTurnPerformanceSummary::from_samples(&samples); + } + } + let mut projects = build_projects(&session_buckets); sort_projects(&mut projects); @@ -335,6 +351,7 @@ struct SessionBucket { model_tokens: HashMap, started_at: Option>, latest_activity: Option>, + turn_performance: Option, } impl SessionBucket { @@ -351,6 +368,7 @@ impl SessionBucket { model_tokens: HashMap::new(), started_at: None, latest_activity: None, + turn_performance: None, } } @@ -406,6 +424,7 @@ impl SessionBucket { totals: self.totals, cost_estimate: self.cost, top_model: self.top_model(), + turn_performance: self.turn_performance.clone(), } } } @@ -1046,6 +1065,7 @@ mod tests { totals: UsageTotals::from_parts(10, 0, 5), cost_estimate: CostEstimate::default(), top_model: Some("gpt-5".into()), + turn_performance: None, }], projects: vec![ProjectUsage { id: "project-abc".into(), @@ -1066,6 +1086,7 @@ mod tests { totals: UsageTotals::from_parts(10, 0, 5), cost_estimate: CostEstimate::default(), top_model: Some("gpt-5".into()), + turn_performance: None, }], is_projectless: false, }], diff --git a/rust/src/codex_workspaces/mod.rs b/rust/src/codex_workspaces/mod.rs index c9397d7b3e..cd6762bcdd 100644 --- a/rust/src/codex_workspaces/mod.rs +++ b/rust/src/codex_workspaces/mod.rs @@ -9,6 +9,7 @@ mod project_names; mod projectless; mod sidecar; mod thread_names; +mod turn_performance; mod types; #[cfg(test)] diff --git a/rust/src/codex_workspaces/project_names/tests.rs b/rust/src/codex_workspaces/project_names/tests.rs index f57e5005d2..7174f3a463 100644 --- a/rust/src/codex_workspaces/project_names/tests.rs +++ b/rust/src/codex_workspaces/project_names/tests.rs @@ -220,6 +220,7 @@ fn overlay_changes_only_labels_and_keeps_totals_sessions_and_order() { totals: UsageTotals::from_parts(3, 0, 1), cost_estimate: CostEstimate::default(), top_model: None, + turn_performance: None, }; let mut snap = snapshot(vec![ project("p1", &format!("{base}/alpha")), diff --git a/rust/src/codex_workspaces/projectless/tests.rs b/rust/src/codex_workspaces/projectless/tests.rs index d0fd186ec8..2ebb24ddef 100644 --- a/rust/src/codex_workspaces/projectless/tests.rs +++ b/rust/src/codex_workspaces/projectless/tests.rs @@ -63,6 +63,7 @@ fn session(id: &str, project_id: &str, title: &str) -> SessionUsage { totals: UsageTotals::from_parts(10, 0, 5), cost_estimate: CostEstimate::default(), top_model: None, + turn_performance: None, } } diff --git a/rust/src/codex_workspaces/sidecar.rs b/rust/src/codex_workspaces/sidecar.rs index a87aa84f10..5d334c2687 100644 --- a/rust/src/codex_workspaces/sidecar.rs +++ b/rust/src/codex_workspaces/sidecar.rs @@ -11,7 +11,7 @@ use rusqlite::{Connection, OpenFlags, OptionalExtension, params}; use super::types::CodexLocalProjectUsageSnapshot; pub const SCHEMA_VERSION: i32 = 5; -pub const PAYLOAD_FORMAT_VERSION: i32 = 3; +pub const PAYLOAD_FORMAT_VERSION: i32 = 4; #[derive(Debug, thiserror::Error)] pub enum SidecarError { diff --git a/rust/src/codex_workspaces/turn_performance.rs b/rust/src/codex_workspaces/turn_performance.rs new file mode 100644 index 0000000000..abf249d1ae --- /dev/null +++ b/rust/src/codex_workspaces/turn_performance.rs @@ -0,0 +1,682 @@ +//! Native Codex turn performance per conversation (upstream #4304). +//! +//! A completed turn becomes a sample only when its evidence agrees: a +//! successful `task_complete` with a positive duration, owned +//! `token_usage_record` requests whose output sums to the turn's reported +//! total, and no contradictory or unparseable usage for that turn. Turn output +//! includes reasoning, tools and waits, so these figures describe observed +//! turns. They are not a measure of model generation speed. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::fs::File; +use std::io::BufReader; +use std::path::Path; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::codex_costs::identified_requests::{ + JsonlLine, Usage, non_empty_str, read_line, request_usage, timestamp_unix_ms, totals_at_least, + turn_id, +}; +use crate::core::{CostUsageDayRange, CostUsagePricing}; + +/// Nearest-rank P95 needs this many observations (upstream). +const P95_MIN_SAMPLES: usize = 20; +/// Nearest-rank quartiles need this many observations (upstream). +const QUARTILE_MIN_SAMPLES: usize = 4; +/// A line can change turn state only if it mentions one of these event types. +const RELEVANT_MARKERS: [&str; 4] = [ + "task_started", + "task_complete", + "turn_context", + "token_usage_record", +]; + +/// One completed turn whose timing and token evidence agree. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct TurnSample { + pub(crate) turn_id: String, + pub(crate) output_tokens: u64, + pub(crate) duration_ms: u64, + pub(crate) first_token_ms: Option, + pub(crate) model: Option, + pub(crate) reasoning_effort: Option, + pub(crate) input_tokens: Option, + pub(crate) cached_input_tokens: Option, +} + +/// Throughput is weighted by elapsed time. Duration and first-token time are medians. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CodexTurnPerformanceSummary { + pub sample_count: usize, + pub output_tokens_per_second: f64, + pub median_duration_milliseconds: f64, + pub median_first_token_milliseconds: Option, + pub first_token_sample_count: usize, + pub details: CodexTurnPerformanceDetails, +} + +/// Spread and grouping for the performance details disclosure. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CodexTurnPerformanceDetails { + pub p95_duration_milliseconds: Option, + pub p95_first_token_milliseconds: Option, + pub output_rate_lower_quartile: Option, + pub output_rate_upper_quartile: Option, + pub cached_input_fraction: Option, + pub cache_sample_count: usize, + pub groups: Vec, +} + +/// Turns that share one model and reasoning effort. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CodexTurnPerformanceGroup { + pub model: Option, + pub reasoning_effort: Option, + pub sample_count: usize, + pub first_token_sample_count: usize, + pub median_first_token_milliseconds: Option, + pub median_duration_milliseconds: f64, + pub output_tokens_per_second: f64, +} + +impl CodexTurnPerformanceSummary { + /// `None` when there are no samples or the timed duration sums to zero. + pub(crate) fn from_samples(samples: &[TurnSample]) -> Option { + if samples.is_empty() { + return None; + } + let output = checked_sum(samples.iter().map(|sample| sample.output_tokens))?; + let duration = checked_sum(samples.iter().map(|sample| sample.duration_ms))?; + if duration == 0 { + return None; + } + let durations = samples + .iter() + .map(|sample| sample.duration_ms as f64) + .collect::>(); + let first_tokens = first_token_values(samples); + Some(Self { + sample_count: samples.len(), + output_tokens_per_second: rate(output, duration), + median_duration_milliseconds: median(&durations).unwrap_or(0.0), + median_first_token_milliseconds: median(&first_tokens), + first_token_sample_count: first_tokens.len(), + details: CodexTurnPerformanceDetails::from_samples(samples), + }) + } +} + +impl CodexTurnPerformanceDetails { + fn from_samples(samples: &[TurnSample]) -> Self { + let durations = samples + .iter() + .map(|sample| sample.duration_ms as f64) + .collect::>(); + let first_tokens = first_token_values(samples); + let rates = samples + .iter() + .map(|sample| rate(sample.output_tokens, sample.duration_ms)) + .collect::>(); + let cache = samples + .iter() + .filter(|sample| sample.input_tokens.is_some() && sample.cached_input_tokens.is_some()) + .collect::>(); + let cached_input_fraction = match ( + checked_sum(cache.iter().filter_map(|sample| sample.input_tokens)), + checked_sum(cache.iter().filter_map(|sample| sample.cached_input_tokens)), + ) { + (Some(input), Some(cached)) if input > 0 => Some(cached as f64 / input as f64), + _ => None, + }; + Self { + p95_duration_milliseconds: nearest_rank(&durations, 0.95, P95_MIN_SAMPLES), + p95_first_token_milliseconds: nearest_rank(&first_tokens, 0.95, P95_MIN_SAMPLES), + output_rate_lower_quartile: nearest_rank(&rates, 0.25, QUARTILE_MIN_SAMPLES), + output_rate_upper_quartile: nearest_rank(&rates, 0.75, QUARTILE_MIN_SAMPLES), + cached_input_fraction, + cache_sample_count: cache.len(), + groups: group_samples(samples), + } + } +} + +fn group_samples(samples: &[TurnSample]) -> Vec { + // BTreeMap order on `Option` puts unknown model and effort first, as the upstream sort does. + let mut grouped: BTreeMap<(Option, Option), Vec<&TurnSample>> = BTreeMap::new(); + for sample in samples { + grouped + .entry((sample.model.clone(), sample.reasoning_effort.clone())) + .or_default() + .push(sample); + } + grouped + .into_iter() + .filter_map(|((model, reasoning_effort), observations)| { + let output = checked_sum(observations.iter().map(|sample| sample.output_tokens))?; + let duration = checked_sum(observations.iter().map(|sample| sample.duration_ms))?; + if duration == 0 { + return None; + } + let durations = observations + .iter() + .map(|sample| sample.duration_ms as f64) + .collect::>(); + let first_tokens = observations + .iter() + .filter_map(|sample| sample.first_token_ms.map(|ms| ms as f64)) + .collect::>(); + Some(CodexTurnPerformanceGroup { + model, + reasoning_effort, + sample_count: observations.len(), + first_token_sample_count: first_tokens.len(), + median_first_token_milliseconds: median(&first_tokens), + median_duration_milliseconds: median(&durations).unwrap_or(0.0), + output_tokens_per_second: rate(output, duration), + }) + }) + .collect() +} + +fn first_token_values(samples: &[TurnSample]) -> Vec { + samples + .iter() + .filter_map(|sample| sample.first_token_ms.map(|ms| ms as f64)) + .collect() +} + +fn checked_sum(mut values: impl Iterator) -> Option { + values.try_fold(0_u64, |total, value| total.checked_add(value)) +} + +fn rate(output: u64, duration_ms: u64) -> f64 { + output as f64 / duration_ms as f64 * 1000.0 +} + +fn sorted(values: &[f64]) -> Vec { + let mut sorted = values.to_vec(); + sorted.sort_by(f64::total_cmp); + sorted +} + +/// Even counts average the two middle values, matching the upstream halving. +fn median(values: &[f64]) -> Option { + if values.is_empty() { + return None; + } + let sorted = sorted(values); + let middle = sorted.len() / 2; + Some(if sorted.len().is_multiple_of(2) { + sorted[middle - 1] / 2.0 + sorted[middle] / 2.0 + } else { + sorted[middle] + }) +} + +/// Nearest-rank percentile; `None` below the minimum observation count. +fn nearest_rank(values: &[f64], fraction: f64, minimum: usize) -> Option { + if values.len() < minimum { + return None; + } + let sorted = sorted(values); + #[expect( + clippy::cast_possible_truncation, + reason = "ceil(len * fraction) with 0 < fraction <= 1 is at most len" + )] + let rank = (sorted.len() as f64 * fraction).ceil() as usize; + sorted.get(rank.checked_sub(1)?).copied() +} + +/// Completed-turn samples from one rollout file that belongs to `session_id`. +/// +/// Returns nothing when the file cannot be read or when a turn-relevant line is +/// too large to inspect: a partial record could change the outcome of any turn. +pub(crate) fn scan_session_turns( + path: &Path, + session_id: &str, + range: &CostUsageDayRange, +) -> Vec { + let Ok(file) = File::open(path) else { + return Vec::new(); + }; + let mut reader = BufReader::new(file); + let mut consumed = 0_u64; + let mut scan = TurnScan::new(session_id); + loop { + match read_line(&mut reader, &mut consumed) { + Ok(Some(JsonlLine::Complete(bytes))) => scan.observe_line(&bytes), + Ok(Some(JsonlLine::Oversized(prefix))) => { + let prefix = String::from_utf8_lossy(&prefix); + if RELEVANT_MARKERS + .iter() + .any(|marker| prefix.contains(marker)) + { + return Vec::new(); + } + } + Ok(None) => break, + Err(_) => return Vec::new(), + } + } + scan.finish(range) +} + +#[derive(Default)] +struct TurnState { + reported_output: Option, + rejected: bool, + completion: Option, + effort: Option, + conflicting_effort: bool, +} + +impl TurnState { + /// Mirrors upstream: a later absent effort is a conflict once an effort was seen. + fn observe_effort(&mut self, effort: Option) { + if self.effort.is_some() && self.effort != effort { + self.conflicting_effort = true; + } + self.effort = effort; + } +} + +#[derive(Clone)] +struct Completion { + completed_at_ms: Option, + started_at_ms: Option, + duration_ms: Option, + first_token_ms: Option, +} + +/// Owned request rows for one turn, summed across unique responses. +struct TurnRows { + output: u64, + input: u64, + cached: u64, + valid: bool, + valid_cache: bool, + first_timestamp_ms: Option, + last_timestamp_ms: Option, + models: BTreeSet, +} + +impl TurnRows { + fn new() -> Self { + Self { + output: 0, + input: 0, + cached: 0, + valid: true, + valid_cache: true, + first_timestamp_ms: None, + last_timestamp_ms: None, + models: BTreeSet::new(), + } + } + + fn add(&mut self, usage: Usage, timestamp_ms: Option, model: String) { + let Ok(output) = u64::try_from(usage.2) else { + self.valid = false; + return; + }; + match self.output.checked_add(output) { + Some(total) => self.output = total, + None => { + self.valid = false; + return; + } + } + match (u64::try_from(usage.0), u64::try_from(usage.1)) { + (Ok(input), Ok(cached)) if cached <= input => { + match ( + self.input.checked_add(input), + self.cached.checked_add(cached), + ) { + (Some(total_input), Some(total_cached)) => { + self.input = total_input; + self.cached = total_cached; + } + _ => self.valid_cache = false, + } + } + _ => self.valid_cache = false, + } + match timestamp_ms { + Some(timestamp) => { + self.first_timestamp_ms = Some( + self.first_timestamp_ms + .map_or(timestamp, |first| first.min(timestamp)), + ); + self.last_timestamp_ms = Some( + self.last_timestamp_ms + .map_or(timestamp, |last| last.max(timestamp)), + ); + } + None => self.valid = false, + } + self.models.insert(model); + } +} + +struct UsageRecord { + response_id: String, + timestamp_ms: Option, + usage: Usage, + turn_total: Option, + model: Option, +} + +fn parse_usage_record(object: &Value, payload: &Value) -> Option { + let response_id = non_empty_str(payload.get("response_id"))?; + let timestamp = object.get("timestamp")?.as_str()?; + let usage = request_usage(payload.get("usage"))?; + let thread_total = request_usage(payload.get("thread_token_usage"))?; + if !totals_at_least(thread_total, usage) { + return None; + } + let turn_total = request_usage(payload.get("turn_token_usage")) + .filter(|total| totals_at_least(*total, usage)); + Some(UsageRecord { + response_id: response_id.to_string(), + timestamp_ms: timestamp_unix_ms(timestamp), + usage, + turn_total, + model: non_empty_str(payload.get("model")).and_then(model_evidence), + }) +} + +struct TurnScan<'a> { + session_id: &'a str, + partial: bool, + current_turn: Option, + active_turn: Option, + current_model: Option, + turn_models: HashMap, + states: HashMap, + rows: HashMap, + responses: HashMap, +} + +static EMPTY_VALUE: Value = Value::Null; + +fn payload_of(object: &Value) -> &Value { + object.get("payload").unwrap_or(&EMPTY_VALUE) +} + +impl<'a> TurnScan<'a> { + fn new(session_id: &'a str) -> Self { + Self { + session_id, + partial: false, + current_turn: None, + active_turn: None, + current_model: None, + turn_models: HashMap::new(), + states: HashMap::new(), + rows: HashMap::new(), + responses: HashMap::new(), + } + } + + fn observe_line(&mut self, bytes: &[u8]) { + let Ok(text) = std::str::from_utf8(bytes) else { + return; + }; + if !RELEVANT_MARKERS.iter().any(|marker| text.contains(marker)) { + return; + } + // A malformed line (for example a rollout still being written) is skipped, not fatal. + let Ok(object) = serde_json::from_str::(text) else { + return; + }; + match object.get("type").and_then(Value::as_str) { + Some("turn_context") => self.observe_turn_context(&object), + Some("token_usage_record") => self.observe_usage_record(&object), + Some("event_msg") => { + let payload = payload_of(&object); + match payload.get("type").and_then(Value::as_str) { + Some("task_started") => self.observe_task_started(payload), + Some("task_complete") => self.observe_task_complete(&object), + _ => {} + } + } + _ => {} + } + } + + fn observe_turn_context(&mut self, object: &Value) { + let payload = payload_of(object); + // An explicitly blank model clears stale evidence; an omitted field keeps it. + if let Some(raw) = payload.get("model").or_else(|| payload.get("model_name")) { + self.current_model = raw.as_str().and_then(model_evidence); + } + let Some(turn) = turn_id(payload) else { + return; + }; + if let Some(model) = &self.current_model { + self.turn_models.insert(turn.clone(), model.clone()); + } + let effort = payload + .get("effort") + .and_then(Value::as_str) + .and_then(effort_evidence); + self.states + .entry(turn.clone()) + .or_default() + .observe_effort(effort); + self.active_turn = Some(turn); + } + + fn observe_task_started(&mut self, payload: &Value) { + let turn = turn_id(payload); + if let Some(turn) = &turn + && let Some(state) = self.states.get_mut(turn) + { + // A restarted turn invalidates any completion seen for its earlier run. + state.completion = None; + } + self.current_turn = turn; + } + + fn observe_task_complete(&mut self, object: &Value) { + let payload = payload_of(object); + let Some(turn) = non_empty_str(payload.get("turn_id")).map(str::to_string) else { + return; + }; + let succeeded = payload.get("error").is_none_or(Value::is_null); + let duration = performance_integer(payload.get("duration_ms")); + let started_seconds = performance_integer(payload.get("started_at")); + let started_at_ms = started_seconds + .and_then(|seconds| i64::try_from(seconds).ok()) + .and_then(|seconds| seconds.checked_mul(1000)); + let valid_start = + payload.get("started_at").is_none_or(Value::is_null) || started_at_ms.is_some(); + let duration_ms = (succeeded && valid_start) + .then_some(duration) + .flatten() + .filter(|ms| *ms > 0); + let completion = Completion { + completed_at_ms: object + .get("timestamp") + .and_then(Value::as_str) + .and_then(timestamp_unix_ms), + started_at_ms, + duration_ms, + first_token_ms: performance_integer(payload.get("time_to_first_token_ms")), + }; + self.states.entry(turn).or_default().completion = Some(completion); + } + + fn observe_usage_record(&mut self, object: &Value) { + let payload = payload_of(object); + if non_empty_str(payload.get("thread_id")) != Some(self.session_id) { + return; + } + if payload + .get("session_id") + .is_some_and(|raw| non_empty_str(Some(raw)) != Some(self.session_id)) + { + return; + } + let turn = turn_id(payload) + .or_else(|| self.current_turn.clone()) + .or_else(|| self.active_turn.clone()); + let Some(record) = parse_usage_record(object, payload) else { + // Owned but unusable usage makes its turn unprovable. + if let Some(turn) = turn { + self.states.entry(turn).or_default().rejected = true; + } + return; + }; + self.accept(record, turn); + } + + fn accept(&mut self, record: UsageRecord, turn: Option) { + let Some(turn) = turn else { + return; + }; + if let Some((seen_turn, seen_usage)) = self.responses.get(&record.response_id) { + if *seen_turn == turn && *seen_usage == record.usage { + // A replayed copy of the same response is counted once. + return; + } + // Contradictory copies of one response make both turns unprovable. + let seen_turn = seen_turn.clone(); + self.states.entry(seen_turn).or_default().rejected = true; + self.states.entry(turn).or_default().rejected = true; + return; + } + self.responses + .insert(record.response_id.clone(), (turn.clone(), record.usage)); + + let model = record + .model + .clone() + .or_else(|| self.turn_models.get(&turn).cloned()) + .unwrap_or_else(|| CostUsagePricing::CODEX_UNATTRIBUTED_MODEL.to_string()); + self.rows + .entry(turn.clone()) + .or_insert_with(TurnRows::new) + .add(record.usage, record.timestamp_ms, model); + // Upstream: the last accepted record's turn total is the reported output. + let reported = record + .turn_total + .and_then(|total| u64::try_from(total.2).ok()); + self.states.entry(turn).or_default().reported_output = reported; + } + + fn finish(self, range: &CostUsageDayRange) -> Vec { + if self.partial { + return Vec::new(); + } + let mut turns = self.states.keys().cloned().collect::>(); + turns.sort(); + turns + .into_iter() + .filter_map(|turn| self.finish_turn(&turn, range)) + .collect() + } + + fn finish_turn(&self, turn: &str, range: &CostUsageDayRange) -> Option { + let state = self.states.get(turn)?; + if state.rejected { + return None; + } + let completion = state.completion.as_ref()?; + let duration_ms = completion.duration_ms?; + let rows = self.rows.get(turn)?; + if !rows.valid || rows.output != state.reported_output? { + return None; + } + let completed_at_ms = completion.completed_at_ms?; + let last = rows.last_timestamp_ms?; + let first = rows.first_timestamp_ms?; + if last > completed_at_ms { + return None; + } + if completion + .started_at_ms + .is_some_and(|started| first < started) + { + return None; + } + let day = bucket_day(completed_at_ms)?; + if !CostUsageDayRange::is_in_range(&day, &range.since_key, &range.until_key) { + return None; + } + let model = match rows.models.len() { + 1 => rows + .models + .iter() + .next() + .filter(|model| model.as_str() != CostUsagePricing::CODEX_UNATTRIBUTED_MODEL) + .cloned(), + _ => None, + }; + let (input_tokens, cached_input_tokens) = if rows.valid_cache { + (Some(rows.input), Some(rows.cached)) + } else { + (None, None) + }; + Some(TurnSample { + turn_id: turn.to_string(), + output_tokens: rows.output, + duration_ms, + first_token_ms: completion.first_token_ms.filter(|ms| *ms <= duration_ms), + model, + reasoning_effort: if state.conflicting_effort { + None + } else { + state.effort.clone() + }, + input_tokens, + cached_input_tokens, + }) + } +} + +/// Local calendar day of a completion, using the same zone as the cost buckets. +fn bucket_day(milliseconds: i64) -> Option { + let instant: DateTime = DateTime::from_timestamp_millis(milliseconds)?; + Some( + crate::cost_reporting_period::cost_bucket_zone() + .date(instant) + .format("%Y-%m-%d") + .to_string(), + ) +} + +fn model_evidence(raw: &str) -> Option { + let trimmed = raw.trim(); + (!trimmed.is_empty()).then(|| CostUsagePricing::normalize_codex_model(trimmed)) +} + +fn effort_evidence(raw: &str) -> Option { + let trimmed = raw.trim(); + (!trimmed.is_empty()).then(|| trimmed.to_string()) +} + +/// Non-negative whole number; booleans and fractions are rejected, whole floats accepted. +fn performance_integer(value: Option<&Value>) -> Option { + let number = value?.as_number()?; + if let Some(whole) = number.as_u64() { + return Some(whole); + } + let float = number.as_f64()?; + #[expect( + clippy::cast_possible_truncation, + reason = "finite, whole, non-negative, and below i64::MAX before the cast" + )] + let whole = float as u64; + (float.is_finite() && float >= 0.0 && float.fract() == 0.0 && float < i64::MAX as f64) + .then_some(whole) +} + +#[cfg(test)] +mod tests; diff --git a/rust/src/codex_workspaces/turn_performance/tests.rs b/rust/src/codex_workspaces/turn_performance/tests.rs new file mode 100644 index 0000000000..3df8368fd7 --- /dev/null +++ b/rust/src/codex_workspaces/turn_performance/tests.rs @@ -0,0 +1,477 @@ +use std::fs; + +use chrono::DateTime; +use serde_json::{Value, json}; + +use super::{CodexTurnPerformanceSummary, TurnSample, TurnScan, scan_session_turns}; +use crate::core::{CostUsageDayRange, CostUsagePricing}; + +const SESSION: &str = "sess-1"; +const T0: &str = "2026-06-01T10:00:00.000Z"; +const T_FIRST_RECORD: &str = "2026-06-01T10:00:05.000Z"; +const T_SECOND_RECORD: &str = "2026-06-01T10:00:20.000Z"; +const T_COMPLETE: &str = "2026-06-01T10:00:21.000Z"; + +fn wide_range() -> CostUsageDayRange { + CostUsageDayRange { + since_key: "2000-01-01".to_string(), + until_key: "2100-12-31".to_string(), + scan_since_key: "1999-12-31".to_string(), + scan_until_key: "2101-01-01".to_string(), + } +} + +fn range_from(since: &str) -> CostUsageDayRange { + CostUsageDayRange { + since_key: since.to_string(), + until_key: "2100-12-31".to_string(), + scan_since_key: since.to_string(), + scan_until_key: "2101-01-01".to_string(), + } +} + +fn epoch_seconds(timestamp: &str) -> i64 { + DateTime::parse_from_rfc3339(timestamp) + .expect("fixture timestamp") + .timestamp() +} + +fn scan_lines(lines: &[Value], range: &CostUsageDayRange) -> Vec { + let mut scan = TurnScan::new(SESSION); + for line in lines { + scan.observe_line(line.to_string().as_bytes()); + } + scan.finish(range) +} + +fn turn_context(turn: &str, model: Option<&str>, effort: Option<&str>) -> Value { + let mut payload = json!({ "turn_id": turn }); + if let Some(model) = model { + payload["model"] = json!(model); + } + if let Some(effort) = effort { + payload["effort"] = json!(effort); + } + json!({ "type": "turn_context", "timestamp": T0, "payload": payload }) +} + +fn task_started(turn: &str) -> Value { + json!({ + "type": "event_msg", + "timestamp": T0, + "payload": { "type": "task_started", "turn_id": turn } + }) +} + +fn task_complete( + turn: &str, + duration_ms: u64, + first_token_ms: Option, + started_at: &str, +) -> Value { + json!({ + "type": "event_msg", + "timestamp": T_COMPLETE, + "payload": { + "type": "task_complete", + "turn_id": turn, + "duration_ms": duration_ms, + "time_to_first_token_ms": first_token_ms, + "started_at": epoch_seconds(started_at), + "error": null + } + }) +} + +/// One `token_usage_record`. `turn_output` is the turn's reported running total. +fn usage_record( + response: &str, + turn: &str, + at: &str, + model: &str, + usage: (u64, u64, u64), + turn_output: u64, +) -> Value { + let (input, cached, output) = usage; + json!({ + "type": "token_usage_record", + "timestamp": at, + "payload": { + "thread_id": SESSION, + "response_id": response, + "turn_id": turn, + "model": model, + "usage": { + "input_tokens": input, + "cached_input_tokens": cached, + "output_tokens": output, + "reasoning_output_tokens": 0 + }, + "thread_token_usage": { + "input_tokens": 1_000_000, + "cached_input_tokens": 1_000_000, + "output_tokens": 1_000_000, + "reasoning_output_tokens": 0 + }, + "turn_token_usage": { + "input_tokens": input, + "cached_input_tokens": cached, + "output_tokens": turn_output, + "reasoning_output_tokens": 0 + } + } + }) +} + +/// A turn with two requests (40 + 60 output) whose reported total is 100. +fn two_request_turn(turn: &str) -> Vec { + vec![ + turn_context(turn, Some("gpt-5.5"), Some("high")), + task_started(turn), + usage_record( + &format!("{turn}-a"), + turn, + T_FIRST_RECORD, + "gpt-5.5", + (1000, 250, 40), + 40, + ), + usage_record( + &format!("{turn}-b"), + turn, + T_SECOND_RECORD, + "gpt-5.5", + (1000, 250, 60), + 100, + ), + task_complete(turn, 20_000, Some(1500), T0), + ] +} + +fn sample(turn: &str, output: u64, duration_ms: u64, first_token_ms: Option) -> TurnSample { + TurnSample { + turn_id: turn.to_string(), + output_tokens: output, + duration_ms, + first_token_ms, + model: None, + reasoning_effort: None, + input_tokens: None, + cached_input_tokens: None, + } +} + +#[test] +fn timed_turn_with_agreeing_requests_becomes_a_sample() { + let samples = scan_lines(&two_request_turn("t1"), &wide_range()); + + assert_eq!(samples.len(), 1); + let turn = &samples[0]; + assert_eq!(turn.turn_id, "t1"); + assert_eq!(turn.output_tokens, 100); + assert_eq!(turn.duration_ms, 20_000); + assert_eq!(turn.first_token_ms, Some(1500)); + assert_eq!( + turn.model.as_deref(), + Some(CostUsagePricing::normalize_codex_model("gpt-5.5").as_str()) + ); + assert_eq!(turn.reasoning_effort.as_deref(), Some("high")); + assert_eq!(turn.input_tokens, Some(2000)); + assert_eq!(turn.cached_input_tokens, Some(500)); +} + +#[test] +fn failed_turn_is_rejected() { + let mut lines = two_request_turn("t1"); + let last = lines.len() - 1; + lines[last]["payload"]["error"] = json!({ "message": "boom" }); + + assert!(scan_lines(&lines, &wide_range()).is_empty()); +} + +#[test] +fn reported_output_that_disagrees_with_requests_is_rejected() { + let lines = vec![ + turn_context("t1", Some("gpt-5.5"), Some("high")), + task_started("t1"), + usage_record("t1-a", "t1", T_FIRST_RECORD, "gpt-5.5", (1000, 250, 40), 40), + usage_record( + "t1-b", + "t1", + T_SECOND_RECORD, + "gpt-5.5", + (1000, 250, 60), + 90, + ), + task_complete("t1", 20_000, Some(1500), T0), + ]; + + assert!(scan_lines(&lines, &wide_range()).is_empty()); +} + +#[test] +fn replayed_identical_response_is_counted_once() { + let lines = vec![ + turn_context("t1", Some("gpt-5.5"), Some("high")), + task_started("t1"), + usage_record("t1-a", "t1", T_FIRST_RECORD, "gpt-5.5", (1000, 250, 40), 40), + usage_record("t1-a", "t1", T_FIRST_RECORD, "gpt-5.5", (1000, 250, 40), 40), + task_complete("t1", 20_000, Some(1500), T0), + ]; + + let samples = scan_lines(&lines, &wide_range()); + + assert_eq!(samples.len(), 1); + assert_eq!(samples[0].output_tokens, 40); +} + +#[test] +fn contradictory_copies_of_one_response_reject_both_turns() { + let lines = vec![ + turn_context("t1", Some("gpt-5.5"), Some("high")), + task_started("t1"), + usage_record( + "shared", + "t1", + T_FIRST_RECORD, + "gpt-5.5", + (1000, 250, 40), + 40, + ), + task_complete("t1", 20_000, Some(1500), T0), + turn_context("t2", Some("gpt-5.5"), Some("high")), + task_started("t2"), + usage_record( + "shared", + "t2", + T_FIRST_RECORD, + "gpt-5.5", + (1000, 250, 41), + 41, + ), + task_complete("t2", 20_000, Some(1500), T0), + ]; + + assert!(scan_lines(&lines, &wide_range()).is_empty()); +} + +#[test] +fn first_token_longer_than_duration_is_dropped_but_turn_is_kept() { + let mut lines = two_request_turn("t1"); + let last = lines.len() - 1; + lines[last] = task_complete("t1", 20_000, Some(25_000), T0); + + let samples = scan_lines(&lines, &wide_range()); + + assert_eq!(samples.len(), 1); + assert_eq!(samples[0].first_token_ms, None); + assert_eq!(samples[0].duration_ms, 20_000); +} + +#[test] +fn turn_with_requests_from_two_models_has_no_single_model() { + let lines = vec![ + turn_context("t1", None, Some("high")), + task_started("t1"), + usage_record("t1-a", "t1", T_FIRST_RECORD, "gpt-5.5", (1000, 250, 40), 40), + usage_record( + "t1-b", + "t1", + T_SECOND_RECORD, + "gpt-5.4", + (1000, 250, 60), + 100, + ), + task_complete("t1", 20_000, Some(1500), T0), + ]; + + let samples = scan_lines(&lines, &wide_range()); + + assert_eq!(samples.len(), 1); + assert_eq!(samples[0].model, None); +} + +#[test] +fn conflicting_effort_for_one_turn_is_dropped() { + let mut lines = two_request_turn("t1"); + // A later context for the same turn without an effort is a conflict upstream. + lines.insert(1, turn_context("t1", Some("gpt-5.5"), None)); + + let samples = scan_lines(&lines, &wide_range()); + + assert_eq!(samples.len(), 1); + assert_eq!(samples[0].reasoning_effort, None); +} + +#[test] +fn usage_for_another_thread_does_not_count_toward_the_turn() { + let mut other = usage_record("t1-a", "t1", T_FIRST_RECORD, "gpt-5.5", (1000, 250, 40), 40); + other["payload"]["thread_id"] = json!("sess-2"); + let lines = vec![ + turn_context("t1", Some("gpt-5.5"), Some("high")), + task_started("t1"), + other, + task_complete("t1", 20_000, Some(1500), T0), + ]; + + assert!(scan_lines(&lines, &wide_range()).is_empty()); +} + +#[test] +fn turn_outside_the_day_range_is_excluded() { + let lines = two_request_turn("t1"); + + assert!(scan_lines(&lines, &range_from("2030-01-01")).is_empty()); + assert_eq!(scan_lines(&lines, &wide_range()).len(), 1); +} + +#[test] +fn oversized_turn_relevant_line_drops_the_whole_file() { + let dir = tempfile::TempDir::new().expect("temp dir"); + let file = dir.path().join("rollout.jsonl"); + let padding = "x".repeat(512 * 1024); + let oversized = format!( + "{{\"type\":\"event_msg\",\"payload\":{{\"type\":\"task_complete\",\"pad\":\"{padding}\"}}}}" + ); + + let clean = two_request_turn("t1") + .iter() + .map(Value::to_string) + .collect::>() + .join("\n"); + fs::write(&file, format!("{clean}\n")).expect("write clean rollout"); + assert_eq!(scan_session_turns(&file, SESSION, &wide_range()).len(), 1); + + fs::write(&file, format!("{clean}\n{oversized}\n")).expect("write oversized rollout"); + assert!(scan_session_turns(&file, SESSION, &wide_range()).is_empty()); +} + +#[test] +fn oversized_line_that_is_not_turn_relevant_keeps_the_file() { + let dir = tempfile::TempDir::new().expect("temp dir"); + let file = dir.path().join("rollout.jsonl"); + let padding = "x".repeat(512 * 1024); + let unrelated = format!("{{\"type\":\"response_item\",\"pad\":\"{padding}\"}}"); + let clean = two_request_turn("t1") + .iter() + .map(Value::to_string) + .collect::>() + .join("\n"); + fs::write(&file, format!("{clean}\n{unrelated}\n")).expect("write rollout"); + + assert_eq!(scan_session_turns(&file, SESSION, &wide_range()).len(), 1); +} + +#[test] +fn p95_needs_twenty_samples() { + let nineteen = (1..=19) + .map(|ms| sample(&format!("t{ms}"), 100, ms * 1000, None)) + .collect::>(); + let twenty = (1..=20) + .map(|ms| sample(&format!("t{ms}"), 100, ms * 1000, None)) + .collect::>(); + + let below = CodexTurnPerformanceSummary::from_samples(&nineteen).expect("summary"); + assert_eq!(below.details.p95_duration_milliseconds, None); + + let at = CodexTurnPerformanceSummary::from_samples(&twenty).expect("summary"); + // Nearest rank: ceil(20 * 0.95) = 19th of 20 ascending values. + assert_eq!(at.details.p95_duration_milliseconds, Some(19_000.0)); +} + +#[test] +fn quartiles_need_four_samples() { + let three = [1000, 2000, 3000] + .iter() + .enumerate() + .map(|(index, duration)| sample(&format!("t{index}"), 1000, *duration, None)) + .collect::>(); + let four = [1000, 2000, 3000, 4000] + .iter() + .enumerate() + .map(|(index, duration)| sample(&format!("t{index}"), 1000, *duration, None)) + .collect::>(); + + let below = CodexTurnPerformanceSummary::from_samples(&three).expect("summary"); + assert_eq!(below.details.output_rate_lower_quartile, None); + assert_eq!(below.details.output_rate_upper_quartile, None); + + // Rates are 1000, 500, 333.3 and 250 tok/s; nearest ranks pick 250 and 500. + let at = CodexTurnPerformanceSummary::from_samples(&four).expect("summary"); + assert_eq!(at.details.output_rate_lower_quartile, Some(250.0)); + assert_eq!(at.details.output_rate_upper_quartile, Some(500.0)); +} + +#[test] +fn even_sample_count_median_averages_the_middle_pair() { + let samples = vec![ + sample("a", 100, 1000, Some(100)), + sample("b", 100, 3000, Some(300)), + ]; + + let summary = CodexTurnPerformanceSummary::from_samples(&samples).expect("summary"); + + assert_eq!(summary.median_duration_milliseconds, 2000.0); + assert_eq!(summary.median_first_token_milliseconds, Some(200.0)); +} + +#[test] +fn output_rate_is_weighted_by_elapsed_time_and_first_token_by_its_own_count() { + let samples = vec![ + sample("a", 100, 10_000, Some(1000)), + sample("b", 300, 10_000, None), + ]; + + let summary = CodexTurnPerformanceSummary::from_samples(&samples).expect("summary"); + + // 400 output tokens over 20 seconds, not the mean of the two per-turn rates. + assert_eq!(summary.sample_count, 2); + assert_eq!(summary.output_tokens_per_second, 20.0); + assert_eq!(summary.first_token_sample_count, 1); + assert_eq!(summary.median_first_token_milliseconds, Some(1000.0)); +} + +#[test] +fn cached_input_fraction_uses_only_samples_with_cache_evidence() { + let mut with_cache = sample("a", 100, 10_000, None); + with_cache.input_tokens = Some(1000); + with_cache.cached_input_tokens = Some(250); + let mut more_cache = sample("b", 100, 10_000, None); + more_cache.input_tokens = Some(1000); + more_cache.cached_input_tokens = Some(750); + let no_evidence = sample("c", 100, 10_000, None); + + let details = CodexTurnPerformanceSummary::from_samples(&[with_cache, more_cache, no_evidence]) + .expect("summary") + .details; + + assert_eq!(details.cache_sample_count, 2); + assert_eq!(details.cached_input_fraction, Some(0.5)); +} + +#[test] +fn groups_separate_model_and_effort_with_unknown_first() { + let mut high = sample("a", 100, 10_000, None); + high.model = Some("gpt-5.5".to_string()); + high.reasoning_effort = Some("high".to_string()); + let unknown = sample("b", 100, 10_000, None); + + let groups = CodexTurnPerformanceSummary::from_samples(&[high, unknown]) + .expect("summary") + .details + .groups; + + assert_eq!(groups.len(), 2); + assert_eq!(groups[0].model, None); + assert_eq!(groups[1].model.as_deref(), Some("gpt-5.5")); + assert_eq!(groups[1].reasoning_effort.as_deref(), Some("high")); +} + +#[test] +fn empty_or_untimed_samples_produce_no_summary() { + assert_eq!(CodexTurnPerformanceSummary::from_samples(&[]), None); + assert_eq!( + CodexTurnPerformanceSummary::from_samples(&[sample("a", 100, 0, None)]), + None + ); +} diff --git a/rust/src/codex_workspaces/types.rs b/rust/src/codex_workspaces/types.rs index 3b1fafa75d..6ba0abb756 100644 --- a/rust/src/codex_workspaces/types.rs +++ b/rust/src/codex_workspaces/types.rs @@ -5,6 +5,8 @@ use std::cmp::Ordering; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; +use super::turn_performance::CodexTurnPerformanceSummary; + /// Progress phases emitted while building a workspaces snapshot. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -132,6 +134,10 @@ pub struct SessionUsage { pub totals: UsageTotals, pub cost_estimate: CostEstimate, pub top_model: Option, + /// Native Codex turn latency and throughput (upstream #4304). `None` when no + /// completed turn in the window has agreeing timing and token evidence. + #[serde(default)] + pub turn_performance: Option, } impl SessionUsage { diff --git a/rust/src/codex_workspaces/types/tests.rs b/rust/src/codex_workspaces/types/tests.rs index 25df8bbdd2..eff2179b3c 100644 --- a/rust/src/codex_workspaces/types/tests.rs +++ b/rust/src/codex_workspaces/types/tests.rs @@ -20,6 +20,7 @@ fn session(id: &str, cost: Option, tokens: u64, minutes: i64) -> SessionUsa }, }, top_model: None, + turn_performance: None, } } From bea4861df50c8cae141677def2d15b611f6c4bc1 Mon Sep 17 00:00:00 2001 From: Factory Agent Date: Fri, 16 Oct 2026 09:49:05 +0700 Subject: [PATCH 3/5] Add Codex turn performance locale strings Add the Usage & Spend performance labels, help text, and caveats to every locale and the TypeScript key list in parity. --- apps/desktop-tauri/src/i18n/keys.ts | 19 +++++++++++++++++++ rust/src/locale.rs | 19 +++++++++++++++++++ rust/src/locale/en-US.ftl | 19 +++++++++++++++++++ rust/src/locale/es-MX.ftl | 19 +++++++++++++++++++ rust/src/locale/ja-JP.ftl | 19 +++++++++++++++++++ rust/src/locale/ko-KR.ftl | 19 +++++++++++++++++++ rust/src/locale/pt-BR.ftl | 19 +++++++++++++++++++ rust/src/locale/ru-RU.ftl | 19 +++++++++++++++++++ rust/src/locale/tr-TR.ftl | 19 +++++++++++++++++++ rust/src/locale/uk-UA.ftl | 19 +++++++++++++++++++ rust/src/locale/zh-CN.ftl | 19 +++++++++++++++++++ rust/src/locale/zh-TW.ftl | 19 +++++++++++++++++++ 12 files changed, 228 insertions(+) diff --git a/apps/desktop-tauri/src/i18n/keys.ts b/apps/desktop-tauri/src/i18n/keys.ts index 57c0f2296a..021c4a7fe9 100644 --- a/apps/desktop-tauri/src/i18n/keys.ts +++ b/apps/desktop-tauri/src/i18n/keys.ts @@ -555,6 +555,25 @@ export const ALL_LOCALE_KEYS = [ "UsageSpendCustomPricingActive", "UsageSpendDefaultPricing", "UsageSpendHourlyActivity", + "UsageSpendPerformanceDetails", + "UsageSpendTurnOutput", + "UsageSpendFirstToken", + "UsageSpendTurnDuration", + "UsageSpendTokensPerSecond", + "UsageSpendSeconds", + "UsageSpendPerformanceCaveat", + "UsageSpendTurnOutputHelp", + "UsageSpendFirstTokenHelp", + "UsageSpendP95Duration", + "UsageSpendP95FirstToken", + "UsageSpendNeedsTwentyTurns", + "UsageSpendMiddleHalfRate", + "UsageSpendNeedsFourTurns", + "UsageSpendCachedInput", + "UsageSpendCacheUnavailable", + "UsageSpendByModelEffort", + "UsageSpendUnknownModel", + "UsageSpendUnknownEffort", "TokenActivityTitle", "TokenActivityInLastYear", "TokenActivityUnavailable", diff --git a/rust/src/locale.rs b/rust/src/locale.rs index a6b2a385b9..9000f26dba 100644 --- a/rust/src/locale.rs +++ b/rust/src/locale.rs @@ -805,6 +805,25 @@ locale_keys! { UsageSpendCustomPricingActive, UsageSpendDefaultPricing, UsageSpendHourlyActivity, + UsageSpendPerformanceDetails, + UsageSpendTurnOutput, + UsageSpendFirstToken, + UsageSpendTurnDuration, + UsageSpendTokensPerSecond, + UsageSpendSeconds, + UsageSpendPerformanceCaveat, + UsageSpendTurnOutputHelp, + UsageSpendFirstTokenHelp, + UsageSpendP95Duration, + UsageSpendP95FirstToken, + UsageSpendNeedsTwentyTurns, + UsageSpendMiddleHalfRate, + UsageSpendNeedsFourTurns, + UsageSpendCachedInput, + UsageSpendCacheUnavailable, + UsageSpendByModelEffort, + UsageSpendUnknownModel, + UsageSpendUnknownEffort, TokenActivityTitle, TokenActivityInLastYear, TokenActivityUnavailable, diff --git a/rust/src/locale/en-US.ftl b/rust/src/locale/en-US.ftl index b83dbe84c0..3d83f1332e 100644 --- a/rust/src/locale/en-US.ftl +++ b/rust/src/locale/en-US.ftl @@ -430,6 +430,25 @@ UsageSpendHistoryCovered = History covered UsageSpendCustomPricingActive = Custom pricing active UsageSpendDefaultPricing = Default pricing UsageSpendHourlyActivity = Hourly activity +UsageSpendPerformanceDetails = Performance details +UsageSpendTurnOutput = Turn output +UsageSpendFirstToken = First token +UsageSpendTurnDuration = Turn duration +UsageSpendTokensPerSecond = tok/s +UsageSpendSeconds = s +UsageSpendPerformanceCaveat = Observed turns; workload and tools affect these results. +UsageSpendTurnOutputHelp = Turn output includes reasoning, tools and waits. It is not model generation speed. +UsageSpendFirstTokenHelp = Model first token may be reasoning, before visible answer text. +UsageSpendP95Duration = P95 duration +UsageSpendP95FirstToken = P95 first token +UsageSpendNeedsTwentyTurns = Needs 20 timed turns +UsageSpendMiddleHalfRate = Middle 50% of turns +UsageSpendNeedsFourTurns = Speed range needs 4 completed turns. +UsageSpendCachedInput = Cached input +UsageSpendCacheUnavailable = unavailable +UsageSpendByModelEffort = By model and reasoning effort +UsageSpendUnknownModel = Unknown model +UsageSpendUnknownEffort = Unknown reasoning effort TokenActivityTitle = Token activity TokenActivityInLastYear = in the last year TokenActivityUnavailable = Unavailable diff --git a/rust/src/locale/es-MX.ftl b/rust/src/locale/es-MX.ftl index b8bce674f9..cbe76d88a9 100644 --- a/rust/src/locale/es-MX.ftl +++ b/rust/src/locale/es-MX.ftl @@ -398,6 +398,25 @@ UsageSpendHistoryCovered = Historial cubierto UsageSpendCustomPricingActive = Precios personalizados activos UsageSpendDefaultPricing = Precios predeterminados UsageSpendHourlyActivity = Actividad por hora +UsageSpendPerformanceDetails = Detalles de rendimiento +UsageSpendTurnOutput = Salida por turno +UsageSpendFirstToken = Primer token +UsageSpendTurnDuration = Duración del turno +UsageSpendTokensPerSecond = tok/s +UsageSpendSeconds = s +UsageSpendPerformanceCaveat = Solo turnos observados; la carga de trabajo y las herramientas afectan estos resultados. +UsageSpendTurnOutputHelp = La salida por turno incluye razonamiento, herramientas y esperas. No es la velocidad de generación del modelo. +UsageSpendFirstTokenHelp = El primer token del modelo puede ser razonamiento, antes del texto visible de la respuesta. +UsageSpendP95Duration = Duración P95 +UsageSpendP95FirstToken = Primer token P95 +UsageSpendNeedsTwentyTurns = Se necesitan 20 turnos medidos +UsageSpendMiddleHalfRate = 50% central de los turnos +UsageSpendNeedsFourTurns = El rango de velocidad necesita 4 turnos completados. +UsageSpendCachedInput = Entrada en caché +UsageSpendCacheUnavailable = no disponible +UsageSpendByModelEffort = Por modelo y esfuerzo de razonamiento +UsageSpendUnknownModel = Modelo desconocido +UsageSpendUnknownEffort = Esfuerzo de razonamiento desconocido TokenActivityTitle = Actividad de tokens TokenActivityInLastYear = en el último año TokenActivityUnavailable = No disponible diff --git a/rust/src/locale/ja-JP.ftl b/rust/src/locale/ja-JP.ftl index b79d46f0b2..dbbc74cbae 100644 --- a/rust/src/locale/ja-JP.ftl +++ b/rust/src/locale/ja-JP.ftl @@ -398,6 +398,25 @@ UsageSpendHistoryCovered = 対象履歴 UsageSpendCustomPricingActive = カスタム価格を適用中 UsageSpendDefaultPricing = 既定の価格 UsageSpendHourlyActivity = 時間別アクティビティ +UsageSpendPerformanceDetails = パフォーマンスの詳細 +UsageSpendTurnOutput = ターン出力 +UsageSpendFirstToken = 最初のトークン +UsageSpendTurnDuration = ターン所要時間 +UsageSpendTokensPerSecond = tok/s +UsageSpendSeconds = 秒 +UsageSpendPerformanceCaveat = 観測されたターンのみが対象です。ワークロードとツールがこの結果に影響します。 +UsageSpendTurnOutputHelp = ターン出力には推論、ツール、待機時間が含まれます。モデルの生成速度ではありません。 +UsageSpendFirstTokenHelp = 最初のトークンは、表示される回答の前の推論を含む場合があります。 +UsageSpendP95Duration = P95 所要時間 +UsageSpendP95FirstToken = P95 最初のトークン +UsageSpendNeedsTwentyTurns = 計測ターンが 20 件必要です +UsageSpendMiddleHalfRate = 中央 50% のターン +UsageSpendNeedsFourTurns = 速度の範囲には完了したターンが 4 件必要です。 +UsageSpendCachedInput = キャッシュ入力 +UsageSpendCacheUnavailable = 利用不可 +UsageSpendByModelEffort = モデルと推論の強度別 +UsageSpendUnknownModel = 不明なモデル +UsageSpendUnknownEffort = 不明な推論の強度 TokenActivityTitle = トークンアクティビティ TokenActivityInLastYear = 過去1年 TokenActivityUnavailable = 利用不可 diff --git a/rust/src/locale/ko-KR.ftl b/rust/src/locale/ko-KR.ftl index 23b1fad68b..8b2d6815f7 100644 --- a/rust/src/locale/ko-KR.ftl +++ b/rust/src/locale/ko-KR.ftl @@ -398,6 +398,25 @@ UsageSpendHistoryCovered = 포함된 기록 UsageSpendCustomPricingActive = 사용자 지정 가격 적용 중 UsageSpendDefaultPricing = 기본 가격 UsageSpendHourlyActivity = 시간별 활동 +UsageSpendPerformanceDetails = 성능 세부 정보 +UsageSpendTurnOutput = 턴 출력 +UsageSpendFirstToken = 첫 토큰 +UsageSpendTurnDuration = 턴 소요 시간 +UsageSpendTokensPerSecond = tok/s +UsageSpendSeconds = 초 +UsageSpendPerformanceCaveat = 관찰된 턴만 포함합니다. 작업량과 도구가 이 결과에 영향을 줍니다. +UsageSpendTurnOutputHelp = 턴 출력에는 추론, 도구, 대기 시간이 포함됩니다. 모델 생성 속도가 아닙니다. +UsageSpendFirstTokenHelp = 모델의 첫 토큰에는 보이는 답변 전의 추론이 포함될 수 있습니다. +UsageSpendP95Duration = P95 소요 시간 +UsageSpendP95FirstToken = P95 첫 토큰 +UsageSpendNeedsTwentyTurns = 측정된 턴 20개가 필요합니다 +UsageSpendMiddleHalfRate = 중간 50% 턴 +UsageSpendNeedsFourTurns = 속도 범위에는 완료된 턴 4개가 필요합니다. +UsageSpendCachedInput = 캐시된 입력 +UsageSpendCacheUnavailable = 사용 불가 +UsageSpendByModelEffort = 모델 및 추론 강도별 +UsageSpendUnknownModel = 알 수 없는 모델 +UsageSpendUnknownEffort = 알 수 없는 추론 강도 TokenActivityTitle = 토큰 활동 TokenActivityInLastYear = 지난 1년 TokenActivityUnavailable = 사용할 수 없음 diff --git a/rust/src/locale/pt-BR.ftl b/rust/src/locale/pt-BR.ftl index 76bcab0ed7..3c0eaed5f1 100644 --- a/rust/src/locale/pt-BR.ftl +++ b/rust/src/locale/pt-BR.ftl @@ -423,6 +423,25 @@ UsageSpendHistoryCovered = História coberta UsageSpendCustomPricingActive = Preço personalizado ativo UsageSpendDefaultPricing = Preço padrão UsageSpendHourlyActivity = Atividade horária +UsageSpendPerformanceDetails = Detalhes de desempenho +UsageSpendTurnOutput = Saída do turno +UsageSpendFirstToken = Primeiro token +UsageSpendTurnDuration = Duração do turno +UsageSpendTokensPerSecond = tok/s +UsageSpendSeconds = s +UsageSpendPerformanceCaveat = Apenas turnos observados; a carga de trabalho e as ferramentas afetam estes resultados. +UsageSpendTurnOutputHelp = A saída do turno inclui raciocínio, ferramentas e esperas. Não é a velocidade de geração do modelo. +UsageSpendFirstTokenHelp = O primeiro token do modelo pode ser raciocínio, antes do texto visível da resposta. +UsageSpendP95Duration = Duração P95 +UsageSpendP95FirstToken = Primeiro token P95 +UsageSpendNeedsTwentyTurns = São necessários 20 turnos medidos +UsageSpendMiddleHalfRate = 50% centrais dos turnos +UsageSpendNeedsFourTurns = A faixa de velocidade precisa de 4 turnos concluídos. +UsageSpendCachedInput = Entrada em cache +UsageSpendCacheUnavailable = indisponível +UsageSpendByModelEffort = Por modelo e esforço de raciocínio +UsageSpendUnknownModel = Modelo desconhecido +UsageSpendUnknownEffort = Esforço de raciocínio desconhecido TokenActivityTitle = Atividade de tokens TokenActivityInLastYear = no último ano TokenActivityUnavailable = Indisponível diff --git a/rust/src/locale/ru-RU.ftl b/rust/src/locale/ru-RU.ftl index c8673d5fb8..cc3e77308c 100644 --- a/rust/src/locale/ru-RU.ftl +++ b/rust/src/locale/ru-RU.ftl @@ -430,6 +430,25 @@ UsageSpendHistoryCovered = Охват истории UsageSpendCustomPricingActive = Свои цены включены UsageSpendDefaultPricing = Цены по умолчанию UsageSpendHourlyActivity = Активность по часам +UsageSpendPerformanceDetails = Подробности производительности +UsageSpendTurnOutput = Вывод за ход +UsageSpendFirstToken = Первый токен +UsageSpendTurnDuration = Длительность хода +UsageSpendTokensPerSecond = ток/с +UsageSpendSeconds = с +UsageSpendPerformanceCaveat = Учитываются только наблюдаемые ходы; нагрузка и инструменты влияют на результаты. +UsageSpendTurnOutputHelp = Вывод за ход включает рассуждения, инструменты и ожидание. Это не скорость генерации модели. +UsageSpendFirstTokenHelp = Первый токен модели может приходиться на рассуждения до видимого текста ответа. +UsageSpendP95Duration = P95 длительности +UsageSpendP95FirstToken = P95 первого токена +UsageSpendNeedsTwentyTurns = Нужно 20 измеренных ходов +UsageSpendMiddleHalfRate = Средние 50% ходов +UsageSpendNeedsFourTurns = Для диапазона скорости нужно 4 завершённых хода. +UsageSpendCachedInput = Кэшированный ввод +UsageSpendCacheUnavailable = недоступно +UsageSpendByModelEffort = По модели и уровню рассуждений +UsageSpendUnknownModel = Неизвестная модель +UsageSpendUnknownEffort = Неизвестный уровень рассуждений TokenActivityTitle = Активность токенов TokenActivityInLastYear = за последний год TokenActivityUnavailable = Недоступно diff --git a/rust/src/locale/tr-TR.ftl b/rust/src/locale/tr-TR.ftl index 71698e3c5b..a1a8f8e0cd 100644 --- a/rust/src/locale/tr-TR.ftl +++ b/rust/src/locale/tr-TR.ftl @@ -430,6 +430,25 @@ UsageSpendHistoryCovered = Kapsanan geçmiş UsageSpendCustomPricingActive = Özel fiyatlandırma etkin UsageSpendDefaultPricing = Varsayılan fiyatlandırma UsageSpendHourlyActivity = Saatlik etkinlik +UsageSpendPerformanceDetails = Performans ayrıntıları +UsageSpendTurnOutput = Tur çıktısı +UsageSpendFirstToken = İlk belirteç +UsageSpendTurnDuration = Tur süresi +UsageSpendTokensPerSecond = tok/sn +UsageSpendSeconds = sn +UsageSpendPerformanceCaveat = Yalnızca gözlenen turlar; iş yükü ve araçlar bu sonuçları etkiler. +UsageSpendTurnOutputHelp = Tur çıktısı akıl yürütme, araçlar ve bekleme sürelerini içerir. Model üretim hızı değildir. +UsageSpendFirstTokenHelp = Modelin ilk belirteci, görünür yanıt metninden önce akıl yürütme olabilir. +UsageSpendP95Duration = P95 süre +UsageSpendP95FirstToken = P95 ilk belirteç +UsageSpendNeedsTwentyTurns = 20 ölçülmüş tur gerekir +UsageSpendMiddleHalfRate = Turların ortanca %50'si +UsageSpendNeedsFourTurns = Hız aralığı için 4 tamamlanmış tur gerekir. +UsageSpendCachedInput = Önbelleğe alınmış girdi +UsageSpendCacheUnavailable = kullanılamıyor +UsageSpendByModelEffort = Model ve akıl yürütme çabasına göre +UsageSpendUnknownModel = Bilinmeyen model +UsageSpendUnknownEffort = Bilinmeyen akıl yürütme çabası TokenActivityTitle = Token etkinliği TokenActivityInLastYear = son yılda TokenActivityUnavailable = Kullanılamıyor diff --git a/rust/src/locale/uk-UA.ftl b/rust/src/locale/uk-UA.ftl index faa9aefef5..eec03ad3f0 100644 --- a/rust/src/locale/uk-UA.ftl +++ b/rust/src/locale/uk-UA.ftl @@ -430,6 +430,25 @@ UsageSpendHistoryCovered = Історія повна UsageSpendCustomPricingActive = Власні ціни ввімкнено UsageSpendDefaultPricing = Ціни за замовчуванням UsageSpendHourlyActivity = Активність за годинами +UsageSpendPerformanceDetails = Деталі продуктивності +UsageSpendTurnOutput = Вивід за хід +UsageSpendFirstToken = Перший токен +UsageSpendTurnDuration = Тривалість ходу +UsageSpendTokensPerSecond = ток/с +UsageSpendSeconds = с +UsageSpendPerformanceCaveat = Враховуються лише спостережувані ходи; навантаження та інструменти впливають на результати. +UsageSpendTurnOutputHelp = Вивід за хід включає міркування, інструменти та очікування. Це не швидкість генерації моделі. +UsageSpendFirstTokenHelp = Перший токен моделі може припадати на міркування до видимого тексту відповіді. +UsageSpendP95Duration = P95 тривалості +UsageSpendP95FirstToken = P95 першого токена +UsageSpendNeedsTwentyTurns = Потрібно 20 вимірюваних ходів +UsageSpendMiddleHalfRate = Середні 50% ходів +UsageSpendNeedsFourTurns = Для діапазону швидкості потрібно 4 завершені ходи. +UsageSpendCachedInput = Кешований вхід +UsageSpendCacheUnavailable = недоступно +UsageSpendByModelEffort = За моделлю та рівнем міркування +UsageSpendUnknownModel = Невідома модель +UsageSpendUnknownEffort = Невідомий рівень міркування TokenActivityTitle = Активність токенів TokenActivityInLastYear = за останній рік TokenActivityUnavailable = Недоступний diff --git a/rust/src/locale/zh-CN.ftl b/rust/src/locale/zh-CN.ftl index 5711873033..47c609de28 100644 --- a/rust/src/locale/zh-CN.ftl +++ b/rust/src/locale/zh-CN.ftl @@ -398,6 +398,25 @@ UsageSpendHistoryCovered = 已覆盖历史 UsageSpendCustomPricingActive = 自定义定价已启用 UsageSpendDefaultPricing = 默认定价 UsageSpendHourlyActivity = 每小时活动 +UsageSpendPerformanceDetails = 性能详情 +UsageSpendTurnOutput = 轮次输出 +UsageSpendFirstToken = 首个 token +UsageSpendTurnDuration = 轮次耗时 +UsageSpendTokensPerSecond = tok/s +UsageSpendSeconds = 秒 +UsageSpendPerformanceCaveat = 仅统计已观察到的轮次；工作负载和工具会影响这些结果。 +UsageSpendTurnOutputHelp = 轮次输出包括推理、工具调用和等待时间，并非模型生成速度。 +UsageSpendFirstTokenHelp = 模型的首个 token 可能包含推理过程，早于可见的回答文本。 +UsageSpendP95Duration = P95 耗时 +UsageSpendP95FirstToken = P95 首个 token +UsageSpendNeedsTwentyTurns = 需要 20 个计时轮次 +UsageSpendMiddleHalfRate = 中间 50% 的轮次 +UsageSpendNeedsFourTurns = 速度区间需要 4 个已完成轮次。 +UsageSpendCachedInput = 缓存输入 +UsageSpendCacheUnavailable = 不可用 +UsageSpendByModelEffort = 按模型和推理强度 +UsageSpendUnknownModel = 未知模型 +UsageSpendUnknownEffort = 未知推理强度 TokenActivityTitle = Token 活动 TokenActivityInLastYear = 近一年 TokenActivityUnavailable = 不可用 diff --git a/rust/src/locale/zh-TW.ftl b/rust/src/locale/zh-TW.ftl index b519c31e71..11551a818a 100644 --- a/rust/src/locale/zh-TW.ftl +++ b/rust/src/locale/zh-TW.ftl @@ -398,6 +398,25 @@ UsageSpendHistoryCovered = 涵蓋歷史 UsageSpendCustomPricingActive = 自訂定價已啟用 UsageSpendDefaultPricing = 預設定價 UsageSpendHourlyActivity = 每小時活動 +UsageSpendPerformanceDetails = 效能詳情 +UsageSpendTurnOutput = 輪次輸出 +UsageSpendFirstToken = 首個 token +UsageSpendTurnDuration = 輪次耗時 +UsageSpendTokensPerSecond = tok/s +UsageSpendSeconds = 秒 +UsageSpendPerformanceCaveat = 僅統計已觀察到的輪次；工作負載與工具會影響這些結果。 +UsageSpendTurnOutputHelp = 輪次輸出包含推理、工具與等待時間，並非模型生成速度。 +UsageSpendFirstTokenHelp = 模型的首個 token 可能包含推理過程，早於可見的回答文字。 +UsageSpendP95Duration = P95 耗時 +UsageSpendP95FirstToken = P95 首個 token +UsageSpendNeedsTwentyTurns = 需要 20 個計時輪次 +UsageSpendMiddleHalfRate = 中間 50% 的輪次 +UsageSpendNeedsFourTurns = 速度區間需要 4 個已完成輪次。 +UsageSpendCachedInput = 快取輸入 +UsageSpendCacheUnavailable = 不可用 +UsageSpendByModelEffort = 依模型與推理強度 +UsageSpendUnknownModel = 未知模型 +UsageSpendUnknownEffort = 未知推理強度 TokenActivityTitle = Token 活動 TokenActivityInLastYear = 近一年 TokenActivityUnavailable = 無法使用 From fe2ee81901ff106f371d70b301f6e4a36deb823e Mon Sep 17 00:00:00 2001 From: Factory Agent Date: Fri, 16 Oct 2026 09:49:21 +0700 Subject: [PATCH 4/5] Show Codex turn performance per conversation Render turn output rate, first-token and turn duration under each Codex conversation in Usage & Spend, with a Performance details disclosure for P95 latency, the middle-50% rate range, cached input share, and per-model and effort groups. Values appear only once the minimum sample counts are met. --- .../tabs/ProjectConversations.test.tsx | 98 +++++++++++++- .../settings/tabs/ProjectConversations.tsx | 122 +++++++++++++++--- .../settings/tabs/UsageSpendTab.test.tsx | 1 + apps/desktop-tauri/src/types/bridge.ts | 32 +++++ 4 files changed, 233 insertions(+), 20 deletions(-) diff --git a/apps/desktop-tauri/src/surfaces/settings/tabs/ProjectConversations.test.tsx b/apps/desktop-tauri/src/surfaces/settings/tabs/ProjectConversations.test.tsx index 96edcf268f..c36fd4815f 100644 --- a/apps/desktop-tauri/src/surfaces/settings/tabs/ProjectConversations.test.tsx +++ b/apps/desktop-tauri/src/surfaces/settings/tabs/ProjectConversations.test.tsx @@ -4,12 +4,19 @@ import ProjectConversations, { CONVERSATION_INITIAL_ROWS, CONVERSATION_MAX_ROWS, } from "./ProjectConversations"; -import type { CodexWorkspacesSessionUsage } from "../../../types/bridge"; +import type { + CodexWorkspacesSessionUsage, + CodexWorkspacesTurnPerformance, +} from "../../../types/bridge"; import type { LocaleKey } from "../../../i18n/keys"; const t = (key: LocaleKey) => key; -function session(index: number, unknownTokens = 0): CodexWorkspacesSessionUsage { +function session( + index: number, + unknownTokens = 0, + turnPerformance: CodexWorkspacesTurnPerformance | null = null, +): CodexWorkspacesSessionUsage { return { id: `session-${index}`, projectId: "project-a", @@ -20,6 +27,27 @@ function session(index: number, unknownTokens = 0): CodexWorkspacesSessionUsage totals: { inputTokens: 10, cachedInputTokens: 0, outputTokens: 0, totalTokens: 10 }, costEstimate: { knownUsd: 100 - index, unknownTokens }, topModel: null, + turnPerformance, + }; +} + +function performance(overrides: Partial = {}): CodexWorkspacesTurnPerformance { + return { + sampleCount: 9, + outputTokensPerSecond: 45.4, + medianDurationMilliseconds: 8100, + medianFirstTokenMilliseconds: 1200, + firstTokenSampleCount: 9, + details: { + p95DurationMilliseconds: null, + p95FirstTokenMilliseconds: null, + outputRateLowerQuartile: null, + outputRateUpperQuartile: null, + cachedInputFraction: null, + cacheSampleCount: 0, + groups: [], + }, + ...overrides, }; } @@ -47,4 +75,70 @@ describe("ProjectConversations", () => { fireEvent.click(screen.getByRole("button", { name: "UsageSpendShowLess" })); expect(screen.getAllByText(/^#\d+$/)).toHaveLength(CONVERSATION_INITIAL_ROWS); }); + + it("shows no performance line for a conversation without timed turns", () => { + render(); + + expect(screen.queryByText(/UsageSpendTurnOutput/)).toBeNull(); + expect(screen.queryByText("UsageSpendPerformanceDetails")).toBeNull(); + }); + + it("shows turn output, first token and duration with their upstream labels", () => { + render(); + + expect(screen.getByText(/UsageSpendTurnOutput: 45\.4 UsageSpendTokensPerSecond/)).toBeTruthy(); + expect(screen.getByText(/UsageSpendFirstToken: 1\.2 UsageSpendSeconds/)).toBeTruthy(); + expect(screen.getByText(/UsageSpendTurnDuration: 8\.1 UsageSpendSeconds/)).toBeTruthy(); + expect(screen.getByTitle("UsageSpendTurnOutputHelp")).toBeTruthy(); + expect(screen.getByText("UsageSpendPerformanceDetails")).toBeTruthy(); + }); + + it("shows spread, cache and per-model details when the sample minimums are met", () => { + const rich = performance({ + details: { + p95DurationMilliseconds: 19000, + p95FirstTokenMilliseconds: 2500, + outputRateLowerQuartile: 250, + outputRateUpperQuartile: 500, + cachedInputFraction: 0.5, + cacheSampleCount: 4, + groups: [ + { + model: "gpt-5.5", + reasoningEffort: "high", + sampleCount: 5, + firstTokenSampleCount: 5, + medianFirstTokenMilliseconds: 1200, + medianDurationMilliseconds: 8100, + outputTokensPerSecond: 40, + }, + { + model: null, + reasoningEffort: null, + sampleCount: 4, + firstTokenSampleCount: 0, + medianFirstTokenMilliseconds: null, + medianDurationMilliseconds: 9000, + outputTokensPerSecond: 30, + }, + ], + }, + }); + render(); + + expect(screen.getByText(/UsageSpendP95Duration: 19\.0 UsageSpendSeconds/)).toBeTruthy(); + expect(screen.getByText(/UsageSpendP95FirstToken: 2\.5 UsageSpendSeconds/)).toBeTruthy(); + expect(screen.getByText(/UsageSpendMiddleHalfRate: 250\.0–500\.0 UsageSpendTokensPerSecond/)).toBeTruthy(); + expect(screen.getByText("UsageSpendCachedInput: 50%")).toBeTruthy(); + expect(screen.getByText(/gpt-5\.5 \/ high: 40\.0 UsageSpendTokensPerSecond/)).toBeTruthy(); + expect(screen.getByText(/UsageSpendUnknownModel \/ UsageSpendUnknownEffort: 30\.0/)).toBeTruthy(); + }); + + it("explains missing spread and cache evidence instead of showing a number", () => { + render(); + + expect(screen.getAllByText(/UsageSpendNeedsTwentyTurns/)).toHaveLength(2); + expect(screen.getByText(/UsageSpendNeedsFourTurns/)).toBeTruthy(); + expect(screen.getByText("UsageSpendCachedInput: UsageSpendCacheUnavailable")).toBeTruthy(); + }); }); diff --git a/apps/desktop-tauri/src/surfaces/settings/tabs/ProjectConversations.tsx b/apps/desktop-tauri/src/surfaces/settings/tabs/ProjectConversations.tsx index 948d1b0bc4..7feb3cf912 100644 --- a/apps/desktop-tauri/src/surfaces/settings/tabs/ProjectConversations.tsx +++ b/apps/desktop-tauri/src/surfaces/settings/tabs/ProjectConversations.tsx @@ -1,6 +1,9 @@ import { useState } from "react"; import type { LocaleKey } from "../../../i18n/keys"; -import type { CodexWorkspacesSessionUsage } from "../../../types/bridge"; +import type { + CodexWorkspacesSessionUsage, + CodexWorkspacesTurnPerformance, +} from "../../../types/bridge"; /** Conversations shown before the expand control, and the most ever shown. */ export const CONVERSATION_INITIAL_ROWS = 8; @@ -20,23 +23,25 @@ export default function ProjectConversations({ conversations, t }: ProjectConver return (
{visible.map((session, index) => ( -
- - #{index + 1} - - - {session.displayTitle} - - - {session.costEstimate.unknownTokens > 0 ? "~" : ""} - ${session.costEstimate.knownUsd.toFixed(2)} - +
+
+ + #{index + 1} + + + {session.displayTitle} + + + {session.costEstimate.unknownTokens > 0 ? "~" : ""} + ${session.costEstimate.knownUsd.toFixed(2)} + +
+ {session.turnPerformance && ( + + )}
))} {capped.length > CONVERSATION_INITIAL_ROWS && ( @@ -53,3 +58,84 @@ export default function ProjectConversations({ conversations, t }: ProjectConver
); } + +function formatRate(value: number): string { + return value.toFixed(1); +} + +function formatSeconds(milliseconds: number): string { + return (milliseconds / 1000).toFixed(1); +} + +function formatOptionalSeconds(milliseconds: number | null, t: (key: LocaleKey) => string): string { + return milliseconds === null ? "—" : `${formatSeconds(milliseconds)} ${t("UsageSpendSeconds")}`; +} + +interface TurnPerformanceProps { + performance: CodexWorkspacesTurnPerformance; + t: (key: LocaleKey) => string; +} + +/** Observed Codex turn latency and throughput for one conversation (upstream #4304). */ +function TurnPerformance({ performance, t }: TurnPerformanceProps) { + const unit = t("UsageSpendTokensPerSecond"); + const { details } = performance; + const cached = + details.cachedInputFraction === null + ? `${t("UsageSpendCachedInput")}: ${t("UsageSpendCacheUnavailable")}` + : `${t("UsageSpendCachedInput")}: ${Math.round(details.cachedInputFraction * 100)}%`; + + return ( +
+ + {t("UsageSpendTurnOutput")}: {formatRate(performance.outputTokensPerSecond)} {unit} + {" · "} + + {t("UsageSpendFirstToken")}: {formatOptionalSeconds(performance.medianFirstTokenMilliseconds, t)} + + {" · "} + {t("UsageSpendTurnDuration")}: {formatOptionalSeconds(performance.medianDurationMilliseconds, t)} + +
+ {t("UsageSpendPerformanceDetails")} +
+ {t("UsageSpendPerformanceCaveat")} + + {t("UsageSpendP95Duration")}:{" "} + {details.p95DurationMilliseconds === null + ? t("UsageSpendNeedsTwentyTurns") + : formatOptionalSeconds(details.p95DurationMilliseconds, t)} + + + {t("UsageSpendP95FirstToken")}:{" "} + {details.p95FirstTokenMilliseconds === null + ? t("UsageSpendNeedsTwentyTurns") + : formatOptionalSeconds(details.p95FirstTokenMilliseconds, t)} + + + {t("UsageSpendMiddleHalfRate")}:{" "} + {details.outputRateLowerQuartile === null || details.outputRateUpperQuartile === null + ? t("UsageSpendNeedsFourTurns") + : `${formatRate(details.outputRateLowerQuartile)}–${formatRate(details.outputRateUpperQuartile)} ${unit}`} + + {cached} + {details.groups.length > 0 && ( +
+ {t("UsageSpendByModelEffort")} + {details.groups.map((group) => ( + + {group.model ?? t("UsageSpendUnknownModel")} /{" "} + {group.reasoningEffort ?? t("UsageSpendUnknownEffort")}: {formatRate(group.outputTokensPerSecond)}{" "} + {unit} + + ))} +
+ )} +
+
+
+ ); +} diff --git a/apps/desktop-tauri/src/surfaces/settings/tabs/UsageSpendTab.test.tsx b/apps/desktop-tauri/src/surfaces/settings/tabs/UsageSpendTab.test.tsx index d77bee08e6..acea02baf3 100644 --- a/apps/desktop-tauri/src/surfaces/settings/tabs/UsageSpendTab.test.tsx +++ b/apps/desktop-tauri/src/surfaces/settings/tabs/UsageSpendTab.test.tsx @@ -453,5 +453,6 @@ function chatSession(id: string, projectId: string, displayTitle: string): Codex totals: { inputTokens: 10, cachedInputTokens: 0, outputTokens: 5, totalTokens: 15 }, costEstimate: { knownUsd: 0.5, unknownTokens: 0 }, topModel: "gpt-5", + turnPerformance: null, }; } diff --git a/apps/desktop-tauri/src/types/bridge.ts b/apps/desktop-tauri/src/types/bridge.ts index 427b0362fe..51506fa1ad 100644 --- a/apps/desktop-tauri/src/types/bridge.ts +++ b/apps/desktop-tauri/src/types/bridge.ts @@ -694,6 +694,37 @@ export interface CodexWorkspacesDailyPoint { estimatedCostUsd: number | null; } +/** Turns that share one model and reasoning effort (upstream #4304). */ +export interface CodexWorkspacesTurnPerformanceGroup { + model: string | null; + reasoningEffort: string | null; + sampleCount: number; + firstTokenSampleCount: number; + medianFirstTokenMilliseconds: number | null; + medianDurationMilliseconds: number; + outputTokensPerSecond: number; +} + +export interface CodexWorkspacesTurnPerformanceDetails { + p95DurationMilliseconds: number | null; + p95FirstTokenMilliseconds: number | null; + outputRateLowerQuartile: number | null; + outputRateUpperQuartile: number | null; + cachedInputFraction: number | null; + cacheSampleCount: number; + groups: CodexWorkspacesTurnPerformanceGroup[]; +} + +/** Native Codex turn latency and throughput for one conversation. */ +export interface CodexWorkspacesTurnPerformance { + sampleCount: number; + outputTokensPerSecond: number; + medianDurationMilliseconds: number; + medianFirstTokenMilliseconds: number | null; + firstTokenSampleCount: number; + details: CodexWorkspacesTurnPerformanceDetails; +} + export interface CodexWorkspacesSessionUsage { id: string; projectId: string; @@ -704,6 +735,7 @@ export interface CodexWorkspacesSessionUsage { totals: CodexWorkspacesUsageTotals; costEstimate: CodexWorkspacesCostEstimate; topModel: string | null; + turnPerformance: CodexWorkspacesTurnPerformance | null; } export interface CodexWorkspacesProjectUsage { From 5555015fbe593cb092684c3e97204100ff1a3f45 Mon Sep 17 00:00:00 2001 From: Factory Agent Date: Fri, 16 Oct 2026 09:49:25 +0700 Subject: [PATCH 5/5] Note Codex turn performance in changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index aeed216403..71b78fb751 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,7 @@ - Use expanded, collapsible Claude and Codex account sections with Add account below the rows; keep the floating bar's usage footprint when authentication expires. - Let the tray panel be moved and resized again: drag the strip along its top to move it (the spot is remembered; double-click returns it next to the tray), resize from any edge or corner, and stop it from closing when a resize starts on its outer frame. Its remembered size is now kept in logical pixels, so it fits monitors with different scaling (a size saved by an earlier version is reset to the automatic size once). - Add Check provider status (Settings > General, on by default as on the Mac). Provider refreshes also read the public status pages of Claude, OpenAI (Codex and OpenAI API), GitHub (Copilot), Cursor, Factory, Augment, Hugging Face and Zoom, and the Google Workspace feed (Gemini and Antigravity). Incidents show as a coloured dot in the Providers sidebar, a line in the tray panel menu and lines in the tray tooltip. In the CLI `--json` output, `status.level` keeps its values, except that a minor incident now reads `degraded` instead of `unknown`. +- Codex usage by session: each conversation in Usage & Spend shows its turn output tokens per second, first-token time, and turn duration from local rollout logs. Expand **Performance details** for P95 latency, the middle-50% throughput range, cached input share, and per-model and reasoning-effort groups; a value appears only once enough completed turns exist, and turns that failed, were interrupted, or have contradictory records are left out (upstream #4304). ## [Windows] 0.70.0 - 2026-10-03