From 1e105fc046f1ccc8dfc9e8f128e57a4488bfc540 Mon Sep 17 00:00:00 2001 From: Giovanni Marzot Date: Tue, 17 Nov 2026 09:04:53 -0800 Subject: [PATCH] perf: record and chart tail latency in the nightly run The nightly run only recorded avg_latency_ms, and perf-compare.py did not track latency at all, so a change in tail latency went unnoticed. - perf-results-to-json.sh adds latency_p50/p90/p95/p99/p999_ms by summing the client's per-second [LATENCY_HIST] buckets over the steady-state window. latency_max_ms is the largest per-second interval max from [AGGREGATE], which today's client already prints. A client without the histogram lines gives null percentiles and a warning. Schema version 5. - perf-compare.py tracks average, p95 and p99 latency at 10%, 15% and 20%, and reports max without flagging it: over the last seven nights the window max moved by up to 2x with no code change. The latency metrics ignore the workflow's --threshold 5 override. - The dashboard gets a p99 chart per profile, p95/p99/max charts per stack, and a p99 column in both latest-result tables. Percentiles need moxygen with the per-second latency histogram (openmoq/moxygen#630); they appear once a moxygen sync brings it in. --- changelog.d/added/perf-tail-latency.md | 1 + docs/perf-tracking.md | 14 ++++++- scripts/perf/perf-compare.py | 24 ++++++++++-- scripts/perf/perf-results-to-json.sh | 52 +++++++++++++++++++++++++- status/index.html | 32 +++++++++++++++- 5 files changed, 116 insertions(+), 7 deletions(-) create mode 100644 changelog.d/added/perf-tail-latency.md diff --git a/changelog.d/added/perf-tail-latency.md b/changelog.d/added/perf-tail-latency.md new file mode 100644 index 000000000..23bbf6129 --- /dev/null +++ b/changelog.d/added/perf-tail-latency.md @@ -0,0 +1 @@ +- Performance dashboard: nightly runs record p50, p90, p95, p99, p99.9 and max object latency. The dashboard charts p99 for each profile and p95, p99 and max for each stack, and comparison reports flag increases in average, p95 and p99 latency. See [perf tracking](/docs/perf-tracking.md#tracked-metrics). diff --git a/docs/perf-tracking.md b/docs/perf-tracking.md index 56206fa73..51932e12d 100644 --- a/docs/perf-tracking.md +++ b/docs/perf-tracking.md @@ -135,9 +135,14 @@ Stack colors and line styles are consistent across modes. | RSS per Session | KB | ✗ | 10% | | Total Resets | count | ✗ | 50% | | UDP Errors/s | count | ✗ | 50% | +| Avg Latency | ms | ✗ | 10% | +| p95 Latency | ms | ✗ | 15% | +| p99 Latency | ms | ✗ | 20% | +| Max Latency | ms | ✗ | reported, not flagged | Regression detection uses a rolling 10-run average on `main` for the same QUIC -stack and workload profile. Missing `schema_version`, `params.quic_stack`, and +stack and workload profile. The latency metrics keep their own thresholds +even when `--threshold` overrides the others. Missing `schema_version`, `params.quic_stack`, and `params.profile` in historical results are treated as version 1, mvfst, and `1mbps` respectively. Warnings are non-blocking (PRs are not failed). @@ -153,6 +158,13 @@ Client-stat anomaly detection checks full-run totals against measured throughput and flags peaks above ten times the full-run throughput; it no longer uses an assumed aggregate throughput target. +Schema version 5 adds `latency_p50_ms`, `latency_p90_ms`, `latency_p95_ms`, +`latency_p99_ms`, `latency_p999_ms`, `latency_max_ms` and `latency_samples`. +The percentiles add up the client's per-second `[LATENCY_HIST]` lines over the +steady-state window. They are exact to 1 ms below 128 ms and within 1/32 of +the value above that. They are null when the client doesn't print those +lines. `latency_max_ms` is the largest per-second maximum in the window. + On the first nightly run with the profile matrix, only `mvfst/1mbps` has a compatible published baseline: legacy runs are interpreted as that series. The other profiles and picoquic start without a baseline and become comparable diff --git a/scripts/perf/perf-compare.py b/scripts/perf/perf-compare.py index 53751afb4..2edd9e04a 100755 --- a/scripts/perf/perf-compare.py +++ b/scripts/perf/perf-compare.py @@ -18,6 +18,7 @@ # Metrics to track with display configuration. # (key, display_name, unit, higher_is_better, threshold_pct) +# A threshold of None reports the metric without flagging it. TRACKED_METRICS = [ ("peak_subscribers", "Peak Subscribers", "", True, 5), ("throughput_mbps", "Throughput", "Mbps", True, 5), @@ -29,8 +30,21 @@ ("rss_per_session_kb", "RSS/Session", "KB", False, 10), ("total_resets", "Total Resets", "", False, 50), ("udp_errors_per_sec", "UDP Errors/s", "", False, 50), + ("avg_latency_ms", "Avg Latency", "ms", False, 10), + ("latency_p95_ms", "p95 Latency", "ms", False, 15), + ("latency_p99_ms", "p99 Latency", "ms", False, 20), + ("latency_max_ms", "Max Latency (not flagged)", "ms", False, None), ] +# Latency varies more from run to run than the --threshold override allows for, +# so these metrics ignore it. +OWN_THRESHOLD_METRICS = { + "avg_latency_ms", + "latency_p95_ms", + "latency_p99_ms", + "latency_max_ms", +} + def load_results_file(path: Path) -> dict: """Load a single results JSON file.""" @@ -147,14 +161,16 @@ def compare_results( current_val = float(current_val) baseline_val = float(baseline_val) - threshold = ( - threshold_override if threshold_override is not None else default_threshold - ) + threshold = default_threshold + if threshold_override is not None and key not in OWN_THRESHOLD_METRICS: + threshold = threshold_override delta_pct = ((current_val - baseline_val) / baseline_val) * 100 # Determine if this is a regression - if higher_is_better: + if threshold is None: + is_regression = False + elif higher_is_better: is_regression = delta_pct < -threshold else: is_regression = delta_pct > threshold diff --git a/scripts/perf/perf-results-to-json.sh b/scripts/perf/perf-results-to-json.sh index 3512fb82b..973ef2d2f 100755 --- a/scripts/perf/perf-results-to-json.sh +++ b/scripts/perf/perf-results-to-json.sh @@ -33,7 +33,8 @@ set -euo pipefail # Bumped whenever the emitted shape changes; consumers (perf-compare.py, the # dashboard) treat missing stack/profile fields as mvfst/1mbps. # v4: CPU calibration target replaces aggregate throughput target. -SCHEMA_VERSION=4 +# v5: latency percentiles, max and sample count. +SCHEMA_VERSION=5 # ── Defaults ─────────────────────────────────────────────────────────────────── CLIENT_OUTPUT="" @@ -151,6 +152,48 @@ if [[ -z "$AVG_LATENCY_MS" ]]; then fi [[ -z "$AVG_LATENCY_MS" ]] && AVG_LATENCY_MS="0" +# Latency percentiles: sum the client's per-second [LATENCY_HIST] buckets +# (":", keyed by the bucket's largest value) over the client window +# and take nearest-rank quantiles. Max is the largest per-second interval max +# from [AGGREGATE], which older clients also print. +read -r LAT_P50 LAT_P90 LAT_P95 LAT_P99 LAT_P999 LAT_MAX LAT_SAMPLES < <( + awk -v s="$CLIENT_WINDOW_START" -v e="$CLIENT_WINDOW_END" ' + function inwin(sec) { return s == "" || e == "" || (sec >= s + 0 && sec <= e + 0) } + /\[LATENCY_HIST\]/ { + if (!match($0, /\[LATENCY_HIST\] \[([0-9]+)s\](.*)$/, m) || !inwin(m[1] + 0)) next + nf = split(m[2], kv, " ") + for (i = 1; i <= nf; i++) { + if (split(kv[i], p, ":") == 2 && p[1] ~ /^[0-9]+$/ && p[2] ~ /^[0-9]+$/) { + cnt[p[1] + 0] += p[2]; total += p[2] + } + } + next + } + /\[AGGREGATE\]/ { + if (!match($0, /\[([0-9]+)s\]/, t) || !inwin(t[1] + 0)) next + if (match($0, /interval min\/avg\/max\): [0-9]+\/[0-9.]+\/([0-9]+) ms/, x) && x[1] + 0 > max) max = x[1] + 0 + } + END { + n = asorti(cnt, keys, "@ind_num_asc") + split("0.5 0.9 0.95 0.99 0.999", qs, " ") + for (j = 1; j <= 5; j++) { + out = "null" + if (total > 0) { + r = qs[j] * total - 1e-9; rank = int(r); if (rank < r) rank++ + if (rank < 1) rank = 1 + if (rank > total) rank = total + run = 0 + for (k = 1; k <= n; k++) { run += cnt[keys[k]]; if (run >= rank) { out = keys[k]; break } } + } + printf "%s ", out + } + printf "%s %d\n", (max > 0 ? max : "null"), total + } + ' "$CLIENT_OUTPUT") +if [[ "$LAT_SAMPLES" -eq 0 ]]; then + echo "::warning::$PROFILE/$QUIC_STACK client log has no [LATENCY_HIST] samples in the window; latency percentiles are null" +fi + # Parse per-second aggregate lines for latency/delivery stats # Format: [AGGREGATE] [Ns] Subs: N | Obj/s: N | Mbps: N.NN | ... # Get peak subscribers from aggregate lines @@ -316,6 +359,13 @@ cat > "$OUTPUT" <4 Mbps · Average latency (ms)

16 Mbps · Average latency (ms)

+
+

1 Mbps · p99 latency (ms)

+ +
+
+

4 Mbps · p99 latency (ms)

+ +
+
+

16 Mbps · p99 latency (ms)

+ +

1 Mbps · RSS per session (KB)

@@ -179,6 +191,18 @@

Throughput per Core (Mbps)

Avg Latency (ms)

+
+

p95 Latency (ms)

+ +
+
+

p99 Latency (ms)

+ +
+
+

Max Latency (ms)

+ +

Delivery Success (%)

@@ -222,6 +246,9 @@

RSS per Session (KB)

['chart-throughput', 'throughput_mbps', CHART_COLORS.primary], ['chart-throughput-core', 'throughput_per_core_mbps', CHART_COLORS.primary], ['chart-latency', 'avg_latency_ms', CHART_COLORS.secondary], + ['chart-latency-p95', 'latency_p95_ms', CHART_COLORS.secondary], + ['chart-latency-p99', 'latency_p99_ms', CHART_COLORS.secondary], + ['chart-latency-max', 'latency_max_ms', CHART_COLORS.danger], ['chart-delivery', 'delivery_success_pct', CHART_COLORS.secondary], ['chart-resets', 'total_resets', CHART_COLORS.danger], ['chart-cpu', 'relay_cpu_pct', CHART_COLORS.warning], @@ -398,6 +425,7 @@

RSS per Session (KB)

['-cpu', 'relay_cpu_pct'], ['', 'throughput_per_core_mbps'], ['-latency', 'avg_latency_ms'], + ['-p99', 'latency_p99_ms'], ['-rss-session', 'rss_per_session_kb'], ]) { for (const profile of PROFILE_IDS) { @@ -457,6 +485,7 @@

RSS per Session (KB)

${results.relay_cpu_pct == null ? '—' : `${results.relay_cpu_pct}%`} ${results.relay_rss_mb ?? '—'} MB ${results.avg_latency_ms ?? '—'} ms + ${results.latency_p99_ms ?? '—'} ms ${results.rss_per_session_kb ?? '—'} KB ${commit ? `${commitLabel}` : commitLabel} @@ -464,7 +493,7 @@

RSS per Session (KB)

}).join(''); container.innerHTML = ` - + ${rows}
Stack / CCThroughputMbps / corePeak subscribersRelay CPURSSAverage latencyRSS / sessionLatest commit
Stack / CCThroughputMbps / corePeak subscribersRelay CPURSSAverage latencyp99 latencyRSS / sessionLatest commit
`; @@ -477,6 +506,7 @@

RSS per Session (KB)

['relay_cpu_pct', 'Relay CPU', '%', false], ['throughput_per_core_mbps', 'TP/core', 'Mbps', true], ['avg_latency_ms', 'Average latency', 'ms', false], + ['latency_p99_ms', 'p99 latency', 'ms', false], ['rss_per_session_kb', 'RSS per session', 'KB', false], ]; const latestRunsByProfile = new Map(PROFILE_IDS.map(profile => [