Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/added/perf-tail-latency.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Performance dashboard: nightly runs record p50, p90, p95, p99, p99.9 and max object latency. The dashboard charts p99 for each profile and p95, p99 and max for each stack, and comparison reports flag increases in average, p95 and p99 latency. See [perf tracking](/docs/perf-tracking.md#tracked-metrics).
14 changes: 13 additions & 1 deletion docs/perf-tracking.md
Original file line number Diff line number Diff line change
Expand Up @@ -135,9 +135,14 @@ Stack colors and line styles are consistent across modes.
| RSS per Session | KB | ✗ | 10% |
| Total Resets | count | ✗ | 50% |
| UDP Errors/s | count | ✗ | 50% |
| Avg Latency | ms | ✗ | 10% |
| p95 Latency | ms | ✗ | 15% |
| p99 Latency | ms | ✗ | 20% |
| Max Latency | ms | ✗ | reported, not flagged |

Regression detection uses a rolling 10-run average on `main` for the same QUIC
stack and workload profile. Missing `schema_version`, `params.quic_stack`, and
stack and workload profile. The latency metrics keep their own thresholds
even when `--threshold` overrides the others. Missing `schema_version`, `params.quic_stack`, and
`params.profile` in historical results are treated as version 1, mvfst, and
`1mbps` respectively. Warnings are non-blocking (PRs are not failed).

Expand All @@ -153,6 +158,13 @@ Client-stat anomaly detection checks full-run totals
against measured throughput and flags peaks above ten times the full-run
throughput; it no longer uses an assumed aggregate throughput target.

Schema version 5 adds `latency_p50_ms`, `latency_p90_ms`, `latency_p95_ms`,
`latency_p99_ms`, `latency_p999_ms`, `latency_max_ms` and `latency_samples`.
The percentiles add up the client's per-second `[LATENCY_HIST]` lines over the
steady-state window. They are exact to 1 ms below 128 ms and within 1/32 of
the value above that. They are null when the client doesn't print those
lines. `latency_max_ms` is the largest per-second maximum in the window.

On the first nightly run with the profile matrix, only `mvfst/1mbps` has a
compatible published baseline: legacy runs are interpreted as that series.
The other profiles and picoquic start without a baseline and become comparable
Expand Down
24 changes: 20 additions & 4 deletions scripts/perf/perf-compare.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

# Metrics to track with display configuration.
# (key, display_name, unit, higher_is_better, threshold_pct)
# A threshold of None reports the metric without flagging it.
TRACKED_METRICS = [
("peak_subscribers", "Peak Subscribers", "", True, 5),
("throughput_mbps", "Throughput", "Mbps", True, 5),
Expand All @@ -29,8 +30,21 @@
("rss_per_session_kb", "RSS/Session", "KB", False, 10),
("total_resets", "Total Resets", "", False, 50),
("udp_errors_per_sec", "UDP Errors/s", "", False, 50),
("avg_latency_ms", "Avg Latency", "ms", False, 10),
("latency_p95_ms", "p95 Latency", "ms", False, 15),
("latency_p99_ms", "p99 Latency", "ms", False, 20),
("latency_max_ms", "Max Latency (not flagged)", "ms", False, None),
]

# Latency varies more from run to run than the --threshold override allows for,
# so these metrics ignore it.
OWN_THRESHOLD_METRICS = {
"avg_latency_ms",
"latency_p95_ms",
"latency_p99_ms",
"latency_max_ms",
}


def load_results_file(path: Path) -> dict:
"""Load a single results JSON file."""
Expand Down Expand Up @@ -147,14 +161,16 @@ def compare_results(

current_val = float(current_val)
baseline_val = float(baseline_val)
threshold = (
threshold_override if threshold_override is not None else default_threshold
)
threshold = default_threshold
if threshold_override is not None and key not in OWN_THRESHOLD_METRICS:
threshold = threshold_override

delta_pct = ((current_val - baseline_val) / baseline_val) * 100

# Determine if this is a regression
if higher_is_better:
if threshold is None:
is_regression = False
elif higher_is_better:
is_regression = delta_pct < -threshold
else:
is_regression = delta_pct > threshold
Expand Down
52 changes: 51 additions & 1 deletion scripts/perf/perf-results-to-json.sh
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,8 @@ set -euo pipefail
# Bumped whenever the emitted shape changes; consumers (perf-compare.py, the
# dashboard) treat missing stack/profile fields as mvfst/1mbps.
# v4: CPU calibration target replaces aggregate throughput target.
SCHEMA_VERSION=4
# v5: latency percentiles, max and sample count.
SCHEMA_VERSION=5

# ── Defaults ───────────────────────────────────────────────────────────────────
CLIENT_OUTPUT=""
Expand Down Expand Up @@ -151,6 +152,48 @@ if [[ -z "$AVG_LATENCY_MS" ]]; then
fi
[[ -z "$AVG_LATENCY_MS" ]] && AVG_LATENCY_MS="0"

# Latency percentiles: sum the client's per-second [LATENCY_HIST] buckets
# ("<ms>:<count>", keyed by the bucket's largest value) over the client window
# and take nearest-rank quantiles. Max is the largest per-second interval max
# from [AGGREGATE], which older clients also print.
read -r LAT_P50 LAT_P90 LAT_P95 LAT_P99 LAT_P999 LAT_MAX LAT_SAMPLES < <(
awk -v s="$CLIENT_WINDOW_START" -v e="$CLIENT_WINDOW_END" '
function inwin(sec) { return s == "" || e == "" || (sec >= s + 0 && sec <= e + 0) }
/\[LATENCY_HIST\]/ {
if (!match($0, /\[LATENCY_HIST\] \[([0-9]+)s\](.*)$/, m) || !inwin(m[1] + 0)) next
nf = split(m[2], kv, " ")
for (i = 1; i <= nf; i++) {
if (split(kv[i], p, ":") == 2 && p[1] ~ /^[0-9]+$/ && p[2] ~ /^[0-9]+$/) {
cnt[p[1] + 0] += p[2]; total += p[2]
}
}
next
}
/\[AGGREGATE\]/ {
if (!match($0, /\[([0-9]+)s\]/, t) || !inwin(t[1] + 0)) next
if (match($0, /interval min\/avg\/max\): [0-9]+\/[0-9.]+\/([0-9]+) ms/, x) && x[1] + 0 > max) max = x[1] + 0

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[LOW] If the client's [AGGREGATE] interval max latency is genuinely 0 ms (which is technically possible in loopback tests), max (which defaults to 0 in a numeric context) will not be updated because 0 > 0 is false. Consequently, the END block will emit "null" instead of 0.

Consider initializing max = -1 in a BEGIN block and checking max >= 0 to ensure a legitimate 0 ms max latency is correctly recorded.

}
END {
n = asorti(cnt, keys, "@ind_num_asc")
split("0.5 0.9 0.95 0.99 0.999", qs, " ")
for (j = 1; j <= 5; j++) {
out = "null"
if (total > 0) {
r = qs[j] * total - 1e-9; rank = int(r); if (rank < r) rank++
if (rank < 1) rank = 1
if (rank > total) rank = total
run = 0
for (k = 1; k <= n; k++) { run += cnt[keys[k]]; if (run >= rank) { out = keys[k]; break } }
}
printf "%s ", out
}
printf "%s %d\n", (max > 0 ? max : "null"), total
}
' "$CLIENT_OUTPUT")
if [[ "$LAT_SAMPLES" -eq 0 ]]; then
echo "::warning::$PROFILE/$QUIC_STACK client log has no [LATENCY_HIST] samples in the window; latency percentiles are null"
fi

# Parse per-second aggregate lines for latency/delivery stats
# Format: [AGGREGATE] [Ns] Subs: N | Obj/s: N | Mbps: N.NN | ...
# Get peak subscribers from aggregate lines
Expand Down Expand Up @@ -316,6 +359,13 @@ cat > "$OUTPUT" <<EOF
"throughput_mbps": $THROUGHPUT_MBPS,
"throughput_full_run_mbps": $RUN_THROUGHPUT_MBPS,
"avg_latency_ms": $AVG_LATENCY_MS,
"latency_p50_ms": $LAT_P50,
"latency_p90_ms": $LAT_P90,
"latency_p95_ms": $LAT_P95,
"latency_p99_ms": $LAT_P99,
"latency_p999_ms": $LAT_P999,
"latency_max_ms": $LAT_MAX,
"latency_samples": $LAT_SAMPLES,
"peak_throughput_mbps": $PEAK_MBPS,
"throughput_per_core_mbps": $THROUGHPUT_PER_CORE,
"subscribers_per_core": $SUBS_PER_CORE,
Expand Down
32 changes: 31 additions & 1 deletion status/index.html
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,18 @@ <h3>4 Mbps · Average latency (ms)</h3>
<h3>16 Mbps · Average latency (ms)</h3>
<canvas id="chart-overview-latency-16mbps"></canvas>
</div>
<div class="card">
<h3>1 Mbps · p99 latency (ms)</h3>
<canvas id="chart-overview-p99-1mbps"></canvas>
</div>
<div class="card">
<h3>4 Mbps · p99 latency (ms)</h3>
<canvas id="chart-overview-p99-4mbps"></canvas>
</div>
<div class="card">
<h3>16 Mbps · p99 latency (ms)</h3>
<canvas id="chart-overview-p99-16mbps"></canvas>
</div>
<div class="card">
<h3>1 Mbps · RSS per session (KB)</h3>
<canvas id="chart-overview-rss-session-1mbps"></canvas>
Expand Down Expand Up @@ -179,6 +191,18 @@ <h3>Throughput per Core (Mbps)</h3>
<h3>Avg Latency (ms)</h3>
<canvas id="chart-latency"></canvas>
</div>
<div class="card">
<h3>p95 Latency (ms)</h3>
<canvas id="chart-latency-p95"></canvas>
</div>
<div class="card">
<h3>p99 Latency (ms)</h3>
<canvas id="chart-latency-p99"></canvas>
</div>
<div class="card">
<h3>Max Latency (ms)</h3>
<canvas id="chart-latency-max"></canvas>
</div>
<div class="card">
<h3>Delivery Success (%)</h3>
<canvas id="chart-delivery"></canvas>
Expand Down Expand Up @@ -222,6 +246,9 @@ <h3>RSS per Session (KB)</h3>
['chart-throughput', 'throughput_mbps', CHART_COLORS.primary],
['chart-throughput-core', 'throughput_per_core_mbps', CHART_COLORS.primary],
['chart-latency', 'avg_latency_ms', CHART_COLORS.secondary],
['chart-latency-p95', 'latency_p95_ms', CHART_COLORS.secondary],
['chart-latency-p99', 'latency_p99_ms', CHART_COLORS.secondary],
['chart-latency-max', 'latency_max_ms', CHART_COLORS.danger],
['chart-delivery', 'delivery_success_pct', CHART_COLORS.secondary],
['chart-resets', 'total_resets', CHART_COLORS.danger],
['chart-cpu', 'relay_cpu_pct', CHART_COLORS.warning],
Expand Down Expand Up @@ -398,6 +425,7 @@ <h3>RSS per Session (KB)</h3>
['-cpu', 'relay_cpu_pct'],
['', 'throughput_per_core_mbps'],
['-latency', 'avg_latency_ms'],
['-p99', 'latency_p99_ms'],
['-rss-session', 'rss_per_session_kb'],
]) {
for (const profile of PROFILE_IDS) {
Expand Down Expand Up @@ -457,14 +485,15 @@ <h3>RSS per Session (KB)</h3>
<td class="${betterClass(stack, 'relay_cpu_pct', false)}">${results.relay_cpu_pct == null ? '—' : `${results.relay_cpu_pct}%`}</td>
<td class="${betterClass(stack, 'relay_rss_mb', false)}">${results.relay_rss_mb ?? '—'} MB</td>
<td class="${betterClass(stack, 'avg_latency_ms', false)}">${results.avg_latency_ms ?? '—'} ms</td>
<td class="${betterClass(stack, 'latency_p99_ms', false)}">${results.latency_p99_ms ?? '—'} ms</td>
<td class="${betterClass(stack, 'rss_per_session_kb', false)}">${results.rss_per_session_kb ?? '—'} KB</td>
<td>${commit ? `<a class="commit-link" href="${REPO_URL}/commit/${commit}">${commitLabel}</a>` : commitLabel}</td>
</tr>
`;
}).join('');
container.innerHTML = `
<table class="stack-comparison">
<thead><tr><th scope="col">Stack / CC</th><th scope="col">Throughput</th><th scope="col">Mbps / core</th><th scope="col">Peak subscribers</th><th scope="col">Relay CPU</th><th scope="col">RSS</th><th scope="col">Average latency</th><th scope="col">RSS / session</th><th scope="col">Latest commit</th></tr></thead>
<thead><tr><th scope="col">Stack / CC</th><th scope="col">Throughput</th><th scope="col">Mbps / core</th><th scope="col">Peak subscribers</th><th scope="col">Relay CPU</th><th scope="col">RSS</th><th scope="col">Average latency</th><th scope="col">p99 latency</th><th scope="col">RSS / session</th><th scope="col">Latest commit</th></tr></thead>
<tbody>${rows}</tbody>
</table>
`;
Expand All @@ -477,6 +506,7 @@ <h3>RSS per Session (KB)</h3>
['relay_cpu_pct', 'Relay CPU', '%', false],
['throughput_per_core_mbps', 'TP/core', 'Mbps', true],
['avg_latency_ms', 'Average latency', 'ms', false],
['latency_p99_ms', 'p99 latency', 'ms', false],
['rss_per_session_kb', 'RSS per session', 'KB', false],
];
const latestRunsByProfile = new Map(PROFILE_IDS.map(profile => [
Expand Down
Loading