From 26543b05347972f6c58615a632472b4db0e89792 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 18:55:21 -0800 Subject: [PATCH 01/36] chore(quest): claim quest/m1/release-perf/relay-cpu-bisect.md Quest-Claim: 8c4be0b7ab86d470 From ac33818539d5cabeff666bbc13d177fc718966a4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 19:11:00 -0800 Subject: [PATCH 02/36] bench(relay): count pinned instructions over matched delivery windows Co-Authored-By: GPT-6.1 Sol --- bench/README.md | 39 +++ bench/instructions.py | 313 ++++++++++++++++++++++ bench/metadata.py | 16 +- bench/profile.py | 2 +- bench/profile.test.py | 65 +++++ bench/relay.sh | 24 +- bench/run.sh | 46 +++- justfile | 3 +- quest/m1/release-perf/relay-cpu-bisect.md | 1 + 9 files changed, 496 insertions(+), 13 deletions(-) create mode 100644 bench/instructions.py diff --git a/bench/README.md b/bench/README.md index e976bc84dc2..b2fb50afc81 100644 --- a/bench/README.md +++ b/bench/README.md @@ -234,6 +234,45 @@ keeps `kernel.perf_event_paranoid=-1`. Profile there only while it is quiet, with no other agents running and load near zero, since other work on the host shows up as lock waits and CPU the relay did not cause. +### Instructions + +Count instructions and cycles (user and kernel) per delivered frame, with +fixed-period instruction stacks for source attribution: + +```bash +BENCH_RELAY_CPUS=4-7 BENCH_GENERATOR_CPUS=0-3 BENCH_RUNTIME_ROUNDS=5 \ + BENCH_INSTRUCTION_PERIOD=10000019 \ + nix develop --command just bench-profile instructions chat 4 io-uring-workers cdff3c8f95 +BENCH_RELAY_CPUS=4-7 BENCH_GENERATOR_CPUS=0-3 \ + nix develop --command just bench-profile instructions video 4 tokio-workers cdff3c8f95 +``` + +Choose disjoint allowed CPU sets, with at least one relay CPU per worker. +Hybrid CPUs require performance cores, and count their `cpu_core` events. +The relay and generator keep these affinities in both the control and measured +runs. Builds stay in separate targets per revision. Base/current pairs alternate +order across `BENCH_RUNTIME_ROUNDS` (default 3); the current generator drives both. + +After seven seconds of warmup, acknowledged perf enable/disable commands bound +five seconds of relay counters. `instructions/instructions.json` records both +acknowledgement brackets, their uncertainty, and the delivered frames linearly +interpolated between the generator's surrounding 500 ms reports at those same +boundaries. This is a sampled delivery estimate, not exact frame accounting. +Missing, uncounted, or multiplexed counters and unbracketed delivery fail the run. +Lost or throttled instruction samples also fail; choose a larger fixed period +for the whole comparison when the host's sample-rate limit requires it. +The host sampler lasts fourteen seconds so delivery reports bracket both edges. +The normal `summary.json` still describes its own final throughput window; +use `instructions.json` for instruction comparisons. + +`BENCH_INSTRUCTION_PERIOD` defaults to 1,000,003 retired instructions per sample. +The fixed-period recording encloses the counter window; `instructions.folded` +weights each stack by that period. `perf.data`, raw `stat.json`, `perf.log`, +thread names, CPU sets, and delivery reports remain with the evidence. Both +counters and samples cover only the relay. Compare equivalent delivery and loss, +and retain controls to expose sampling overhead. Instructions reduce frequency +and scheduling noise; pinning does not remove interference from other host work. + ### Heap The heap mode builds the relay with `moq-relay/jemalloc` (the other modes keep diff --git a/bench/instructions.py b/bench/instructions.py new file mode 100644 index 00000000000..c6e6b4ee224 --- /dev/null +++ b/bench/instructions.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 +"""Count relay instructions and sample their callers on disjoint pinned cores.""" + +import argparse +import bisect +import json +import math +import os +import profile as profiles +import signal +import struct +import subprocess +import time +from pathlib import Path + + +def cpus(value): + if not value: + raise ValueError("instructions mode requires BENCH_RELAY_CPUS and BENCH_GENERATOR_CPUS") + result = set() + for part in value.split(","): + first, _, last = part.partition("-") + first, last = int(first), int(last or first) + if first < 0 or last < first: + raise ValueError(f"invalid CPU range: {part}") + result.update(range(first, last + 1)) + return result + + +def settings(workers): + relay = cpus(os.environ.get("BENCH_RELAY_CPUS", "")) + generator = cpus(os.environ.get("BENCH_GENERATOR_CPUS", "")) + allowed = os.sched_getaffinity(0) + if relay & generator or not (relay | generator) <= allowed: + raise ValueError("relay and generator CPU sets must be disjoint subsets of the allowed CPUs") + if len(relay) < max(map(int, workers.split())): + raise ValueError("relay CPU set has fewer CPUs than workers") + # A hybrid event silently counts nothing when its task runs on the other core type. + core = Path("/sys/devices/cpu_core/cpus") + if core.exists() and not (relay | generator) <= cpus(core.read_text().strip()): + raise ValueError("instructions mode requires performance cores on a hybrid CPU") + period = int(os.environ.get("BENCH_INSTRUCTION_PERIOD", "1000003")) + if period <= 0: + raise ValueError("BENCH_INSTRUCTION_PERIOD must be positive") + return dict(relay_cpus=sorted(relay), generator_cpus=sorted(generator), period=period) + + +def events(): + prefix = "cpu_core/" if Path("/sys/devices/cpu_core").exists() else "" + return [f"{prefix}{name}{'/' if prefix else ''}" for name in ("instructions", "cycles")] + + +class Controlled: + """An acknowledged perf control channel bounds the actual counter window.""" + + def __init__(self, command, log): + self.record = command[1] == "record" + read, self.write = os.pipe() + self.read, write = os.pipe() + try: + self.process = subprocess.Popen( + [*command, "--delay", "-1", "--control", f"fd:{read},{write}"], + pass_fds=(read, write), + stdout=log, + stderr=log, + env=dict(os.environ, LC_ALL="C", DEBUGINFOD_URLS=""), + ) + finally: + os.close(read) + os.close(write) + + def command(self, command): + before = time.time_ns() + os.write(self.write, command.encode() + b"\n") + ack = b"" + while not ack.endswith(b"\0"): + if not (part := os.read(self.read, 1)): + raise ValueError(f"perf exited before acknowledging {command}; see perf.log") + ack += part + after = time.time_ns() + if ack != b"ack\n\0": + raise ValueError(f"unexpected perf acknowledgement: {ack!r}") + return [before / 1e6, after / 1e6] + + def close(self): + try: + if self.process.poll() is None: + if self.record: + # Record must finish its header before exiting; stat exits on SIGINT. + self.command("stop") + else: + self.process.send_signal(signal.SIGINT) + except (OSError, ValueError): + self.process.kill() + raise + finally: + os.close(self.write) + os.close(self.read) + self.process.wait() + status = self.process.wait() + if status != 0 and (self.record or status != -signal.SIGINT): + raise ValueError(f"perf failed ({status}); see perf.log") + + +def capture(pid, directory, delay, duration): + if not math.isfinite(delay) or delay < 0 or not math.isfinite(duration) or duration <= 0: + raise ValueError("capture needs a nonnegative delay and a positive duration") + config = settings("1") + for task in (Path("/proc") / str(pid) / "task").iterdir(): + if os.sched_getaffinity(int(task.name)) != set(config["relay_cpus"]): + raise ValueError(f"relay thread {task.name} has unexpected affinity") + directory.mkdir(parents=True, exist_ok=True) + counters = events() + names = profiles.threads(pid) + running = [] + try: + with (directory / "perf.log").open("w") as log: + stat = Controlled( + [ + "perf", + "stat", + "-j", + "-o", + str(directory / "stat.json"), + "-p", + str(pid), + "-e", + ",".join([*counters, "task-clock"]), + ], + log, + ) + running.append(stat) + record = Controlled( + [ + "perf", + "record", + "--quiet", + "--no-buildid-cache", + "-p", + str(pid), + "-e", + counters[0], + "-c", + str(config["period"]), + "--call-graph", + "fp", + "-k", + "monotonic", + "-o", + str(directory / "perf.data"), + ], + log, + ) + running.append(record) + # These also prove both processes have attached before the warmup starts. + stat.command("disable") + record.command("disable") + time.sleep(delay) + record.command("enable") + start = stat.command("enable") + time.sleep(duration) + end = stat.command("disable") + record.command("disable") + finally: + errors = [] + for process in reversed(running): + try: + process.close() + except (OSError, ValueError) as error: + errors.append(str(error)) + if errors: + raise ValueError("; ".join(errors)) + names.update(profiles.threads(pid)) + (directory / "window.json").write_text( + json.dumps( + dict( + **config, + start_ms=start, + end_ms=end, + threads=names, + events=counters, + ), + indent=2, + ) + + "\n" + ) + + +def delivered(rows, at): + """Interpolate delivery, never extrapolate past the generator's reports.""" + index = bisect.bisect_left([row["timestamp_ms"] for row in rows], at) + if index == len(rows) or at < rows[0]["timestamp_ms"]: + raise ValueError("generator reports do not bracket the counter window") + if rows[index]["timestamp_ms"] == at: + return rows[index]["frames_recv"] + first, last = rows[index - 1], rows[index] + weight = (at - first["timestamp_ms"]) / (last["timestamp_ms"] - first["timestamp_ms"]) + return first["frames_recv"] + weight * (last["frames_recv"] - first["frames_recv"]) + + +def reduce(window, stats, rows): + if len(rows) < 2: + raise ValueError("fewer than two generator reports") + for row in rows: + for key in ("timestamp_ms", "frames_recv"): + if not isinstance(row[key], (int, float)) or not math.isfinite(row[key]) or row[key] < 0: + raise ValueError(f"invalid generator counter: {key}") + for first, last in zip(rows, rows[1:]): + if last["timestamp_ms"] <= first["timestamp_ms"] or last["frames_recv"] < first["frames_recv"]: + raise ValueError("generator counters moved backwards") + start, end = [sum(window[key]) / 2 for key in ("start_ms", "end_ms")] + if end <= start: + raise ValueError("counter window did not advance") + frames = delivered(rows, end) - delivered(rows, start) + if frames <= 0: + raise ValueError("no frames delivered in the counter window") + counts = {} + for event in [*window["events"], "task-clock"]: + matches = [row for row in stats if row["event"] == event] + if len(matches) != 1: + raise ValueError(f"missing or duplicate perf counter: {event}") + row = matches[0] + value = float(row["counter-value"]) + if not math.isfinite(value) or value <= 0 or float(row["pcnt-running"]) != 100: + raise ValueError(f"invalid or multiplexed perf counter: {event}") + counts[event] = value + return dict( + version=1, + **window, + window_start_ms=start, + window_end_ms=end, + boundary_uncertainty_ms=[b - a for a, b in (window["start_ms"], window["end_ms"])], + delivered_frames=frames, + recv_fps=frames * 1000 / (end - start), + counters=counts, + instructions_per_frame=counts[window["events"][0]] / frames, + cycles_per_frame=counts[window["events"][1]] / frames, + scope="all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", + ) + + +def report(directory, load): + validate_data(directory / "perf.data") + window = json.loads((directory / "window.json").read_text()) + stats = [ + json.loads(line.rstrip(",")) + for line in (directory / "stat.json").read_text().splitlines() + if line.strip().startswith("{") + ] + result = reduce(window, stats, [json.loads(line) for line in load.read_text().splitlines()]) + records = profiles.parse(profiles.perf_script(directory / "perf.data"), kernel=True) + records = [row for row in records if row[2].startswith(window["events"][0])] + if not records or not any(profiles.RESOLVED.search(frame) for _, _, _, frames in records for frame in frames): + raise ValueError("no instruction samples have resolved MoQ frames") + from collections import Counter + + folded = Counter() + for tid, _, _, frames in records: + folded[";".join([window["threads"].get(tid, f"tid-{tid}"), *reversed(frames)])] += window["period"] + result["instruction_samples"] = len(records) + (directory / "instructions.folded").write_text( + "".join(f"{stack} {count}\n" for stack, count in folded.most_common()) + ) + (directory / "instructions.json").write_text(json.dumps(result, indent=2) + "\n") + + +def validate_data(path): + """Reject lost or throttled samples before expensive symbolization.""" + data = path.read_bytes() + if data[:8] != b"PERFILE2" or len(data) < 56: + raise ValueError("unsupported or incomplete perf.data header") + offset, size = struct.unpack_from(" len(data): + raise ValueError("incomplete perf.data recording") + while offset < end: + kind, _, size = struct.unpack_from(" end: + raise ValueError("invalid perf.data record") + # perf_event_header types: LOST, THROTTLE, LOST_SAMPLES. perf script's + # --show-lost-events omits THROTTLE, which would bias fixed-period attribution. + if kind in (2, 5, 13): + raise ValueError(f"perf lost or throttled samples (record type {kind}); increase BENCH_INSTRUCTION_PERIOD") + offset += size + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + check = sub.add_parser("check") + check.add_argument("workers") + capture_args = sub.add_parser("capture") + capture_args.add_argument("pid", type=int) + capture_args.add_argument("directory", type=Path) + capture_args.add_argument("--delay", type=float, default=7) + capture_args.add_argument("--duration", type=float, default=5) + report_args = sub.add_parser("report") + report_args.add_argument("directory", type=Path) + report_args.add_argument("load", type=Path) + args = parser.parse_args() + try: + if args.command == "check": + settings(args.workers) + elif args.command == "capture": + capture(args.pid, args.directory, args.delay, args.duration) + else: + report(args.directory, args.load) + except (ValueError, OSError, subprocess.CalledProcessError) as error: + parser.exit(1, f"instructions: {error}\n") + + +if __name__ == "__main__": + main() diff --git a/bench/metadata.py b/bench/metadata.py index 7bf6e731f25..700d14e9ad7 100644 --- a/bench/metadata.py +++ b/bench/metadata.py @@ -73,7 +73,21 @@ def command(*args): } (out / "metadata.json").write_text(json.dumps(metadata, indent=2) + "\n") subprocess.run( - ["cargo", "tree", "--locked", "-p", "moq-relay", "-e", "features"], + [ + "cargo", + "--config", + str(Path.cwd() / ".cargo/config.toml"), + "tree", + "--manifest-path", + str(Path.cwd() / "Cargo.toml"), + "--locked", + "-p", + "moq-relay", + "-e", + "features", + ], + # Nested revision checkouts must not inherit the current tree's Cargo config. + cwd="/", stdout=(out / "features.txt").open("w"), check=True, ) diff --git a/bench/profile.py b/bench/profile.py index d9c5381ef43..9e7b86c6931 100644 --- a/bench/profile.py +++ b/bench/profile.py @@ -432,7 +432,7 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) commands = parser.add_subparsers(dest="command", required=True) check = commands.add_parser("check", help="refuse a capture this host cannot take") - check.add_argument("mode", choices=["lock", "cpu", "heap"]) + check.add_argument("mode", choices=["lock", "cpu", "heap", "instructions"]) for name, help in [ ("lock", "record worker off-CPU stacks of PID into DIRECTORY"), ("cpu", "sample CPU stacks of every PID thread into DIRECTORY"), diff --git a/bench/profile.test.py b/bench/profile.test.py index 3896329606b..ce4cfa46fc9 100644 --- a/bench/profile.test.py +++ b/bench/profile.test.py @@ -2,6 +2,7 @@ import json import os import signal +import struct import subprocess import tempfile import unittest @@ -16,6 +17,70 @@ runtime = importlib.util.module_from_spec(spec) spec.loader.exec_module(runtime) +spec = importlib.util.spec_from_file_location("instructions", Path(__file__).with_name("instructions.py")) +instructions = importlib.util.module_from_spec(spec) +spec.loader.exec_module(instructions) + + +class Instructions(unittest.TestCase): + def setUp(self): + self.window = dict(start_ms=[1250, 1252], end_ms=[2250, 2252], events=["instructions", "cycles"]) + self.stats = [ + {"event": event, "counter-value": count, "pcnt-running": 100} + for event, count in [("instructions", 20000), ("cycles", 10000), ("task-clock", 10)] + ] + self.load = [dict(timestamp_ms=t, frames_recv=t * 2) for t in [1000, 1500, 2000, 2500]] + + def test_frames_match_the_acknowledged_counter_window(self): + result = instructions.reduce(self.window, self.stats, self.load) + self.assertEqual(result["delivered_frames"], 2000) + self.assertEqual(result["instructions_per_frame"], 10) + self.assertEqual(result["window_start_ms"], 1251) + self.assertEqual(result["boundary_uncertainty_ms"], [2, 2]) + with self.assertRaisesRegex(ValueError, "bracket"): + instructions.reduce(self.window, self.stats, self.load[:-1]) + self.load[2]["frames_recv"] = 0 + with self.assertRaisesRegex(ValueError, "backwards"): + instructions.reduce(self.window, self.stats, self.load) + + def test_partial_or_multiplexed_counts_are_refused(self): + for value in ["", "nan", "0"]: + with self.subTest(value=value), self.assertRaises(ValueError): + instructions.reduce( + self.window, [dict(self.stats[0], **{"counter-value": value}), *self.stats[1:]], self.load + ) + with self.assertRaisesRegex(ValueError, "multiplexed"): + instructions.reduce(self.window, [dict(self.stats[0], **{"pcnt-running": 99}), *self.stats[1:]], self.load) + with self.assertRaisesRegex(ValueError, "missing"): + instructions.reduce(self.window, self.stats[1:], self.load) + + def test_cpu_sets_cannot_overlap_or_escape_the_allowed_set(self): + with patch.object(os, "sched_getaffinity", return_value={0, 1, 2, 3}): + for relay, generator in [("0-1", "1-2"), ("4", "0"), ("0", ""), ("0-1", "3-2")]: + with ( + self.subTest(relay=relay, generator=generator), + patch.dict(os.environ, BENCH_RELAY_CPUS=relay, BENCH_GENERATOR_CPUS=generator), + self.assertRaises(ValueError), + ): + instructions.settings("1") + + def test_lost_or_throttled_samples_refuse_attribution(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "perf.data" + for kind in [2, 5, 13, 9]: + header = bytearray(56) + header[:8] = b"PERFILE2" + struct.pack_into(""$log" 2>&1 & + local -a affinity=() + if [[ ${INSTRUCTION_AFFINITY:-false} == true ]]; then + affinity=(taskset -c "$BENCH_RELAY_CPUS") + fi + env "${environment[@]}" "${affinity[@]}" "$relay" "$config" >"$log" 2>&1 & RELAY_PID=$! for _ in {1..100}; do @@ -296,16 +300,27 @@ run_workload() { if [[ $(uname -s) == Linux ]]; then # The sampler owns the measurement lifetime. The generator runs until # interrupted after sampling, so its normal exit cannot race /proc reads. - "${load[@]}" >"$directory/load.log" 2>&1 & + local -a affinity=() + if [[ ${INSTRUCTION_AFFINITY:-false} == true ]]; then + affinity=(taskset -c "$BENCH_GENERATOR_CPUS") + fi + "${affinity[@]}" "${load[@]}" >"$directory/load.log" 2>&1 & LOAD_PID=$! generator_pid=$LOAD_PID + local duration=10s + if [[ $PROFILE == instructions ]]; then duration=14s; fi "$HOST_BIN" --pid "$relay_pid" --pid "$generator_pid" --threads \ - --interval 500ms --duration 10s --output "$host" \ + --interval 500ms --duration "$duration" --output "$host" \ >"$directory/host.log" 2>&1 & HOST_PID=$! # The second half of the host window, matching the summary's throughput window. # Its own process group, so cleanup also stops perf and its workload. case $PROFILE in + instructions) + setsid python3 "$ROOT/bench/instructions.py" capture "$relay_pid" "$profile" \ + --delay 7 --duration 5 >"$directory/profile.log" 2>&1 & + PROFILE_PID=$! + ;; lock | cpu) setsid python3 "$ROOT/bench/profile.py" "$PROFILE" "$relay_pid" "$profile" --delay 5 --duration 5 \ >"$directory/profile.log" 2>&1 & @@ -348,6 +363,9 @@ run_workload() { --settled "http://127.0.0.1:$((RELAY_PORT + 1))/sessions" fi stop_relay + if [[ $PROFILE == instructions ]]; then + python3 "$ROOT/bench/instructions.py" report "$profile" "$stats" + fi if [[ $PROFILE == heap ]]; then python3 "$ROOT/bench/profile.py" heap "$relay" "$profile" fi diff --git a/bench/run.sh b/bench/run.sh index fde8c5e7151..170c7943670 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -88,12 +88,12 @@ case $MODE in fi PROFILE_WORKERS=${BENCH_PROFILE_WORKERS:-1 4 16} if [[ -z $PROFILE_MODES ]]; then - printf 'BENCH_PROFILE_MODES must name one or more of: lock cpu heap\n' >&2 + printf 'BENCH_PROFILE_MODES must name one or more of: lock cpu heap instructions\n' >&2 exit 1 fi for value in $PROFILE_MODES; do - if [[ ! $value =~ ^(lock|cpu|heap)$ ]]; then - printf 'unknown profile mode %q; expected lock, cpu, or heap\n' "$value" >&2 + if [[ ! $value =~ ^(lock|cpu|heap|instructions)$ ]]; then + printf 'unknown profile mode %q; expected lock, cpu, heap, or instructions\n' "$value" >&2 exit 1 fi done @@ -733,8 +733,10 @@ build_workers() { fi printf 'Building worker runtimes: %s (%s)\n' "$checkout" "$profile" ( - cd "$checkout" - CARGO_TARGET_DIR=$target cargo build --locked --profile "$profile" \ + # Read just this revision's configuration, even in a nested scratch worktree. + cd / + CARGO_TARGET_DIR=$target cargo --config "$checkout/.cargo/config.toml" build \ + --manifest-path "$checkout/Cargo.toml" --locked --profile "$profile" \ -p moq-relay -p moq-bench --features "$features" ) } @@ -843,15 +845,24 @@ run_profiles() { local mode for mode in $PROFILE_MODES; do python3 "$ROOT/bench/profile.py" check "$mode" + if [[ $mode == instructions ]]; then + python3 "$ROOT/bench/instructions.py" check "$PROFILE_WORKERS" + fi done generate_certificate + if [[ " $PROFILE_MODES " == *' instructions '* ]]; then + # Keep each bisect revision's artifacts instead of overwriting one base target. + CURRENT_TARGET=$TARGET/revisions/$(git rev-parse HEAD) + BASE_TARGET=$TARGET/revisions/$BASE + fi + if [[ -n $BASE ]]; then checkout_base "$BASE" mkdir -p "$RUN/base-context" (cd "$WORKTREE" && python3 "$ROOT/bench/metadata.py" "$RUN/base-context" "$BASE" --profile) fi - local label checkout target + local label checkout target round index local -a revisions=(current) if [[ -n $BASE ]]; then revisions=(base current); fi for mode in $PROFILE_MODES; do @@ -869,8 +880,21 @@ run_profiles() { if [[ $PROFILE_RUNTIME == io-uring-workers ]]; then check_uring "$target/profiling" fi - run_profile "$mode" "$label" "$target/profiling/moq-relay" + if [[ $mode != instructions ]]; then + run_profile "$mode" "$label" "$target/profiling/moq-relay" + fi done + if [[ $mode == instructions ]]; then + # Pair in alternating order rather than measuring every base run before current. + for ((round = 1; round <= ROUNDS; round++)); do + for ((index = 0; index < ${#revisions[@]}; index++)); do + label=${revisions[$(((index + round - 1) % ${#revisions[@]}))]} + target=$CURRENT_TARGET/profile + if [[ $label == base ]]; then target=$BASE_TARGET/profile; fi + run_profile "$mode" "$label/round-$round" "$target/profiling/moq-relay" + done + done + fi done } @@ -879,6 +903,8 @@ run_profiles() { run_profile() { local mode=$1 label=$2 relay=$3 local workers control profiled + local INSTRUCTION_AFFINITY=false + if [[ $mode == instructions ]]; then INSTRUCTION_AFFINITY=true; fi TRANSPORT=$PROFILE_TRANSPORT case $mode in lock) @@ -897,6 +923,9 @@ run_profile() { workers control-fps profiled-fps control-CPU profiled-CPU control-RSS profiled-RSS \ before steady teardown churn-MiB/s allocs/s ;; + instructions) + printf '\nInstructions: %s over %s, %s, %s\n' "$PROFILE_WORKLOAD" "$TRANSPORT" "$PROFILE_RUNTIME" "$label" + ;; esac for workers in $PROFILE_WORKERS; do run_workload "$label/control-$mode-$workers" "$relay" "$PROFILE_WORKLOAD" "$PROFILE_RUNTIME" "$workers" @@ -906,6 +935,9 @@ run_profile() { control=$RUN/relay/$label/control-$mode-$workers/$PROFILE_WORKLOAD/summary.json profiled=$RUN/relay/$label/$mode-$workers/$PROFILE_WORKLOAD case $mode in + instructions) + cat "$profiled/instructions/instructions.json" + ;; lock) jq -r -s '[ .[0].recv_fps, .[1].recv_fps, .[0].cpu_cores, .[1].cpu_cores, (.[2].idle | length), diff --git a/justfile b/justfile index e9a52a9ac78..ce5f727999a 100644 --- a/justfile +++ b/justfile @@ -70,7 +70,8 @@ bench-shard $SHARDS="0 1 4 16" $CONNECTIONS="16 64 256": BENCH_SHARD_COUNTS="$SHARDS" BENCH_SHARD_CONNECTIONS="$CONNECTIONS" bench/run.sh --shard # Profile the relay at each worker count: std mutex waits by stack (lock), CPU -# stacks (cpu), or jemalloc heap and churn (heap), over one transport (quic, ws, wss, tcp, or tls). +# stacks (cpu), hardware instructions and fixed-period stacks (instructions), or +# jemalloc heap and churn (heap), over one transport (quic, ws, wss, tcp, or tls). # Example: `just bench-profile "cpu heap" fanout-heavy 4`, or `just bench-profile cpu chat 4 tokio-workers "" ws`. bench-profile $MODES $WORKLOAD="fanout" $WORKERS="1 4 16" $RUNTIME="tokio-workers" $BASE="" $TRANSPORT="quic": BENCH_PROFILE_MODES="$MODES" BENCH_PROFILE_WORKLOAD="$WORKLOAD" BENCH_PROFILE_WORKERS="$WORKERS" BENCH_PROFILE_RUNTIME="$RUNTIME" BENCH_BASE="$BASE" BENCH_PROFILE_TRANSPORT="$TRANSPORT" bench/run.sh --profile diff --git a/quest/m1/release-perf/relay-cpu-bisect.md b/quest/m1/release-perf/relay-cpu-bisect.md index b206f31d9c9..fe6d6f908ca 100644 --- a/quest/m1/release-perf/relay-cpu-bisect.md +++ b/quest/m1/release-perf/relay-cpu-bisect.md @@ -23,6 +23,7 @@ Decided 2026-11-01 (maintainer, follow-up planning after the 2026-11-01 /quest-a - Bisect `cdff3c8f95..main` on chat over io_uring first (the largest rise and the tightest ranges), with a step size that keeps each build's cost bounded, then confirm the culprits on video and on the tokio workers. A rise that shows on io_uring but not on the tokio workers is attributed on its own, not folded into a shared culprit. - Attribute each step with `perf record` at a fixed period, as #7505 did, before deciding fix or accept. Record accepted costs in the epic's Plan with the feature they pay for. - Commit the driver this needs as a deterministic mode of `just bench-profile` or `just bench-runtime` (pinned cores, instructions per delivered frame, fixed-period samples) instead of another throwaway script; #7505's driver on `bench/relay.sh` was not committed. +- Implementation checkpoint: `just bench-profile instructions` now owns disjoint relay/generator affinities, acknowledged PMU windows matched to interpolated 500 ms delivery reports, alternating revision pairs, and fixed-period source profiles. Missing or multiplexed counters, lost/throttled samples, and unbracketed delivery refuse the result. The historical builds explicitly load only their own Cargo configuration, avoiding ancestor alias conflicts in nested scratch checkouts. The bisect, attribution, and any resulting fixes remain to be completed; the first chat capture hit the host's 1,000 Hz sample-rate limit and is not attribution evidence. Use a larger fixed period consistently across the pair, rather than accepting a throttled profile. - [Fanout lock contention](/quest/m1/release-perf/fanout-lock.md) covers the bimodal futex cost on the fanout workload; don't fix it twice. A bisect step that lands on the same lock goes there. Public API: none expected. Wire: none. From 9c19f0a6cda900407ee7202b23e8633e95f87857 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 19:20:20 -0800 Subject: [PATCH 03/36] chore(quest): checkpoint the profiling host blocker Co-Authored-By: GPT-6.1 Sol --- bench/README.md | 5 +++-- bench/instructions.py | 8 +++++++- bench/metadata.py | 4 +++- bench/profile.test.py | 2 +- quest/m1/release-perf/relay-cpu-bisect.md | 3 ++- 5 files changed, 16 insertions(+), 6 deletions(-) diff --git a/bench/README.md b/bench/README.md index b2fb50afc81..343dbd3adfd 100644 --- a/bench/README.md +++ b/bench/README.md @@ -259,8 +259,9 @@ acknowledgement brackets, their uncertainty, and the delivered frames linearly interpolated between the generator's surrounding 500 ms reports at those same boundaries. This is a sampled delivery estimate, not exact frame accounting. Missing, uncounted, or multiplexed counters and unbracketed delivery fail the run. -Lost or throttled instruction samples also fail; choose a larger fixed period -for the whole comparison when the host's sample-rate limit requires it. +Lost or throttled instruction samples also fail. Inspect the host's sample-rate +limit and buffers before choosing a larger fixed period for the whole comparison; +some sample-rate limits throttle on every interrupt, regardless of the period. The host sampler lasts fourteen seconds so delivery reports bracket both edges. The normal `summary.json` still describes its own final throughput window; use `instructions.json` for instruction comparisons. diff --git a/bench/instructions.py b/bench/instructions.py index c6e6b4ee224..95e5fc0485e 100644 --- a/bench/instructions.py +++ b/bench/instructions.py @@ -179,6 +179,7 @@ def capture(pid, directory, delay, duration): end_ms=end, threads=names, events=counters, + sample_rate_limit=int(Path("/proc/sys/kernel/perf_event_max_sample_rate").read_text()), ), indent=2, ) @@ -274,13 +275,18 @@ def validate_data(path): if not size or end > len(data): raise ValueError("incomplete perf.data recording") while offset < end: + if end - offset < 8: + raise ValueError("incomplete perf.data record header") kind, _, size = struct.unpack_from(" end: raise ValueError("invalid perf.data record") # perf_event_header types: LOST, THROTTLE, LOST_SAMPLES. perf script's # --show-lost-events omits THROTTLE, which would bias fixed-period attribution. if kind in (2, 5, 13): - raise ValueError(f"perf lost or throttled samples (record type {kind}); increase BENCH_INSTRUCTION_PERIOD") + raise ValueError( + f"perf lost or throttled samples (record type {kind}); inspect the host's " + "kernel.perf_event_max_sample_rate and capture buffers before choosing a fixed period" + ) offset += size diff --git a/bench/metadata.py b/bench/metadata.py index 700d14e9ad7..ff5e8883793 100644 --- a/bench/metadata.py +++ b/bench/metadata.py @@ -69,7 +69,9 @@ def command(*args): "profilers": {tool: command(tool, "--version").splitlines()[0] for tool in ("perf", "jeprof") if shutil.which(tool)} if mode == "--profile" else None, - "workload": None if mode == "--criterion" else {"startup": "2s", "duration": "10s", "report": "500ms"}, + "workload": None + if mode == "--criterion" + else {"startup": "2s", "duration": "10s", "report": "500ms", "instructions_duration": "14s"}, } (out / "metadata.json").write_text(json.dumps(metadata, indent=2) + "\n") subprocess.run( diff --git a/bench/profile.test.py b/bench/profile.test.py index ce4cfa46fc9..00e2ecda523 100644 --- a/bench/profile.test.py +++ b/bench/profile.test.py @@ -55,7 +55,7 @@ def test_partial_or_multiplexed_counts_are_refused(self): instructions.reduce(self.window, self.stats[1:], self.load) def test_cpu_sets_cannot_overlap_or_escape_the_allowed_set(self): - with patch.object(os, "sched_getaffinity", return_value={0, 1, 2, 3}): + with patch.object(os, "sched_getaffinity", return_value={0, 1, 2, 3}, create=True): for relay, generator in [("0-1", "1-2"), ("4", "0"), ("0", ""), ("0-1", "3-2")]: with ( self.subTest(relay=relay, generator=generator), diff --git a/quest/m1/release-perf/relay-cpu-bisect.md b/quest/m1/release-perf/relay-cpu-bisect.md index fe6d6f908ca..9387b3dd21b 100644 --- a/quest/m1/release-perf/relay-cpu-bisect.md +++ b/quest/m1/release-perf/relay-cpu-bisect.md @@ -23,7 +23,8 @@ Decided 2026-11-01 (maintainer, follow-up planning after the 2026-11-01 /quest-a - Bisect `cdff3c8f95..main` on chat over io_uring first (the largest rise and the tightest ranges), with a step size that keeps each build's cost bounded, then confirm the culprits on video and on the tokio workers. A rise that shows on io_uring but not on the tokio workers is attributed on its own, not folded into a shared culprit. - Attribute each step with `perf record` at a fixed period, as #7505 did, before deciding fix or accept. Record accepted costs in the epic's Plan with the feature they pay for. - Commit the driver this needs as a deterministic mode of `just bench-profile` or `just bench-runtime` (pinned cores, instructions per delivered frame, fixed-period samples) instead of another throwaway script; #7505's driver on `bench/relay.sh` was not committed. -- Implementation checkpoint: `just bench-profile instructions` now owns disjoint relay/generator affinities, acknowledged PMU windows matched to interpolated 500 ms delivery reports, alternating revision pairs, and fixed-period source profiles. Missing or multiplexed counters, lost/throttled samples, and unbracketed delivery refuse the result. The historical builds explicitly load only their own Cargo configuration, avoiding ancestor alias conflicts in nested scratch checkouts. The bisect, attribution, and any resulting fixes remain to be completed; the first chat capture hit the host's 1,000 Hz sample-rate limit and is not attribution evidence. Use a larger fixed period consistently across the pair, rather than accepting a throttled profile. +- Implementation checkpoint in [moq-dev/moq#7605](https://github.com/moq-dev/moq/pull/7605): `just bench-profile instructions` owns disjoint relay/generator affinities, acknowledged PMU windows matched to interpolated 500 ms delivery reports, alternating revision pairs, and fixed-period source profiles. Missing or multiplexed counters, lost/throttled samples, and unbracketed delivery refuse the result. Historical profiling builds explicitly load only their own Cargo configuration, avoiding ancestor alias conflicts in nested scratch checkouts. +- The bisect, attribution, and resulting fixes remain blocked on the profiling host's sample-rate configuration. With `kernel.perf_event_max_sample_rate=1000`, both a 1,000,003-instruction current-tree capture and a 10,000,019-instruction historical capture throttled. The historical run had 957 instruction samples and 957 THROTTLE records. Increasing the period did not produce complete fixed-period attribution. Decide in the foreground whether to configure a higher host sample-rate limit temporarily and restore it afterward, or use another profiling host. No host sysctl was changed, and no feature cost has been accepted or optimization attributed. Re-run both endpoints with identical capture settings before bisecting; then confirm each measured step on video and Tokio workers. Keep the separately owned [moq-dev/moq#7493](https://github.com/moq-dev/moq/pull/7493) out of this fix. - [Fanout lock contention](/quest/m1/release-perf/fanout-lock.md) covers the bimodal futex cost on the fanout workload; don't fix it twice. A bisect step that lands on the same lock goes there. Public API: none expected. Wire: none. From dad136c30e0b2a7c8e05d106b675cac2619cbae5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 20:53:11 -0800 Subject: [PATCH 04/36] chore(quest): checkpoint relay CPU bisect evidence and remaining scope Co-Authored-By: GPT-6.1 Sol --- .github/relay-cpu-bisect.json | 6625 +++++++++++++++++++++ .github/relay-cpu-bisect.md | 48 + bench/README.md | 4 + bench/instructions.py | 6 +- bench/profile.test.py | 8 + bench/run.sh | 7 +- quest/m1/release-perf/relay-cpu-bisect.md | 27 +- 7 files changed, 6710 insertions(+), 15 deletions(-) create mode 100644 .github/relay-cpu-bisect.json create mode 100644 .github/relay-cpu-bisect.md diff --git a/.github/relay-cpu-bisect.json b/.github/relay-cpu-bisect.json new file mode 100644 index 00000000000..82826747a2e --- /dev/null +++ b/.github/relay-cpu-bisect.json @@ -0,0 +1,6625 @@ +{ + "version": 1, + "status": "Unfinished investigation, checkpointed at the user request. No runtime fix or accepted feature cost is claimed.", + "production_control_revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "method": { + "workload": { + "connections": 2001, + "publishers": 1, + "viewers": 2000, + "fps": 10, + "frame_bytes": 300, + "frames_per_group": 1 + }, + "runtime": "io-uring-workers", + "workers": 4, + "relay_cpus": [ + 4, + 5, + 6, + 7 + ], + "generator_cpus": [ + 0, + 1, + 2, + 3 + ], + "profile": "profiling", + "rustflags": "-C force-frame-pointers=yes", + "features": "moq-relay/io-uring,moq-bench/uring", + "warmup_s": 7, + "counter_window_s": 5, + "host_window_s": 14, + "generator_report_ms": 500, + "instruction_period": 10000019, + "counter_scope": "all relay threads, user and kernel", + "delivery_scope": "cumulative frames linearly interpolated at acknowledged perf enable/disable boundaries", + "source_scope": "fixed-period frame-pointer instruction samples; inclusive marker counts overlap and are not additive", + "source_markers": { + "resume_learn_end": "learn_end", + "publisher_forward_drop": "poll_forward_drop", + "track_state_on_stack": "model::track::TrackState", + "resume_reader_on_stack": "model::resume::Reader", + "waiter_list_on_stack": "WaiterList" + }, + "host_setup": "Original sample-rate limit 1000 throttled every interrupt. Each successful gated command temporarily requested 100000 and restored 1000 on exit. The kernel sometimes lowered the active limit; each capture records it. No sysctl changes in the committed driver.", + "limitations": [ + "Exploratory three-round comparisons; no performance acceptance threshold was introduced.", + "A shared loaded host and interpolated delivery are not exact deterministic accounting.", + "Every historical revision is paired with the same current source, rather than direct adjacent historical pairs.", + "No video or Tokio-worker confirmation has been run in this attempt.", + "No production optimization, benchmark sweep for a fix, or complete attribution of all increases has been completed.", + "The final resume-end after-change command completed capture but had materially different delivery and confirmed loss; its performance comparison is inconclusive and all six cells remain archived." + ], + "cpu_model": "Intel(R) Core(TM) Ultra 7 270K Plus", + "perf_version": "perf version 7.3.0" + }, + "runs": [ + { + "artifact": "run.IkXbDf", + "description": "merged", + "base_revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "comparison_status": "exploratory comparable-delivery capture", + "harness_head": "9c19f0a6cda900407ee7202b23e8633e95f87857", + "machine": { + "os": "Linux-7.0.0-42-generic-x86_64-with-glibc2.44", + "machine": "x86_64", + "rustc": "rustc 1.98.1 (48a229cea 2026-09-01)\nbinary: rustc\ncommit-hash: 48a229ceaefd4985c50990b14116b6d856af0985\ncommit-date: 2026-09-01\nhost: x86_64-unknown-linux-gnu\nrelease: 1.98.1\nLLVM version: 22.1.8", + "cargo": "cargo 1.98.1 (797e8a9bc 2026-08-05)", + "affinity": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7 + ] + }, + "harness_dirty_patch_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "cells": [ + { + "variant": "base", + "round": 1, + "capture": { + "version": 1, + "relay_cpus": [ + 4, + 5, + 6, + 7 + ], + "generator_cpus": [ + 0, + 1, + 2, + 3 + ], + "period": 10000019, + "start_ms": [ + 1793590071225.251, + 1793590071225.4373 + ], + "end_ms": [ + 1793590076225.6614, + 1793590076225.9832 + ], + "threads": { + "2082145": "moq-relay", + "2082153": "tokio-rt-worker", + "2082154": "notify-rs inoti", + "2082159": "moq-uring-0", + "2082162": "moq-uring-1", + "2082163": "moq-uring-2", + "2082164": "moq-uring-3" + }, + "events": [ + "cpu_core/instructions/", + "cpu_core/cycles/" + ], + "sample_rate_limit": 33000, + "window_start_ms": 1793590071225.3442, + "window_end_ms": 1793590076225.8223, + "boundary_uncertainty_ms": [ + 0.186279296875, + 0.32177734375 + ], + "delivered_frames": 100021.45055736473, + "recv_fps": 20002.377774769677, + "counters": { + "cpu_core/instructions/": 11832327091.0, + "cpu_core/cycles/": 22214916545.0, + "task-clock": 4375.270177 + }, + "instructions_per_frame": 118297.89535209622, + "cycles_per_frame": 222101.52343530755, + "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", + "instruction_samples": 1176 + }, + "delivery_boundary_reports": { + "window_start_ms": [ + { + "timestamp_ms": 1793590071023, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 139981, + "frames_sent": 71, + "bytes_recv": 41994300, + "groups_lost": 0, + "groups_expected": 133513, + "groups_present": 133513 + }, + { + "timestamp_ms": 1793590071522, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 149981, + "frames_sent": 76, + "bytes_recv": 44994300, + "groups_lost": 0, + "groups_expected": 143513, + "groups_present": 143513 + } + ], + "window_end_ms": [ + { + "timestamp_ms": 1793590076022, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 239981, + "frames_sent": 121, + "bytes_recv": 71994300, + "groups_lost": 0, + "groups_expected": 233513, + "groups_present": 233513 + }, + { + "timestamp_ms": 1793590076522, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 249981, + "frames_sent": 126, + "bytes_recv": 74994300, + "groups_lost": 0, + "groups_expected": 243513, + "groups_present": 243513 + } + ] + }, + "source_inclusive_samples": { + "resume_learn_end": 0, + "publisher_forward_drop": 109, + "track_state_on_stack": 103, + "resume_reader_on_stack": 181, + "waiter_list_on_stack": 104 + }, + "artifact_sha256": { + "load.jsonl": "20cdf65e356081638b9e66b4afe73c67ed4c1faf9a0cf39b9d2ca0b0704d0e5b", + "stat.json": "8f9d1cd4830704baefbed4c40d1a7fc39abd8aab604b683e88e11b2204085ee9", + "window.json": "d08f30e7a140f397e9892381626ddbfd83ba27849da3b4d803e656ecb05f94fb", + "perf.data": "6e7d7f4d3f69199bad33f8a4656d7e295252932b8d4178d799f688b39681e4b4", + "instructions.folded": "f0aaa8eac9610cdbeef61dfc37670c89de7e0a0a4e60a267fd68adc680238c3c" + } + }, + { + "variant": "base", + "round": 2, + "capture": { + "version": 1, + "relay_cpus": [ + 4, + 5, + 6, + 7 + ], + "generator_cpus": [ + 0, + 1, + 2, + 3 + ], + "period": 10000019, + "start_ms": [ + 1793590246597.0916, + 1793590246597.5728 + ], + "end_ms": [ + 1793590251597.7349, + 1793590251597.8665 + ], + "threads": { + "2238199": "moq-relay", + "2238202": "tokio-rt-worker", + "2238203": "notify-rs inoti", + "2238204": "moq-uring-0", + "2238205": "moq-uring-1", + "2238206": "moq-uring-2", + "2238207": "moq-uring-3" + }, + "events": [ + "cpu_core/instructions/", + "cpu_core/cycles/" + ], + "sample_rate_limit": 33000, + "window_start_ms": 1793590246597.332, + "window_end_ms": 1793590251597.8008, + "boundary_uncertainty_ms": [ + 0.481201171875, + 0.131591796875 + ], + "delivered_frames": 100009.375, + "recv_fps": 20000.0, + "counters": { + "cpu_core/instructions/": 10871014863.0, + "cpu_core/cycles/": 19624006962.0, + "task-clock": 3769.699453 + }, + "instructions_per_frame": 108699.95800893666, + "cycles_per_frame": 196221.67383807767, + "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", + "instruction_samples": 1080 + }, + "delivery_boundary_reports": { + "window_start_ms": [ + { + "timestamp_ms": 1793590246413, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 139986, + "frames_sent": 71, + "bytes_recv": 41995800, + "groups_lost": 0, + "groups_expected": 133272, + "groups_present": 133272 + }, + { + "timestamp_ms": 1793590246913, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 149986, + "frames_sent": 76, + "bytes_recv": 44995800, + "groups_lost": 0, + "groups_expected": 143272, + "groups_present": 143272 + } + ], + "window_end_ms": [ + { + "timestamp_ms": 1793590251413, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 239986, + "frames_sent": 121, + "bytes_recv": 71995800, + "groups_lost": 0, + "groups_expected": 233272, + "groups_present": 233272 + }, + { + "timestamp_ms": 1793590251913, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 249986, + "frames_sent": 126, + "bytes_recv": 74995800, + "groups_lost": 0, + "groups_expected": 243272, + "groups_present": 243272 + } + ] + }, + "source_inclusive_samples": { + "resume_learn_end": 0, + "publisher_forward_drop": 60, + "track_state_on_stack": 86, + "resume_reader_on_stack": 143, + "waiter_list_on_stack": 88 + }, + "artifact_sha256": { + "load.jsonl": "631a46b79e6ff611c5d674430633f108546a84f83831b61aff84c2e4164fdda5", + "stat.json": "f777ccd81c6c64c052a017591838c830e1d85dd4e59faf9a06aa157cfb6fdea2", + "window.json": "e80e117d68f839ee2ac7fd1fad68404127a3e1323e275e3ae8a84dc2b17fb88a", + "perf.data": "ef649b30d819f0c999ad6e638296df2bc5b18a819953066177c34c8a4f82e56c", + "instructions.folded": "b1c3d3b0a9463f2421ad0274a339af2ea3b58607687d3b7911013fb1005c744b" + } + }, + { + "variant": "base", + "round": 3, + "capture": { + "version": 1, + "relay_cpus": [ + 4, + 5, + 6, + 7 + ], + "generator_cpus": [ + 0, + 1, + 2, + 3 + ], + "period": 10000019, + "start_ms": [ + 1793590286315.6274, + 1793590286315.6765 + ], + "end_ms": [ + 1793590291315.8625, + 1793590291316.802 + ], + "threads": { + "2247444": "moq-relay", + "2247449": "tokio-rt-worker", + "2247451": "notify-rs inoti", + "2247453": "moq-uring-0", + "2247454": "moq-uring-1", + "2247455": "moq-uring-2", + "2247456": "moq-uring-3" + }, + "events": [ + "cpu_core/instructions/", + "cpu_core/cycles/" + ], + "sample_rate_limit": 33000, + "window_start_ms": 1793590286315.6519, + "window_end_ms": 1793590291316.3323, + "boundary_uncertainty_ms": [ + 0.049072265625, + 0.939453125 + ], + "delivered_frames": 100033.6083984375, + "recv_fps": 20003.99945573813, + "counters": { + "cpu_core/instructions/": 10806432745.0, + "cpu_core/cycles/": 19674669867.0, + "task-clock": 3774.105446 + }, + "instructions_per_frame": 108028.02096229084, + "cycles_per_frame": 196680.59747115264, + "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", + "instruction_samples": 1075 + }, + "delivery_boundary_reports": { + "window_start_ms": [ + { + "timestamp_ms": 1793590286136, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 139981, + "frames_sent": 71, + "bytes_recv": 41994300, + "groups_lost": 0, + "groups_expected": 133270, + "groups_present": 133270 + }, + { + "timestamp_ms": 1793590286636, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 149981, + "frames_sent": 76, + "bytes_recv": 44994300, + "groups_lost": 0, + "groups_expected": 143270, + "groups_present": 143270 + } + ], + "window_end_ms": [ + { + "timestamp_ms": 1793590291135, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 239981, + "frames_sent": 121, + "bytes_recv": 71994300, + "groups_lost": 0, + "groups_expected": 233270, + "groups_present": 233270 + }, + { + "timestamp_ms": 1793590291635, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 249981, + "frames_sent": 126, + "bytes_recv": 74994300, + "groups_lost": 0, + "groups_expected": 243270, + "groups_present": 243270 + } + ] + }, + "source_inclusive_samples": { + "resume_learn_end": 0, + "publisher_forward_drop": 69, + "track_state_on_stack": 80, + "resume_reader_on_stack": 126, + "waiter_list_on_stack": 105 + }, + "artifact_sha256": { + "load.jsonl": "7b2954c51f439f6d82526ae842dde2eaa8f500a3b645819582a9faa41ceb7532", + "stat.json": "7012723323d3555cbad0bcf7ea10d25db27629f5f7eb2407e5326a63aaf08301", + "window.json": "1a0a50c0fd2d2d0df217844bb1981f42d27d4a8c495e9bcd267462a8c02b64ac", + "perf.data": "ed4b3d745707e3460e125d96481d1fd3f969234168db95d565a604e9f2e87803", + "instructions.folded": "5c854cf6102c5872e02ee821666012e1a1066f9fea2a88acfcfccd89dd0079cf" + } + }, + { + "variant": "current", + "round": 1, + "capture": { + "version": 1, + "relay_cpus": [ + 4, + 5, + 6, + 7 + ], + "generator_cpus": [ + 0, + 1, + 2, + 3 + ], + "period": 10000019, + "start_ms": [ + 1793590112027.7512, + 1793590112028.049 + ], + "end_ms": [ + 1793590117028.1233, + 1793590117029.06 + ], + "threads": { + "2142408": "moq-relay", + "2142413": "tokio-rt-worker", + "2142415": "notify-rs inoti", + "2142417": "moq-uring-0", + "2142419": "moq-uring-1", + "2142420": "moq-uring-2", + "2142421": "moq-uring-3" + }, + "events": [ + "cpu_core/instructions/", + "cpu_core/cycles/" + ], + "sample_rate_limit": 33000, + "window_start_ms": 1793590112027.9001, + "window_end_ms": 1793590117028.5918, + "boundary_uncertainty_ms": [ + 0.2978515625, + 0.936767578125 + ], + "delivered_frames": 100033.8330078125, + "recv_fps": 20003.999446756217, + "counters": { + "cpu_core/instructions/": 15779052332.0, + "cpu_core/cycles/": 28362193024.0, + "task-clock": 5418.859292 + }, + "instructions_per_frame": 157737.15609565494, + "cycles_per_frame": 283526.00486462365, + "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", + "instruction_samples": 1572 + }, + "delivery_boundary_reports": { + "window_start_ms": [ + { + "timestamp_ms": 1793590111842, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 139968, + "frames_sent": 71, + "bytes_recv": 41990400, + "groups_lost": 0, + "groups_expected": 135340, + "groups_present": 135340 + }, + { + "timestamp_ms": 1793590112342, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 149968, + "frames_sent": 75, + "bytes_recv": 44990400, + "groups_lost": 0, + "groups_expected": 145340, + "groups_present": 145340 + } + ], + "window_end_ms": [ + { + "timestamp_ms": 1793590116841, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 239968, + "frames_sent": 120, + "bytes_recv": 71990400, + "groups_lost": 0, + "groups_expected": 235340, + "groups_present": 235340 + }, + { + "timestamp_ms": 1793590117341, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 249968, + "frames_sent": 125, + "bytes_recv": 74990400, + "groups_lost": 0, + "groups_expected": 245340, + "groups_present": 245340 + } + ] + }, + "source_inclusive_samples": { + "resume_learn_end": 14, + "publisher_forward_drop": 300, + "track_state_on_stack": 242, + "resume_reader_on_stack": 238, + "waiter_list_on_stack": 148 + }, + "artifact_sha256": { + "load.jsonl": "40855e1f8e4d738f01ab4aa8fed6777b4b3d0881071fb2ca98b4c653b718f3fc", + "stat.json": "f8405f70e88b8ed4bc626370248df9225fb6af0a8f5b90c4aec688061749ce57", + "window.json": "b96e0f20cec54ecdcf1bf86553300e81faf972f1dd6b2e9567bcdcbb31fb3b9a", + "perf.data": "8c748b60f4dc42e2da2d414e62e3a66079211b16520eb65bb97e6ae4a6f7fe3e", + "instructions.folded": "4229cefd0ebee590cfb4bb4447d1b07d6c71dc53bae873ade6509e12578676c4" + } + }, + { + "variant": "current", + "round": 2, + "capture": { + "version": 1, + "relay_cpus": [ + 4, + 5, + 6, + 7 + ], + "generator_cpus": [ + 0, + 1, + 2, + 3 + ], + "period": 10000019, + "start_ms": [ + 1793590177857.9124, + 1793590177858.239 + ], + "end_ms": [ + 1793590182858.3975, + 1793590182858.6152 + ], + "threads": { + "2220019": "moq-relay", + "2220039": "tokio-rt-worker", + "2220041": "notify-rs inoti", + "2220046": "moq-uring-0", + "2220047": "moq-uring-1", + "2220048": "moq-uring-2", + "2220049": "moq-uring-3" + }, + "events": [ + "cpu_core/instructions/", + "cpu_core/cycles/" + ], + "sample_rate_limit": 33000, + "window_start_ms": 1793590177858.0757, + "window_end_ms": 1793590182858.5063, + "boundary_uncertainty_ms": [ + 0.32666015625, + 0.2177734375 + ], + "delivered_frames": 100015.80792444263, + "recv_fps": 20001.43880471023, + "counters": { + "cpu_core/instructions/": 15681117586.0, + "cpu_core/cycles/": 32301932612.0, + "task-clock": 6087.315496 + }, + "instructions_per_frame": 156786.39118574501, + "cycles_per_frame": 322968.27153966133, + "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", + "instruction_samples": 1560 + }, + "delivery_boundary_reports": { + "window_start_ms": [ + { + "timestamp_ms": 1793590177679, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 139981, + "frames_sent": 70, + "bytes_recv": 41994300, + "groups_lost": 0, + "groups_expected": 136499, + "groups_present": 136499 + }, + { + "timestamp_ms": 1793590178179, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 149981, + "frames_sent": 75, + "bytes_recv": 44994300, + "groups_lost": 0, + "groups_expected": 146499, + "groups_present": 146499 + } + ], + "window_end_ms": [ + { + "timestamp_ms": 1793590182679, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 239981, + "frames_sent": 120, + "bytes_recv": 71994300, + "groups_lost": 0, + "groups_expected": 236499, + "groups_present": 236499 + }, + { + "timestamp_ms": 1793590183178, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 249981, + "frames_sent": 125, + "bytes_recv": 74994300, + "groups_lost": 0, + "groups_expected": 246499, + "groups_present": 246499 + } + ] + }, + "source_inclusive_samples": { + "resume_learn_end": 12, + "publisher_forward_drop": 290, + "track_state_on_stack": 248, + "resume_reader_on_stack": 215, + "waiter_list_on_stack": 146 + }, + "artifact_sha256": { + "load.jsonl": "8f4586a395f61c85d53dcf827370c72976dce874184aae772983351319a9b2ef", + "stat.json": "1ebc50981d14534d69827c3a497235bf2c5f8be8bb2edc61f6753ef1b52775cd", + "window.json": "1cc71083ce565f1f01fb6150d14ee6906ef68f43432360f07560eb963b053cd9", + "perf.data": "30043f4a5f7460afd7f66da8eafc9dc08a6029d1a7a2181cbdb0d38dbe274baa", + "instructions.folded": "4cc71a4480a54aab1f8c8e2a96702bae8f05245d235fab9a4ef3c2cdd9a267c4" + } + }, + { + "variant": "current", + "round": 3, + "capture": { + "version": 1, + "relay_cpus": [ + 4, + 5, + 6, + 7 + ], + "generator_cpus": [ + 0, + 1, + 2, + 3 + ], + "period": 10000019, + "start_ms": [ + 1793590325452.0142, + 1793590325452.4946 + ], + "end_ms": [ + 1793590330452.6287, + 1793590330453.953 + ], + "threads": { + "2258914": "moq-relay", + "2258917": "tokio-rt-worker", + "2258918": "notify-rs inoti", + "2258919": "moq-uring-0", + "2258920": "moq-uring-1", + "2258921": "moq-uring-2", + "2258922": "moq-uring-3" + }, + "events": [ + "cpu_core/instructions/", + "cpu_core/cycles/" + ], + "sample_rate_limit": 33000, + "window_start_ms": 1793590325452.2544, + "window_end_ms": 1793590330453.2908, + "boundary_uncertainty_ms": [ + 0.48046875, + 1.32421875 + ], + "delivered_frames": 100008.08300391404, + "recv_fps": 19997.471617041872, + "counters": { + "cpu_core/instructions/": 15597902854.0, + "cpu_core/cycles/": 27915802473.0, + "task-clock": 5318.083006 + }, + "instructions_per_frame": 155966.4217680239, + "cycles_per_frame": 279135.4621996649, + "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", + "instruction_samples": 1554 + }, + "delivery_boundary_reports": { + "window_start_ms": [ + { + "timestamp_ms": 1793590325268, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 139978, + "frames_sent": 71, + "bytes_recv": 41993400, + "groups_lost": 0, + "groups_expected": 136739, + "groups_present": 136739 + }, + { + "timestamp_ms": 1793590325769, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 149978, + "frames_sent": 76, + "bytes_recv": 44993400, + "groups_lost": 0, + "groups_expected": 146739, + "groups_present": 146739 + } + ], + "window_end_ms": [ + { + "timestamp_ms": 1793590330269, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 239978, + "frames_sent": 121, + "bytes_recv": 71993400, + "groups_lost": 0, + "groups_expected": 236739, + "groups_present": 236739 + }, + { + "timestamp_ms": 1793590330769, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 249978, + "frames_sent": 126, + "bytes_recv": 74993400, + "groups_lost": 0, + "groups_expected": 246739, + "groups_present": 246739 + } + ] + }, + "source_inclusive_samples": { + "resume_learn_end": 12, + "publisher_forward_drop": 307, + "track_state_on_stack": 252, + "resume_reader_on_stack": 212, + "waiter_list_on_stack": 152 + }, + "artifact_sha256": { + "load.jsonl": "fdf60c2ebe2a9ae694cf8fbd64ca399447559baca98c03298c254264d91d1cab", + "stat.json": "11ef9c5a545ea170ba8ed0a73737de37f350fc9cfb1a6ecc4efb023b455a16df", + "window.json": "afdb3d75cc3a51faedd37fd72d7471e9518b1c35a94fdcded5ca09d176c7466f", + "perf.data": "fa889f9ed771c6df1d49b697e576047ca6d61a1c6983efad6f248fa17c663ee8", + "instructions.folded": "fb6d9baf6b828d705a250553d6a9d986d89b5f65b96ac4fe11a736f48acea6ab" + } + } + ], + "summary": { + "base": { + "completed_cells": 3, + "median_instructions_per_frame": 108699.95800893666, + "min_instructions_per_frame": 108028.02096229084, + "max_instructions_per_frame": 118297.89535209622 + }, + "current": { + "completed_cells": 3, + "median_instructions_per_frame": 156786.39118574501, + "min_instructions_per_frame": 155966.4217680239, + "max_instructions_per_frame": 157737.15609565494 + } + }, + "pull_request_url": "https://github.com/moq-dev/moq/pull/5926" + }, + { + "artifact": "run.vgmg4w", + "description": "Immediately before dynamic tracks", + "base_revision": "55f6ed229250d17fc99389237659906d011d8a94", + "comparison_status": "exploratory comparable-delivery capture", + "harness_head": "9c19f0a6cda900407ee7202b23e8633e95f87857", + "machine": { + "os": "Linux-7.0.0-42-generic-x86_64-with-glibc2.44", + "machine": "x86_64", + "rustc": "rustc 1.98.1 (48a229cea 2026-09-01)\nbinary: rustc\ncommit-hash: 48a229ceaefd4985c50990b14116b6d856af0985\ncommit-date: 2026-09-01\nhost: x86_64-unknown-linux-gnu\nrelease: 1.98.1\nLLVM version: 22.1.8", + "cargo": "cargo 1.98.1 (797e8a9bc 2026-08-05)", + "affinity": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7 + ] + }, + "harness_dirty_patch_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "cells": [ + { + "variant": "base", + "round": 1, + "capture": { + "version": 1, + "relay_cpus": [ + 4, + 5, + 6, + 7 + ], + "generator_cpus": [ + 0, + 1, + 2, + 3 + ], + "period": 10000019, + "start_ms": [ + 1793590706186.8125, + 1793590706186.9048 + ], + "end_ms": [ + 1793590711186.9834, + 1793590711187.879 + ], + "threads": { + "2331187": "moq-relay", + "2331190": "tokio-rt-worker", + "2331191": "notify-rs inoti", + "2331192": "moq-uring-0", + "2331193": "moq-uring-1", + "2331194": "moq-uring-2", + "2331195": "moq-uring-3" + }, + "events": [ + "cpu_core/instructions/", + "cpu_core/cycles/" + ], + "sample_rate_limit": 37000, + "window_start_ms": 1793590706186.8586, + "window_end_ms": 1793590711187.4312, + "boundary_uncertainty_ms": [ + 0.09228515625, + 0.8955078125 + ], + "delivered_frames": 101565.97229003906, + "recv_fps": 20310.868823857814, + "counters": { + "cpu_core/instructions/": 14496066156.0, + "cpu_core/cycles/": 31070291092.0, + "task-clock": 5895.938447 + }, + "instructions_per_frame": 142725.61793239173, + "cycles_per_frame": 305912.4073885046, + "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", + "instruction_samples": 1451 + }, + "delivery_boundary_reports": { + "window_start_ms": [ + { + "timestamp_ms": 1793590705957, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 139981, + "frames_sent": 70, + "bytes_recv": 41994300, + "groups_lost": 0, + "groups_expected": 135783, + "groups_present": 135783 + }, + { + "timestamp_ms": 1793590706459, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 146626, + "frames_sent": 76, + "bytes_recv": 43987800, + "groups_lost": 0, + "groups_expected": 142387, + "groups_present": 142387 + } + ], + "window_end_ms": [ + { + "timestamp_ms": 1793590710957, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 239981, + "frames_sent": 121, + "bytes_recv": 71994300, + "groups_lost": 0, + "groups_expected": 235783, + "groups_present": 235783 + }, + { + "timestamp_ms": 1793590711457, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 249981, + "frames_sent": 126, + "bytes_recv": 74994300, + "groups_lost": 0, + "groups_expected": 245783, + "groups_present": 245783 + } + ] + }, + "source_inclusive_samples": { + "resume_learn_end": 175, + "publisher_forward_drop": 137, + "track_state_on_stack": 264, + "resume_reader_on_stack": 431, + "waiter_list_on_stack": 192 + }, + "artifact_sha256": { + "load.jsonl": "d9387716044598430ad419eea1424ce8b8fa2f4e64d590a53483593dca06ee9c", + "stat.json": "12d024357993e42a51a6595b5751e52d74d8a4ca22bcf3f569f31f495a03a90d", + "window.json": "521f6e686b7ca883c52952938aa8b85f141de246fa8ac5277a39d7f94320d289", + "perf.data": "aa3a98f263a4626af310d51846f4bef54a2c42557dbe211110e7a46fbabb5b95", + "instructions.folded": "468eb686d6eb6a1d33c0e1639cb33ad1f83e75c26ffc3f1f76e38e71ea1d14be" + } + }, + { + "variant": "base", + "round": 2, + "capture": { + "version": 1, + "relay_cpus": [ + 4, + 5, + 6, + 7 + ], + "generator_cpus": [ + 0, + 1, + 2, + 3 + ], + "period": 10000019, + "start_ms": [ + 1793590885648.5505, + 1793590885648.8875 + ], + "end_ms": [ + 1793590890648.9712, + 1793590890649.185 + ], + "threads": { + "2385091": "moq-relay", + "2385094": "tokio-rt-worker", + "2385095": "notify-rs inoti", + "2385096": "moq-uring-0", + "2385097": "moq-uring-1", + "2385098": "moq-uring-2", + "2385099": "moq-uring-3" + }, + "events": [ + "cpu_core/instructions/", + "cpu_core/cycles/" + ], + "sample_rate_limit": 37000, + "window_start_ms": 1793590885648.719, + "window_end_ms": 1793590890649.0781, + "boundary_uncertainty_ms": [ + 0.3369140625, + 0.2138671875 + ], + "delivered_frames": 100007.1826171875, + "recv_fps": 20000.0, + "counters": { + "cpu_core/instructions/": 14395623521.0, + "cpu_core/cycles/": 29627853834.0, + "task-clock": 5626.923651 + }, + "instructions_per_frame": 143945.89612732406, + "cycles_per_frame": 296257.2593151732, + "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", + "instruction_samples": 1434 + }, + "delivery_boundary_reports": { + "window_start_ms": [ + { + "timestamp_ms": 1793590885449, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 139780, + "frames_sent": 70, + "bytes_recv": 41934000, + "groups_lost": 0, + "groups_expected": 134368, + "groups_present": 134368 + }, + { + "timestamp_ms": 1793590885949, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 149780, + "frames_sent": 75, + "bytes_recv": 44934000, + "groups_lost": 0, + "groups_expected": 144368, + "groups_present": 144368 + } + ], + "window_end_ms": [ + { + "timestamp_ms": 1793590890449, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 239780, + "frames_sent": 120, + "bytes_recv": 71934000, + "groups_lost": 0, + "groups_expected": 234368, + "groups_present": 234368 + }, + { + "timestamp_ms": 1793590890949, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 249780, + "frames_sent": 125, + "bytes_recv": 74934000, + "groups_lost": 0, + "groups_expected": 244368, + "groups_present": 244368 + } + ] + }, + "source_inclusive_samples": { + "resume_learn_end": 205, + "publisher_forward_drop": 169, + "track_state_on_stack": 269, + "resume_reader_on_stack": 474, + "waiter_list_on_stack": 201 + }, + "artifact_sha256": { + "load.jsonl": "857d82933db2b1c9e56aab7fd49bba40daf6e057f2edeb534001cd3e04de589f", + "stat.json": "ffb627e0007cf99530ea29492785a87feb91549834871832f746f99f374ffeda", + "window.json": "9f24c5898781597beff538338f8a4af6a227e4c7075318444468c187661bae69", + "perf.data": "62f6b07cb0ec9894315cf83a859a3a58c9bd7aa384d07b4cc7aac3b8c8db9ee1", + "instructions.folded": "1c408aa5dc83ae2300f63e82348567f959a0fa9f62321d363d71b219e26f72bb" + } + }, + { + "variant": "base", + "round": 3, + "capture": { + "version": 1, + "relay_cpus": [ + 4, + 5, + 6, + 7 + ], + "generator_cpus": [ + 0, + 1, + 2, + 3 + ], + "period": 10000019, + "start_ms": [ + 1793590931850.6985, + 1793590931850.786 + ], + "end_ms": [ + 1793590936852.2207, + 1793590936852.305 + ], + "threads": { + "2401372": "moq-relay", + "2401375": "tokio-rt-worker", + "2401376": "notify-rs inoti", + "2401377": "moq-uring-0", + "2401378": "moq-uring-1", + "2401379": "moq-uring-2", + "2401380": "moq-uring-3" + }, + "events": [ + "cpu_core/instructions/", + "cpu_core/cycles/" + ], + "sample_rate_limit": 37000, + "window_start_ms": 1793590931850.7422, + "window_end_ms": 1793590936852.2627, + "boundary_uncertainty_ms": [ + 0.08740234375, + 0.084228515625 + ], + "delivered_frames": 100307.95075888355, + "recv_fps": 20055.49124555223, + "counters": { + "cpu_core/instructions/": 13895968135.0, + "cpu_core/cycles/": 27911394233.0, + "task-clock": 5273.558912 + }, + "instructions_per_frame": 138533.06771666187, + "cycles_per_frame": 278257.0476401452, + "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", + "instruction_samples": 1383 + }, + "delivery_boundary_reports": { + "window_start_ms": [ + { + "timestamp_ms": 1793590931606, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 139126, + "frames_sent": 71, + "bytes_recv": 41737800, + "groups_lost": 0, + "groups_expected": 130733, + "groups_present": 130733 + }, + { + "timestamp_ms": 1793590932102, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 148782, + "frames_sent": 75, + "bytes_recv": 44634600, + "groups_lost": 0, + "groups_expected": 140389, + "groups_present": 140389 + } + ], + "window_end_ms": [ + { + "timestamp_ms": 1793590936603, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 238799, + "frames_sent": 120, + "bytes_recv": 71639700, + "groups_lost": 0, + "groups_expected": 230406, + "groups_present": 230406 + }, + { + "timestamp_ms": 1793590937103, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 249630, + "frames_sent": 125, + "bytes_recv": 74889000, + "groups_lost": 0, + "groups_expected": 241237, + "groups_present": 241237 + } + ] + }, + "source_inclusive_samples": { + "resume_learn_end": 173, + "publisher_forward_drop": 120, + "track_state_on_stack": 259, + "resume_reader_on_stack": 409, + "waiter_list_on_stack": 198 + }, + "artifact_sha256": { + "load.jsonl": "745144aef18decea25e2f35878b5ce25a642e6bb02d761c006fab9b2bd2ba3b9", + "stat.json": "e528efdebbedf5de12d35b8616dde3add346b338a07f9e4d535225cf9bcb8063", + "window.json": "9ed6b1aa64ebc2d10df095c269a95e84ba773037639cb56da39d0c93983a7b22", + "perf.data": "975a84429db8444100a39b096a25da33575a248303a2855415ee8b012305c3ad", + "instructions.folded": "eb0ffee09226991bc1ff401e751586826c21a480d369399f0244b76a952778bd" + } + }, + { + "variant": "current", + "round": 1, + "capture": { + "version": 1, + "relay_cpus": [ + 4, + 5, + 6, + 7 + ], + "generator_cpus": [ + 0, + 1, + 2, + 3 + ], + "period": 10000019, + "start_ms": [ + 1793590752483.7834, + 1793590752483.9976 + ], + "end_ms": [ + 1793590757484.0757, + 1793590757484.4133 + ], + "threads": { + "2337236": "moq-relay", + "2337239": "tokio-rt-worker", + "2337240": "notify-rs inoti", + "2337241": "moq-uring-0", + "2337242": "moq-uring-1", + "2337243": "moq-uring-2", + "2337244": "moq-uring-3" + }, + "events": [ + "cpu_core/instructions/", + "cpu_core/cycles/" + ], + "sample_rate_limit": 37000, + "window_start_ms": 1793590752483.8906, + "window_end_ms": 1793590757484.2446, + "boundary_uncertainty_ms": [ + 0.214111328125, + 0.337646484375 + ], + "delivered_frames": 100007.080078125, + "recv_fps": 20000.0, + "counters": { + "cpu_core/instructions/": 15544423267.0, + "cpu_core/cycles/": 30519449347.0, + "task-clock": 5761.496011 + }, + "instructions_per_frame": 155433.22787603416, + "cycles_per_frame": 305172.8869911847, + "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", + "instruction_samples": 1547 + }, + "delivery_boundary_reports": { + "window_start_ms": [ + { + "timestamp_ms": 1793590752294, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 139982, + "frames_sent": 71, + "bytes_recv": 41994600, + "groups_lost": 0, + "groups_expected": 135591, + "groups_present": 135591 + }, + { + "timestamp_ms": 1793590752794, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 149982, + "frames_sent": 76, + "bytes_recv": 44994600, + "groups_lost": 0, + "groups_expected": 145591, + "groups_present": 145591 + } + ], + "window_end_ms": [ + { + "timestamp_ms": 1793590757294, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 239982, + "frames_sent": 121, + "bytes_recv": 71994600, + "groups_lost": 0, + "groups_expected": 235591, + "groups_present": 235591 + }, + { + "timestamp_ms": 1793590757794, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 249982, + "frames_sent": 126, + "bytes_recv": 74994600, + "groups_lost": 0, + "groups_expected": 245591, + "groups_present": 245591 + } + ] + }, + "source_inclusive_samples": { + "resume_learn_end": 20, + "publisher_forward_drop": 298, + "track_state_on_stack": 245, + "resume_reader_on_stack": 218, + "waiter_list_on_stack": 152 + }, + "artifact_sha256": { + "load.jsonl": "f7c1587f5349063077acec6cacb26dc48b3c11a8d1e5e215b496b0b7ebc20561", + "stat.json": "82179b288bcd10c0794e475ba9961144f49b08d8ce00b6bf2618b41907756967", + "window.json": "83b6bea0d5f7c463970dcd799986236b3464b7dc75a55f11a4fa6a5a60a1a3c3", + "perf.data": "3bc21650def500654eac69b81e00b86eb96f58287f26aae588824171883bb7f4", + "instructions.folded": "fa454199437af12c23c59523554d4d2564556f3c4ae594ca5936c920defb00ff" + } + }, + { + "variant": "current", + "round": 2, + "capture": { + "version": 1, + "relay_cpus": [ + 4, + 5, + 6, + 7 + ], + "generator_cpus": [ + 0, + 1, + 2, + 3 + ], + "period": 10000019, + "start_ms": [ + 1793590819431.9517, + 1793590819433.8025 + ], + "end_ms": [ + 1793590824433.878, + 1793590824433.97 + ], + "threads": { + "2365412": "moq-relay", + "2365415": "tokio-rt-worker", + "2365416": "notify-rs inoti", + "2365417": "moq-uring-0", + "2365418": "moq-uring-1", + "2365419": "moq-uring-2", + "2365420": "moq-uring-3" + }, + "events": [ + "cpu_core/instructions/", + "cpu_core/cycles/" + ], + "sample_rate_limit": 37000, + "window_start_ms": 1793590819432.877, + "window_end_ms": 1793590824433.9238, + "boundary_uncertainty_ms": [ + 1.850830078125, + 0.092041015625 + ], + "delivered_frames": 100870.07506763568, + "recv_fps": 20169.791963334814, + "counters": { + "cpu_core/instructions/": 15436734420.0, + "cpu_core/cycles/": 26185777424.0, + "task-clock": 4925.804928 + }, + "instructions_per_frame": 153035.81770559127, + "cycles_per_frame": 259599.06747805869, + "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", + "instruction_samples": 1544 + }, + "delivery_boundary_reports": { + "window_start_ms": [ + { + "timestamp_ms": 1793590819224, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 138501, + "frames_sent": 70, + "bytes_recv": 41550300, + "groups_lost": 0, + "groups_expected": 130221, + "groups_present": 130221 + }, + { + "timestamp_ms": 1793590819726, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 148598, + "frames_sent": 75, + "bytes_recv": 44579400, + "groups_lost": 0, + "groups_expected": 140318, + "groups_present": 140318 + } + ], + "window_end_ms": [ + { + "timestamp_ms": 1793590824224, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 239745, + "frames_sent": 120, + "bytes_recv": 71923500, + "groups_lost": 0, + "groups_expected": 231465, + "groups_present": 231465 + }, + { + "timestamp_ms": 1793590824724, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 248861, + "frames_sent": 125, + "bytes_recv": 74658300, + "groups_lost": 0, + "groups_expected": 240581, + "groups_present": 240581 + } + ] + }, + "source_inclusive_samples": { + "resume_learn_end": 10, + "publisher_forward_drop": 306, + "track_state_on_stack": 224, + "resume_reader_on_stack": 228, + "waiter_list_on_stack": 171 + }, + "artifact_sha256": { + "load.jsonl": "f6d77ab1853d5d37da53766d1346c400fac71128a0a57df15316b5715b328721", + "stat.json": "ba757b4f96753169b8cce398f96e9f4ff7560fa399e6869c68b50570c581a915", + "window.json": "bfcfbb2276a0a141f97f07972452e47b20e24ddb3200402b71fdec0668454528", + "perf.data": "979b1f5c44adab78538f20e40def8373aca2a5f7227e02236010628307ecb11b", + "instructions.folded": "cf35020b74f6d70598c8210ea0ec588e12a616585835eb5d79d1c6bac4755e74" + } + }, + { + "variant": "current", + "round": 3, + "capture": { + "version": 1, + "relay_cpus": [ + 4, + 5, + 6, + 7 + ], + "generator_cpus": [ + 0, + 1, + 2, + 3 + ], + "period": 10000019, + "start_ms": [ + 1793590977976.9817, + 1793590977977.159 + ], + "end_ms": [ + 1793590982977.8079, + 1793590982977.8906 + ], + "threads": { + "2423267": "moq-relay", + "2423270": "tokio-rt-worker", + "2423271": "notify-rs inoti", + "2423272": "moq-uring-0", + "2423273": "moq-uring-1", + "2423274": "moq-uring-2", + "2423275": "moq-uring-3" + }, + "events": [ + "cpu_core/instructions/", + "cpu_core/cycles/" + ], + "sample_rate_limit": 37000, + "window_start_ms": 1793590977977.0703, + "window_end_ms": 1793590982977.849, + "boundary_uncertainty_ms": [ + 0.17724609375, + 0.082763671875 + ], + "delivered_frames": 100023.58615668837, + "recv_fps": 20001.601747471734, + "counters": { + "cpu_core/instructions/": 15430577805.0, + "cpu_core/cycles/": 28288190709.0, + "task-clock": 5362.83851 + }, + "instructions_per_frame": 154269.3918295209, + "cycles_per_frame": 282815.20185335237, + "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", + "instruction_samples": 1539 + }, + "delivery_boundary_reports": { + "window_start_ms": [ + { + "timestamp_ms": 1793590977778, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 139249, + "frames_sent": 71, + "bytes_recv": 41774700, + "groups_lost": 0, + "groups_expected": 131569, + "groups_present": 131569 + }, + { + "timestamp_ms": 1793590978278, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 149249, + "frames_sent": 76, + "bytes_recv": 44774700, + "groups_lost": 0, + "groups_expected": 141569, + "groups_present": 141569 + } + ], + "window_end_ms": [ + { + "timestamp_ms": 1793590982778, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 239249, + "frames_sent": 121, + "bytes_recv": 71774700, + "groups_lost": 0, + "groups_expected": 231569, + "groups_present": 231569 + }, + { + "timestamp_ms": 1793590983277, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 249249, + "frames_sent": 126, + "bytes_recv": 74774700, + "groups_lost": 0, + "groups_expected": 241569, + "groups_present": 241569 + } + ] + }, + "source_inclusive_samples": { + "resume_learn_end": 10, + "publisher_forward_drop": 284, + "track_state_on_stack": 254, + "resume_reader_on_stack": 224, + "waiter_list_on_stack": 129 + }, + "artifact_sha256": { + "load.jsonl": "00f4df2d4f298559a956bc73f8b9ad30e4f158e9650cccb1d5089fad690815c7", + "stat.json": "6f8a89d9bb2ed54a37bbc751ec68481c1f405e09922d3d48d011d47db868e348", + "window.json": "e9908747a036dde3c8025313281c7a13afb3daeb19bb05e8fdc64ea3abbbd5c9", + "perf.data": "fce504ebb39c8daa3939b72a573d8bb11d87b126467de40bbcc3ca4fbd873072", + "instructions.folded": "6517dfd50294a7831f5439cbe87a141430265a06175c3c992c9bfae31cef2e91" + } + } + ], + "summary": { + "base": { + "completed_cells": 3, + "median_instructions_per_frame": 142725.61793239173, + "min_instructions_per_frame": 138533.06771666187, + "max_instructions_per_frame": 143945.89612732406 + }, + "current": { + "completed_cells": 3, + "median_instructions_per_frame": 154269.3918295209, + "min_instructions_per_frame": 153035.81770559127, + "max_instructions_per_frame": 155433.22787603416 + } + }, + "pull_request_url": "https://github.com/moq-dev/moq/pull/6171" + }, + { + "artifact": "run.FlzdYt", + "description": "Immediately before per-reader ranges", + "base_revision": "dcd7d2cbb891dc1cf72c68295ba70fbd17a4ec15", + "comparison_status": "exploratory comparable-delivery capture", + "harness_head": "9c19f0a6cda900407ee7202b23e8633e95f87857", + "machine": { + "os": "Linux-7.0.0-42-generic-x86_64-with-glibc2.44", + "machine": "x86_64", + "rustc": "rustc 1.98.1 (48a229cea 2026-09-01)\nbinary: rustc\ncommit-hash: 48a229ceaefd4985c50990b14116b6d856af0985\ncommit-date: 2026-09-01\nhost: x86_64-unknown-linux-gnu\nrelease: 1.98.1\nLLVM version: 22.1.8", + "cargo": "cargo 1.98.1 (797e8a9bc 2026-08-05)", + "affinity": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7 + ] + }, + "harness_dirty_patch_sha256": "d06a5759b069b0f7905e2c48b2b0a66c62495be6edd89e6d222a1ba7363be5ee", + "cells": [ + { + "variant": "base", + "round": 1, + "capture": { + "version": 1, + "relay_cpus": [ + 4, + 5, + 6, + 7 + ], + "generator_cpus": [ + 0, + 1, + 2, + 3 + ], + "period": 10000019, + "start_ms": [ + 1793591429532.2722, + 1793591429532.3203 + ], + "end_ms": [ + 1793591434532.479, + 1793591434532.7402 + ], + "threads": { + "2602269": "moq-relay", + "2602272": "tokio-rt-worker", + "2602273": "notify-rs inoti", + "2602274": "moq-uring-0", + "2602275": "moq-uring-1", + "2602276": "moq-uring-2", + "2602277": "moq-uring-3" + }, + "events": [ + "cpu_core/instructions/", + "cpu_core/cycles/" + ], + "sample_rate_limit": 40000, + "window_start_ms": 1793591429532.2964, + "window_end_ms": 1793591434532.6096, + "boundary_uncertainty_ms": [ + 0.048095703125, + 0.26123046875 + ], + "delivered_frames": 100006.2646484375, + "recv_fps": 20000.0, + "counters": { + "cpu_core/instructions/": 10514596796.0, + "cpu_core/cycles/": 19722345269.0, + "task-clock": 3737.32059 + }, + "instructions_per_frame": 105139.38134738922, + "cycles_per_frame": 197211.0981080238, + "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", + "instruction_samples": 1044 + }, + "delivery_boundary_reports": { + "window_start_ms": [ + { + "timestamp_ms": 1793591429332, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 139979, + "frames_sent": 71, + "bytes_recv": 41993700, + "groups_lost": 0, + "groups_expected": 133243, + "groups_present": 133243 + }, + { + "timestamp_ms": 1793591429832, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 149979, + "frames_sent": 75, + "bytes_recv": 44993700, + "groups_lost": 0, + "groups_expected": 143243, + "groups_present": 143243 + } + ], + "window_end_ms": [ + { + "timestamp_ms": 1793591434332, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 239979, + "frames_sent": 120, + "bytes_recv": 71993700, + "groups_lost": 0, + "groups_expected": 233243, + "groups_present": 233243 + }, + { + "timestamp_ms": 1793591434832, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 249979, + "frames_sent": 125, + "bytes_recv": 74993700, + "groups_lost": 0, + "groups_expected": 243243, + "groups_present": 243243 + } + ] + }, + "source_inclusive_samples": { + "resume_learn_end": 0, + "publisher_forward_drop": 65, + "track_state_on_stack": 103, + "resume_reader_on_stack": 115, + "waiter_list_on_stack": 117 + }, + "artifact_sha256": { + "load.jsonl": "18900ab161b6b103a6838c59a1ab282f4d0ed95a4506bdebde572fb8d430640d", + "stat.json": "79a43e7a88d738d8f67eef81c74179fa506b9fc434b8892513e8a25ad87b1e4e", + "window.json": "8032ba2ed5355ea28bcc20dff7a4beaab3d05453600bc99d24d82fc4dbd5b614", + "perf.data": "bc4bd163880aeb6c287f1973c232655b2f31b2654582fecc3a00e75a0fe9ae90", + "instructions.folded": "7ae8ac62057da779e04cbf2dbbe2770de2fc593e4897f8e431418811c41639a1" + } + }, + { + "variant": "base", + "round": 2, + "capture": { + "version": 1, + "relay_cpus": [ + 4, + 5, + 6, + 7 + ], + "generator_cpus": [ + 0, + 1, + 2, + 3 + ], + "period": 10000019, + "start_ms": [ + 1793591599197.893, + 1793591599198.4832 + ], + "end_ms": [ + 1793591604198.6462, + 1793591604199.917 + ], + "threads": { + "2803976": "moq-relay", + "2803981": "tokio-rt-worker", + "2803982": "notify-rs inoti", + "2803983": "moq-uring-0", + "2803984": "moq-uring-1", + "2803985": "moq-uring-2", + "2803986": "moq-uring-3" + }, + "events": [ + "cpu_core/instructions/", + "cpu_core/cycles/" + ], + "sample_rate_limit": 40000, + "window_start_ms": 1793591599198.188, + "window_end_ms": 1793591604199.2817, + "boundary_uncertainty_ms": [ + 0.590087890625, + 1.270751953125 + ], + "delivered_frames": 100009.62176305737, + "recv_fps": 19997.549888573347, + "counters": { + "cpu_core/instructions/": 10578214542.0, + "cpu_core/cycles/": 20004162999.0, + "task-clock": 3805.836228 + }, + "instructions_per_frame": 105771.96829182985, + "cycles_per_frame": 200022.38431012, + "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", + "instruction_samples": 1051 + }, + "delivery_boundary_reports": { + "window_start_ms": [ + { + "timestamp_ms": 1793591599005, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 139284, + "frames_sent": 70, + "bytes_recv": 41785200, + "groups_lost": 0, + "groups_expected": 130545, + "groups_present": 130545 + }, + { + "timestamp_ms": 1793591599505, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 149284, + "frames_sent": 75, + "bytes_recv": 44785200, + "groups_lost": 0, + "groups_expected": 140545, + "groups_present": 140545 + } + ], + "window_end_ms": [ + { + "timestamp_ms": 1793591604006, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 239284, + "frames_sent": 121, + "bytes_recv": 71785200, + "groups_lost": 0, + "groups_expected": 230545, + "groups_present": 230545 + }, + { + "timestamp_ms": 1793591604505, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 249284, + "frames_sent": 125, + "bytes_recv": 74785200, + "groups_lost": 0, + "groups_expected": 240545, + "groups_present": 240545 + } + ] + }, + "source_inclusive_samples": { + "resume_learn_end": 0, + "publisher_forward_drop": 65, + "track_state_on_stack": 73, + "resume_reader_on_stack": 127, + "waiter_list_on_stack": 124 + }, + "artifact_sha256": { + "load.jsonl": "b6a9e9fea814022133474d2c335ed406fe8f1d0e131ff4607a896d9bb4b22d91", + "stat.json": "8caff7a9e8ad0c632ffe28872d69d2341f3f2781be4c941f1973d8411ea298be", + "window.json": "c3324e3a4c2770dba5081272cc953c5d10b0364b2935574e7fa3e44ce50c1d5c", + "perf.data": "00763bf2de5d0c9a836baa7ffda3b649196f60dec428979f51637df88ffd1946", + "instructions.folded": "f4f6b20f2484c48922e21f8f36fcfc3c5cd869af8954c298d8b29728487bace7" + } + }, + { + "variant": "base", + "round": 3, + "capture": { + "version": 1, + "relay_cpus": [ + 4, + 5, + 6, + 7 + ], + "generator_cpus": [ + 0, + 1, + 2, + 3 + ], + "period": 10000019, + "start_ms": [ + 1793591639413.9666, + 1793591639414.2297 + ], + "end_ms": [ + 1793591644414.3, + 1793591644415.768 + ], + "threads": { + "2826068": "moq-relay", + "2826073": "tokio-rt-worker", + "2826074": "notify-rs inoti", + "2826076": "moq-uring-0", + "2826077": "moq-uring-1", + "2826078": "moq-uring-2", + "2826079": "moq-uring-3" + }, + "events": [ + "cpu_core/instructions/", + "cpu_core/cycles/" + ], + "sample_rate_limit": 40000, + "window_start_ms": 1793591639414.0981, + "window_end_ms": 1793591644415.0342, + "boundary_uncertainty_ms": [ + 0.26318359375, + 1.468017578125 + ], + "delivered_frames": 100031.09458816741, + "recv_fps": 20002.474313799543, + "counters": { + "cpu_core/instructions/": 10918608415.0, + "cpu_core/cycles/": 19942807466.0, + "task-clock": 3816.712397 + }, + "instructions_per_frame": 109152.14374042802, + "cycles_per_frame": 199366.08259767076, + "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", + "instruction_samples": 1086 + }, + "delivery_boundary_reports": { + "window_start_ms": [ + { + "timestamp_ms": 1793591639225, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 139984, + "frames_sent": 71, + "bytes_recv": 41995200, + "groups_lost": 0, + "groups_expected": 132784, + "groups_present": 132784 + }, + { + "timestamp_ms": 1793591639725, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 149984, + "frames_sent": 76, + "bytes_recv": 44995200, + "groups_lost": 0, + "groups_expected": 142784, + "groups_present": 142784 + } + ], + "window_end_ms": [ + { + "timestamp_ms": 1793591644224, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 239984, + "frames_sent": 121, + "bytes_recv": 71995200, + "groups_lost": 0, + "groups_expected": 232784, + "groups_present": 232784 + }, + { + "timestamp_ms": 1793591644725, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 249984, + "frames_sent": 126, + "bytes_recv": 74995200, + "groups_lost": 0, + "groups_expected": 242784, + "groups_present": 242784 + } + ] + }, + "source_inclusive_samples": { + "resume_learn_end": 0, + "publisher_forward_drop": 61, + "track_state_on_stack": 76, + "resume_reader_on_stack": 122, + "waiter_list_on_stack": 107 + }, + "artifact_sha256": { + "load.jsonl": "472fe13808cba4aefb9e0d312b1a74968118656a0ec2a97ff9bddad24ff88b7b", + "stat.json": "73fa91b9a99d0fed5074fa8aca44d9afaea4bea38bcca59d22941c141dc526f5", + "window.json": "0d12ad459a77b1b71ccb69d91e48c0ec0f76625e9c20c11499a305041e0040ae", + "perf.data": "01e1d99f7660e2e290c18f21e7c9c219cdaf7bfc659579b74bd80220cbd6c85d", + "instructions.folded": "aa738de3508117eae48aa0b5a95d0359d3170cb3d80c1195a7a7c3dc5f2c8da4" + } + }, + { + "variant": "current", + "round": 1, + "capture": { + "version": 1, + "relay_cpus": [ + 4, + 5, + 6, + 7 + ], + "generator_cpus": [ + 0, + 1, + 2, + 3 + ], + "period": 10000019, + "start_ms": [ + 1793591468080.8552, + 1793591468080.9146 + ], + "end_ms": [ + 1793591473080.9873, + 1793591473081.6592 + ], + "threads": { + "2614759": "moq-relay", + "2614770": "tokio-rt-worker", + "2614774": "notify-rs inoti", + "2614776": "moq-uring-0", + "2614777": "moq-uring-1", + "2614779": "moq-uring-2", + "2614780": "moq-uring-3" + }, + "events": [ + "cpu_core/instructions/", + "cpu_core/cycles/" + ], + "sample_rate_limit": 40000, + "window_start_ms": 1793591468080.8848, + "window_end_ms": 1793591473081.3232, + "boundary_uncertainty_ms": [ + 0.059326171875, + 0.671875 + ], + "delivered_frames": 100008.76953125, + "recv_fps": 20000.0, + "counters": { + "cpu_core/instructions/": 15745630432.0, + "cpu_core/cycles/": 29240252466.0, + "task-clock": 5579.799099 + }, + "instructions_per_frame": 157442.49735099403, + "cycles_per_frame": 292376.88457773917, + "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", + "instruction_samples": 1569 + }, + "delivery_boundary_reports": { + "window_start_ms": [ + { + "timestamp_ms": 1793591467895, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 139973, + "frames_sent": 70, + "bytes_recv": 41991900, + "groups_lost": 0, + "groups_expected": 135570, + "groups_present": 135570 + }, + { + "timestamp_ms": 1793591468395, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 149973, + "frames_sent": 75, + "bytes_recv": 44991900, + "groups_lost": 0, + "groups_expected": 145570, + "groups_present": 145570 + } + ], + "window_end_ms": [ + { + "timestamp_ms": 1793591472895, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 239973, + "frames_sent": 121, + "bytes_recv": 71991900, + "groups_lost": 0, + "groups_expected": 235570, + "groups_present": 235570 + }, + { + "timestamp_ms": 1793591473395, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 249973, + "frames_sent": 125, + "bytes_recv": 74991900, + "groups_lost": 0, + "groups_expected": 245570, + "groups_present": 245570 + } + ] + }, + "source_inclusive_samples": { + "resume_learn_end": 17, + "publisher_forward_drop": 321, + "track_state_on_stack": 256, + "resume_reader_on_stack": 234, + "waiter_list_on_stack": 155 + }, + "artifact_sha256": { + "load.jsonl": "92e49d1c7d0a81ffebe9c28d86ecfb80917afef109a3847907d2bd487cc4e23e", + "stat.json": "34efb2855ff09efbb27ed589ecc25359ff827b64dd9cd2cd97f837fff75a7cb5", + "window.json": "ab234eb58e797c67fa6e89fd5774eb3b0ef08aed1397dd49f6b78c226c173f69", + "perf.data": "66e5eebf77a530a074faf7be3e7605c6035cebb76a98bd79aa44bf65f9613060", + "instructions.folded": "80769aa1222e4fc20cea327ba70f98bd73bf03fc81f098e2358bcfed3ee47dd6" + } + }, + { + "variant": "current", + "round": 2, + "capture": { + "version": 1, + "relay_cpus": [ + 4, + 5, + 6, + 7 + ], + "generator_cpus": [ + 0, + 1, + 2, + 3 + ], + "period": 10000019, + "start_ms": [ + 1793591532508.8257, + 1793591532508.883 + ], + "end_ms": [ + 1793591537508.9658, + 1793591537510.3801 + ], + "threads": { + "2714794": "moq-relay", + "2714801": "tokio-rt-worker", + "2714806": "notify-rs inoti", + "2714807": "moq-uring-0", + "2714808": "moq-uring-1", + "2714810": "moq-uring-2", + "2714812": "moq-uring-3" + }, + "events": [ + "cpu_core/instructions/", + "cpu_core/cycles/" + ], + "sample_rate_limit": 40000, + "window_start_ms": 1793591532508.8545, + "window_end_ms": 1793591537509.6729, + "boundary_uncertainty_ms": [ + 0.057373046875, + 1.414306640625 + ], + "delivered_frames": 100016.39998747496, + "recv_fps": 20000.006558921483, + "counters": { + "cpu_core/instructions/": 16096769995.0, + "cpu_core/cycles/": 32341038169.0, + "task-clock": 6182.029905 + }, + "instructions_per_frame": 160941.3055960402, + "cycles_per_frame": 323357.35112491617, + "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", + "instruction_samples": 1603 + }, + "delivery_boundary_reports": { + "window_start_ms": [ + { + "timestamp_ms": 1793591532328, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 139962, + "frames_sent": 70, + "bytes_recv": 41988600, + "groups_lost": 0, + "groups_expected": 135340, + "groups_present": 135340 + }, + { + "timestamp_ms": 1793591532827, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 149962, + "frames_sent": 75, + "bytes_recv": 44988600, + "groups_lost": 0, + "groups_expected": 145340, + "groups_present": 145340 + } + ], + "window_end_ms": [ + { + "timestamp_ms": 1793591537328, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 239962, + "frames_sent": 120, + "bytes_recv": 71988600, + "groups_lost": 0, + "groups_expected": 235340, + "groups_present": 235340 + }, + { + "timestamp_ms": 1793591537827, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 249962, + "frames_sent": 125, + "bytes_recv": 74988600, + "groups_lost": 0, + "groups_expected": 245340, + "groups_present": 245340 + } + ] + }, + "source_inclusive_samples": { + "resume_learn_end": 12, + "publisher_forward_drop": 355, + "track_state_on_stack": 280, + "resume_reader_on_stack": 242, + "waiter_list_on_stack": 145 + }, + "artifact_sha256": { + "load.jsonl": "9bcd7ca7519beeb948685fc6b70604fb379e76d2a8f829cac879baf9456041a4", + "stat.json": "d8b9216a860adb4284d21b95dbfc1d2087a7567c4177344228c4810a04e02ce5", + "window.json": "a2722113b86f554bc3f8647f5de5f08c53a26388d76b0aad8134eedf6122e2db", + "perf.data": "5b46ef6d2c8733f7b497e059d3b821f55a9c81ee097ac4afa778eb3cf690641c", + "instructions.folded": "e9cafe61fd5a4c5d1d94221be4627140e114e009bf7ff7eaa17b6170953d193b" + } + }, + { + "variant": "current", + "round": 3, + "capture": { + "version": 1, + "relay_cpus": [ + 4, + 5, + 6, + 7 + ], + "generator_cpus": [ + 0, + 1, + 2, + 3 + ], + "period": 10000019, + "start_ms": [ + 1793591678816.9011, + 1793591678816.9712 + ], + "end_ms": [ + 1793591683817.052, + 1793591683817.2087 + ], + "threads": { + "2834707": "moq-relay", + "2834710": "tokio-rt-worker", + "2834711": "notify-rs inoti", + "2834712": "moq-uring-0", + "2834713": "moq-uring-1", + "2834714": "moq-uring-2", + "2834715": "moq-uring-3" + }, + "events": [ + "cpu_core/instructions/", + "cpu_core/cycles/" + ], + "sample_rate_limit": 40000, + "window_start_ms": 1793591678816.936, + "window_end_ms": 1793591683817.1304, + "boundary_uncertainty_ms": [ + 0.070068359375, + 0.15673828125 + ], + "delivered_frames": 99126.18189792291, + "recv_fps": 19824.465858352978, + "counters": { + "cpu_core/instructions/": 15546877923.0, + "cpu_core/cycles/": 31677863143.0, + "task-clock": 5994.939972 + }, + "instructions_per_frame": 156839.26915504216, + "cycles_per_frame": 319571.10156447755, + "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", + "instruction_samples": 1546 + }, + "delivery_boundary_reports": { + "window_start_ms": [ + { + "timestamp_ms": 1793591678634, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 139983, + "frames_sent": 70, + "bytes_recv": 41994900, + "groups_lost": 0, + "groups_expected": 135950, + "groups_present": 135950 + }, + { + "timestamp_ms": 1793591679134, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 149983, + "frames_sent": 75, + "bytes_recv": 44994900, + "groups_lost": 0, + "groups_expected": 145950, + "groups_present": 145950 + } + ], + "window_end_ms": [ + { + "timestamp_ms": 1793591683634, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 238943, + "frames_sent": 120, + "bytes_recv": 71682900, + "groups_lost": 0, + "groups_expected": 234910, + "groups_present": 234910 + }, + { + "timestamp_ms": 1793591684135, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 249407, + "frames_sent": 125, + "bytes_recv": 74822100, + "groups_lost": 0, + "groups_expected": 245374, + "groups_present": 245374 + } + ] + }, + "source_inclusive_samples": { + "resume_learn_end": 14, + "publisher_forward_drop": 273, + "track_state_on_stack": 245, + "resume_reader_on_stack": 181, + "waiter_list_on_stack": 157 + }, + "artifact_sha256": { + "load.jsonl": "8f7fc5db17c0fe9791dd998525a2197963ece459f8a0a368ca70bff5deaa342a", + "stat.json": "623cb58ccf32bbf612a62b333f636024e78843e5393eef394f97c63070f16820", + "window.json": "6537835011e0e20d9b2d0f8734b3f1e6f2a6961a7d98dd16f3a0180dc693e30d", + "perf.data": "f7c49a286fc4c6f8090b7a9de2670fa2736d4c2e1d7fd40aae4664e95efc33ad", + "instructions.folded": "8ba42bd224b9579db2b12345fe8d77683bd2e200e7b8a80500fb3ba79a2271d9" + } + } + ], + "summary": { + "base": { + "completed_cells": 3, + "median_instructions_per_frame": 105771.96829182985, + "min_instructions_per_frame": 105139.38134738922, + "max_instructions_per_frame": 109152.14374042802 + }, + "current": { + "completed_cells": 3, + "median_instructions_per_frame": 157442.49735099403, + "min_instructions_per_frame": 156839.26915504216, + "max_instructions_per_frame": 160941.3055960402 + } + }, + "pull_request_url": "https://github.com/moq-dev/moq/pull/5929" + }, + { + "artifact": "run.2bOgus", + "description": "Per-reader ranges", + "base_revision": "fe722cec9f8432e0e46cb633cc5e79e59e7724cb", + "comparison_status": "exploratory comparable-delivery capture", + "harness_head": "9c19f0a6cda900407ee7202b23e8633e95f87857", + "machine": { + "os": "Linux-7.0.0-42-generic-x86_64-with-glibc2.44", + "machine": "x86_64", + "rustc": "rustc 1.98.1 (48a229cea 2026-09-01)\nbinary: rustc\ncommit-hash: 48a229ceaefd4985c50990b14116b6d856af0985\ncommit-date: 2026-09-01\nhost: x86_64-unknown-linux-gnu\nrelease: 1.98.1\nLLVM version: 22.1.8", + "cargo": "cargo 1.98.1 (797e8a9bc 2026-08-05)", + "affinity": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7 + ] + }, + "harness_dirty_patch_sha256": "1fb2250488672e6005088a1008ec311575d40c0a8ef4546a592f4ca2acea68a0", + "cells": [ + { + "variant": "base", + "round": 1, + "capture": { + "version": 1, + "relay_cpus": [ + 4, + 5, + 6, + 7 + ], + "generator_cpus": [ + 0, + 1, + 2, + 3 + ], + "period": 10000019, + "start_ms": [ + 1793592726175.0137, + 1793592726175.0774 + ], + "end_ms": [ + 1793592731175.1494, + 1793592731176.1135 + ], + "threads": { + "3403256": "moq-relay", + "3403259": "tokio-rt-worker", + "3403260": "notify-rs inoti", + "3403261": "moq-uring-0", + "3403262": "moq-uring-1", + "3403263": "moq-uring-2", + "3403264": "moq-uring-3" + }, + "events": [ + "cpu_core/instructions/", + "cpu_core/cycles/" + ], + "sample_rate_limit": 28000, + "window_start_ms": 1793592726175.0454, + "window_end_ms": 1793592731175.6313, + "boundary_uncertainty_ms": [ + 0.063720703125, + 0.964111328125 + ], + "delivered_frames": 99996.17166437415, + "recv_fps": 19996.890947216954, + "counters": { + "cpu_core/instructions/": 11314632417.0, + "cpu_core/cycles/": 21542189727.0, + "task-clock": 4131.846034 + }, + "instructions_per_frame": 113150.65595687287, + "cycles_per_frame": 215430.1446589768, + "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", + "instruction_samples": 1123 + }, + "delivery_boundary_reports": { + "window_start_ms": [ + { + "timestamp_ms": 1793592725981, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 139982, + "frames_sent": 70, + "bytes_recv": 41994600, + "groups_lost": 0, + "groups_expected": 133966, + "groups_present": 133966 + }, + { + "timestamp_ms": 1793592726480, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 149982, + "frames_sent": 75, + "bytes_recv": 44994600, + "groups_lost": 0, + "groups_expected": 143966, + "groups_present": 143966 + } + ], + "window_end_ms": [ + { + "timestamp_ms": 1793592730981, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 239982, + "frames_sent": 121, + "bytes_recv": 71994600, + "groups_lost": 0, + "groups_expected": 233966, + "groups_present": 233966 + }, + { + "timestamp_ms": 1793592731482, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 249982, + "frames_sent": 126, + "bytes_recv": 74994600, + "groups_lost": 0, + "groups_expected": 243966, + "groups_present": 243966 + } + ] + }, + "source_inclusive_samples": { + "resume_learn_end": 0, + "publisher_forward_drop": 75, + "track_state_on_stack": 118, + "resume_reader_on_stack": 137, + "waiter_list_on_stack": 111 + }, + "artifact_sha256": { + "load.jsonl": "dd5f67f709e6c35dbcfb529d1f45523d14a87983cf086221e937185e7d523227", + "stat.json": "7159bb7b77f8233904824d9dc4c2f2e276b718306c1ed2be6f7c23130c3d39eb", + "window.json": "2e417fc62091c4383c0b9ff0e68798c044c60445fd9c56fc96f44a25baa99909", + "perf.data": "8d3014e68d9f71df3b60c037ad04c6e321f63fabfd1ae6071baa440d45e77e81", + "instructions.folded": "9eade372a947195f6deb137fab1ea7f6d42667d0cca7ce819290baa9237dd91e" + } + }, + { + "variant": "base", + "round": 2, + "capture": { + "version": 1, + "relay_cpus": [ + 4, + 5, + 6, + 7 + ], + "generator_cpus": [ + 0, + 1, + 2, + 3 + ], + "period": 10000019, + "start_ms": [ + 1793592897069.7192, + 1793592897069.8096 + ], + "end_ms": [ + 1793592902069.882, + 1793592902070.324 + ], + "threads": { + "3458033": "moq-relay", + "3458036": "tokio-rt-worker", + "3458037": "notify-rs inoti", + "3458038": "moq-uring-0", + "3458039": "moq-uring-1", + "3458040": "moq-uring-2", + "3458041": "moq-uring-3" + }, + "events": [ + "cpu_core/instructions/", + "cpu_core/cycles/" + ], + "sample_rate_limit": 28000, + "window_start_ms": 1793592897069.7644, + "window_end_ms": 1793592902070.103, + "boundary_uncertainty_ms": [ + 0.09033203125, + 0.44189453125 + ], + "delivered_frames": 100006.7724609375, + "recv_fps": 20000.0, + "counters": { + "cpu_core/instructions/": 11211395662.0, + "cpu_core/cycles/": 21548382019.0, + "task-clock": 4110.358241 + }, + "instructions_per_frame": 112106.36426027202, + "cycles_per_frame": 215469.22762072706, + "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", + "instruction_samples": 1116 + }, + "delivery_boundary_reports": { + "window_start_ms": [ + { + "timestamp_ms": 1793592896879, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 139986, + "frames_sent": 70, + "bytes_recv": 41995800, + "groups_lost": 0, + "groups_expected": 132685, + "groups_present": 132685 + }, + { + "timestamp_ms": 1793592897379, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 149986, + "frames_sent": 75, + "bytes_recv": 44995800, + "groups_lost": 0, + "groups_expected": 142685, + "groups_present": 142685 + } + ], + "window_end_ms": [ + { + "timestamp_ms": 1793592901879, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 239986, + "frames_sent": 120, + "bytes_recv": 71995800, + "groups_lost": 0, + "groups_expected": 232685, + "groups_present": 232685 + }, + { + "timestamp_ms": 1793592902379, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 249986, + "frames_sent": 125, + "bytes_recv": 74995800, + "groups_lost": 0, + "groups_expected": 242685, + "groups_present": 242685 + } + ] + }, + "source_inclusive_samples": { + "resume_learn_end": 0, + "publisher_forward_drop": 84, + "track_state_on_stack": 110, + "resume_reader_on_stack": 148, + "waiter_list_on_stack": 115 + }, + "artifact_sha256": { + "load.jsonl": "636e399eabec69a817967b8c175519d31170de69b1a14092a2a97ccf259dbc8d", + "stat.json": "5f83462da4826283b2207df484c53cc0f07ba4ffb4169d59bd6eaffb84872431", + "window.json": "ac0e663e767e2c28da162cc5b969d468facc4f9c7126ccc5fde6d34ea2e8f0b0", + "perf.data": "12e271b74d443587a22b2c1e90fb262bd0c5d16e78b2658a1697da01cafa8eac", + "instructions.folded": "38cd2b29e52051621eed50956c71081e9fbbd13a7f7f3cf2bd25f79a8f47432c" + } + }, + { + "variant": "base", + "round": 3, + "capture": { + "version": 1, + "relay_cpus": [ + 4, + 5, + 6, + 7 + ], + "generator_cpus": [ + 0, + 1, + 2, + 3 + ], + "period": 10000019, + "start_ms": [ + 1793592937340.704, + 1793592937340.7566 + ], + "end_ms": [ + 1793592942340.8152, + 1793592942340.9216 + ], + "threads": { + "3463175": "moq-relay", + "3463178": "tokio-rt-worker", + "3463179": "notify-rs inoti", + "3463180": "moq-uring-0", + "3463181": "moq-uring-1", + "3463182": "moq-uring-2", + "3463183": "moq-uring-3" + }, + "events": [ + "cpu_core/instructions/", + "cpu_core/cycles/" + ], + "sample_rate_limit": 28000, + "window_start_ms": 1793592937340.7305, + "window_end_ms": 1793592942340.8684, + "boundary_uncertainty_ms": [ + 0.052490234375, + 0.1064453125 + ], + "delivered_frames": 99990.28858498245, + "recv_fps": 19997.50602798742, + "counters": { + "cpu_core/instructions/": 11190626305.0, + "cpu_core/cycles/": 21607081135.0, + "task-clock": 4097.69505 + }, + "instructions_per_frame": 111917.13178714359, + "cycles_per_frame": 216091.7969212179, + "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", + "instruction_samples": 1113 + }, + "delivery_boundary_reports": { + "window_start_ms": [ + { + "timestamp_ms": 1793592937152, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 140000, + "frames_sent": 70, + "bytes_recv": 42000000, + "groups_lost": 0, + "groups_expected": 132879, + "groups_present": 132879 + }, + { + "timestamp_ms": 1793592937652, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 150000, + "frames_sent": 75, + "bytes_recv": 45000000, + "groups_lost": 0, + "groups_expected": 142879, + "groups_present": 142879 + } + ], + "window_end_ms": [ + { + "timestamp_ms": 1793592942153, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 240000, + "frames_sent": 120, + "bytes_recv": 72000000, + "groups_lost": 0, + "groups_expected": 232879, + "groups_present": 232879 + }, + { + "timestamp_ms": 1793592942652, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 250000, + "frames_sent": 125, + "bytes_recv": 75000000, + "groups_lost": 0, + "groups_expected": 242879, + "groups_present": 242879 + } + ] + }, + "source_inclusive_samples": { + "resume_learn_end": 0, + "publisher_forward_drop": 78, + "track_state_on_stack": 97, + "resume_reader_on_stack": 140, + "waiter_list_on_stack": 87 + }, + "artifact_sha256": { + "load.jsonl": "7a6d9e0130d91ce0e8441bded0444a0e4e1db1878d5162c60a4c61f8ef905c63", + "stat.json": "07fa293123ba4198fa9377c53ddb3937d81aa354ea56e3c209b92883f6d584b1", + "window.json": "f421bf6b4a1e522ed5c2dc3bd93235f9179e7a4fac7a9d0ec96c45caed841d33", + "perf.data": "55985e651bb8ffff4a72ed340112fc373fefc5674f4fd78780cecd47ebb0554a", + "instructions.folded": "ab14407a6cdd7b41f6fbd07f206e74ba217fda1ab2460dbfc0b7e74b3d456211" + } + }, + { + "variant": "current", + "round": 1, + "capture": { + "version": 1, + "relay_cpus": [ + 4, + 5, + 6, + 7 + ], + "generator_cpus": [ + 0, + 1, + 2, + 3 + ], + "period": 10000019, + "start_ms": [ + 1793592766105.59, + 1793592766105.6628 + ], + "end_ms": [ + 1793592771105.8254, + 1793592771107.3223 + ], + "threads": { + "3415357": "moq-relay", + "3415362": "tokio-rt-worker", + "3415363": "notify-rs inoti", + "3415365": "moq-uring-0", + "3415366": "moq-uring-1", + "3415367": "moq-uring-2", + "3415368": "moq-uring-3" + }, + "events": [ + "cpu_core/instructions/", + "cpu_core/cycles/" + ], + "sample_rate_limit": 28000, + "window_start_ms": 1793592766105.6265, + "window_end_ms": 1793592771106.5737, + "boundary_uncertainty_ms": [ + 0.07275390625, + 1.496826171875 + ], + "delivered_frames": 100018.9453125, + "recv_fps": 20000.0, + "counters": { + "cpu_core/instructions/": 15711169720.0, + "cpu_core/cycles/": 28031733524.0, + "task-clock": 5358.969932 + }, + "instructions_per_frame": 157081.93753605275, + "cycles_per_frame": 280264.23830422753, + "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", + "instruction_samples": 1564 + }, + "delivery_boundary_reports": { + "window_start_ms": [ + { + "timestamp_ms": 1793592765920, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 139980, + "frames_sent": 71, + "bytes_recv": 41994000, + "groups_lost": 0, + "groups_expected": 136160, + "groups_present": 136160 + }, + { + "timestamp_ms": 1793592766420, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 149980, + "frames_sent": 75, + "bytes_recv": 44994000, + "groups_lost": 0, + "groups_expected": 146160, + "groups_present": 146160 + } + ], + "window_end_ms": [ + { + "timestamp_ms": 1793592770920, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 239980, + "frames_sent": 120, + "bytes_recv": 71994000, + "groups_lost": 0, + "groups_expected": 236160, + "groups_present": 236160 + }, + { + "timestamp_ms": 1793592771420, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 249980, + "frames_sent": 125, + "bytes_recv": 74994000, + "groups_lost": 0, + "groups_expected": 246160, + "groups_present": 246160 + } + ] + }, + "source_inclusive_samples": { + "resume_learn_end": 11, + "publisher_forward_drop": 318, + "track_state_on_stack": 260, + "resume_reader_on_stack": 222, + "waiter_list_on_stack": 171 + }, + "artifact_sha256": { + "load.jsonl": "bcbfc7727bc9bd01bb59d33ef9002e8db4e518b04b59d627e4c2344d6853b993", + "stat.json": "1cef97e8ac64b5763d8f1ce1029dc682d03ef7b043c02eaeb90bf6d42a94bb08", + "window.json": "1864085828f6ddcff3f0da2960d72f739059d2e8340d8cb45c1c48822db6ff07", + "perf.data": "270b786d8279bca3f2d63bb7f1adbbc7eba0f476051965bf1713d4caad7be6ac", + "instructions.folded": "1af64c071fcb65c83f2747e683c8d66dfa416b4ef72bf66817b7778df0efd0c2" + } + }, + { + "variant": "current", + "round": 2, + "capture": { + "version": 1, + "relay_cpus": [ + 4, + 5, + 6, + 7 + ], + "generator_cpus": [ + 0, + 1, + 2, + 3 + ], + "period": 10000019, + "start_ms": [ + 1793592831708.72, + 1793592831708.791 + ], + "end_ms": [ + 1793592836708.8792, + 1793592836710.3916 + ], + "threads": { + "3447207": "moq-relay", + "3447213": "tokio-rt-worker", + "3447215": "notify-rs inoti", + "3447217": "moq-uring-0", + "3447218": "moq-uring-1", + "3447219": "moq-uring-2", + "3447220": "moq-uring-3" + }, + "events": [ + "cpu_core/instructions/", + "cpu_core/cycles/" + ], + "sample_rate_limit": 28000, + "window_start_ms": 1793592831708.7554, + "window_end_ms": 1793592836709.6353, + "boundary_uncertainty_ms": [ + 0.071044921875, + 1.512451171875 + ], + "delivered_frames": 100017.59765625, + "recv_fps": 20000.0, + "counters": { + "cpu_core/instructions/": 16477354135.0, + "cpu_core/cycles/": 31166122602.0, + "task-clock": 6008.284902 + }, + "instructions_per_frame": 164744.5501703704, + "cycles_per_frame": 311606.3905985294, + "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", + "instruction_samples": 1637 + }, + "delivery_boundary_reports": { + "window_start_ms": [ + { + "timestamp_ms": 1793592831520, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 139971, + "frames_sent": 70, + "bytes_recv": 41991300, + "groups_lost": 0, + "groups_expected": 136123, + "groups_present": 136123 + }, + { + "timestamp_ms": 1793592832020, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 149971, + "frames_sent": 75, + "bytes_recv": 44991300, + "groups_lost": 0, + "groups_expected": 146123, + "groups_present": 146123 + } + ], + "window_end_ms": [ + { + "timestamp_ms": 1793592836520, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 239971, + "frames_sent": 120, + "bytes_recv": 71991300, + "groups_lost": 0, + "groups_expected": 236123, + "groups_present": 236123 + }, + { + "timestamp_ms": 1793592837020, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 249971, + "frames_sent": 125, + "bytes_recv": 74991300, + "groups_lost": 0, + "groups_expected": 246123, + "groups_present": 246123 + } + ] + }, + "source_inclusive_samples": { + "resume_learn_end": 18, + "publisher_forward_drop": 318, + "track_state_on_stack": 253, + "resume_reader_on_stack": 248, + "waiter_list_on_stack": 150 + }, + "artifact_sha256": { + "load.jsonl": "d976e8be43a5aea576bd68852a7ec3ba2c9962e00ac40a7facf35e9a00533713", + "stat.json": "c64bac5a7448524b4c6c13779b5a4ec0ac63077d025c0f3ba69e6a56c2508685", + "window.json": "970554cc01826cc3d020ab305155874062652f84b8860f9edd18702e2c89a6e1", + "perf.data": "d4fd6439d364fc468c07937048b55b5ceafe5732b9d2b97fc194a5e5e3b30797", + "instructions.folded": "c8afed451d97b180fcf9b579eb6d7a5e993c6716882ff209469ce7d5fe8ef586" + } + }, + { + "variant": "current", + "round": 3, + "capture": { + "version": 1, + "relay_cpus": [ + 4, + 5, + 6, + 7 + ], + "generator_cpus": [ + 0, + 1, + 2, + 3 + ], + "period": 10000019, + "start_ms": [ + 1793592977142.9905, + 1793592977143.1973 + ], + "end_ms": [ + 1793592982143.2788, + 1793592982143.3743 + ], + "threads": { + "3470232": "moq-relay", + "3470235": "tokio-rt-worker", + "3470236": "notify-rs inoti", + "3470237": "moq-uring-0", + "3470238": "moq-uring-1", + "3470239": "moq-uring-2", + "3470240": "moq-uring-3" + }, + "events": [ + "cpu_core/instructions/", + "cpu_core/cycles/" + ], + "sample_rate_limit": 28000, + "window_start_ms": 1793592977143.0938, + "window_end_ms": 1793592982143.3267, + "boundary_uncertainty_ms": [ + 0.206787109375, + 0.095458984375 + ], + "delivered_frames": 99442.91801408317, + "recv_fps": 19887.657195347674, + "counters": { + "cpu_core/instructions/": 15483800976.0, + "cpu_core/cycles/": 31019643084.0, + "task-clock": 5846.34583 + }, + "instructions_per_frame": 155705.41658690237, + "cycles_per_frame": 311934.15985245904, + "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", + "instruction_samples": 1543 + }, + "delivery_boundary_reports": { + "window_start_ms": [ + { + "timestamp_ms": 1793592976959, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 139959, + "frames_sent": 71, + "bytes_recv": 41987700, + "groups_lost": 0, + "groups_expected": 135047, + "groups_present": 135047 + }, + { + "timestamp_ms": 1793592977460, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 149348, + "frames_sent": 75, + "bytes_recv": 44804400, + "groups_lost": 0, + "groups_expected": 144436, + "groups_present": 144436 + } + ], + "window_end_ms": [ + { + "timestamp_ms": 1793592981959, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 238702, + "frames_sent": 120, + "bytes_recv": 71610600, + "groups_lost": 0, + "groups_expected": 233790, + "groups_present": 233790 + }, + { + "timestamp_ms": 1793592982459, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 249959, + "frames_sent": 126, + "bytes_recv": 74987700, + "groups_lost": 0, + "groups_expected": 245047, + "groups_present": 245047 + } + ] + }, + "source_inclusive_samples": { + "resume_learn_end": 4, + "publisher_forward_drop": 288, + "track_state_on_stack": 254, + "resume_reader_on_stack": 238, + "waiter_list_on_stack": 144 + }, + "artifact_sha256": { + "load.jsonl": "3915b34852bc9c694f2ca177bd52fcea6f0dffecad5b070030731ad7a26bd1d9", + "stat.json": "9f51ef443296da6fec4d138bdc430f37920947f7b73241054cc1f908ad9d7fc1", + "window.json": "aeb0d534b8bad83ba4c5b0c73bd01ac48fd34b95b06eb455f5d4f24642d8fe5b", + "perf.data": "db28232f4f94f8b8d33315c6dc6a6c7344a109af9efc2ebea73cb38743d7b8bb", + "instructions.folded": "211cbe80cbe847e693fc16edd69565cefd5435a7c5b512ceb7bd502b96530be7" + } + } + ], + "summary": { + "base": { + "completed_cells": 3, + "median_instructions_per_frame": 112106.36426027202, + "min_instructions_per_frame": 111917.13178714359, + "max_instructions_per_frame": 113150.65595687287 + }, + "current": { + "completed_cells": 3, + "median_instructions_per_frame": 157081.93753605275, + "min_instructions_per_frame": 155705.41658690237, + "max_instructions_per_frame": 164744.5501703704 + } + }, + "pull_request_url": "https://github.com/moq-dev/moq/pull/5929" + }, + { + "artifact": "run.iFmfVR", + "description": "Immediately before resume end tracking", + "base_revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "comparison_status": "exploratory comparable-delivery capture", + "harness_head": "9c19f0a6cda900407ee7202b23e8633e95f87857", + "machine": { + "os": "Linux-7.0.0-42-generic-x86_64-with-glibc2.44", + "machine": "x86_64", + "rustc": "rustc 1.98.1 (48a229cea 2026-09-01)\nbinary: rustc\ncommit-hash: 48a229ceaefd4985c50990b14116b6d856af0985\ncommit-date: 2026-09-01\nhost: x86_64-unknown-linux-gnu\nrelease: 1.98.1\nLLVM version: 22.1.8", + "cargo": "cargo 1.98.1 (797e8a9bc 2026-08-05)", + "affinity": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7 + ] + }, + "harness_dirty_patch_sha256": "1fb2250488672e6005088a1008ec311575d40c0a8ef4546a592f4ca2acea68a0", + "cells": [ + { + "variant": "base", + "round": 1, + "capture": { + "version": 1, + "relay_cpus": [ + 4, + 5, + 6, + 7 + ], + "generator_cpus": [ + 0, + 1, + 2, + 3 + ], + "period": 10000019, + "start_ms": [ + 1793593639871.8943, + 1793593639871.9539 + ], + "end_ms": [ + 1793593644872.0403, + 1793593644872.16 + ], + "threads": { + "3853743": "moq-relay", + "3853746": "tokio-rt-worker", + "3853747": "notify-rs inoti", + "3853748": "moq-uring-0", + "3853749": "moq-uring-1", + "3853750": "moq-uring-2", + "3853751": "moq-uring-3" + }, + "events": [ + "cpu_core/instructions/", + "cpu_core/cycles/" + ], + "sample_rate_limit": 34000, + "window_start_ms": 1793593639871.924, + "window_end_ms": 1793593644872.1, + "boundary_uncertainty_ms": [ + 0.0595703125, + 0.11962890625 + ], + "delivered_frames": 100011.54142886103, + "recv_fps": 20001.604127736264, + "counters": { + "cpu_core/instructions/": 12769024351.0, + "cpu_core/cycles/": 26050400250.0, + "task-clock": 4972.217921 + }, + "instructions_per_frame": 127675.50793207905, + "cycles_per_frame": 260473.9400855035, + "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", + "instruction_samples": 1271 + }, + "delivery_boundary_reports": { + "window_start_ms": [ + { + "timestamp_ms": 1793593639671, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 139982, + "frames_sent": 70, + "bytes_recv": 41994600, + "groups_lost": 0, + "groups_expected": 133374, + "groups_present": 133374 + }, + { + "timestamp_ms": 1793593640172, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 149982, + "frames_sent": 75, + "bytes_recv": 44994600, + "groups_lost": 0, + "groups_expected": 143374, + "groups_present": 143374 + } + ], + "window_end_ms": [ + { + "timestamp_ms": 1793593644671, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 239982, + "frames_sent": 120, + "bytes_recv": 71994600, + "groups_lost": 0, + "groups_expected": 233374, + "groups_present": 233374 + }, + { + "timestamp_ms": 1793593645171, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 249982, + "frames_sent": 125, + "bytes_recv": 74994600, + "groups_lost": 0, + "groups_expected": 243374, + "groups_present": 243374 + } + ] + }, + "source_inclusive_samples": { + "resume_learn_end": 0, + "publisher_forward_drop": 76, + "track_state_on_stack": 121, + "resume_reader_on_stack": 236, + "waiter_list_on_stack": 123 + }, + "artifact_sha256": { + "load.jsonl": "1596865acaad0a890e1f47e22a42e43bcdc3f524fa58f935aedd9306fbe2b805", + "stat.json": "429263049c8d76b81daba010f9ea863ee60de0c85193ad08fd24eaf0938b3618", + "window.json": "c0266e3c8571a67fc691b8edb7595fa325fe3b6c543801e1cff7c53bc9447923", + "perf.data": "00ec413ee9809d3b968545996a4dc1021cb0e7b5fd7956d4c7b0e369c57f98aa", + "instructions.folded": "2b020f70432710eb4a264d1e7ae8fb244097f9f2ee79c066eef8b1a3ed85472c" + } + }, + { + "variant": "base", + "round": 2, + "capture": { + "version": 1, + "relay_cpus": [ + 4, + 5, + 6, + 7 + ], + "generator_cpus": [ + 0, + 1, + 2, + 3 + ], + "period": 10000019, + "start_ms": [ + 1793593814922.583, + 1793593814922.6758 + ], + "end_ms": [ + 1793593819922.7593, + 1793593819923.0583 + ], + "threads": { + "3882011": "moq-relay", + "3882014": "tokio-rt-worker", + "3882015": "notify-rs inoti", + "3882016": "moq-uring-0", + "3882017": "moq-uring-1", + "3882018": "moq-uring-2", + "3882019": "moq-uring-3" + }, + "events": [ + "cpu_core/instructions/", + "cpu_core/cycles/" + ], + "sample_rate_limit": 34000, + "window_start_ms": 1793593814922.6294, + "window_end_ms": 1793593819922.9087, + "boundary_uncertainty_ms": [ + 0.0927734375, + 0.299072265625 + ], + "delivered_frames": 100017.52571029816, + "recv_fps": 20002.38782117743, + "counters": { + "cpu_core/instructions/": 12654788263.0, + "cpu_core/cycles/": 29591227591.0, + "task-clock": 5609.588089 + }, + "instructions_per_frame": 126525.70810094554, + "cycles_per_frame": 295860.4242691557, + "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", + "instruction_samples": 1261 + }, + "delivery_boundary_reports": { + "window_start_ms": [ + { + "timestamp_ms": 1793593814722, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 139223, + "frames_sent": 71, + "bytes_recv": 41766900, + "groups_lost": 0, + "groups_expected": 129070, + "groups_present": 129070 + }, + { + "timestamp_ms": 1793593815222, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 149223, + "frames_sent": 76, + "bytes_recv": 44766900, + "groups_lost": 0, + "groups_expected": 139070, + "groups_present": 139070 + } + ], + "window_end_ms": [ + { + "timestamp_ms": 1793593819721, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 239223, + "frames_sent": 121, + "bytes_recv": 71766900, + "groups_lost": 0, + "groups_expected": 229070, + "groups_present": 229070 + }, + { + "timestamp_ms": 1793593820222, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 249223, + "frames_sent": 126, + "bytes_recv": 74766900, + "groups_lost": 0, + "groups_expected": 239070, + "groups_present": 239070 + } + ] + }, + "source_inclusive_samples": { + "resume_learn_end": 0, + "publisher_forward_drop": 89, + "track_state_on_stack": 102, + "resume_reader_on_stack": 240, + "waiter_list_on_stack": 126 + }, + "artifact_sha256": { + "load.jsonl": "01b91704669d71b4dcd5180ff722f58500f77e344cc0509e0922988d9a0cd4fd", + "stat.json": "b1aa02b423fa575ae62fc88b1f924c06573e582cf4bd7a2544df793de4605137", + "window.json": "3d9db44b9640e7dd7a821310e3af250e0ae3ccf8b6e7bd5e176791c8f9144abf", + "perf.data": "b6fc6e6db4a76da34bcc68d49b2df449ac3d9cf4b8a84b508f75c0b929a82e4c", + "instructions.folded": "b9d2bf3318a06d22175f8502f8e4638fc51aadc765b973f05ecf95cb931670bd" + } + }, + { + "variant": "base", + "round": 3, + "capture": { + "version": 1, + "relay_cpus": [ + 4, + 5, + 6, + 7 + ], + "generator_cpus": [ + 0, + 1, + 2, + 3 + ], + "period": 10000019, + "start_ms": [ + 1793593860507.2256, + 1793593860507.2815 + ], + "end_ms": [ + 1793593865507.3655, + 1793593865507.817 + ], + "threads": { + "3933959": "moq-relay", + "3933966": "tokio-rt-worker", + "3933968": "notify-rs inoti", + "3933972": "moq-uring-0", + "3933973": "moq-uring-1", + "3933974": "moq-uring-2", + "3933976": "moq-uring-3" + }, + "events": [ + "cpu_core/instructions/", + "cpu_core/cycles/" + ], + "sample_rate_limit": 34000, + "window_start_ms": 1793593860507.2534, + "window_end_ms": 1793593865507.5913, + "boundary_uncertainty_ms": [ + 0.055908203125, + 0.451416015625 + ], + "delivered_frames": 100006.7578125, + "recv_fps": 20000.0, + "counters": { + "cpu_core/instructions/": 13416194801.0, + "cpu_core/cycles/": 31466803079.0, + "task-clock": 6035.26755 + }, + "instructions_per_frame": 134152.8822097569, + "cycles_per_frame": 314646.7675514116, + "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", + "instruction_samples": 1333 + }, + "delivery_boundary_reports": { + "window_start_ms": [ + { + "timestamp_ms": 1793593860324, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 139979, + "frames_sent": 70, + "bytes_recv": 41993700, + "groups_lost": 0, + "groups_expected": 133884, + "groups_present": 133884 + }, + { + "timestamp_ms": 1793593860824, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 149979, + "frames_sent": 75, + "bytes_recv": 44993700, + "groups_lost": 0, + "groups_expected": 143884, + "groups_present": 143884 + } + ], + "window_end_ms": [ + { + "timestamp_ms": 1793593865324, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 239979, + "frames_sent": 120, + "bytes_recv": 71993700, + "groups_lost": 0, + "groups_expected": 233884, + "groups_present": 233884 + }, + { + "timestamp_ms": 1793593865824, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 249979, + "frames_sent": 125, + "bytes_recv": 74993700, + "groups_lost": 0, + "groups_expected": 243884, + "groups_present": 243884 + } + ] + }, + "source_inclusive_samples": { + "resume_learn_end": 0, + "publisher_forward_drop": 104, + "track_state_on_stack": 137, + "resume_reader_on_stack": 260, + "waiter_list_on_stack": 145 + }, + "artifact_sha256": { + "load.jsonl": "510731dc8d872b37d5948235c19fe62a1bdd31d59945edb0a607fb72379b05d6", + "stat.json": "32617342cd466bfad5e8beaa7ed6e5fadbb2a43063ceece96158ba268aac7057", + "window.json": "8f7168be3faf6104825296b8aee38773c3f75b5b01a619127dc27a2c2c5b6726", + "perf.data": "f4fc58b07e321395fca58b222877b31bf7fc95793365baa30ea28f9f60be3ac5", + "instructions.folded": "26a1fcda3d14edddd7937d7c1d037c922f0095b5a3ab1367e8e1c22dae90699a" + } + }, + { + "variant": "current", + "round": 1, + "capture": { + "version": 1, + "relay_cpus": [ + 4, + 5, + 6, + 7 + ], + "generator_cpus": [ + 0, + 1, + 2, + 3 + ], + "period": 10000019, + "start_ms": [ + 1793593685029.5342, + 1793593685029.6401 + ], + "end_ms": [ + 1793593690029.722, + 1793593690030.6287 + ], + "threads": { + "3860718": "moq-relay", + "3860729": "tokio-rt-worker", + "3860730": "notify-rs inoti", + "3860731": "moq-uring-0", + "3860732": "moq-uring-1", + "3860733": "moq-uring-2", + "3860734": "moq-uring-3" + }, + "events": [ + "cpu_core/instructions/", + "cpu_core/cycles/" + ], + "sample_rate_limit": 34000, + "window_start_ms": 1793593685029.5872, + "window_end_ms": 1793593690030.1753, + "boundary_uncertainty_ms": [ + 0.10595703125, + 0.90673828125 + ], + "delivered_frames": 100011.7626953125, + "recv_fps": 20000.0, + "counters": { + "cpu_core/instructions/": 15530949172.0, + "cpu_core/cycles/": 31052115126.0, + "task-clock": 5863.975766 + }, + "instructions_per_frame": 155291.22528632253, + "cycles_per_frame": 310484.62989899283, + "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", + "instruction_samples": 1552 + }, + "delivery_boundary_reports": { + "window_start_ms": [ + { + "timestamp_ms": 1793593684820, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 139114, + "frames_sent": 70, + "bytes_recv": 41734200, + "groups_lost": 0, + "groups_expected": 130750, + "groups_present": 130750 + }, + { + "timestamp_ms": 1793593685320, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 149114, + "frames_sent": 75, + "bytes_recv": 44734200, + "groups_lost": 0, + "groups_expected": 140750, + "groups_present": 140750 + } + ], + "window_end_ms": [ + { + "timestamp_ms": 1793593689820, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 239114, + "frames_sent": 120, + "bytes_recv": 71734200, + "groups_lost": 0, + "groups_expected": 230750, + "groups_present": 230750 + }, + { + "timestamp_ms": 1793593690320, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 249114, + "frames_sent": 125, + "bytes_recv": 74734200, + "groups_lost": 0, + "groups_expected": 240750, + "groups_present": 240750 + } + ] + }, + "source_inclusive_samples": { + "resume_learn_end": 17, + "publisher_forward_drop": 268, + "track_state_on_stack": 233, + "resume_reader_on_stack": 222, + "waiter_list_on_stack": 179 + }, + "artifact_sha256": { + "load.jsonl": "a1104b94382832194337330502439389218c7f0792d07f36ef73dd6299865fa3", + "stat.json": "31b9de14451c433738e4a04007ed552e17f6a92d975adffad721e76f2adc7030", + "window.json": "7f5b1e10790a269afa2890fac0cda062f77da7cad2e8f982fd269a281af875ca", + "perf.data": "a9b95e6a7017ed2f431fabac649b2cdea220f7fc445bc933a603166ccb6791dc", + "instructions.folded": "56e0437cf2b89c0c781234942cfd38ae2e6a9cca630b1d5c72748b0a03d83783" + } + }, + { + "variant": "current", + "round": 2, + "capture": { + "version": 1, + "relay_cpus": [ + 4, + 5, + 6, + 7 + ], + "generator_cpus": [ + 0, + 1, + 2, + 3 + ], + "period": 10000019, + "start_ms": [ + 1793593748793.664, + 1793593748793.9766 + ], + "end_ms": [ + 1793593753794.0137, + 1793593753795.1243 + ], + "threads": { + "3872892": "moq-relay", + "3872895": "tokio-rt-worker", + "3872896": "notify-rs inoti", + "3872897": "moq-uring-0", + "3872898": "moq-uring-1", + "3872899": "moq-uring-2", + "3872900": "moq-uring-3" + }, + "events": [ + "cpu_core/instructions/", + "cpu_core/cycles/" + ], + "sample_rate_limit": 34000, + "window_start_ms": 1793593748793.8203, + "window_end_ms": 1793593753794.5688, + "boundary_uncertainty_ms": [ + 0.3125, + 1.110595703125 + ], + "delivered_frames": 100007.08372317866, + "recv_fps": 19998.422840122654, + "counters": { + "cpu_core/instructions/": 15643159353.0, + "cpu_core/cycles/": 29421932916.0, + "task-clock": 5613.358136 + }, + "instructions_per_frame": 156420.51313385495, + "cycles_per_frame": 294198.48895344674, + "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", + "instruction_samples": 1557 + }, + "delivery_boundary_reports": { + "window_start_ms": [ + { + "timestamp_ms": 1793593748597, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 139490, + "frames_sent": 71, + "bytes_recv": 41847000, + "groups_lost": 0, + "groups_expected": 132312, + "groups_present": 132312 + }, + { + "timestamp_ms": 1793593749097, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 149490, + "frames_sent": 76, + "bytes_recv": 44847000, + "groups_lost": 0, + "groups_expected": 142312, + "groups_present": 142312 + } + ], + "window_end_ms": [ + { + "timestamp_ms": 1793593753597, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 239490, + "frames_sent": 121, + "bytes_recv": 71847000, + "groups_lost": 0, + "groups_expected": 232312, + "groups_present": 232312 + }, + { + "timestamp_ms": 1793593754098, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 249490, + "frames_sent": 126, + "bytes_recv": 74847000, + "groups_lost": 0, + "groups_expected": 242312, + "groups_present": 242312 + } + ] + }, + "source_inclusive_samples": { + "resume_learn_end": 17, + "publisher_forward_drop": 270, + "track_state_on_stack": 240, + "resume_reader_on_stack": 223, + "waiter_list_on_stack": 151 + }, + "artifact_sha256": { + "load.jsonl": "878cd6efd67ce31d406c8e8868aa5e09e3a00d35e2e531339493e0054adfc472", + "stat.json": "8dadbbc62284d5d8ad178ad4d5596a546426b7d5f75182cff25e153b5342c26a", + "window.json": "75a13755b3ed0b5c914358287bd2abf7134bc8f1a9d719d4494d2d8d6f99c79e", + "perf.data": "8cd1acaba3743f6e60d4162ec03b0a561c91a43d3a96ddfd19565a66a5d92edb", + "instructions.folded": "5f3adec772d21f451b0f1341c2314cc0d976de0648196ba666fa0acfd8bb9957" + } + }, + { + "variant": "current", + "round": 3, + "capture": { + "version": 1, + "relay_cpus": [ + 4, + 5, + 6, + 7 + ], + "generator_cpus": [ + 0, + 1, + 2, + 3 + ], + "period": 10000019, + "start_ms": [ + 1793593908674.517, + 1793593908674.5664 + ], + "end_ms": [ + 1793593913674.7961, + 1793593913675.5957 + ], + "threads": { + "3999994": "moq-relay", + "4000003": "tokio-rt-worker", + "4000005": "notify-rs inoti", + "4000009": "moq-uring-0", + "4000010": "moq-uring-1", + "4000011": "moq-uring-2", + "4000012": "moq-uring-3" + }, + "events": [ + "cpu_core/instructions/", + "cpu_core/cycles/" + ], + "sample_rate_limit": 34000, + "window_start_ms": 1793593908674.5417, + "window_end_ms": 1793593913675.1958, + "boundary_uncertainty_ms": [ + 0.04931640625, + 0.799560546875 + ], + "delivered_frames": 100005.44848779004, + "recv_fps": 19998.473686278438, + "counters": { + "cpu_core/instructions/": 15660840892.0, + "cpu_core/cycles/": 31134166368.0, + "task-clock": 5931.797298 + }, + "instructions_per_frame": 156599.8765948445, + "cycles_per_frame": 311324.7011916682, + "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", + "instruction_samples": 1559 + }, + "delivery_boundary_reports": { + "window_start_ms": [ + { + "timestamp_ms": 1793593908484, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 139977, + "frames_sent": 70, + "bytes_recv": 41993100, + "groups_lost": 0, + "groups_expected": 135246, + "groups_present": 135246 + }, + { + "timestamp_ms": 1793593908984, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 149977, + "frames_sent": 75, + "bytes_recv": 44993100, + "groups_lost": 0, + "groups_expected": 145246, + "groups_present": 145246 + } + ], + "window_end_ms": [ + { + "timestamp_ms": 1793593913484, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 239977, + "frames_sent": 120, + "bytes_recv": 71993100, + "groups_lost": 0, + "groups_expected": 235246, + "groups_present": 235246 + }, + { + "timestamp_ms": 1793593913985, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 249977, + "frames_sent": 126, + "bytes_recv": 74993100, + "groups_lost": 0, + "groups_expected": 245246, + "groups_present": 245246 + } + ] + }, + "source_inclusive_samples": { + "resume_learn_end": 11, + "publisher_forward_drop": 325, + "track_state_on_stack": 254, + "resume_reader_on_stack": 224, + "waiter_list_on_stack": 153 + }, + "artifact_sha256": { + "load.jsonl": "11c2cbc9dd3a74b7977886e1b4aa4e1dd0697d8cb20265598490e2d079803c84", + "stat.json": "7ca078952f185a2c00f77c48c4bc2b9f4e210f43e44015d989e449eb27e62758", + "window.json": "c7163d8126dcb546788bc51abb7142bc6d886f9c80c61514d82492b1992585ed", + "perf.data": "7408f1467e4bc481cda97f57026835c3adc0fdb51163548a4a209468334641e4", + "instructions.folded": "10abb2084945602304d0ec47bb25e78c7d658583fd75fb15f83549a487763c04" + } + } + ], + "summary": { + "base": { + "completed_cells": 3, + "median_instructions_per_frame": 127675.50793207905, + "min_instructions_per_frame": 126525.70810094554, + "max_instructions_per_frame": 134152.8822097569 + }, + "current": { + "completed_cells": 3, + "median_instructions_per_frame": 156420.51313385495, + "min_instructions_per_frame": 155291.22528632253, + "max_instructions_per_frame": 156599.8765948445 + } + }, + "pull_request_url": "https://github.com/moq-dev/moq/pull/6224" + }, + { + "artifact": "run.9iDBWq", + "description": "Resume end tracking", + "base_revision": "fd59eb18a50f6e0ac819e73d941a3832cbdac91f", + "comparison_status": "inconclusive: delivery changed materially and confirmed loss was reported; do not infer a regression from this run", + "harness_head": "9c19f0a6cda900407ee7202b23e8633e95f87857", + "machine": { + "os": "Linux-7.0.0-42-generic-x86_64-with-glibc2.44", + "machine": "x86_64", + "rustc": "rustc 1.98.1 (48a229cea 2026-09-01)\nbinary: rustc\ncommit-hash: 48a229ceaefd4985c50990b14116b6d856af0985\ncommit-date: 2026-09-01\nhost: x86_64-unknown-linux-gnu\nrelease: 1.98.1\nLLVM version: 22.1.8", + "cargo": "cargo 1.98.1 (797e8a9bc 2026-08-05)", + "affinity": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7 + ] + }, + "harness_dirty_patch_sha256": "1fb2250488672e6005088a1008ec311575d40c0a8ef4546a592f4ca2acea68a0", + "cells": [ + { + "variant": "base", + "round": 1, + "capture": { + "version": 1, + "relay_cpus": [ + 4, + 5, + 6, + 7 + ], + "generator_cpus": [ + 0, + 1, + 2, + 3 + ], + "period": 10000019, + "start_ms": [ + 1793594381594.4658, + 1793594381594.559 + ], + "end_ms": [ + 1793594386594.65, + 1793594386594.7551 + ], + "threads": { + "331621": "moq-relay", + "331624": "tokio-rt-worker", + "331625": "notify-rs inoti", + "331627": "moq-uring-0", + "331628": "moq-uring-1", + "331629": "moq-uring-2", + "331630": "moq-uring-3" + }, + "events": [ + "cpu_core/instructions/", + "cpu_core/cycles/" + ], + "sample_rate_limit": 25000, + "window_start_ms": 1793594381594.5125, + "window_end_ms": 1793594386594.7026, + "boundary_uncertainty_ms": [ + 0.09326171875, + 0.105224609375 + ], + "delivered_frames": 100095.05315240734, + "recv_fps": 20018.249194147375, + "counters": { + "cpu_core/instructions/": 14399429625.0, + "cpu_core/cycles/": 39461160782.0, + "task-clock": 7432.597766 + }, + "instructions_per_frame": 143857.55510889288, + "cycles_per_frame": 394236.8732440294, + "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", + "instruction_samples": 1433 + }, + "delivery_boundary_reports": { + "window_start_ms": [ + { + "timestamp_ms": 1793594381226, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 136838, + "frames_sent": 70, + "bytes_recv": 41051400, + "groups_lost": 0, + "groups_expected": 125638, + "groups_present": 125638 + }, + { + "timestamp_ms": 1793594381726, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 147205, + "frames_sent": 76, + "bytes_recv": 44161500, + "groups_lost": 0, + "groups_expected": 136005, + "groups_present": 136005 + } + ], + "window_end_ms": [ + { + "timestamp_ms": 1793594386227, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 237205, + "frames_sent": 121, + "bytes_recv": 71161500, + "groups_lost": 0, + "groups_expected": 226005, + "groups_present": 226005 + }, + { + "timestamp_ms": 1793594386726, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 247205, + "frames_sent": 126, + "bytes_recv": 74161500, + "groups_lost": 0, + "groups_expected": 236005, + "groups_present": 236005 + } + ] + }, + "source_inclusive_samples": { + "resume_learn_end": 149, + "publisher_forward_drop": 135, + "track_state_on_stack": 253, + "resume_reader_on_stack": 427, + "waiter_list_on_stack": 220 + }, + "artifact_sha256": { + "load.jsonl": "c6635a2341f6e5d82258b8cc8cca1eb770d5cdf9bd21383dcb145a30ce943a0c", + "stat.json": "762933d9c68eea7e5fcfc50d4e1537654b50155b14c917dca7ae775fc38cf455", + "window.json": "e44d573b80528c963c803a2079ec5913919a5d6838587bf82b49d1c16911cf40", + "perf.data": "16c0b3a22610ccaf5d67903a405db41a339c052c7872200629244951e825fcb2", + "instructions.folded": "3f92bd25686b357f470abb8cfbced60fca3e7ccd4125380befcdef48d430f850" + } + }, + { + "variant": "base", + "round": 2, + "capture": { + "version": 1, + "relay_cpus": [ + 4, + 5, + 6, + 7 + ], + "generator_cpus": [ + 0, + 1, + 2, + 3 + ], + "period": 10000019, + "start_ms": [ + 1793594579283.6018, + 1793594579285.9014 + ], + "end_ms": [ + 1793594584285.9883, + 1793594584288.9255 + ], + "threads": { + "542687": "moq-relay", + "542709": "tokio-rt-worker", + "542713": "notify-rs inoti", + "542719": "moq-uring-0", + "542720": "moq-uring-1", + "542721": "moq-uring-2", + "542725": "moq-uring-3" + }, + "events": [ + "cpu_core/instructions/", + "cpu_core/cycles/" + ], + "sample_rate_limit": 25000, + "window_start_ms": 1793594579284.7515, + "window_end_ms": 1793594584287.457, + "boundary_uncertainty_ms": [ + 2.299560546875, + 2.937255859375 + ], + "delivered_frames": 111136.91455377889, + "recv_fps": 22215.3618833929, + "counters": { + "cpu_core/instructions/": 14496953653.0, + "cpu_core/cycles/": 38139777320.0, + "task-clock": 7159.595114 + }, + "instructions_per_frame": 130442.29013560529, + "cycles_per_frame": 343178.29924587527, + "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", + "instruction_samples": 1445 + }, + "delivery_boundary_reports": { + "window_start_ms": [ + { + "timestamp_ms": 1793594578941, + "ready": false, + "connections": { + "target": 2001, + "established": 1976 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 1916, + "established": 1841 + }, + "frames_recv": 103716, + "frames_sent": 71, + "bytes_recv": 31114800, + "groups_lost": 8698, + "groups_expected": 93167, + "groups_present": 93057 + }, + { + "timestamp_ms": 1793594579444, + "ready": false, + "connections": { + "target": 2001, + "established": 1990 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 1946, + "established": 1901 + }, + "frames_recv": 111655, + "frames_sent": 76, + "bytes_recv": 33496500, + "groups_lost": 8699, + "groups_expected": 100671, + "groups_present": 100561 + } + ], + "window_end_ms": [ + { + "timestamp_ms": 1793594583940, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 214150, + "frames_sent": 120, + "bytes_recv": 64245000, + "groups_lost": 10872, + "groups_expected": 201838, + "groups_present": 201724 + }, + { + "timestamp_ms": 1793594584440, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 222969, + "frames_sent": 125, + "bytes_recv": 66890700, + "groups_lost": 10872, + "groups_expected": 210657, + "groups_present": 210543 + } + ] + }, + "source_inclusive_samples": { + "resume_learn_end": 149, + "publisher_forward_drop": 109, + "track_state_on_stack": 259, + "resume_reader_on_stack": 388, + "waiter_list_on_stack": 214 + }, + "artifact_sha256": { + "load.jsonl": "8461651165eab4a26d5eed290af87252c60158767a987c41752e43408ecc32fe", + "stat.json": "c6038808d665bdbfbe91dfee959b9fc95f2c2e0631c2b97d6e6c58f1fcd82b67", + "window.json": "218faaba4fde6cb55a5e5d694be51726d30a1d55381bb3490b59a127bf88c79a", + "perf.data": "984910e6ec332d86f1386845c0f865d65d8c75e46b95c6c6f6a1309111a3c609", + "instructions.folded": "5005c77d20bd8631f6c96d205e7db6f234620136a8f820cebade588065d2d0b7" + } + }, + { + "variant": "base", + "round": 3, + "capture": { + "version": 1, + "relay_cpus": [ + 4, + 5, + 6, + 7 + ], + "generator_cpus": [ + 0, + 1, + 2, + 3 + ], + "period": 10000019, + "start_ms": [ + 1793594647413.7844, + 1793594647413.8716 + ], + "end_ms": [ + 1793594652413.9639, + 1793594652414.0789 + ], + "threads": { + "659791": "moq-relay", + "659808": "tokio-rt-worker", + "659812": "notify-rs inoti", + "659842": "moq-uring-0", + "659844": "moq-uring-1", + "659845": "moq-uring-2", + "659847": "moq-uring-3" + }, + "events": [ + "cpu_core/instructions/", + "cpu_core/cycles/" + ], + "sample_rate_limit": 25000, + "window_start_ms": 1793594647413.8281, + "window_end_ms": 1793594652414.0215, + "boundary_uncertainty_ms": [ + 0.087158203125, + 0.114990234375 + ], + "delivered_frames": 100245.75456898703, + "recv_fps": 20048.375605522036, + "counters": { + "cpu_core/instructions/": 14290822931.0, + "cpu_core/cycles/": 31924206244.0, + "task-clock": 6028.561508 + }, + "instructions_per_frame": 142557.8867897628, + "cycles_per_frame": 318459.4338309901, + "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", + "instruction_samples": 1423 + }, + "delivery_boundary_reports": { + "window_start_ms": [ + { + "timestamp_ms": 1793594647006, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 138869, + "frames_sent": 70, + "bytes_recv": 41660700, + "groups_lost": 0, + "groups_expected": 126705, + "groups_present": 126705 + }, + { + "timestamp_ms": 1793594647507, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 148840, + "frames_sent": 75, + "bytes_recv": 44652000, + "groups_lost": 0, + "groups_expected": 136676, + "groups_present": 136676 + } + ], + "window_end_ms": [ + { + "timestamp_ms": 1793594652006, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 239071, + "frames_sent": 120, + "bytes_recv": 71721300, + "groups_lost": 0, + "groups_expected": 226907, + "groups_present": 226907 + }, + { + "timestamp_ms": 1793594652506, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 249071, + "frames_sent": 125, + "bytes_recv": 74721300, + "groups_lost": 0, + "groups_expected": 236907, + "groups_present": 236907 + } + ] + }, + "source_inclusive_samples": { + "resume_learn_end": 163, + "publisher_forward_drop": 121, + "track_state_on_stack": 251, + "resume_reader_on_stack": 399, + "waiter_list_on_stack": 218 + }, + "artifact_sha256": { + "load.jsonl": "1874080e711ba7b934680da35586a5c48fbbabfa0fb155ef435a9bb037d598eb", + "stat.json": "150b5289d32f0a5570d6ade4c3b1d196397336acbfb54c3d91a2f24e695a37f6", + "window.json": "745d17d3675f91db7d481f4de11a8bb1354e062e9ec707c1e631b8b9b512c2d4", + "perf.data": "cb849a6cfdfc5e4a28ac731b6c5d5283ec90ee68c273fbeed6eb542128fe2300", + "instructions.folded": "bc70c79e82e28992f5bdaf996e6c96d1de787ef94618b52c653e8a5f249d966f" + } + }, + { + "variant": "current", + "round": 1, + "capture": { + "version": 1, + "relay_cpus": [ + 4, + 5, + 6, + 7 + ], + "generator_cpus": [ + 0, + 1, + 2, + 3 + ], + "period": 10000019, + "start_ms": [ + 1793594430209.9126, + 1793594430209.982 + ], + "end_ms": [ + 1793594435210.0576, + 1793594435210.8723 + ], + "threads": { + "379928": "moq-relay", + "379931": "tokio-rt-worker", + "379932": "notify-rs inoti", + "379933": "moq-uring-0", + "379934": "moq-uring-1", + "379935": "moq-uring-2", + "379936": "moq-uring-3" + }, + "events": [ + "cpu_core/instructions/", + "cpu_core/cycles/" + ], + "sample_rate_limit": 25000, + "window_start_ms": 1793594430209.9473, + "window_end_ms": 1793594435210.4648, + "boundary_uncertainty_ms": [ + 0.0693359375, + 0.814697265625 + ], + "delivered_frames": 101235.58537766157, + "recv_fps": 20245.021399489007, + "counters": { + "cpu_core/instructions/": 15529380560.0, + "cpu_core/cycles/": 32771189013.0, + "task-clock": 6142.396911 + }, + "instructions_per_frame": 153398.43694356392, + "cycles_per_frame": 323712.150137191, + "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", + "instruction_samples": 1552 + }, + "delivery_boundary_reports": { + "window_start_ms": [ + { + "timestamp_ms": 1793594429990, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 136127, + "frames_sent": 70, + "bytes_recv": 40838100, + "groups_lost": 0, + "groups_expected": 125410, + "groups_present": 125410 + }, + { + "timestamp_ms": 1793594430490, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 144795, + "frames_sent": 75, + "bytes_recv": 43438500, + "groups_lost": 0, + "groups_expected": 134078, + "groups_present": 134078 + } + ], + "window_end_ms": [ + { + "timestamp_ms": 1793594434990, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 236909, + "frames_sent": 121, + "bytes_recv": 71072700, + "groups_lost": 0, + "groups_expected": 226192, + "groups_present": 226192 + }, + { + "timestamp_ms": 1793594435489, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 246566, + "frames_sent": 125, + "bytes_recv": 73969800, + "groups_lost": 0, + "groups_expected": 235849, + "groups_present": 235849 + } + ] + }, + "source_inclusive_samples": { + "resume_learn_end": 6, + "publisher_forward_drop": 267, + "track_state_on_stack": 272, + "resume_reader_on_stack": 229, + "waiter_list_on_stack": 156 + }, + "artifact_sha256": { + "load.jsonl": "1018c7927375b1d99d229b29e1e602e1b9f14b0c8b245bbd8d1242def6a0faf1", + "stat.json": "6cec61573391d19c9323ed1efbfe97089c40bdf77b2956051d9282d46a9a8b19", + "window.json": "8f7eeaef1ba10e088ca3ee39084bba4e45c1806a65735d9a71a86b079a9fdc8b", + "perf.data": "1d4832ca0788473db120d9d9ccd4ce9adf8bf798c76709cdc9b9cd1c8fcdbb66", + "instructions.folded": "9a81ea5fe9a63b1348b4f46beda24ef01f84fc9226c71e90980804d7105c5303" + } + }, + { + "variant": "current", + "round": 2, + "capture": { + "version": 1, + "relay_cpus": [ + 4, + 5, + 6, + 7 + ], + "generator_cpus": [ + 0, + 1, + 2, + 3 + ], + "period": 10000019, + "start_ms": [ + 1793594500464.6292, + 1793594500464.705 + ], + "end_ms": [ + 1793594505464.9849, + 1793594505465.2336 + ], + "threads": { + "413374": "moq-relay", + "413378": "tokio-rt-worker", + "413381": "notify-rs inoti", + "413382": "moq-uring-0", + "413383": "moq-uring-1", + "413384": "moq-uring-2", + "413386": "moq-uring-3" + }, + "events": [ + "cpu_core/instructions/", + "cpu_core/cycles/" + ], + "sample_rate_limit": 25000, + "window_start_ms": 1793594500464.667, + "window_end_ms": 1793594505465.1094, + "boundary_uncertainty_ms": [ + 0.075927734375, + 0.248779296875 + ], + "delivered_frames": 99433.38824336545, + "recv_fps": 19884.91829945636, + "counters": { + "cpu_core/instructions/": 15418054843.0, + "cpu_core/cycles/": 30441872584.0, + "task-clock": 5712.46671 + }, + "instructions_per_frame": 155059.13169994735, + "cycles_per_frame": 306153.42715158046, + "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", + "instruction_samples": 1540 + }, + "delivery_boundary_reports": { + "window_start_ms": [ + { + "timestamp_ms": 1793594500244, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 139053, + "frames_sent": 70, + "bytes_recv": 41715900, + "groups_lost": 0, + "groups_expected": 131563, + "groups_present": 131563 + }, + { + "timestamp_ms": 1793594500745, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 149053, + "frames_sent": 76, + "bytes_recv": 44715900, + "groups_lost": 0, + "groups_expected": 141563, + "groups_present": 141563 + } + ], + "window_end_ms": [ + { + "timestamp_ms": 1793594505246, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 238067, + "frames_sent": 120, + "bytes_recv": 71420100, + "groups_lost": 0, + "groups_expected": 230577, + "groups_present": 230577 + }, + { + "timestamp_ms": 1793594505745, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 249053, + "frames_sent": 125, + "bytes_recv": 74715900, + "groups_lost": 0, + "groups_expected": 241563, + "groups_present": 241563 + } + ] + }, + "source_inclusive_samples": { + "resume_learn_end": 11, + "publisher_forward_drop": 258, + "track_state_on_stack": 226, + "resume_reader_on_stack": 233, + "waiter_list_on_stack": 168 + }, + "artifact_sha256": { + "load.jsonl": "f040c49aa1ae11940e2f9e9b16a0cbd9e07150e0ff481f79621d19aae4088b00", + "stat.json": "80c531a0880a9f1cae9d79ac5bf71b8ad85871d96ecd7b87cfefcd7d3cec888d", + "window.json": "f3ba3629de9b2d9b55dcf8584ac892e1b2f7ef003462bad971701bf3b1902334", + "perf.data": "87d1822b7515516f4cd55aa40f94abd8283b86c1701060566aa3c05a8609822d", + "instructions.folded": "441062103cb7eb8bc213981bbc82155494e86aba26caefd57f5f5d914d1ae3e2" + } + }, + { + "variant": "current", + "round": 3, + "capture": { + "version": 1, + "relay_cpus": [ + 4, + 5, + 6, + 7 + ], + "generator_cpus": [ + 0, + 1, + 2, + 3 + ], + "period": 10000019, + "start_ms": [ + 1793594694838.9155, + 1793594694839.8674 + ], + "end_ms": [ + 1793594699840.0994, + 1793594699840.213 + ], + "threads": { + "778541": "moq-relay", + "778554": "tokio-rt-worker", + "778558": "notify-rs inoti", + "778561": "moq-uring-0", + "778563": "moq-uring-1", + "778564": "moq-uring-2", + "778566": "moq-uring-3" + }, + "events": [ + "cpu_core/instructions/", + "cpu_core/cycles/" + ], + "sample_rate_limit": 25000, + "window_start_ms": 1793594694839.3916, + "window_end_ms": 1793594699840.1562, + "boundary_uncertainty_ms": [ + 0.951904296875, + 0.113525390625 + ], + "delivered_frames": 91833.47595923272, + "recv_fps": 18363.886808375653, + "counters": { + "cpu_core/instructions/": 14142104401.0, + "cpu_core/cycles/": 31545899811.0, + "task-clock": 5912.347715 + }, + "instructions_per_frame": 153997.26791652807, + "cycles_per_frame": 343511.98712116753, + "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", + "instruction_samples": 1411 + }, + "delivery_boundary_reports": { + "window_start_ms": [ + { + "timestamp_ms": 1793594694631, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 137055, + "frames_sent": 70, + "bytes_recv": 41116500, + "groups_lost": 68, + "groups_expected": 127560, + "groups_present": 127560 + }, + { + "timestamp_ms": 1793594695121, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 147270, + "frames_sent": 75, + "bytes_recv": 44181000, + "groups_lost": 68, + "groups_expected": 137775, + "groups_present": 137775 + } + ], + "window_end_ms": [ + { + "timestamp_ms": 1793594699622, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 226762, + "frames_sent": 120, + "bytes_recv": 68028600, + "groups_lost": 2959, + "groups_expected": 217263, + "groups_present": 217263 + }, + { + "timestamp_ms": 1793594700121, + "ready": true, + "connections": { + "target": 2001, + "established": 2001 + }, + "broadcasts": { + "target": 1, + "active": 1, + "established": 1 + }, + "subscriptions": { + "target": 2000, + "active": 2000, + "established": 2000 + }, + "frames_recv": 241563, + "frames_sent": 125, + "bytes_recv": 72468900, + "groups_lost": 4201, + "groups_expected": 232072, + "groups_present": 232068 + } + ] + }, + "source_inclusive_samples": { + "resume_learn_end": 15, + "publisher_forward_drop": 229, + "track_state_on_stack": 235, + "resume_reader_on_stack": 171, + "waiter_list_on_stack": 158 + }, + "artifact_sha256": { + "load.jsonl": "3077124b8edc46b3e23990e7bc30de6f33e4de9905677bdbf8d8736963d5a950", + "stat.json": "13a6eeee58035eaeb02c55853f5eeb2aa722aac2ff58e8531758762385e1a973", + "window.json": "d22e332093899df276d460548df681e5cea963fecaa56a865c000825dc260483", + "perf.data": "33f1b24821c432eab674a46976db0ea43a46b9fe1cb05f2ae269e7582c3926ff", + "instructions.folded": "0a430966175cc3889e469ce255213762a887cf789d2e4ba59320ad8564bdec3d" + } + } + ], + "summary": { + "base": { + "completed_cells": 3, + "median_instructions_per_frame": 142557.8867897628, + "min_instructions_per_frame": 130442.29013560529, + "max_instructions_per_frame": 143857.55510889288 + }, + "current": { + "completed_cells": 3, + "median_instructions_per_frame": 153997.26791652807, + "min_instructions_per_frame": 153398.43694356392, + "max_instructions_per_frame": 155059.13169994735 + } + }, + "pull_request_url": "https://github.com/moq-dev/moq/pull/6224" + } + ], + "excluded_attempts": [ + { + "artifact": "run.RBmaZ8", + "reason": "Current-tree samples throttled at period 1000003; predates the throttle validator. Any existing derived JSON is excluded." + }, + { + "artifact": "run.JuOoVh", + "reason": "Historical capture throttled at period 10000019 (957 samples and 957 THROTTLE records); command failed." + }, + { + "artifact": "run.hDJ7BQ", + "reason": "Nested Cargo alias configuration conflict; fixed by explicit isolated configuration." + }, + { + "artifact": "run.T2mHXL", + "reason": "New parser incorrectly rejected identical cumulative snapshots during teardown; fixed and the whole comparison repeated. This partial attempt is excluded." + } + ], + "production_control_url": "https://github.com/moq-dev/moq/commit/21d6da9fc108b48373a961f34e43c3fcd38b0146" +} diff --git a/.github/relay-cpu-bisect.md b/.github/relay-cpu-bisect.md new file mode 100644 index 00000000000..e6fad74f482 --- /dev/null +++ b/.github/relay-cpu-bisect.md @@ -0,0 +1,48 @@ +# Relay CPU bisect checkpoint + +Stopped at the user request on 2026-11-01. The investigation is unfinished. No production optimization, complete culprit attribution, or feature-cost acceptance is claimed. + +The reusable driver is `just bench-profile instructions`. Exact per-cell counts, boundary reports, source sample markers, artifact hashes, and excluded attempts are in [the evidence file](relay-cpu-bisect.json). Raw captures remain in this checkout’s ignored `.tmp/bench/`. + +## Method + +Each historical relay is paired with the same production control, [21d6da9fc1](https://github.com/moq-dev/moq/commit/21d6da9fc108b48373a961f34e43c3fcd38b0146), driven by the current generator. Builds use the profiling profile, frame pointers, and `moq-relay/io-uring,moq-bench/uring`, with a target per revision. Four io\_uring workers use cores 4-7; the generator uses 0-3. The chat workload has one publisher, 2,000 viewers, 10 fps, and one 300-byte frame per group. + +Three alternating historical/current pairs count user and kernel instructions over acknowledged five-second PMU windows after seven seconds of warmup. Delivery is interpolated from the generator’s surrounding 500 ms reports at those same boundaries. This is sampled delivery accounting. The normal host summaries cover a different final window and are not substituted for these counts. + +Fixed-period instruction profiles use 10,000,019 instructions/sample. The host’s original 1,000 Hz sample limit throttled every interrupt. Each successful gated run temporarily requested a 100,000 Hz limit and restored the original on exit. The kernel sometimes lowered the active limit itself; captures record it and refuse lost or throttled samples. The committed driver changes no sysctls. + +## Measurements + +Cells are medians of three, with min-max in brackets, in thousands of instructions per delivered frame. Each row has its own interleaved current control; this is not a directly interleaved pair of adjacent historical commits. Host load and scheduling still affect instruction counts. No confidence interval or arbitrary performance threshold is asserted. + +| Historical revision | Historical kinstr/frame | Paired current kinstr/frame | Evidence | +| --- | ---: | ---: | --- | +| [cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95), [moq-dev/moq#5926](https://github.com/moq-dev/moq/pull/5926) merged | 108.7 \[108.0-118.3] | 156.8 \[156.0-157.7] | `run.IkXbDf` | +| [55f6ed2292](https://github.com/moq-dev/moq/commit/55f6ed229250d17fc99389237659906d011d8a94), Immediately before dynamic tracks ([moq-dev/moq#6171](https://github.com/moq-dev/moq/pull/6171)) | 142.7 \[138.5-143.9] | 154.3 \[153.0-155.4] | `run.vgmg4w` | +| [dcd7d2cbb8](https://github.com/moq-dev/moq/commit/dcd7d2cbb891dc1cf72c68295ba70fbd17a4ec15), Immediately before per-reader ranges ([moq-dev/moq#5929](https://github.com/moq-dev/moq/pull/5929)) | 105.8 \[105.1-109.2] | 157.4 \[156.8-160.9] | `run.FlzdYt` | +| [fe722cec9f](https://github.com/moq-dev/moq/commit/fe722cec9f8432e0e46cb633cc5e79e59e7724cb), Per-reader ranges ([moq-dev/moq#5929](https://github.com/moq-dev/moq/pull/5929)) | 112.1 \[111.9-113.2] | 157.1 \[155.7-164.7] | `run.2bOgus` | +| [e84ffb3de3](https://github.com/moq-dev/moq/commit/e84ffb3de3139f7245bed5697754fd6add9a022e), Immediately before resume end tracking ([moq-dev/moq#6224](https://github.com/moq-dev/moq/pull/6224)) | 127.7 \[126.5-134.2] | 156.4 \[155.3-156.6] | `run.iFmfVR` | + +The five comparisons above delivered approximately 20,000 frames/s and their boundary reports recorded zero confirmed lost groups. The evidence retains exact delivery estimates, including variation; no outlier cells were trimmed. Failed attempts are excluded explicitly rather than counted as completed comparisons. The final completed command, `run.9iDBWq` at [fd59eb18a5](https://github.com/moq-dev/moq/commit/fd59eb18a50f6e0ac819e73d941a3832cbdac91f), is retained separately in the JSON as **inconclusive**: delivery estimates ranged from approximately 18,364 to 22,215 frames/s and boundary reports included confirmed loss. Its counters and stacks were captured, but those differing workloads cannot establish the instruction effect of the resume-end change. No repeat was started after the stop request. + +## Source attribution and ownership + +- The overall chat increase is reproduced, but the complete regression has not been bisected. +- The range-serving boundary [moq-dev/moq#5929](https://github.com/moq-dev/moq/pull/5929) shows a smaller historical increase. It does not explain the whole current gap. Its cost has not been accepted as a feature cost in the release-perf epic. +- [moq-dev/moq#6224](https://github.com/moq-dev/moq/pull/6224) introduced `Reader::learn_end`, called by synchronization, retention, and finish polling. Its attempted after-change comparison was inconclusive because delivery changed and losses were reported; the source introduction is verified, but its instruction effect has not been confirmed. +- The current control’s `poll_forward_drop` path accounts for roughly 17-21% of sampled instructions across these runs, versus about 6-7% at the old endpoint. This is inclusive stack attribution, not an isolated cost or a claim that all of the path is avoidable. +- The historical pre-dynamic capture puts about 12.5% of samples under `learn_end`, while current captures put roughly 0.6-1.1% there. The later [moq-dev/moq#7532](https://github.com/moq-dev/moq/pull/7532) watch changes already alter this path. Do not attribute the whole current increase to the historical end-polling step or remove its correctness behavior. +- [moq-dev/moq#7493](https://github.com/moq-dev/moq/pull/7493) owns the overlapping lite publisher and track/resume/cache polling fixes, including the end-declared flag and fewer cursor locks. Those production paths were not edited here. The build-cost audit [moq-dev/moq#7602](https://github.com/moq-dev/moq/pull/7602) and archive epic [moq-dev/moq#7604](https://github.com/moq-dev/moq/pull/7604) are also untouched. + +The source markers in the evidence are inclusive, can overlap, and are not additive. They identify where to continue reading, not a finished attribution of deliberate versus avoidable costs. + +## Remaining scope + +The existing [quest](../quest/m1/release-perf/relay-cpu-bisect.md) retains the work. Obtain a comparable-delivery measurement of the resume-end boundary. Resolve the unbisected growth between per-reader ranges and the parent of that change, then the later interval from the pre-dynamic point to the pinned control. Account for intervening improvements as well as regressions. Coordinate overlapping fixes with the fanout-lock owner before changing those files. Confirm attributed steps on video and Tokio workers; implement any remaining avoidable costs with publisher/viewer benchmark sweeps. Record justified feature-cost acceptances in the release-perf epic, then complete checks and local other-vendor review. None of this work was started after the stop request. + +## Validation + +- Focused profile, relay lifecycle, and summary tests pass, along with Ruff and shellcheck. The profile tests cover matching windows, unsupported counters, CPU sets, loss/throttling, and identical cumulative snapshots at one timestamp. +- Two full `quest gate just ci` attempts failed on the known mbx `E2BIG` defect ([moq-dev/moq#7594](https://github.com/moq-dev/moq/pull/7594)) and runner-image `/proc/self/uid_map` namespace denial, including downstream build and tooling-test failures. The first also failed an unchanged JS announcement-sync test; the second passed JS tests. These are not full-gate passes. +- No other-vendor review or ready-for-review transition has been completed. The PR remains a draft. Public API and wire: no changes. diff --git a/bench/README.md b/bench/README.md index 343dbd3adfd..016d3c03f10 100644 --- a/bench/README.md +++ b/bench/README.md @@ -262,6 +262,10 @@ Missing, uncounted, or multiplexed counters and unbracketed delivery fail the ru Lost or throttled instruction samples also fail. Inspect the host's sample-rate limit and buffers before choosing a larger fixed period for the whole comparison; some sample-rate limits throttle on every interrupt, regardless of the period. +On the profiling host, a 1,000 Hz limit throttled every sample even at a period +of 10,000,019. Temporarily raising `kernel.perf_event_max_sample_rate` to 100,000 +allowed complete captures. Save and restore the original setting around the +whole gated run; the driver records the observed limit and changes no sysctls. The host sampler lasts fourteen seconds so delivery reports bracket both edges. The normal `summary.json` still describes its own final throughput window; use `instructions.json` for instruction comparisons. diff --git a/bench/instructions.py b/bench/instructions.py index 95e5fc0485e..a8cc16e2f83 100644 --- a/bench/instructions.py +++ b/bench/instructions.py @@ -207,8 +207,12 @@ def reduce(window, stats, rows): if not isinstance(row[key], (int, float)) or not math.isfinite(row[key]) or row[key] < 0: raise ValueError(f"invalid generator counter: {key}") for first, last in zip(rows, rows[1:]): - if last["timestamp_ms"] <= first["timestamp_ms"] or last["frames_recv"] < first["frames_recv"]: + if last["timestamp_ms"] < first["timestamp_ms"] or last["frames_recv"] < first["frames_recv"]: raise ValueError("generator counters moved backwards") + # Catch-up ticks can observe the same cumulative snapshot at millisecond + # resolution. Different counters at that timestamp cannot be interpolated. + if last["timestamp_ms"] == first["timestamp_ms"] and last["frames_recv"] != first["frames_recv"]: + raise ValueError("conflicting generator counters at the same timestamp") start, end = [sum(window[key]) / 2 for key in ("start_ms", "end_ms")] if end <= start: raise ValueError("counter window did not advance") diff --git a/bench/profile.test.py b/bench/profile.test.py index 00e2ecda523..2c61d728fae 100644 --- a/bench/profile.test.py +++ b/bench/profile.test.py @@ -54,6 +54,14 @@ def test_partial_or_multiplexed_counts_are_refused(self): with self.assertRaisesRegex(ValueError, "missing"): instructions.reduce(self.window, self.stats[1:], self.load) + def test_identical_cumulative_snapshots_remain_interpolatable(self): + self.load.insert(2, dict(self.load[1])) + result = instructions.reduce(self.window, self.stats, self.load) + self.assertEqual(result["delivered_frames"], 2000) + self.load[2]["frames_recv"] += 1 + with self.assertRaisesRegex(ValueError, "same timestamp"): + instructions.reduce(self.window, self.stats, self.load) + def test_cpu_sets_cannot_overlap_or_escape_the_allowed_set(self): with patch.object(os, "sched_getaffinity", return_value={0, 1, 2, 3}, create=True): for relay, generator in [("0-1", "1-2"), ("4", "0"), ("0", ""), ("0-1", "3-2")]: diff --git a/bench/run.sh b/bench/run.sh index 170c7943670..bbce75e64d0 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -225,8 +225,8 @@ export CRITERION_HOME=$RUN/criterion checkout_base() { # Cargo merges every ancestor's .cargo/config.toml. A nested worktree would - # inherit this checkout's settings instead of measuring the base in isolation. - WORKTREE=$(mktemp -d "${TMPDIR:-/tmp}/moq-bench-base.XXXXXX") + # inherit this checkout's settings unless the caller loads only its own config. + WORKTREE=$(mktemp -d "${2:-${TMPDIR:-/tmp}}/moq-bench-base.XXXXXX") git worktree add --detach "$WORKTREE" "$1" >/dev/null } @@ -858,7 +858,8 @@ run_profiles() { fi if [[ -n $BASE ]]; then - checkout_base "$BASE" + mkdir -p "$ROOT/.tmp" + checkout_base "$BASE" "$ROOT/.tmp" mkdir -p "$RUN/base-context" (cd "$WORKTREE" && python3 "$ROOT/bench/metadata.py" "$RUN/base-context" "$BASE" --profile) fi diff --git a/quest/m1/release-perf/relay-cpu-bisect.md b/quest/m1/release-perf/relay-cpu-bisect.md index 9387b3dd21b..28dd49555a5 100644 --- a/quest/m1/release-perf/relay-cpu-bisect.md +++ b/quest/m1/release-perf/relay-cpu-bisect.md @@ -1,30 +1,35 @@ -# [M] Find and fix the relay's per-frame CPU growth since #5926 +# [M] Find and fix the relay's per-frame CPU growth since [moq-dev/moq#5926](https://github.com/moq-dev/moq/pull/5926) ## Goal -The commits that raised the relay's instructions per delivered frame between [`cdff3c8f95`](https://github.com/moq-dev/moq/commit/cdff3c8f95) ([moq-dev/moq#5926](https://github.com/moq-dev/moq/pull/5926) as merged) and `main` are found, each rise is attributed to a cause, and the avoidable costs are fixed. A rise that pays for a deliberate feature is accepted with its reason in the [release-perf epic](/quest/m1/release-perf/README.md). +The commits that raised the relay's instructions per delivered frame between [[cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95)](https://github.com/moq-dev/moq/commit/cdff3c8f95) ([moq-dev/moq#5926](https://github.com/moq-dev/moq/pull/5926) as merged) and `main` are found, each rise is attributed to a cause, and the avoidable costs are fixed. A rise that pays for a deliberate feature is accepted with its reason in the [release-perf epic](/quest/m1/release-perf/README.md). ## Plan -Found by [moq-dev/moq#7505](https://github.com/moq-dev/moq/pull/7505), which profiled the io_uring clock read. At `main` [`30f23c3f8a`](https://github.com/moq-dev/moq/commit/30f23c3f8a), relay instructions per delivered frame had risen 25-56% on io_uring since `cdff3c8f95` and 18-38% on the tokio workers since `1dd9b83d1`, with the same delivery: +Found by [moq-dev/moq#7505](https://github.com/moq-dev/moq/pull/7505), which profiled the io_uring clock read. At `main` [`30f23c3f8a`](https://github.com/moq-dev/moq/commit/30f23c3f8a), relay instructions per delivered frame had risen 25-56% on io_uring since [cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95) and 18-38% on the tokio workers since `1dd9b83d1`, with the same delivery: | workload | runtime | before, kinstr/frame | `30f23c3f8a`, kinstr/frame | |---|---|---|---| -| chat | io_uring | 86.3 [84.5-87.4] at `cdff3c8f95` | 134.8 [128.5-151.3] | -| video | io_uring | 108.7 [106.6-116.2] at `cdff3c8f95` | 136.1 [132.6-152.8] | +| chat | io_uring | 86.3 [84.5-87.4] at [cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95) | 134.8 [128.5-151.3] | +| video | io_uring | 108.7 [106.6-116.2] at [cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95) | 136.1 [132.6-152.8] | | chat | tokio workers | 104.0 [101.4-105.0] at `1dd9b83d1` | 143.1 [127.5-205.3] | | video | tokio workers | 141.6 [139.3-148.3] at `1dd9b83d1` | 167.1 [164.9-173.6] | -Medians of 5 interleaved reps, min-max in brackets. The tokio rows start at [`1dd9b83d1`](https://github.com/moq-dev/moq/commit/1dd9b83d19), #5926's base, since #7505 measured the tokio workers only there and at `main`. Both runtimes moved, so at least one cause is shared code (moq-net's model, kio, or the session layer). That does not rule out a further io_uring-only cost, such as the clock read #7505 profiled. The tokio range also spans #5926 itself, so the two rises can't be subtracted to find one. +Medians of 5 interleaved reps, min-max in brackets. The tokio rows start at [`1dd9b83d1`](https://github.com/moq-dev/moq/commit/1dd9b83d19), [moq-dev/moq#5926](https://github.com/moq-dev/moq/pull/5926)'s base, since [moq-dev/moq#7505](https://github.com/moq-dev/moq/pull/7505) measured the tokio workers only there and at `main`. Both runtimes moved, so at least one cause is shared code (moq-net's model, kio, or the session layer). That does not rule out a further io_uring-only cost, such as the clock read [moq-dev/moq#7505](https://github.com/moq-dev/moq/pull/7505) profiled. The tokio range also spans [moq-dev/moq#5926](https://github.com/moq-dev/moq/pull/5926) itself, so the two rises can't be subtracted to find one. Decided 2026-11-01 (maintainer, follow-up planning after the 2026-11-01 /quest-afk run): m1, in this epic, since `main` must not reach the cut carrying accidental per-frame cost. -- Measure as #7505 did: `--profile profiling` builds with frame pointers and `moq-relay/io-uring,moq-bench/uring`, one target directory per revision, `perf stat` `cpu_core` instructions (user and kernel) over the steady window, divided by the frames the generator's 500 ms reports count over the same window. 4 io_uring workers, not 1, so chat does not saturate a worker. Pin the relay and generator cores so a loaded host still reads instruction deltas. +- Measure as [moq-dev/moq#7505](https://github.com/moq-dev/moq/pull/7505) did: `--profile profiling` builds with frame pointers and `moq-relay/io-uring,moq-bench/uring`, one target directory per revision, `perf stat` `cpu_core` instructions (user and kernel) over the steady window, divided by the frames the generator's 500 ms reports count over the same window. 4 io_uring workers, not 1, so chat does not saturate a worker. Pin the relay and generator cores so a loaded host still reads instruction deltas. - Bisect `cdff3c8f95..main` on chat over io_uring first (the largest rise and the tightest ranges), with a step size that keeps each build's cost bounded, then confirm the culprits on video and on the tokio workers. A rise that shows on io_uring but not on the tokio workers is attributed on its own, not folded into a shared culprit. -- Attribute each step with `perf record` at a fixed period, as #7505 did, before deciding fix or accept. Record accepted costs in the epic's Plan with the feature they pay for. -- Commit the driver this needs as a deterministic mode of `just bench-profile` or `just bench-runtime` (pinned cores, instructions per delivered frame, fixed-period samples) instead of another throwaway script; #7505's driver on `bench/relay.sh` was not committed. -- Implementation checkpoint in [moq-dev/moq#7605](https://github.com/moq-dev/moq/pull/7605): `just bench-profile instructions` owns disjoint relay/generator affinities, acknowledged PMU windows matched to interpolated 500 ms delivery reports, alternating revision pairs, and fixed-period source profiles. Missing or multiplexed counters, lost/throttled samples, and unbracketed delivery refuse the result. Historical profiling builds explicitly load only their own Cargo configuration, avoiding ancestor alias conflicts in nested scratch checkouts. -- The bisect, attribution, and resulting fixes remain blocked on the profiling host's sample-rate configuration. With `kernel.perf_event_max_sample_rate=1000`, both a 1,000,003-instruction current-tree capture and a 10,000,019-instruction historical capture throttled. The historical run had 957 instruction samples and 957 THROTTLE records. Increasing the period did not produce complete fixed-period attribution. Decide in the foreground whether to configure a higher host sample-rate limit temporarily and restore it afterward, or use another profiling host. No host sysctl was changed, and no feature cost has been accepted or optimization attributed. Re-run both endpoints with identical capture settings before bisecting; then confirm each measured step on video and Tokio workers. Keep the separately owned [moq-dev/moq#7493](https://github.com/moq-dev/moq/pull/7493) out of this fix. +- Attribute each step with `perf record` at a fixed period, as [moq-dev/moq#7505](https://github.com/moq-dev/moq/pull/7505) did, before deciding fix or accept. Record accepted costs in the epic's Plan with the feature they pay for. +- Commit the driver this needs as a deterministic mode of `just bench-profile` or `just bench-runtime` (pinned cores, instructions per delivered frame, fixed-period samples) instead of another throwaway script; [moq-dev/moq#7505](https://github.com/moq-dev/moq/pull/7505)'s driver on `bench/relay.sh` was not committed. +- Checkpointed at the maintainer's stop request in [moq-dev/moq#7605](https://github.com/moq-dev/moq/pull/7605), without a production fix or completed attribution. The [measurement report](/.github/relay-cpu-bisect.md) and [per-cell evidence](/.github/relay-cpu-bisect.json) retain counts, matching delivery boundaries, source sample markers, artifact hashes, and excluded attempts. Raw captures remain in that checkout's `.tmp/bench/`. +- `just bench-profile instructions` now provides disjoint relay/generator CPU sets, acknowledged PMU windows, delivery interpolated from surrounding 500 ms reports, alternating revision pairs, isolated revision builds, and fixed-period instruction stacks. It refuses unavailable/multiplexed counters and lost/throttled samples. Historical profiling checkouts live under `.tmp/` and load only their own Cargo configuration. +- The host's original 1,000 Hz sample-rate limit throttled every sample. Gated measurements temporarily requested 100,000 Hz and restored the original on exit; the driver changes no sysctls. A parser defect discovered during collection is fixed: identical cumulative snapshots at one millisecond remain valid, while contradictory observations are refused. Failed attempts are excluded explicitly. +- Completed exploratory chat comparisons use the same [21d6da9fc1](https://github.com/moq-dev/moq/commit/21d6da9fc108b48373a961f34e43c3fcd38b0146) production control, four io_uring workers, cores 4-7 for the relay and 0-3 for the generator, three alternating pairs, and a fixed period of 10,000,019 instructions. The old endpoint's median is 108.7k instructions/frame [108.0-118.3k], versus its paired current control's 156.8k [156.0-157.7k], at approximately 20,000 frames/s with zero confirmed loss in the boundary reports. No outlier cells were trimmed. These are sampled delivery estimates on a shared host, not confidence intervals. +- Attribution remains partial. The range-serving boundary [moq-dev/moq#5929](https://github.com/moq-dev/moq/pull/5929) accounts for a smaller measured historical step. [moq-dev/moq#6224](https://github.com/moq-dev/moq/pull/6224) introduced repeated `Reader::learn_end` polling; its after-change capture is retained but inconclusive because delivery changed and confirmed losses were reported. The source introduction is verified; its instruction effect is not yet confirmed. Later [moq-dev/moq#7532](https://github.com/moq-dev/moq/pull/7532) watch changes already reduce that path on the current control. Current `poll_forward_drop` stacks remain prominent and overlap the separately owned [moq-dev/moq#7493](https://github.com/moq-dev/moq/pull/7493); those production files were not edited here. No feature cost has been accepted in the epic from these incomplete measurements. +- Remaining: obtain a comparable-delivery measurement of the resume-end boundary, then attribute the growth between [fe722cec9f](https://github.com/moq-dev/moq/commit/fe722cec9f8432e0e46cb633cc5e79e59e7724cb) and the parent of [fd59eb18a5](https://github.com/moq-dev/moq/commit/fd59eb18a50f6e0ac819e73d941a3832cbdac91f), then account for changes through the pre-dynamic point [55f6ed2292](https://github.com/moq-dev/moq/commit/55f6ed229250d17fc99389237659906d011d8a94) and the pinned current control [21d6da9fc1](https://github.com/moq-dev/moq/commit/21d6da9fc108b48373a961f34e43c3fcd38b0146), including improvements that offset rises. Coordinate track/resume/publisher changes with [moq-dev/moq#7493](https://github.com/moq-dev/moq/pull/7493). Confirm attributed steps on video and Tokio workers, fix remaining avoidable costs with publisher/viewer benchmark sweeps, record justified feature-cost acceptances in the epic, and complete local review. None of those remaining experiments or fixes was started after the stop request. +- Validation: focused measurement tests and lints pass, but two full local gates failed on the known mbx E2BIG and runner namespace denial; an unchanged JS test also failed on the first attempt and passed on the second. The PR remains a draft and other-vendor review has not run. Preserve those limits when resuming. - [Fanout lock contention](/quest/m1/release-perf/fanout-lock.md) covers the bimodal futex cost on the fanout workload; don't fix it twice. A bisect step that lands on the same lock goes there. Public API: none expected. Wire: none. From 8c62afd3195a3307054120226b54e3309a5dbc1f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 20:54:15 -0800 Subject: [PATCH 05/36] docs(quest): keep checkpoint references valid Co-Authored-By: GPT-6.1 Sol --- quest/m1/release-perf/relay-cpu-bisect.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/quest/m1/release-perf/relay-cpu-bisect.md b/quest/m1/release-perf/relay-cpu-bisect.md index 28dd49555a5..387c7c87123 100644 --- a/quest/m1/release-perf/relay-cpu-bisect.md +++ b/quest/m1/release-perf/relay-cpu-bisect.md @@ -1,8 +1,8 @@ -# [M] Find and fix the relay's per-frame CPU growth since [moq-dev/moq#5926](https://github.com/moq-dev/moq/pull/5926) +# [M] Find and fix the relay's per-frame CPU growth ## Goal -The commits that raised the relay's instructions per delivered frame between [[cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95)](https://github.com/moq-dev/moq/commit/cdff3c8f95) ([moq-dev/moq#5926](https://github.com/moq-dev/moq/pull/5926) as merged) and `main` are found, each rise is attributed to a cause, and the avoidable costs are fixed. A rise that pays for a deliberate feature is accepted with its reason in the [release-perf epic](/quest/m1/release-perf/README.md). +The commits that raised the relay's instructions per delivered frame between [cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95) ([moq-dev/moq#5926](https://github.com/moq-dev/moq/pull/5926) as merged) and `main` are found, each rise is attributed to a cause, and the avoidable costs are fixed. A rise that pays for a deliberate feature is accepted with its reason in the [release-perf epic](/quest/m1/release-perf/README.md). ## Plan From 015d23057ac24fa5b2316e447ae4491f16109b3c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 16:51:58 -0800 Subject: [PATCH 06/36] bench(relay): rotate several bases with current in one instructions run Instructions mode now accepts space-separated BENCH_BASE revisions, so a bisect step measures its interior points side by side in each round. The paired revisions are the control, so instructions skip the unprofiled run. Co-Authored-By: Claude Opus 5.5 --- bench/instructions.py | 52 ++++++++++++++++++++ bench/run.sh | 108 ++++++++++++++++++++++++++++++------------ 2 files changed, 129 insertions(+), 31 deletions(-) diff --git a/bench/instructions.py b/bench/instructions.py index a8cc16e2f83..0fafdc09d98 100644 --- a/bench/instructions.py +++ b/bench/instructions.py @@ -269,6 +269,53 @@ def report(directory, load): (directory / "instructions.json").write_text(json.dumps(result, indent=2) + "\n") +def lost(rows, start, end): + """Groups the generator confirmed lost between the reports bracketing the window.""" + before = [row for row in rows if row["timestamp_ms"] <= start] + after = [row for row in rows if row["timestamp_ms"] >= end] + if not before or not after: + raise ValueError("generator reports do not bracket the counter window") + return after[0]["groups_lost"] - before[-1]["groups_lost"] + + +def summarize(run, labels): + """Print each revision's median and range across rounds, and keep them beside the evidence.""" + from statistics import median + + result = {} + for label in labels: + cells = [] + for path in sorted((run / "relay" / label).glob("round-*/instructions-*/*/instructions/instructions.json")): + cell = json.loads(path.read_text()) + rows = [json.loads(line) for line in (path.parent.parent / "load.jsonl").read_text().splitlines()] + cells.append( + dict( + cell=str(path.relative_to(run)), + instructions_per_frame=cell["instructions_per_frame"], + cycles_per_frame=cell["cycles_per_frame"], + recv_fps=cell["recv_fps"], + groups_lost=lost(rows, cell["window_start_ms"], cell["window_end_ms"]), + ) + ) + if not cells: + raise ValueError(f"no instruction cells for {label}") + result[label] = dict( + cells=cells, + **{ + key: dict(median=median(c[key] for c in cells), min=min(c[key] for c in cells), max=max(c[key] for c in cells)) + for key in ("instructions_per_frame", "cycles_per_frame", "recv_fps", "groups_lost") + }, + ) + (run / "instructions-summary.json").write_text(json.dumps(result, indent=2) + "\n") + print(f"\n{'revision':<20} {'kinstr/frame [min-max]':>26} {'kcycles/frame':>14} {'recv fps [min-max]':>24} {'lost':>5}") + for label, row in result.items(): + i, c, f = row["instructions_per_frame"], row["cycles_per_frame"], row["recv_fps"] + print( + f"{label:<20} {i['median'] / 1e3:>9.1f} [{i['min'] / 1e3:.1f}-{i['max'] / 1e3:.1f}] " + f"{c['median'] / 1e3:>14.1f} {f['median']:>9.0f} [{f['min']:.0f}-{f['max']:.0f}] {row['groups_lost']['max']:>5}" + ) + + def validate_data(path): """Reject lost or throttled samples before expensive symbolization.""" data = path.read_bytes() @@ -307,12 +354,17 @@ def main(): report_args = sub.add_parser("report") report_args.add_argument("directory", type=Path) report_args.add_argument("load", type=Path) + summarize_args = sub.add_parser("summarize") + summarize_args.add_argument("run", type=Path) + summarize_args.add_argument("labels", nargs="+") args = parser.parse_args() try: if args.command == "check": settings(args.workers) elif args.command == "capture": capture(args.pid, args.directory, args.delay, args.duration) + elif args.command == "summarize": + summarize(args.run, args.labels) else: report(args.directory, args.load) except (ValueError, OSError, subprocess.CalledProcessError) as error: diff --git a/bench/run.sh b/bench/run.sh index bbce75e64d0..e720fe1f114 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -70,10 +70,12 @@ case $MODE in BASE= ;; --profile) - BASE=${BENCH_BASE:-} - if [[ -n $BASE ]]; then - BASE=$(git rev-parse --verify "$BASE^{commit}") - fi + # Instructions mode accepts several space-separated bases, rotated with current in each round. + BASES=() + for value in ${BENCH_BASE:-}; do + BASES+=("$(git rev-parse --verify "$value^{commit}")") + done + BASE=${BASES[*]} PROFILE_RUNTIME=${BENCH_PROFILE_RUNTIME:-tokio-workers} if [[ ! $PROFILE_RUNTIME =~ ^(tokio-workers|io-uring-workers)$ ]]; then printf 'profile runtime must be tokio-workers or io-uring-workers, got %q\n' "$PROFILE_RUNTIME" >&2 @@ -97,6 +99,15 @@ case $MODE in exit 1 fi done + # Instruction counts pair revisions in rotated rounds, so they run on their own. + if [[ " $PROFILE_MODES " == *' instructions '* && $PROFILE_MODES != instructions ]]; then + printf 'BENCH_PROFILE_MODES=instructions runs alone, not with %q\n' "$PROFILE_MODES" >&2 + exit 1 + fi + if ((${#BASES[@]} > 1)) && [[ $PROFILE_MODES != instructions ]]; then + printf 'only BENCH_PROFILE_MODES=instructions accepts more than one BENCH_BASE revision\n' >&2 + exit 1 + fi for value in $PROFILE_WORKERS; do if [[ ! $value =~ ^[1-9][0-9]{0,2}$ ]] || ((value > 256)); then printf 'profile workers must be integers from 1 to 256, got %q\n' "$value" >&2 @@ -167,6 +178,7 @@ fi RUN=$(mktemp -d "$ARTIFACTS/run.XXXXXX") printf 'Benchmark evidence: %s\n' "$RUN" WORKTREE= +WORKTREES=() RELAY_PID= HOST_PID= LOAD_PID= @@ -192,10 +204,11 @@ cleanup() { kill "$RELAY_PID" 2>/dev/null || true wait "$RELAY_PID" 2>/dev/null || true fi - if [[ -n $WORKTREE ]]; then - git -C "$ROOT" worktree remove --force "$WORKTREE" >/dev/null 2>&1 || true - rmdir "$WORKTREE" 2>/dev/null || true - fi + local worktree + for worktree in ${WORKTREE:+"$WORKTREE"} ${WORKTREES[@]+"${WORKTREES[@]}"}; do + git -C "$ROOT" worktree remove --force "$worktree" >/dev/null 2>&1 || true + rmdir "$worktree" 2>/dev/null || true + done # Close both pipe writers before waiting for the owned log copiers. if [[ -n $STDOUT_PID ]]; then exec 1>&3 2>&4 @@ -851,19 +864,17 @@ run_profiles() { done generate_certificate - if [[ " $PROFILE_MODES " == *' instructions '* ]]; then - # Keep each bisect revision's artifacts instead of overwriting one base target. - CURRENT_TARGET=$TARGET/revisions/$(git rev-parse HEAD) - BASE_TARGET=$TARGET/revisions/$BASE + if [[ $PROFILE_MODES == instructions ]]; then + run_instructions + return fi if [[ -n $BASE ]]; then - mkdir -p "$ROOT/.tmp" - checkout_base "$BASE" "$ROOT/.tmp" + checkout_base "$BASE" mkdir -p "$RUN/base-context" (cd "$WORKTREE" && python3 "$ROOT/bench/metadata.py" "$RUN/base-context" "$BASE" --profile) fi - local label checkout target round index + local label checkout target local -a revisions=(current) if [[ -n $BASE ]]; then revisions=(base current); fi for mode in $PROFILE_MODES; do @@ -881,24 +892,54 @@ run_profiles() { if [[ $PROFILE_RUNTIME == io-uring-workers ]]; then check_uring "$target/profiling" fi - if [[ $mode != instructions ]]; then - run_profile "$mode" "$label" "$target/profiling/moq-relay" - fi + run_profile "$mode" "$label" "$target/profiling/moq-relay" done - if [[ $mode == instructions ]]; then - # Pair in alternating order rather than measuring every base run before current. - for ((round = 1; round <= ROUNDS; round++)); do - for ((index = 0; index < ${#revisions[@]}; index++)); do - label=${revisions[$(((index + round - 1) % ${#revisions[@]}))]} - target=$CURRENT_TARGET/profile - if [[ $label == base ]]; then target=$BASE_TARGET/profile; fi - run_profile "$mode" "$label/round-$round" "$target/profiling/moq-relay" - done - done - fi done } +# Count every base and the current tree in rotated order, so each round measures +# all revisions back to back and no revision always runs first. The current +# generator and sampler drive every relay, including old bases whose own harness +# cannot select the io_uring runtime. +run_instructions() { + local label sha target round index next worktree + local -a labels=() targets=() + # Each revision keeps its own target, so a later bisect step reuses its build. + build_workers "$ROOT" "$TARGET/revisions/$(git rev-parse HEAD)/profile" profiling instructions + LOAD_BIN=$TARGET/revisions/$(git rev-parse HEAD)/profile/profiling/moq-bench + HOST_BIN=$TARGET/revisions/$(git rev-parse HEAD)/profile/profiling/moq-bench-host + for sha in ${BASES[@]+"${BASES[@]}"}; do + label=base + if ((${#BASES[@]} > 1)); then label=base-${sha:0:12}; fi + # Nested in the checkout's ignored scratch space, loading only its own Cargo config. + mkdir -p "$ROOT/.tmp" + checkout_base "$sha" "$ROOT/.tmp" + worktree=$WORKTREE + WORKTREES+=("$worktree") + WORKTREE= + mkdir -p "$RUN/base-context/$label" + (cd "$worktree" && python3 "$ROOT/bench/metadata.py" "$RUN/base-context/$label" "$sha" --profile) + target=$TARGET/revisions/$sha/profile + build_workers "$worktree" "$target" profiling instructions + labels+=("$label") + targets+=("$target") + done + labels+=(current) + targets+=("$TARGET/revisions/$(git rev-parse HEAD)/profile") + if [[ $PROFILE_RUNTIME == io-uring-workers ]]; then + for target in "${targets[@]}"; do + check_uring "$target/profiling" + done + fi + for ((round = 1; round <= ROUNDS; round++)); do + for ((index = 0; index < ${#labels[@]}; index++)); do + next=$(((index + round - 1) % ${#labels[@]})) + run_profile instructions "${labels[$next]}/round-$round" "${targets[$next]}/profiling/moq-relay" + done + done + python3 "$ROOT/bench/instructions.py" summarize "$RUN" "${labels[@]}" +} + # Each profiled run follows an unprofiled control on the same binary, so the # throughput and CPU difference shows the profiler's overhead. run_profile() { @@ -929,7 +970,11 @@ run_profile() { ;; esac for workers in $PROFILE_WORKERS; do - run_workload "$label/control-$mode-$workers" "$relay" "$PROFILE_WORKLOAD" "$PROFILE_RUNTIME" "$workers" + # Instruction counts compare revisions under the same profiler, so the paired + # revisions are the control instead of an unprofiled run of each. + if [[ $mode != instructions ]]; then + run_workload "$label/control-$mode-$workers" "$relay" "$PROFILE_WORKLOAD" "$PROFILE_RUNTIME" "$workers" + fi PROFILE=$mode run_workload "$label/$mode-$workers" "$relay" "$PROFILE_WORKLOAD" "$PROFILE_RUNTIME" "$workers" PROFILE= @@ -937,7 +982,8 @@ run_profile() { profiled=$RUN/relay/$label/$mode-$workers/$PROFILE_WORKLOAD case $mode in instructions) - cat "$profiled/instructions/instructions.json" + jq -r '"\(.instructions_per_frame / 1000 | round) kinstr/frame at \(.recv_fps | round) fps"' \ + "$profiled/instructions/instructions.json" ;; lock) jq -r -s '[ From 82194441a92bb89fbc69f3759ef336d676fad643 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 17:06:16 -0800 Subject: [PATCH 07/36] bench(relay): keep each instructions base's checkout and build them side by side A base checked out at a fresh path rebuilt from scratch every run. Each base now keeps a worktree beside its target, and the revisions build in parallel on every allowed CPU, since fat LTO links on one thread. Co-Authored-By: Claude Opus 5.5 --- bench/README.md | 22 +++++++++++---- bench/instructions.py | 16 +++++------ bench/profile.test.py | 30 ++++++++++++++++++++ bench/run.sh | 65 +++++++++++++++++++++++++++---------------- 4 files changed, 94 insertions(+), 39 deletions(-) diff --git a/bench/README.md b/bench/README.md index 0b9bf7b341d..77f62a973c5 100644 --- a/bench/README.md +++ b/bench/README.md @@ -289,13 +289,23 @@ BENCH_RELAY_CPUS=4-7 BENCH_GENERATOR_CPUS=0-3 BENCH_RUNTIME_ROUNDS=5 \ nix develop --command just bench-profile instructions chat 4 io-uring-workers cdff3c8f95 BENCH_RELAY_CPUS=4-7 BENCH_GENERATOR_CPUS=0-3 \ nix develop --command just bench-profile instructions video 4 tokio-workers cdff3c8f95 +# A bisect step: every listed base and the current tree, rotated in each round. +BENCH_RELAY_CPUS=4-7 BENCH_GENERATOR_CPUS=0-3 \ + nix develop --command just bench-profile instructions chat 4 io-uring-workers "fe722cec9f 2d73eea67d e84ffb3de3" ``` Choose disjoint allowed CPU sets, with at least one relay CPU per worker. Hybrid CPUs require performance cores, and count their `cpu_core` events. -The relay and generator keep these affinities in both the control and measured -runs. Builds stay in separate targets per revision. Base/current pairs alternate -order across `BENCH_RUNTIME_ROUNDS` (default 3); the current generator drives both. +The relay and generator keep these affinities for every revision. Instructions +mode runs alone. Each base keeps a detached worktree and target under +`target/bench/revisions/`, so later runs reuse its build; delete them with +`git worktree remove`. The revisions build side by side on every allowed CPU, +since fat LTO links on one thread. Every round measures each base and the current tree back +to back, rotating which goes first, for `BENCH_RUNTIME_ROUNDS` rounds (default +3); the current generator drives them all. The revisions are each other's +control, so there is no unprofiled run per revision. The run ends with each +revision's median and range in `instructions-summary.json`, including the groups +the generator confirmed lost between the reports bracketing each counter window. After seven seconds of warmup, acknowledged perf enable/disable commands bound five seconds of relay counters. `instructions/instructions.json` records both @@ -318,9 +328,9 @@ use `instructions.json` for instruction comparisons. The fixed-period recording encloses the counter window; `instructions.folded` weights each stack by that period. `perf.data`, raw `stat.json`, `perf.log`, thread names, CPU sets, and delivery reports remain with the evidence. Both -counters and samples cover only the relay. Compare equivalent delivery and loss, -and retain controls to expose sampling overhead. Instructions reduce frequency -and scheduling noise; pinning does not remove interference from other host work. +counters and samples cover only the relay. Compare equivalent delivery and loss. +Instructions reduce frequency and scheduling noise; pinning does not remove +interference from other host work. ### Heap diff --git a/bench/instructions.py b/bench/instructions.py index 0fafdc09d98..3f76eb36b3a 100644 --- a/bench/instructions.py +++ b/bench/instructions.py @@ -299,20 +299,18 @@ def summarize(run, labels): ) if not cells: raise ValueError(f"no instruction cells for {label}") - result[label] = dict( - cells=cells, - **{ - key: dict(median=median(c[key] for c in cells), min=min(c[key] for c in cells), max=max(c[key] for c in cells)) - for key in ("instructions_per_frame", "cycles_per_frame", "recv_fps", "groups_lost") - }, - ) + result[label] = dict(cells=cells) + for key in ("instructions_per_frame", "cycles_per_frame", "recv_fps", "groups_lost"): + values = [cell[key] for cell in cells] + result[label][key] = dict(median=median(values), min=min(values), max=max(values)) (run / "instructions-summary.json").write_text(json.dumps(result, indent=2) + "\n") - print(f"\n{'revision':<20} {'kinstr/frame [min-max]':>26} {'kcycles/frame':>14} {'recv fps [min-max]':>24} {'lost':>5}") + print(f"\n{'revision':<20} {'kinstr/frame [min-max]':>26} {'kcycles/frame':>14} {'fps [min-max]':>24} {'lost':>5}") for label, row in result.items(): i, c, f = row["instructions_per_frame"], row["cycles_per_frame"], row["recv_fps"] print( f"{label:<20} {i['median'] / 1e3:>9.1f} [{i['min'] / 1e3:.1f}-{i['max'] / 1e3:.1f}] " - f"{c['median'] / 1e3:>14.1f} {f['median']:>9.0f} [{f['min']:.0f}-{f['max']:.0f}] {row['groups_lost']['max']:>5}" + f"{c['median'] / 1e3:>14.1f} {f['median']:>9.0f} [{f['min']:.0f}-{f['max']:.0f}] " + f"{row['groups_lost']['max']:>5}" ) diff --git a/bench/profile.test.py b/bench/profile.test.py index 2c61d728fae..0438bd0de30 100644 --- a/bench/profile.test.py +++ b/bench/profile.test.py @@ -72,6 +72,36 @@ def test_cpu_sets_cannot_overlap_or_escape_the_allowed_set(self): ): instructions.settings("1") + def test_summary_reports_each_revision_across_rounds(self): + with tempfile.TemporaryDirectory() as directory: + run = Path(directory) + for label, values in [("base-0123", [12, 10, 11]), ("current", [20, 21])]: + for round, value in enumerate(values, 1): + cell = run / "relay" / label / f"round-{round}" / "instructions-4" / "chat" + (cell / "instructions").mkdir(parents=True) + (cell / "instructions" / "instructions.json").write_text( + json.dumps( + dict( + instructions_per_frame=value, + cycles_per_frame=value * 2, + recv_fps=1000, + window_start_ms=1250, + window_end_ms=2250, + ) + ) + ) + rows = [dict(timestamp_ms=t, groups_lost=lost) for t, lost in [(1000, 1), (1500, 2), (2500, round)]] + (cell / "load.jsonl").write_text("".join(json.dumps(row) + "\n" for row in rows)) + with patch("builtins.print"): + instructions.summarize(run, ["base-0123", "current"]) + summary = json.loads((run / "instructions-summary.json").read_text()) + self.assertEqual(summary["base-0123"]["instructions_per_frame"], dict(median=11, min=10, max=12)) + self.assertEqual(summary["current"]["instructions_per_frame"]["median"], 20.5) + # Loss counts between the reports bracketing the window, not over the lifetime. + self.assertEqual([cell["groups_lost"] for cell in summary["base-0123"]["cells"]], [0, 1, 2]) + with self.assertRaisesRegex(ValueError, "no instruction cells"): + instructions.summarize(run, ["missing"]) + def test_lost_or_throttled_samples_refuse_attribution(self): with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "perf.data" diff --git a/bench/run.sh b/bench/run.sh index e720fe1f114..9b46de1fa35 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -24,6 +24,8 @@ ROUNDS=$((10#$ROUNDS)) # the performance cores it may already use. The kernel only lists them on Intel # hybrid parts. PINNED= +# Builds don't affect what a benchmark measures, so instructions mode compiles on every allowed CPU. +ALLOWED_CPUS=$(python3 -c 'import os; print(",".join(map(str, sorted(os.sched_getaffinity(0)))))') if [[ -r /sys/devices/cpu_core/cpus ]]; then PINNED=$( python3 - "$$" <<'EOF' @@ -178,7 +180,6 @@ fi RUN=$(mktemp -d "$ARTIFACTS/run.XXXXXX") printf 'Benchmark evidence: %s\n' "$RUN" WORKTREE= -WORKTREES=() RELAY_PID= HOST_PID= LOAD_PID= @@ -204,11 +205,10 @@ cleanup() { kill "$RELAY_PID" 2>/dev/null || true wait "$RELAY_PID" 2>/dev/null || true fi - local worktree - for worktree in ${WORKTREE:+"$WORKTREE"} ${WORKTREES[@]+"${WORKTREES[@]}"}; do - git -C "$ROOT" worktree remove --force "$worktree" >/dev/null 2>&1 || true - rmdir "$worktree" 2>/dev/null || true - done + if [[ -n $WORKTREE ]]; then + git -C "$ROOT" worktree remove --force "$WORKTREE" >/dev/null 2>&1 || true + rmdir "$WORKTREE" 2>/dev/null || true + fi # Close both pipe writers before waiting for the owned log copiers. if [[ -n $STDOUT_PID ]]; then exec 1>&3 2>&4 @@ -238,8 +238,8 @@ export CRITERION_HOME=$RUN/criterion checkout_base() { # Cargo merges every ancestor's .cargo/config.toml. A nested worktree would - # inherit this checkout's settings unless the caller loads only its own config. - WORKTREE=$(mktemp -d "${2:-${TMPDIR:-/tmp}}/moq-bench-base.XXXXXX") + # inherit this checkout's settings instead of measuring the base in isolation. + WORKTREE=$(mktemp -d "${TMPDIR:-/tmp}/moq-bench-base.XXXXXX") git worktree add --detach "$WORKTREE" "$1" >/dev/null } @@ -902,30 +902,47 @@ run_profiles() { # generator and sampler drive every relay, including old bases whose own harness # cannot select the io_uring runtime. run_instructions() { - local label sha target round index next worktree - local -a labels=() targets=() - # Each revision keeps its own target, so a later bisect step reuses its build. - build_workers "$ROOT" "$TARGET/revisions/$(git rev-parse HEAD)/profile" profiling instructions - LOAD_BIN=$TARGET/revisions/$(git rev-parse HEAD)/profile/profiling/moq-bench - HOST_BIN=$TARGET/revisions/$(git rev-parse HEAD)/profile/profiling/moq-bench-host + local label sha target round index next checkout + local -a labels=() targets=() checkouts=() builds=() for sha in ${BASES[@]+"${BASES[@]}"}; do label=base if ((${#BASES[@]} > 1)); then label=base-${sha:0:12}; fi - # Nested in the checkout's ignored scratch space, loading only its own Cargo config. - mkdir -p "$ROOT/.tmp" - checkout_base "$sha" "$ROOT/.tmp" - worktree=$WORKTREE - WORKTREES+=("$worktree") - WORKTREE= + # A checkout that stays put keeps Cargo's fingerprints, so a later bisect step + # reuses this revision's build. It loads only its own Cargo configuration. + checkout=$TARGET/revisions/$sha/checkout + if [[ $(git -C "$checkout" rev-parse HEAD 2>/dev/null) != "$sha" ]]; then + rm -rf "$checkout" + git worktree prune + git worktree add --detach "$checkout" "$sha" >/dev/null + fi mkdir -p "$RUN/base-context/$label" - (cd "$worktree" && python3 "$ROOT/bench/metadata.py" "$RUN/base-context/$label" "$sha" --profile) - target=$TARGET/revisions/$sha/profile - build_workers "$worktree" "$target" profiling instructions + (cd "$checkout" && python3 "$ROOT/bench/metadata.py" "$RUN/base-context/$label" "$sha" --profile) labels+=("$label") - targets+=("$target") + checkouts+=("$checkout") + targets+=("$TARGET/revisions/$sha/profile") done labels+=(current) + checkouts+=("$ROOT") targets+=("$TARGET/revisions/$(git rev-parse HEAD)/profile") + LOAD_BIN=${targets[-1]}/profiling/moq-bench + HOST_BIN=${targets[-1]}/profiling/moq-bench-host + + # Fat LTO links on one thread, so revisions build side by side. + for ((index = 0; index < ${#labels[@]}; index++)); do + mkdir -p "$RUN/build" + taskset -c "$ALLOWED_CPUS" bash -c "$(declare -f build_workers); build_workers \"\$@\"" _ \ + "${checkouts[$index]}" "${targets[$index]}" profiling instructions \ + >"$RUN/build/${labels[$index]}.log" 2>&1 & + builds+=($!) + done + for ((index = 0; index < ${#labels[@]}; index++)); do + if ! wait "${builds[$index]}"; then + printf 'build failed for %s:\n' "${labels[$index]}" >&2 + tail -n 40 "$RUN/build/${labels[$index]}.log" >&2 + return 1 + fi + tail -n 1 "$RUN/build/${labels[$index]}.log" + done if [[ $PROFILE_RUNTIME == io-uring-workers ]]; then for target in "${targets[@]}"; do check_uring "$target/profiling" From b68d9989abcac9ced05afd18391251ca7ebf9b2d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 17:55:23 -0800 Subject: [PATCH 08/36] bench(relay): record the host load each instruction window ran under Co-Authored-By: Claude Opus 5.5 --- bench/README.md | 4 +++- bench/instructions.py | 18 +++++++++++++++--- bench/profile.test.py | 2 ++ 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/bench/README.md b/bench/README.md index 77f62a973c5..a37d48fe035 100644 --- a/bench/README.md +++ b/bench/README.md @@ -305,7 +305,9 @@ to back, rotating which goes first, for `BENCH_RUNTIME_ROUNDS` rounds (default 3); the current generator drives them all. The revisions are each other's control, so there is no unprofiled run per revision. The run ends with each revision's median and range in `instructions-summary.json`, including the groups -the generator confirmed lost between the reports bracketing each counter window. +the generator confirmed lost between the reports bracketing each counter window +and the host's highest one-minute load during it. Admission only waits for room +when the run starts, so judge cells by the load they actually ran under. After seven seconds of warmup, acknowledged perf enable/disable commands bound five seconds of relay counters. `instructions/instructions.json` records both diff --git a/bench/instructions.py b/bench/instructions.py index 3f76eb36b3a..072c5575ae4 100644 --- a/bench/instructions.py +++ b/bench/instructions.py @@ -158,8 +158,10 @@ def capture(pid, directory, delay, duration): time.sleep(delay) record.command("enable") start = stat.command("enable") + load = [host_load()] time.sleep(duration) end = stat.command("disable") + load.append(host_load()) record.command("disable") finally: errors = [] @@ -180,6 +182,7 @@ def capture(pid, directory, delay, duration): threads=names, events=counters, sample_rate_limit=int(Path("/proc/sys/kernel/perf_event_max_sample_rate").read_text()), + host_load_1m=load, ), indent=2, ) @@ -187,6 +190,11 @@ def capture(pid, directory, delay, duration): ) +def host_load(): + """Other host work can change after admission, so each window keeps the load it ran under.""" + return float(Path("/proc/loadavg").read_text().split()[0]) + + def delivered(rows, at): """Interpolate delivery, never extrapolate past the generator's reports.""" index = bisect.bisect_left([row["timestamp_ms"] for row in rows], at) @@ -295,22 +303,26 @@ def summarize(run, labels): cycles_per_frame=cell["cycles_per_frame"], recv_fps=cell["recv_fps"], groups_lost=lost(rows, cell["window_start_ms"], cell["window_end_ms"]), + host_load_1m=max(cell.get("host_load_1m", [math.nan])), ) ) if not cells: raise ValueError(f"no instruction cells for {label}") result[label] = dict(cells=cells) - for key in ("instructions_per_frame", "cycles_per_frame", "recv_fps", "groups_lost"): + for key in ("instructions_per_frame", "cycles_per_frame", "recv_fps", "groups_lost", "host_load_1m"): values = [cell[key] for cell in cells] result[label][key] = dict(median=median(values), min=min(values), max=max(values)) (run / "instructions-summary.json").write_text(json.dumps(result, indent=2) + "\n") - print(f"\n{'revision':<20} {'kinstr/frame [min-max]':>26} {'kcycles/frame':>14} {'fps [min-max]':>24} {'lost':>5}") + print( + f"\n{'revision':<20} {'kinstr/frame [min-max]':>26} {'kcycles/frame':>14} {'fps [min-max]':>24} " + f"{'lost':>6} {'load':>5}" + ) for label, row in result.items(): i, c, f = row["instructions_per_frame"], row["cycles_per_frame"], row["recv_fps"] print( f"{label:<20} {i['median'] / 1e3:>9.1f} [{i['min'] / 1e3:.1f}-{i['max'] / 1e3:.1f}] " f"{c['median'] / 1e3:>14.1f} {f['median']:>9.0f} [{f['min']:.0f}-{f['max']:.0f}] " - f"{row['groups_lost']['max']:>5}" + f"{row['groups_lost']['max']:>6} {row['host_load_1m']['max']:>5.1f}" ) diff --git a/bench/profile.test.py b/bench/profile.test.py index 0438bd0de30..63333f14896 100644 --- a/bench/profile.test.py +++ b/bench/profile.test.py @@ -87,6 +87,7 @@ def test_summary_reports_each_revision_across_rounds(self): recv_fps=1000, window_start_ms=1250, window_end_ms=2250, + host_load_1m=[round, 0.5], ) ) ) @@ -99,6 +100,7 @@ def test_summary_reports_each_revision_across_rounds(self): self.assertEqual(summary["current"]["instructions_per_frame"]["median"], 20.5) # Loss counts between the reports bracketing the window, not over the lifetime. self.assertEqual([cell["groups_lost"] for cell in summary["base-0123"]["cells"]], [0, 1, 2]) + self.assertEqual(summary["base-0123"]["host_load_1m"], dict(median=2, min=1, max=3)) with self.assertRaisesRegex(ValueError, "no instruction cells"): instructions.summarize(run, ["missing"]) From d935d13e8dba7d36e06f836a04984d940dbf0a57 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 17:55:56 -0800 Subject: [PATCH 09/36] bench(relay): build the current tree in one stable instructions target Co-Authored-By: Claude Opus 5.5 --- bench/run.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bench/run.sh b/bench/run.sh index 9b46de1fa35..07750468efb 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -923,7 +923,8 @@ run_instructions() { done labels+=(current) checkouts+=("$ROOT") - targets+=("$TARGET/revisions/$(git rev-parse HEAD)/profile") + # The working tree changes under one target; Cargo's fingerprints track it. + targets+=("$TARGET/revisions/current/profile") LOAD_BIN=${targets[-1]}/profiling/moq-bench HOST_BIN=${targets[-1]}/profiling/moq-bench-host From a6cd6f91b6a0c79a81f2a012a2683bc77bd6c7d7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 18:28:41 -0800 Subject: [PATCH 10/36] bench(relay): hold the instructions generator at one revision A newer client moved what historical relays spend per frame: with main's generator, 21d6da9fc1's relay read 176k instructions per frame instead of the 156k it read under its own. BENCH_GENERATOR pins the client across a bisect's runs. Co-Authored-By: Claude Opus 5.5 --- bench/README.md | 7 +++-- bench/run.sh | 82 +++++++++++++++++++++++++++++++------------------ 2 files changed, 57 insertions(+), 32 deletions(-) diff --git a/bench/README.md b/bench/README.md index a37d48fe035..e7b465a0969 100644 --- a/bench/README.md +++ b/bench/README.md @@ -290,7 +290,7 @@ BENCH_RELAY_CPUS=4-7 BENCH_GENERATOR_CPUS=0-3 BENCH_RUNTIME_ROUNDS=5 \ BENCH_RELAY_CPUS=4-7 BENCH_GENERATOR_CPUS=0-3 \ nix develop --command just bench-profile instructions video 4 tokio-workers cdff3c8f95 # A bisect step: every listed base and the current tree, rotated in each round. -BENCH_RELAY_CPUS=4-7 BENCH_GENERATOR_CPUS=0-3 \ +BENCH_RELAY_CPUS=4-7 BENCH_GENERATOR_CPUS=0-3 BENCH_GENERATOR=21d6da9fc1 \ nix develop --command just bench-profile instructions chat 4 io-uring-workers "fe722cec9f 2d73eea67d e84ffb3de3" ``` @@ -302,7 +302,10 @@ mode runs alone. Each base keeps a detached worktree and target under `git worktree remove`. The revisions build side by side on every allowed CPU, since fat LTO links on one thread. Every round measures each base and the current tree back to back, rotating which goes first, for `BENCH_RUNTIME_ROUNDS` rounds (default -3); the current generator drives them all. The revisions are each other's +3); one generator drives them all. That is the current tree's unless +`BENCH_GENERATOR` names a revision: hold it fixed across a bisect's runs, since +client changes (flow control, subscription options) move what an old relay +spends per frame. The revisions are each other's control, so there is no unprofiled run per revision. The run ends with each revision's median and range in `instructions-summary.json`, including the groups the generator confirmed lost between the reports bracketing each counter window diff --git a/bench/run.sh b/bench/run.sh index 07750468efb..c9a0c5f0574 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -78,6 +78,12 @@ case $MODE in BASES+=("$(git rev-parse --verify "$value^{commit}")") done BASE=${BASES[*]} + # A client revision held fixed across bisect runs, so a newer client's behavior + # doesn't move the historical relays it drives. + GENERATOR= + if [[ -n ${BENCH_GENERATOR:-} ]]; then + GENERATOR=$(git rev-parse --verify "$BENCH_GENERATOR^{commit}") + fi PROFILE_RUNTIME=${BENCH_PROFILE_RUNTIME:-tokio-workers} if [[ ! $PROFILE_RUNTIME =~ ^(tokio-workers|io-uring-workers)$ ]]; then printf 'profile runtime must be tokio-workers or io-uring-workers, got %q\n' "$PROFILE_RUNTIME" >&2 @@ -106,8 +112,8 @@ case $MODE in printf 'BENCH_PROFILE_MODES=instructions runs alone, not with %q\n' "$PROFILE_MODES" >&2 exit 1 fi - if ((${#BASES[@]} > 1)) && [[ $PROFILE_MODES != instructions ]]; then - printf 'only BENCH_PROFILE_MODES=instructions accepts more than one BENCH_BASE revision\n' >&2 + if { ((${#BASES[@]} > 1)) || [[ -n $GENERATOR ]]; } && [[ $PROFILE_MODES != instructions ]]; then + printf 'only BENCH_PROFILE_MODES=instructions accepts several BENCH_BASE revisions or BENCH_GENERATOR\n' >&2 exit 1 fi for value in $PROFILE_WORKERS; do @@ -897,56 +903,72 @@ run_profiles() { done } +# Check out one revision where it stays put, so Cargo's fingerprints survive and a +# later bisect step reuses its build. It loads only its own Cargo configuration. +revision_checkout() { + local sha=$1 label=$2 checkout=$TARGET/revisions/$1/checkout + if [[ $(git -C "$checkout" rev-parse HEAD 2>/dev/null) != "$sha" ]]; then + rm -rf "$checkout" + git worktree prune + git worktree add --detach "$checkout" "$sha" >/dev/null + fi + mkdir -p "$RUN/base-context/$label" + (cd "$checkout" && python3 "$ROOT/bench/metadata.py" "$RUN/base-context/$label" "$sha" --profile) +} + # Count every base and the current tree in rotated order, so each round measures -# all revisions back to back and no revision always runs first. The current -# generator and sampler drive every relay, including old bases whose own harness -# cannot select the io_uring runtime. +# all revisions back to back and no revision always runs first. One generator and +# sampler drive every relay, including old bases whose own harness cannot select +# the io_uring runtime: BENCH_GENERATOR's, or else the current tree's. run_instructions() { - local label sha target round index next checkout + local label sha round index next generator built + # Measured revisions, then what gets built: those plus a generator measured nowhere. local -a labels=() targets=() checkouts=() builds=() for sha in ${BASES[@]+"${BASES[@]}"}; do label=base if ((${#BASES[@]} > 1)); then label=base-${sha:0:12}; fi - # A checkout that stays put keeps Cargo's fingerprints, so a later bisect step - # reuses this revision's build. It loads only its own Cargo configuration. - checkout=$TARGET/revisions/$sha/checkout - if [[ $(git -C "$checkout" rev-parse HEAD 2>/dev/null) != "$sha" ]]; then - rm -rf "$checkout" - git worktree prune - git worktree add --detach "$checkout" "$sha" >/dev/null - fi - mkdir -p "$RUN/base-context/$label" - (cd "$checkout" && python3 "$ROOT/bench/metadata.py" "$RUN/base-context/$label" "$sha" --profile) + revision_checkout "$sha" "$label" labels+=("$label") - checkouts+=("$checkout") + checkouts+=("$TARGET/revisions/$sha/checkout") targets+=("$TARGET/revisions/$sha/profile") done labels+=(current) checkouts+=("$ROOT") # The working tree changes under one target; Cargo's fingerprints track it. targets+=("$TARGET/revisions/current/profile") - LOAD_BIN=${targets[-1]}/profiling/moq-bench - HOST_BIN=${targets[-1]}/profiling/moq-bench-host + local -a build_labels=("${labels[@]}") build_checkouts=("${checkouts[@]}") build_targets=("${targets[@]}") + generator=${targets[-1]} + if [[ -n $GENERATOR ]]; then + generator=$TARGET/revisions/$GENERATOR/profile + if [[ " ${BASES[*]} " != *" $GENERATOR "* ]]; then + revision_checkout "$GENERATOR" generator + build_labels+=(generator) + build_checkouts+=("$TARGET/revisions/$GENERATOR/checkout") + build_targets+=("$generator") + fi + fi + LOAD_BIN=$generator/profiling/moq-bench + HOST_BIN=$generator/profiling/moq-bench-host - # Fat LTO links on one thread, so revisions build side by side. - for ((index = 0; index < ${#labels[@]}; index++)); do - mkdir -p "$RUN/build" + # Fat LTO links on one thread, so revisions build side by side on every allowed CPU. + mkdir -p "$RUN/build" + for ((index = 0; index < ${#build_labels[@]}; index++)); do taskset -c "$ALLOWED_CPUS" bash -c "$(declare -f build_workers); build_workers \"\$@\"" _ \ - "${checkouts[$index]}" "${targets[$index]}" profiling instructions \ - >"$RUN/build/${labels[$index]}.log" 2>&1 & + "${build_checkouts[$index]}" "${build_targets[$index]}" profiling instructions \ + >"$RUN/build/${build_labels[$index]}.log" 2>&1 & builds+=($!) done - for ((index = 0; index < ${#labels[@]}; index++)); do + for ((index = 0; index < ${#build_labels[@]}; index++)); do if ! wait "${builds[$index]}"; then - printf 'build failed for %s:\n' "${labels[$index]}" >&2 - tail -n 40 "$RUN/build/${labels[$index]}.log" >&2 + printf 'build failed for %s:\n' "${build_labels[$index]}" >&2 + tail -n 40 "$RUN/build/${build_labels[$index]}.log" >&2 return 1 fi - tail -n 1 "$RUN/build/${labels[$index]}.log" + tail -n 1 "$RUN/build/${build_labels[$index]}.log" done if [[ $PROFILE_RUNTIME == io-uring-workers ]]; then - for target in "${targets[@]}"; do - check_uring "$target/profiling" + for built in "${build_targets[@]}"; do + check_uring "$built/profiling" done fi for ((round = 1; round <= ROUNDS; round++)); do From 74f46e79b75bb2cbb4c5d4c4d75e390249ba22d5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 19:03:56 -0800 Subject: [PATCH 11/36] bench(relay): count instructions with the sampler off Sampling interrupts run as the relay's kernel instructions, at a cost that follows its stack depth, so counting them biased deep-stacked revisions. Stacks now come from the five seconds after the counter window. Co-Authored-By: Claude Opus 5.5 --- bench/README.md | 9 +++++++-- bench/instructions.py | 5 ++++- bench/metadata.py | 2 +- bench/relay.sh | 2 +- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/bench/README.md b/bench/README.md index e7b465a0969..57ac610c073 100644 --- a/bench/README.md +++ b/bench/README.md @@ -325,12 +325,17 @@ On the profiling host, a 1,000 Hz limit throttled every sample even at a period of 10,000,019. Temporarily raising `kernel.perf_event_max_sample_rate` to 100,000 allowed complete captures. Save and restore the original setting around the whole gated run; the driver records the observed limit and changes no sysctls. -The host sampler lasts fourteen seconds so delivery reports bracket both edges. +The host sampler lasts nineteen seconds so delivery reports bracket both edges +and the sampling window that follows. The normal `summary.json` still describes its own final throughput window; use `instructions.json` for instruction comparisons. `BENCH_INSTRUCTION_PERIOD` defaults to 1,000,003 retired instructions per sample. -The fixed-period recording encloses the counter window; `instructions.folded` +The fixed-period recording covers the five seconds after the counter window, +not the window itself: each sampling interrupt runs as the relay's kernel +instructions, at a cost that follows its stack depth, so at 2,000,003 it added +about 28k instructions per frame to one relay and the kernel lowered its sample +rate limit. Scale stacks by their share of samples. `instructions.folded` weights each stack by that period. `perf.data`, raw `stat.json`, `perf.log`, thread names, CPU sets, and delivery reports remain with the evidence. Both counters and samples cover only the relay. Compare equivalent delivery and loss. diff --git a/bench/instructions.py b/bench/instructions.py index 072c5575ae4..aafddfa83c0 100644 --- a/bench/instructions.py +++ b/bench/instructions.py @@ -156,12 +156,15 @@ def capture(pid, directory, delay, duration): stat.command("disable") record.command("disable") time.sleep(delay) - record.command("enable") + # Sampling interrupts run as the relay's kernel instructions, and their cost + # follows its stack depth, so the stacks come from the next window instead. start = stat.command("enable") load = [host_load()] time.sleep(duration) end = stat.command("disable") load.append(host_load()) + record.command("enable") + time.sleep(duration) record.command("disable") finally: errors = [] diff --git a/bench/metadata.py b/bench/metadata.py index ff5e8883793..d4e24cd2005 100644 --- a/bench/metadata.py +++ b/bench/metadata.py @@ -71,7 +71,7 @@ def command(*args): else None, "workload": None if mode == "--criterion" - else {"startup": "2s", "duration": "10s", "report": "500ms", "instructions_duration": "14s"}, + else {"startup": "2s", "duration": "10s", "report": "500ms", "instructions_duration": "19s"}, } (out / "metadata.json").write_text(json.dumps(metadata, indent=2) + "\n") subprocess.run( diff --git a/bench/relay.sh b/bench/relay.sh index f4b7ef75c6f..5dc5b9e3f56 100755 --- a/bench/relay.sh +++ b/bench/relay.sh @@ -308,7 +308,7 @@ run_workload() { LOAD_PID=$! generator_pid=$LOAD_PID local duration=10s - if [[ $PROFILE == instructions ]]; then duration=14s; fi + if [[ $PROFILE == instructions ]]; then duration=19s; fi "$HOST_BIN" --pid "$relay_pid" --pid "$generator_pid" --threads \ --interval 500ms --duration "$duration" --output "$host" \ >"$directory/host.log" 2>&1 & From bd812a762c149a4345430b53530a597d261144b8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 19:45:09 -0800 Subject: [PATCH 12/36] docs(bench): instruction counts batch down under host load Co-Authored-By: Claude Opus 5.5 --- bench/README.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/bench/README.md b/bench/README.md index 57ac610c073..ecc2386e375 100644 --- a/bench/README.md +++ b/bench/README.md @@ -332,15 +332,17 @@ use `instructions.json` for instruction comparisons. `BENCH_INSTRUCTION_PERIOD` defaults to 1,000,003 retired instructions per sample. The fixed-period recording covers the five seconds after the counter window, -not the window itself: each sampling interrupt runs as the relay's kernel -instructions, at a cost that follows its stack depth, so at 2,000,003 it added -about 28k instructions per frame to one relay and the kernel lowered its sample -rate limit. Scale stacks by their share of samples. `instructions.folded` +not the window itself, so sampling interrupts, which run as the relay's kernel +instructions, stay out of the counts. Scale stacks by their share of samples. +`instructions.folded` weights each stack by that period. `perf.data`, raw `stat.json`, `perf.log`, thread names, CPU sets, and delivery reports remain with the evidence. Both counters and samples cover only the relay. Compare equivalent delivery and loss. -Instructions reduce frequency and scheduling noise; pinning does not remove -interference from other host work. +Instructions reduce frequency noise, but not batching: a relay contending with +other host work handles more frames per wakeup, so each frame costs fewer +instructions. On the profiling host, cells measured under a one-minute load +above about 5 read up to 20% low, so compare cells from quiet stretches, within +one run. Pinning does not remove interference from other host work. ### Heap From adda6448de16df611e6103bd9bbabb16948c0876 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 21:16:01 -0800 Subject: [PATCH 13/36] docs(bench): attribute the relay's per-frame growth and what main still carries The two large rises, moq-dev/moq#6224's learn_end polling and the track lock contention after moq-dev/moq#6171, are already gone from main. The report and evidence replace the first checkpoint's, and the quest keeps the remaining feature-cost decisions. Co-Authored-By: Claude Opus 5.5 --- .github/relay-cpu-bisect.json | 6934 +-------------------- .github/relay-cpu-bisect.md | 82 +- quest/m1/release-perf/relay-cpu-bisect.md | 41 +- 3 files changed, 380 insertions(+), 6677 deletions(-) diff --git a/.github/relay-cpu-bisect.json b/.github/relay-cpu-bisect.json index 82826747a2e..cdd1d3e742b 100644 --- a/.github/relay-cpu-bisect.json +++ b/.github/relay-cpu-bisect.json @@ -1,6625 +1,315 @@ { - "version": 1, - "status": "Unfinished investigation, checkpointed at the user request. No runtime fix or accepted feature cost is claimed.", - "production_control_revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", - "method": { - "workload": { - "connections": 2001, - "publishers": 1, - "viewers": 2000, - "fps": 10, - "frame_bytes": 300, - "frames_per_group": 1 - }, - "runtime": "io-uring-workers", - "workers": 4, - "relay_cpus": [ - 4, - 5, - 6, - 7 - ], - "generator_cpus": [ - 0, - 1, - 2, - 3 - ], - "profile": "profiling", - "rustflags": "-C force-frame-pointers=yes", - "features": "moq-relay/io-uring,moq-bench/uring", - "warmup_s": 7, - "counter_window_s": 5, - "host_window_s": 14, - "generator_report_ms": 500, - "instruction_period": 10000019, - "counter_scope": "all relay threads, user and kernel", - "delivery_scope": "cumulative frames linearly interpolated at acknowledged perf enable/disable boundaries", - "source_scope": "fixed-period frame-pointer instruction samples; inclusive marker counts overlap and are not additive", - "source_markers": { - "resume_learn_end": "learn_end", - "publisher_forward_drop": "poll_forward_drop", - "track_state_on_stack": "model::track::TrackState", - "resume_reader_on_stack": "model::resume::Reader", - "waiter_list_on_stack": "WaiterList" - }, - "host_setup": "Original sample-rate limit 1000 throttled every interrupt. Each successful gated command temporarily requested 100000 and restored 1000 on exit. The kernel sometimes lowered the active limit; each capture records it. No sysctl changes in the committed driver.", - "limitations": [ - "Exploratory three-round comparisons; no performance acceptance threshold was introduced.", - "A shared loaded host and interpolated delivery are not exact deterministic accounting.", - "Every historical revision is paired with the same current source, rather than direct adjacent historical pairs.", - "No video or Tokio-worker confirmation has been run in this attempt.", - "No production optimization, benchmark sweep for a fix, or complete attribution of all increases has been completed.", - "The final resume-end after-change command completed capture but had materially different delivery and confirmed loss; its performance comparison is inconclusive and all six cells remain archived." - ], - "cpu_model": "Intel(R) Core(TM) Ultra 7 270K Plus", - "perf_version": "perf version 7.3.0" + "version": 2, + "metric": "relay instructions (user and kernel, all threads) per delivered frame over a 5 s counter window after 7 s warmup", + "markers": { + "learn_end": "inclusive share of fixed-period instruction samples whose stack matches resume::Reader>::learn_end", + "poll_forward_drop": "inclusive share of fixed-period instruction samples whose stack matches poll_forward_drop", + "cursor_probe": "inclusive share of fixed-period instruction samples whose stack matches model::track::Cursor>::probe", + "syscall_leaf": "inclusive share of fixed-period instruction samples whose stack matches ;syscall$", + "mutex_leaf": "inclusive share of fixed-period instruction samples whose stack matches ;(lock|unlock)$" + }, + "runs": [ + { + "artifact": "run.eVjz2w", + "harness_revision": "82194441a92bb89fbc69f3759ef336d676fad643", + "workload": "chat", + "runtime": "io-uring-workers", + "workers": 4, + "generator_revision": "82194441a92bb89fbc69f3759ef336d676fad643", + "instruction_period": 10000019, + "sampling": "concurrent with the counter window", + "relay_cpus": "4-7", + "generator_cpus": "0-3", + "cells": [ + {"revision": "2d73eea67d083ecf18938f4fc7c4842118eb87cf", "label": "base-2d73eea67d08", "round": 1, "kinstr_per_frame": 121.8, "kcycles_per_frame": 224.8, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 4.81, "samples": 1209, "markers": {"learn_end": 0.0, "poll_forward_drop": 8.9, "cursor_probe": 0.0, "syscall_leaf": 23.2, "mutex_leaf": 16.3}}, + {"revision": "2d73eea67d083ecf18938f4fc7c4842118eb87cf", "label": "base-2d73eea67d08", "round": 2, "kinstr_per_frame": 117.8, "kcycles_per_frame": 232.6, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 5.87, "samples": 1167, "markers": {"learn_end": 0.0, "poll_forward_drop": 6.9, "cursor_probe": 0.0, "syscall_leaf": 20.1, "mutex_leaf": 16.5}}, + {"revision": "2d73eea67d083ecf18938f4fc7c4842118eb87cf", "label": "base-2d73eea67d08", "round": 3, "kinstr_per_frame": 122.6, "kcycles_per_frame": 235.2, "recv_fps": 19999, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 4.2, "samples": 1216, "markers": {"learn_end": 0.0, "poll_forward_drop": 6.6, "cursor_probe": 0.0, "syscall_leaf": 21.7, "mutex_leaf": 16.0}}, + {"revision": "32c1e9940e5519a2bdc7c853d84846f55195cb94", "label": "base-32c1e9940e55", "round": 1, "kinstr_per_frame": 123.4, "kcycles_per_frame": 221.0, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 5.81, "samples": 1229, "markers": {"learn_end": 0.0, "poll_forward_drop": 8.0, "cursor_probe": 0.0, "syscall_leaf": 23.6, "mutex_leaf": 17.8}}, + {"revision": "32c1e9940e5519a2bdc7c853d84846f55195cb94", "label": "base-32c1e9940e55", "round": 2, "kinstr_per_frame": 115.8, "kcycles_per_frame": 220.7, "recv_fps": 20003, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 3.96, "samples": 1150, "markers": {"learn_end": 0.0, "poll_forward_drop": 8.5, "cursor_probe": 0.0, "syscall_leaf": 22.8, "mutex_leaf": 19.7}}, + {"revision": "32c1e9940e5519a2bdc7c853d84846f55195cb94", "label": "base-32c1e9940e55", "round": 3, "kinstr_per_frame": 117.3, "kcycles_per_frame": 223.7, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 3.13, "samples": 1168, "markers": {"learn_end": 0.0, "poll_forward_drop": 8.0, "cursor_probe": 0.0, "syscall_leaf": 21.8, "mutex_leaf": 17.6}}, + {"revision": "cbab2a8922c53a4541f919b89a95c131628f0cb0", "label": "base-cbab2a8922c5", "round": 1, "kinstr_per_frame": 129.2, "kcycles_per_frame": 241.4, "recv_fps": 19997, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 5.81, "samples": 1282, "markers": {"learn_end": 0.0, "poll_forward_drop": 8.7, "cursor_probe": 0.0, "syscall_leaf": 24.1, "mutex_leaf": 16.3}}, + {"revision": "cbab2a8922c53a4541f919b89a95c131628f0cb0", "label": "base-cbab2a8922c5", "round": 2, "kinstr_per_frame": 121.7, "kcycles_per_frame": 233.1, "recv_fps": 19997, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 4.88, "samples": 1210, "markers": {"learn_end": 0.0, "poll_forward_drop": 8.8, "cursor_probe": 0.0, "syscall_leaf": 20.6, "mutex_leaf": 18.1}}, + {"revision": "cbab2a8922c53a4541f919b89a95c131628f0cb0", "label": "base-cbab2a8922c5", "round": 3, "kinstr_per_frame": 124.0, "kcycles_per_frame": 244.0, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 4.37, "samples": 1233, "markers": {"learn_end": 0.0, "poll_forward_drop": 9.3, "cursor_probe": 0.0, "syscall_leaf": 23.5, "mutex_leaf": 16.7}}, + {"revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", "label": "base-e84ffb3de313", "round": 1, "kinstr_per_frame": 133.4, "kcycles_per_frame": 270.7, "recv_fps": 20003, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 4.41, "samples": 1327, "markers": {"learn_end": 0.0, "poll_forward_drop": 6.9, "cursor_probe": 0.0, "syscall_leaf": 22.0, "mutex_leaf": 16.9}}, + {"revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", "label": "base-e84ffb3de313", "round": 2, "kinstr_per_frame": 132.9, "kcycles_per_frame": 282.7, "recv_fps": 20001, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 6.04, "samples": 1320, "markers": {"learn_end": 0.0, "poll_forward_drop": 7.6, "cursor_probe": 0.0, "syscall_leaf": 21.4, "mutex_leaf": 15.4}}, + {"revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", "label": "base-e84ffb3de313", "round": 3, "kinstr_per_frame": 135.4, "kcycles_per_frame": 269.0, "recv_fps": 20002, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 4.05, "samples": 1346, "markers": {"learn_end": 0.0, "poll_forward_drop": 7.6, "cursor_probe": 0.0, "syscall_leaf": 21.1, "mutex_leaf": 16.5}}, + {"revision": "fe722cec9f8432e0e46cb633cc5e79e59e7724cb", "label": "base-fe722cec9f84", "round": 1, "kinstr_per_frame": 117.5, "kcycles_per_frame": 230.1, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 6.52, "samples": 1166, "markers": {"learn_end": 0.0, "poll_forward_drop": 8.1, "cursor_probe": 0.0, "syscall_leaf": 22.4, "mutex_leaf": 16.8}}, + {"revision": "fe722cec9f8432e0e46cb633cc5e79e59e7724cb", "label": "base-fe722cec9f84", "round": 2, "kinstr_per_frame": 114.6, "kcycles_per_frame": 222.0, "recv_fps": 20003, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 4.66, "samples": 1139, "markers": {"learn_end": 0.0, "poll_forward_drop": 8.5, "cursor_probe": 0.0, "syscall_leaf": 20.9, "mutex_leaf": 18.1}}, + {"revision": "fe722cec9f8432e0e46cb633cc5e79e59e7724cb", "label": "base-fe722cec9f84", "round": 3, "kinstr_per_frame": 120.5, "kcycles_per_frame": 220.7, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 3.06, "samples": 1200, "markers": {"learn_end": 0.0, "poll_forward_drop": 9.0, "cursor_probe": 0.0, "syscall_leaf": 25.0, "mutex_leaf": 17.9}}, + {"revision": "82194441a92bb89fbc69f3759ef336d676fad643", "label": "current", "round": 1, "kinstr_per_frame": 146.9, "kcycles_per_frame": 248.6, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 3.8, "samples": 1461, "markers": {"learn_end": 0.5, "poll_forward_drop": 14.9, "cursor_probe": 6.4, "syscall_leaf": 17.2, "mutex_leaf": 17.5}}, + {"revision": "82194441a92bb89fbc69f3759ef336d676fad643", "label": "current", "round": 2, "kinstr_per_frame": 145.2, "kcycles_per_frame": 278.8, "recv_fps": 20004, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 5.49, "samples": 1447, "markers": {"learn_end": 0.5, "poll_forward_drop": 13.3, "cursor_probe": 6.7, "syscall_leaf": 16.6, "mutex_leaf": 18.9}}, + {"revision": "82194441a92bb89fbc69f3759ef336d676fad643", "label": "current", "round": 3, "kinstr_per_frame": 147.1, "kcycles_per_frame": 270.2, "recv_fps": 20001, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 3.47, "samples": 1465, "markers": {"learn_end": 0.5, "poll_forward_drop": 15.1, "cursor_probe": 6.8, "syscall_leaf": 18.3, "mutex_leaf": 17.2}} + ] }, - "runs": [ - { - "artifact": "run.IkXbDf", - "description": "merged", - "base_revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", - "comparison_status": "exploratory comparable-delivery capture", - "harness_head": "9c19f0a6cda900407ee7202b23e8633e95f87857", - "machine": { - "os": "Linux-7.0.0-42-generic-x86_64-with-glibc2.44", - "machine": "x86_64", - "rustc": "rustc 1.98.1 (48a229cea 2026-09-01)\nbinary: rustc\ncommit-hash: 48a229ceaefd4985c50990b14116b6d856af0985\ncommit-date: 2026-09-01\nhost: x86_64-unknown-linux-gnu\nrelease: 1.98.1\nLLVM version: 22.1.8", - "cargo": "cargo 1.98.1 (797e8a9bc 2026-08-05)", - "affinity": [ - 0, - 1, - 2, - 3, - 4, - 5, - 6, - 7 - ] - }, - "harness_dirty_patch_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "cells": [ - { - "variant": "base", - "round": 1, - "capture": { - "version": 1, - "relay_cpus": [ - 4, - 5, - 6, - 7 - ], - "generator_cpus": [ - 0, - 1, - 2, - 3 - ], - "period": 10000019, - "start_ms": [ - 1793590071225.251, - 1793590071225.4373 - ], - "end_ms": [ - 1793590076225.6614, - 1793590076225.9832 - ], - "threads": { - "2082145": "moq-relay", - "2082153": "tokio-rt-worker", - "2082154": "notify-rs inoti", - "2082159": "moq-uring-0", - "2082162": "moq-uring-1", - "2082163": "moq-uring-2", - "2082164": "moq-uring-3" - }, - "events": [ - "cpu_core/instructions/", - "cpu_core/cycles/" - ], - "sample_rate_limit": 33000, - "window_start_ms": 1793590071225.3442, - "window_end_ms": 1793590076225.8223, - "boundary_uncertainty_ms": [ - 0.186279296875, - 0.32177734375 - ], - "delivered_frames": 100021.45055736473, - "recv_fps": 20002.377774769677, - "counters": { - "cpu_core/instructions/": 11832327091.0, - "cpu_core/cycles/": 22214916545.0, - "task-clock": 4375.270177 - }, - "instructions_per_frame": 118297.89535209622, - "cycles_per_frame": 222101.52343530755, - "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", - "instruction_samples": 1176 - }, - "delivery_boundary_reports": { - "window_start_ms": [ - { - "timestamp_ms": 1793590071023, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 139981, - "frames_sent": 71, - "bytes_recv": 41994300, - "groups_lost": 0, - "groups_expected": 133513, - "groups_present": 133513 - }, - { - "timestamp_ms": 1793590071522, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 149981, - "frames_sent": 76, - "bytes_recv": 44994300, - "groups_lost": 0, - "groups_expected": 143513, - "groups_present": 143513 - } - ], - "window_end_ms": [ - { - "timestamp_ms": 1793590076022, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 239981, - "frames_sent": 121, - "bytes_recv": 71994300, - "groups_lost": 0, - "groups_expected": 233513, - "groups_present": 233513 - }, - { - "timestamp_ms": 1793590076522, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 249981, - "frames_sent": 126, - "bytes_recv": 74994300, - "groups_lost": 0, - "groups_expected": 243513, - "groups_present": 243513 - } - ] - }, - "source_inclusive_samples": { - "resume_learn_end": 0, - "publisher_forward_drop": 109, - "track_state_on_stack": 103, - "resume_reader_on_stack": 181, - "waiter_list_on_stack": 104 - }, - "artifact_sha256": { - "load.jsonl": "20cdf65e356081638b9e66b4afe73c67ed4c1faf9a0cf39b9d2ca0b0704d0e5b", - "stat.json": "8f9d1cd4830704baefbed4c40d1a7fc39abd8aab604b683e88e11b2204085ee9", - "window.json": "d08f30e7a140f397e9892381626ddbfd83ba27849da3b4d803e656ecb05f94fb", - "perf.data": "6e7d7f4d3f69199bad33f8a4656d7e295252932b8d4178d799f688b39681e4b4", - "instructions.folded": "f0aaa8eac9610cdbeef61dfc37670c89de7e0a0a4e60a267fd68adc680238c3c" - } - }, - { - "variant": "base", - "round": 2, - "capture": { - "version": 1, - "relay_cpus": [ - 4, - 5, - 6, - 7 - ], - "generator_cpus": [ - 0, - 1, - 2, - 3 - ], - "period": 10000019, - "start_ms": [ - 1793590246597.0916, - 1793590246597.5728 - ], - "end_ms": [ - 1793590251597.7349, - 1793590251597.8665 - ], - "threads": { - "2238199": "moq-relay", - "2238202": "tokio-rt-worker", - "2238203": "notify-rs inoti", - "2238204": "moq-uring-0", - "2238205": "moq-uring-1", - "2238206": "moq-uring-2", - "2238207": "moq-uring-3" - }, - "events": [ - "cpu_core/instructions/", - "cpu_core/cycles/" - ], - "sample_rate_limit": 33000, - "window_start_ms": 1793590246597.332, - "window_end_ms": 1793590251597.8008, - "boundary_uncertainty_ms": [ - 0.481201171875, - 0.131591796875 - ], - "delivered_frames": 100009.375, - "recv_fps": 20000.0, - "counters": { - "cpu_core/instructions/": 10871014863.0, - "cpu_core/cycles/": 19624006962.0, - "task-clock": 3769.699453 - }, - "instructions_per_frame": 108699.95800893666, - "cycles_per_frame": 196221.67383807767, - "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", - "instruction_samples": 1080 - }, - "delivery_boundary_reports": { - "window_start_ms": [ - { - "timestamp_ms": 1793590246413, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 139986, - "frames_sent": 71, - "bytes_recv": 41995800, - "groups_lost": 0, - "groups_expected": 133272, - "groups_present": 133272 - }, - { - "timestamp_ms": 1793590246913, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 149986, - "frames_sent": 76, - "bytes_recv": 44995800, - "groups_lost": 0, - "groups_expected": 143272, - "groups_present": 143272 - } - ], - "window_end_ms": [ - { - "timestamp_ms": 1793590251413, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 239986, - "frames_sent": 121, - "bytes_recv": 71995800, - "groups_lost": 0, - "groups_expected": 233272, - "groups_present": 233272 - }, - { - "timestamp_ms": 1793590251913, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 249986, - "frames_sent": 126, - "bytes_recv": 74995800, - "groups_lost": 0, - "groups_expected": 243272, - "groups_present": 243272 - } - ] - }, - "source_inclusive_samples": { - "resume_learn_end": 0, - "publisher_forward_drop": 60, - "track_state_on_stack": 86, - "resume_reader_on_stack": 143, - "waiter_list_on_stack": 88 - }, - "artifact_sha256": { - "load.jsonl": "631a46b79e6ff611c5d674430633f108546a84f83831b61aff84c2e4164fdda5", - "stat.json": "f777ccd81c6c64c052a017591838c830e1d85dd4e59faf9a06aa157cfb6fdea2", - "window.json": "e80e117d68f839ee2ac7fd1fad68404127a3e1323e275e3ae8a84dc2b17fb88a", - "perf.data": "ef649b30d819f0c999ad6e638296df2bc5b18a819953066177c34c8a4f82e56c", - "instructions.folded": "b1c3d3b0a9463f2421ad0274a339af2ea3b58607687d3b7911013fb1005c744b" - } - }, - { - "variant": "base", - "round": 3, - "capture": { - "version": 1, - "relay_cpus": [ - 4, - 5, - 6, - 7 - ], - "generator_cpus": [ - 0, - 1, - 2, - 3 - ], - "period": 10000019, - "start_ms": [ - 1793590286315.6274, - 1793590286315.6765 - ], - "end_ms": [ - 1793590291315.8625, - 1793590291316.802 - ], - "threads": { - "2247444": "moq-relay", - "2247449": "tokio-rt-worker", - "2247451": "notify-rs inoti", - "2247453": "moq-uring-0", - "2247454": "moq-uring-1", - "2247455": "moq-uring-2", - "2247456": "moq-uring-3" - }, - "events": [ - "cpu_core/instructions/", - "cpu_core/cycles/" - ], - "sample_rate_limit": 33000, - "window_start_ms": 1793590286315.6519, - "window_end_ms": 1793590291316.3323, - "boundary_uncertainty_ms": [ - 0.049072265625, - 0.939453125 - ], - "delivered_frames": 100033.6083984375, - "recv_fps": 20003.99945573813, - "counters": { - "cpu_core/instructions/": 10806432745.0, - "cpu_core/cycles/": 19674669867.0, - "task-clock": 3774.105446 - }, - "instructions_per_frame": 108028.02096229084, - "cycles_per_frame": 196680.59747115264, - "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", - "instruction_samples": 1075 - }, - "delivery_boundary_reports": { - "window_start_ms": [ - { - "timestamp_ms": 1793590286136, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 139981, - "frames_sent": 71, - "bytes_recv": 41994300, - "groups_lost": 0, - "groups_expected": 133270, - "groups_present": 133270 - }, - { - "timestamp_ms": 1793590286636, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 149981, - "frames_sent": 76, - "bytes_recv": 44994300, - "groups_lost": 0, - "groups_expected": 143270, - "groups_present": 143270 - } - ], - "window_end_ms": [ - { - "timestamp_ms": 1793590291135, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 239981, - "frames_sent": 121, - "bytes_recv": 71994300, - "groups_lost": 0, - "groups_expected": 233270, - "groups_present": 233270 - }, - { - "timestamp_ms": 1793590291635, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 249981, - "frames_sent": 126, - "bytes_recv": 74994300, - "groups_lost": 0, - "groups_expected": 243270, - "groups_present": 243270 - } - ] - }, - "source_inclusive_samples": { - "resume_learn_end": 0, - "publisher_forward_drop": 69, - "track_state_on_stack": 80, - "resume_reader_on_stack": 126, - "waiter_list_on_stack": 105 - }, - "artifact_sha256": { - "load.jsonl": "7b2954c51f439f6d82526ae842dde2eaa8f500a3b645819582a9faa41ceb7532", - "stat.json": "7012723323d3555cbad0bcf7ea10d25db27629f5f7eb2407e5326a63aaf08301", - "window.json": "1a0a50c0fd2d2d0df217844bb1981f42d27d4a8c495e9bcd267462a8c02b64ac", - "perf.data": "ed4b3d745707e3460e125d96481d1fd3f969234168db95d565a604e9f2e87803", - "instructions.folded": "5c854cf6102c5872e02ee821666012e1a1066f9fea2a88acfcfccd89dd0079cf" - } - }, - { - "variant": "current", - "round": 1, - "capture": { - "version": 1, - "relay_cpus": [ - 4, - 5, - 6, - 7 - ], - "generator_cpus": [ - 0, - 1, - 2, - 3 - ], - "period": 10000019, - "start_ms": [ - 1793590112027.7512, - 1793590112028.049 - ], - "end_ms": [ - 1793590117028.1233, - 1793590117029.06 - ], - "threads": { - "2142408": "moq-relay", - "2142413": "tokio-rt-worker", - "2142415": "notify-rs inoti", - "2142417": "moq-uring-0", - "2142419": "moq-uring-1", - "2142420": "moq-uring-2", - "2142421": "moq-uring-3" - }, - "events": [ - "cpu_core/instructions/", - "cpu_core/cycles/" - ], - "sample_rate_limit": 33000, - "window_start_ms": 1793590112027.9001, - "window_end_ms": 1793590117028.5918, - "boundary_uncertainty_ms": [ - 0.2978515625, - 0.936767578125 - ], - "delivered_frames": 100033.8330078125, - "recv_fps": 20003.999446756217, - "counters": { - "cpu_core/instructions/": 15779052332.0, - "cpu_core/cycles/": 28362193024.0, - "task-clock": 5418.859292 - }, - "instructions_per_frame": 157737.15609565494, - "cycles_per_frame": 283526.00486462365, - "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", - "instruction_samples": 1572 - }, - "delivery_boundary_reports": { - "window_start_ms": [ - { - "timestamp_ms": 1793590111842, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 139968, - "frames_sent": 71, - "bytes_recv": 41990400, - "groups_lost": 0, - "groups_expected": 135340, - "groups_present": 135340 - }, - { - "timestamp_ms": 1793590112342, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 149968, - "frames_sent": 75, - "bytes_recv": 44990400, - "groups_lost": 0, - "groups_expected": 145340, - "groups_present": 145340 - } - ], - "window_end_ms": [ - { - "timestamp_ms": 1793590116841, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 239968, - "frames_sent": 120, - "bytes_recv": 71990400, - "groups_lost": 0, - "groups_expected": 235340, - "groups_present": 235340 - }, - { - "timestamp_ms": 1793590117341, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 249968, - "frames_sent": 125, - "bytes_recv": 74990400, - "groups_lost": 0, - "groups_expected": 245340, - "groups_present": 245340 - } - ] - }, - "source_inclusive_samples": { - "resume_learn_end": 14, - "publisher_forward_drop": 300, - "track_state_on_stack": 242, - "resume_reader_on_stack": 238, - "waiter_list_on_stack": 148 - }, - "artifact_sha256": { - "load.jsonl": "40855e1f8e4d738f01ab4aa8fed6777b4b3d0881071fb2ca98b4c653b718f3fc", - "stat.json": "f8405f70e88b8ed4bc626370248df9225fb6af0a8f5b90c4aec688061749ce57", - "window.json": "b96e0f20cec54ecdcf1bf86553300e81faf972f1dd6b2e9567bcdcbb31fb3b9a", - "perf.data": "8c748b60f4dc42e2da2d414e62e3a66079211b16520eb65bb97e6ae4a6f7fe3e", - "instructions.folded": "4229cefd0ebee590cfb4bb4447d1b07d6c71dc53bae873ade6509e12578676c4" - } - }, - { - "variant": "current", - "round": 2, - "capture": { - "version": 1, - "relay_cpus": [ - 4, - 5, - 6, - 7 - ], - "generator_cpus": [ - 0, - 1, - 2, - 3 - ], - "period": 10000019, - "start_ms": [ - 1793590177857.9124, - 1793590177858.239 - ], - "end_ms": [ - 1793590182858.3975, - 1793590182858.6152 - ], - "threads": { - "2220019": "moq-relay", - "2220039": "tokio-rt-worker", - "2220041": "notify-rs inoti", - "2220046": "moq-uring-0", - "2220047": "moq-uring-1", - "2220048": "moq-uring-2", - "2220049": "moq-uring-3" - }, - "events": [ - "cpu_core/instructions/", - "cpu_core/cycles/" - ], - "sample_rate_limit": 33000, - "window_start_ms": 1793590177858.0757, - "window_end_ms": 1793590182858.5063, - "boundary_uncertainty_ms": [ - 0.32666015625, - 0.2177734375 - ], - "delivered_frames": 100015.80792444263, - "recv_fps": 20001.43880471023, - "counters": { - "cpu_core/instructions/": 15681117586.0, - "cpu_core/cycles/": 32301932612.0, - "task-clock": 6087.315496 - }, - "instructions_per_frame": 156786.39118574501, - "cycles_per_frame": 322968.27153966133, - "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", - "instruction_samples": 1560 - }, - "delivery_boundary_reports": { - "window_start_ms": [ - { - "timestamp_ms": 1793590177679, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 139981, - "frames_sent": 70, - "bytes_recv": 41994300, - "groups_lost": 0, - "groups_expected": 136499, - "groups_present": 136499 - }, - { - "timestamp_ms": 1793590178179, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 149981, - "frames_sent": 75, - "bytes_recv": 44994300, - "groups_lost": 0, - "groups_expected": 146499, - "groups_present": 146499 - } - ], - "window_end_ms": [ - { - "timestamp_ms": 1793590182679, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 239981, - "frames_sent": 120, - "bytes_recv": 71994300, - "groups_lost": 0, - "groups_expected": 236499, - "groups_present": 236499 - }, - { - "timestamp_ms": 1793590183178, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 249981, - "frames_sent": 125, - "bytes_recv": 74994300, - "groups_lost": 0, - "groups_expected": 246499, - "groups_present": 246499 - } - ] - }, - "source_inclusive_samples": { - "resume_learn_end": 12, - "publisher_forward_drop": 290, - "track_state_on_stack": 248, - "resume_reader_on_stack": 215, - "waiter_list_on_stack": 146 - }, - "artifact_sha256": { - "load.jsonl": "8f4586a395f61c85d53dcf827370c72976dce874184aae772983351319a9b2ef", - "stat.json": "1ebc50981d14534d69827c3a497235bf2c5f8be8bb2edc61f6753ef1b52775cd", - "window.json": "1cc71083ce565f1f01fb6150d14ee6906ef68f43432360f07560eb963b053cd9", - "perf.data": "30043f4a5f7460afd7f66da8eafc9dc08a6029d1a7a2181cbdb0d38dbe274baa", - "instructions.folded": "4cc71a4480a54aab1f8c8e2a96702bae8f05245d235fab9a4ef3c2cdd9a267c4" - } - }, - { - "variant": "current", - "round": 3, - "capture": { - "version": 1, - "relay_cpus": [ - 4, - 5, - 6, - 7 - ], - "generator_cpus": [ - 0, - 1, - 2, - 3 - ], - "period": 10000019, - "start_ms": [ - 1793590325452.0142, - 1793590325452.4946 - ], - "end_ms": [ - 1793590330452.6287, - 1793590330453.953 - ], - "threads": { - "2258914": "moq-relay", - "2258917": "tokio-rt-worker", - "2258918": "notify-rs inoti", - "2258919": "moq-uring-0", - "2258920": "moq-uring-1", - "2258921": "moq-uring-2", - "2258922": "moq-uring-3" - }, - "events": [ - "cpu_core/instructions/", - "cpu_core/cycles/" - ], - "sample_rate_limit": 33000, - "window_start_ms": 1793590325452.2544, - "window_end_ms": 1793590330453.2908, - "boundary_uncertainty_ms": [ - 0.48046875, - 1.32421875 - ], - "delivered_frames": 100008.08300391404, - "recv_fps": 19997.471617041872, - "counters": { - "cpu_core/instructions/": 15597902854.0, - "cpu_core/cycles/": 27915802473.0, - "task-clock": 5318.083006 - }, - "instructions_per_frame": 155966.4217680239, - "cycles_per_frame": 279135.4621996649, - "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", - "instruction_samples": 1554 - }, - "delivery_boundary_reports": { - "window_start_ms": [ - { - "timestamp_ms": 1793590325268, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 139978, - "frames_sent": 71, - "bytes_recv": 41993400, - "groups_lost": 0, - "groups_expected": 136739, - "groups_present": 136739 - }, - { - "timestamp_ms": 1793590325769, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 149978, - "frames_sent": 76, - "bytes_recv": 44993400, - "groups_lost": 0, - "groups_expected": 146739, - "groups_present": 146739 - } - ], - "window_end_ms": [ - { - "timestamp_ms": 1793590330269, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 239978, - "frames_sent": 121, - "bytes_recv": 71993400, - "groups_lost": 0, - "groups_expected": 236739, - "groups_present": 236739 - }, - { - "timestamp_ms": 1793590330769, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 249978, - "frames_sent": 126, - "bytes_recv": 74993400, - "groups_lost": 0, - "groups_expected": 246739, - "groups_present": 246739 - } - ] - }, - "source_inclusive_samples": { - "resume_learn_end": 12, - "publisher_forward_drop": 307, - "track_state_on_stack": 252, - "resume_reader_on_stack": 212, - "waiter_list_on_stack": 152 - }, - "artifact_sha256": { - "load.jsonl": "fdf60c2ebe2a9ae694cf8fbd64ca399447559baca98c03298c254264d91d1cab", - "stat.json": "11ef9c5a545ea170ba8ed0a73737de37f350fc9cfb1a6ecc4efb023b455a16df", - "window.json": "afdb3d75cc3a51faedd37fd72d7471e9518b1c35a94fdcded5ca09d176c7466f", - "perf.data": "fa889f9ed771c6df1d49b697e576047ca6d61a1c6983efad6f248fa17c663ee8", - "instructions.folded": "fb6d9baf6b828d705a250553d6a9d986d89b5f65b96ac4fe11a736f48acea6ab" - } - } - ], - "summary": { - "base": { - "completed_cells": 3, - "median_instructions_per_frame": 108699.95800893666, - "min_instructions_per_frame": 108028.02096229084, - "max_instructions_per_frame": 118297.89535209622 - }, - "current": { - "completed_cells": 3, - "median_instructions_per_frame": 156786.39118574501, - "min_instructions_per_frame": 155966.4217680239, - "max_instructions_per_frame": 157737.15609565494 - } - }, - "pull_request_url": "https://github.com/moq-dev/moq/pull/5926" - }, - { - "artifact": "run.vgmg4w", - "description": "Immediately before dynamic tracks", - "base_revision": "55f6ed229250d17fc99389237659906d011d8a94", - "comparison_status": "exploratory comparable-delivery capture", - "harness_head": "9c19f0a6cda900407ee7202b23e8633e95f87857", - "machine": { - "os": "Linux-7.0.0-42-generic-x86_64-with-glibc2.44", - "machine": "x86_64", - "rustc": "rustc 1.98.1 (48a229cea 2026-09-01)\nbinary: rustc\ncommit-hash: 48a229ceaefd4985c50990b14116b6d856af0985\ncommit-date: 2026-09-01\nhost: x86_64-unknown-linux-gnu\nrelease: 1.98.1\nLLVM version: 22.1.8", - "cargo": "cargo 1.98.1 (797e8a9bc 2026-08-05)", - "affinity": [ - 0, - 1, - 2, - 3, - 4, - 5, - 6, - 7 - ] - }, - "harness_dirty_patch_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "cells": [ - { - "variant": "base", - "round": 1, - "capture": { - "version": 1, - "relay_cpus": [ - 4, - 5, - 6, - 7 - ], - "generator_cpus": [ - 0, - 1, - 2, - 3 - ], - "period": 10000019, - "start_ms": [ - 1793590706186.8125, - 1793590706186.9048 - ], - "end_ms": [ - 1793590711186.9834, - 1793590711187.879 - ], - "threads": { - "2331187": "moq-relay", - "2331190": "tokio-rt-worker", - "2331191": "notify-rs inoti", - "2331192": "moq-uring-0", - "2331193": "moq-uring-1", - "2331194": "moq-uring-2", - "2331195": "moq-uring-3" - }, - "events": [ - "cpu_core/instructions/", - "cpu_core/cycles/" - ], - "sample_rate_limit": 37000, - "window_start_ms": 1793590706186.8586, - "window_end_ms": 1793590711187.4312, - "boundary_uncertainty_ms": [ - 0.09228515625, - 0.8955078125 - ], - "delivered_frames": 101565.97229003906, - "recv_fps": 20310.868823857814, - "counters": { - "cpu_core/instructions/": 14496066156.0, - "cpu_core/cycles/": 31070291092.0, - "task-clock": 5895.938447 - }, - "instructions_per_frame": 142725.61793239173, - "cycles_per_frame": 305912.4073885046, - "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", - "instruction_samples": 1451 - }, - "delivery_boundary_reports": { - "window_start_ms": [ - { - "timestamp_ms": 1793590705957, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 139981, - "frames_sent": 70, - "bytes_recv": 41994300, - "groups_lost": 0, - "groups_expected": 135783, - "groups_present": 135783 - }, - { - "timestamp_ms": 1793590706459, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 146626, - "frames_sent": 76, - "bytes_recv": 43987800, - "groups_lost": 0, - "groups_expected": 142387, - "groups_present": 142387 - } - ], - "window_end_ms": [ - { - "timestamp_ms": 1793590710957, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 239981, - "frames_sent": 121, - "bytes_recv": 71994300, - "groups_lost": 0, - "groups_expected": 235783, - "groups_present": 235783 - }, - { - "timestamp_ms": 1793590711457, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 249981, - "frames_sent": 126, - "bytes_recv": 74994300, - "groups_lost": 0, - "groups_expected": 245783, - "groups_present": 245783 - } - ] - }, - "source_inclusive_samples": { - "resume_learn_end": 175, - "publisher_forward_drop": 137, - "track_state_on_stack": 264, - "resume_reader_on_stack": 431, - "waiter_list_on_stack": 192 - }, - "artifact_sha256": { - "load.jsonl": "d9387716044598430ad419eea1424ce8b8fa2f4e64d590a53483593dca06ee9c", - "stat.json": "12d024357993e42a51a6595b5751e52d74d8a4ca22bcf3f569f31f495a03a90d", - "window.json": "521f6e686b7ca883c52952938aa8b85f141de246fa8ac5277a39d7f94320d289", - "perf.data": "aa3a98f263a4626af310d51846f4bef54a2c42557dbe211110e7a46fbabb5b95", - "instructions.folded": "468eb686d6eb6a1d33c0e1639cb33ad1f83e75c26ffc3f1f76e38e71ea1d14be" - } - }, - { - "variant": "base", - "round": 2, - "capture": { - "version": 1, - "relay_cpus": [ - 4, - 5, - 6, - 7 - ], - "generator_cpus": [ - 0, - 1, - 2, - 3 - ], - "period": 10000019, - "start_ms": [ - 1793590885648.5505, - 1793590885648.8875 - ], - "end_ms": [ - 1793590890648.9712, - 1793590890649.185 - ], - "threads": { - "2385091": "moq-relay", - "2385094": "tokio-rt-worker", - "2385095": "notify-rs inoti", - "2385096": "moq-uring-0", - "2385097": "moq-uring-1", - "2385098": "moq-uring-2", - "2385099": "moq-uring-3" - }, - "events": [ - "cpu_core/instructions/", - "cpu_core/cycles/" - ], - "sample_rate_limit": 37000, - "window_start_ms": 1793590885648.719, - "window_end_ms": 1793590890649.0781, - "boundary_uncertainty_ms": [ - 0.3369140625, - 0.2138671875 - ], - "delivered_frames": 100007.1826171875, - "recv_fps": 20000.0, - "counters": { - "cpu_core/instructions/": 14395623521.0, - "cpu_core/cycles/": 29627853834.0, - "task-clock": 5626.923651 - }, - "instructions_per_frame": 143945.89612732406, - "cycles_per_frame": 296257.2593151732, - "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", - "instruction_samples": 1434 - }, - "delivery_boundary_reports": { - "window_start_ms": [ - { - "timestamp_ms": 1793590885449, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 139780, - "frames_sent": 70, - "bytes_recv": 41934000, - "groups_lost": 0, - "groups_expected": 134368, - "groups_present": 134368 - }, - { - "timestamp_ms": 1793590885949, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 149780, - "frames_sent": 75, - "bytes_recv": 44934000, - "groups_lost": 0, - "groups_expected": 144368, - "groups_present": 144368 - } - ], - "window_end_ms": [ - { - "timestamp_ms": 1793590890449, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 239780, - "frames_sent": 120, - "bytes_recv": 71934000, - "groups_lost": 0, - "groups_expected": 234368, - "groups_present": 234368 - }, - { - "timestamp_ms": 1793590890949, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 249780, - "frames_sent": 125, - "bytes_recv": 74934000, - "groups_lost": 0, - "groups_expected": 244368, - "groups_present": 244368 - } - ] - }, - "source_inclusive_samples": { - "resume_learn_end": 205, - "publisher_forward_drop": 169, - "track_state_on_stack": 269, - "resume_reader_on_stack": 474, - "waiter_list_on_stack": 201 - }, - "artifact_sha256": { - "load.jsonl": "857d82933db2b1c9e56aab7fd49bba40daf6e057f2edeb534001cd3e04de589f", - "stat.json": "ffb627e0007cf99530ea29492785a87feb91549834871832f746f99f374ffeda", - "window.json": "9f24c5898781597beff538338f8a4af6a227e4c7075318444468c187661bae69", - "perf.data": "62f6b07cb0ec9894315cf83a859a3a58c9bd7aa384d07b4cc7aac3b8c8db9ee1", - "instructions.folded": "1c408aa5dc83ae2300f63e82348567f959a0fa9f62321d363d71b219e26f72bb" - } - }, - { - "variant": "base", - "round": 3, - "capture": { - "version": 1, - "relay_cpus": [ - 4, - 5, - 6, - 7 - ], - "generator_cpus": [ - 0, - 1, - 2, - 3 - ], - "period": 10000019, - "start_ms": [ - 1793590931850.6985, - 1793590931850.786 - ], - "end_ms": [ - 1793590936852.2207, - 1793590936852.305 - ], - "threads": { - "2401372": "moq-relay", - "2401375": "tokio-rt-worker", - "2401376": "notify-rs inoti", - "2401377": "moq-uring-0", - "2401378": "moq-uring-1", - "2401379": "moq-uring-2", - "2401380": "moq-uring-3" - }, - "events": [ - "cpu_core/instructions/", - "cpu_core/cycles/" - ], - "sample_rate_limit": 37000, - "window_start_ms": 1793590931850.7422, - "window_end_ms": 1793590936852.2627, - "boundary_uncertainty_ms": [ - 0.08740234375, - 0.084228515625 - ], - "delivered_frames": 100307.95075888355, - "recv_fps": 20055.49124555223, - "counters": { - "cpu_core/instructions/": 13895968135.0, - "cpu_core/cycles/": 27911394233.0, - "task-clock": 5273.558912 - }, - "instructions_per_frame": 138533.06771666187, - "cycles_per_frame": 278257.0476401452, - "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", - "instruction_samples": 1383 - }, - "delivery_boundary_reports": { - "window_start_ms": [ - { - "timestamp_ms": 1793590931606, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 139126, - "frames_sent": 71, - "bytes_recv": 41737800, - "groups_lost": 0, - "groups_expected": 130733, - "groups_present": 130733 - }, - { - "timestamp_ms": 1793590932102, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 148782, - "frames_sent": 75, - "bytes_recv": 44634600, - "groups_lost": 0, - "groups_expected": 140389, - "groups_present": 140389 - } - ], - "window_end_ms": [ - { - "timestamp_ms": 1793590936603, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 238799, - "frames_sent": 120, - "bytes_recv": 71639700, - "groups_lost": 0, - "groups_expected": 230406, - "groups_present": 230406 - }, - { - "timestamp_ms": 1793590937103, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 249630, - "frames_sent": 125, - "bytes_recv": 74889000, - "groups_lost": 0, - "groups_expected": 241237, - "groups_present": 241237 - } - ] - }, - "source_inclusive_samples": { - "resume_learn_end": 173, - "publisher_forward_drop": 120, - "track_state_on_stack": 259, - "resume_reader_on_stack": 409, - "waiter_list_on_stack": 198 - }, - "artifact_sha256": { - "load.jsonl": "745144aef18decea25e2f35878b5ce25a642e6bb02d761c006fab9b2bd2ba3b9", - "stat.json": "e528efdebbedf5de12d35b8616dde3add346b338a07f9e4d535225cf9bcb8063", - "window.json": "9ed6b1aa64ebc2d10df095c269a95e84ba773037639cb56da39d0c93983a7b22", - "perf.data": "975a84429db8444100a39b096a25da33575a248303a2855415ee8b012305c3ad", - "instructions.folded": "eb0ffee09226991bc1ff401e751586826c21a480d369399f0244b76a952778bd" - } - }, - { - "variant": "current", - "round": 1, - "capture": { - "version": 1, - "relay_cpus": [ - 4, - 5, - 6, - 7 - ], - "generator_cpus": [ - 0, - 1, - 2, - 3 - ], - "period": 10000019, - "start_ms": [ - 1793590752483.7834, - 1793590752483.9976 - ], - "end_ms": [ - 1793590757484.0757, - 1793590757484.4133 - ], - "threads": { - "2337236": "moq-relay", - "2337239": "tokio-rt-worker", - "2337240": "notify-rs inoti", - "2337241": "moq-uring-0", - "2337242": "moq-uring-1", - "2337243": "moq-uring-2", - "2337244": "moq-uring-3" - }, - "events": [ - "cpu_core/instructions/", - "cpu_core/cycles/" - ], - "sample_rate_limit": 37000, - "window_start_ms": 1793590752483.8906, - "window_end_ms": 1793590757484.2446, - "boundary_uncertainty_ms": [ - 0.214111328125, - 0.337646484375 - ], - "delivered_frames": 100007.080078125, - "recv_fps": 20000.0, - "counters": { - "cpu_core/instructions/": 15544423267.0, - "cpu_core/cycles/": 30519449347.0, - "task-clock": 5761.496011 - }, - "instructions_per_frame": 155433.22787603416, - "cycles_per_frame": 305172.8869911847, - "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", - "instruction_samples": 1547 - }, - "delivery_boundary_reports": { - "window_start_ms": [ - { - "timestamp_ms": 1793590752294, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 139982, - "frames_sent": 71, - "bytes_recv": 41994600, - "groups_lost": 0, - "groups_expected": 135591, - "groups_present": 135591 - }, - { - "timestamp_ms": 1793590752794, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 149982, - "frames_sent": 76, - "bytes_recv": 44994600, - "groups_lost": 0, - "groups_expected": 145591, - "groups_present": 145591 - } - ], - "window_end_ms": [ - { - "timestamp_ms": 1793590757294, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 239982, - "frames_sent": 121, - "bytes_recv": 71994600, - "groups_lost": 0, - "groups_expected": 235591, - "groups_present": 235591 - }, - { - "timestamp_ms": 1793590757794, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 249982, - "frames_sent": 126, - "bytes_recv": 74994600, - "groups_lost": 0, - "groups_expected": 245591, - "groups_present": 245591 - } - ] - }, - "source_inclusive_samples": { - "resume_learn_end": 20, - "publisher_forward_drop": 298, - "track_state_on_stack": 245, - "resume_reader_on_stack": 218, - "waiter_list_on_stack": 152 - }, - "artifact_sha256": { - "load.jsonl": "f7c1587f5349063077acec6cacb26dc48b3c11a8d1e5e215b496b0b7ebc20561", - "stat.json": "82179b288bcd10c0794e475ba9961144f49b08d8ce00b6bf2618b41907756967", - "window.json": "83b6bea0d5f7c463970dcd799986236b3464b7dc75a55f11a4fa6a5a60a1a3c3", - "perf.data": "3bc21650def500654eac69b81e00b86eb96f58287f26aae588824171883bb7f4", - "instructions.folded": "fa454199437af12c23c59523554d4d2564556f3c4ae594ca5936c920defb00ff" - } - }, - { - "variant": "current", - "round": 2, - "capture": { - "version": 1, - "relay_cpus": [ - 4, - 5, - 6, - 7 - ], - "generator_cpus": [ - 0, - 1, - 2, - 3 - ], - "period": 10000019, - "start_ms": [ - 1793590819431.9517, - 1793590819433.8025 - ], - "end_ms": [ - 1793590824433.878, - 1793590824433.97 - ], - "threads": { - "2365412": "moq-relay", - "2365415": "tokio-rt-worker", - "2365416": "notify-rs inoti", - "2365417": "moq-uring-0", - "2365418": "moq-uring-1", - "2365419": "moq-uring-2", - "2365420": "moq-uring-3" - }, - "events": [ - "cpu_core/instructions/", - "cpu_core/cycles/" - ], - "sample_rate_limit": 37000, - "window_start_ms": 1793590819432.877, - "window_end_ms": 1793590824433.9238, - "boundary_uncertainty_ms": [ - 1.850830078125, - 0.092041015625 - ], - "delivered_frames": 100870.07506763568, - "recv_fps": 20169.791963334814, - "counters": { - "cpu_core/instructions/": 15436734420.0, - "cpu_core/cycles/": 26185777424.0, - "task-clock": 4925.804928 - }, - "instructions_per_frame": 153035.81770559127, - "cycles_per_frame": 259599.06747805869, - "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", - "instruction_samples": 1544 - }, - "delivery_boundary_reports": { - "window_start_ms": [ - { - "timestamp_ms": 1793590819224, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 138501, - "frames_sent": 70, - "bytes_recv": 41550300, - "groups_lost": 0, - "groups_expected": 130221, - "groups_present": 130221 - }, - { - "timestamp_ms": 1793590819726, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 148598, - "frames_sent": 75, - "bytes_recv": 44579400, - "groups_lost": 0, - "groups_expected": 140318, - "groups_present": 140318 - } - ], - "window_end_ms": [ - { - "timestamp_ms": 1793590824224, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 239745, - "frames_sent": 120, - "bytes_recv": 71923500, - "groups_lost": 0, - "groups_expected": 231465, - "groups_present": 231465 - }, - { - "timestamp_ms": 1793590824724, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 248861, - "frames_sent": 125, - "bytes_recv": 74658300, - "groups_lost": 0, - "groups_expected": 240581, - "groups_present": 240581 - } - ] - }, - "source_inclusive_samples": { - "resume_learn_end": 10, - "publisher_forward_drop": 306, - "track_state_on_stack": 224, - "resume_reader_on_stack": 228, - "waiter_list_on_stack": 171 - }, - "artifact_sha256": { - "load.jsonl": "f6d77ab1853d5d37da53766d1346c400fac71128a0a57df15316b5715b328721", - "stat.json": "ba757b4f96753169b8cce398f96e9f4ff7560fa399e6869c68b50570c581a915", - "window.json": "bfcfbb2276a0a141f97f07972452e47b20e24ddb3200402b71fdec0668454528", - "perf.data": "979b1f5c44adab78538f20e40def8373aca2a5f7227e02236010628307ecb11b", - "instructions.folded": "cf35020b74f6d70598c8210ea0ec588e12a616585835eb5d79d1c6bac4755e74" - } - }, - { - "variant": "current", - "round": 3, - "capture": { - "version": 1, - "relay_cpus": [ - 4, - 5, - 6, - 7 - ], - "generator_cpus": [ - 0, - 1, - 2, - 3 - ], - "period": 10000019, - "start_ms": [ - 1793590977976.9817, - 1793590977977.159 - ], - "end_ms": [ - 1793590982977.8079, - 1793590982977.8906 - ], - "threads": { - "2423267": "moq-relay", - "2423270": "tokio-rt-worker", - "2423271": "notify-rs inoti", - "2423272": "moq-uring-0", - "2423273": "moq-uring-1", - "2423274": "moq-uring-2", - "2423275": "moq-uring-3" - }, - "events": [ - "cpu_core/instructions/", - "cpu_core/cycles/" - ], - "sample_rate_limit": 37000, - "window_start_ms": 1793590977977.0703, - "window_end_ms": 1793590982977.849, - "boundary_uncertainty_ms": [ - 0.17724609375, - 0.082763671875 - ], - "delivered_frames": 100023.58615668837, - "recv_fps": 20001.601747471734, - "counters": { - "cpu_core/instructions/": 15430577805.0, - "cpu_core/cycles/": 28288190709.0, - "task-clock": 5362.83851 - }, - "instructions_per_frame": 154269.3918295209, - "cycles_per_frame": 282815.20185335237, - "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", - "instruction_samples": 1539 - }, - "delivery_boundary_reports": { - "window_start_ms": [ - { - "timestamp_ms": 1793590977778, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 139249, - "frames_sent": 71, - "bytes_recv": 41774700, - "groups_lost": 0, - "groups_expected": 131569, - "groups_present": 131569 - }, - { - "timestamp_ms": 1793590978278, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 149249, - "frames_sent": 76, - "bytes_recv": 44774700, - "groups_lost": 0, - "groups_expected": 141569, - "groups_present": 141569 - } - ], - "window_end_ms": [ - { - "timestamp_ms": 1793590982778, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 239249, - "frames_sent": 121, - "bytes_recv": 71774700, - "groups_lost": 0, - "groups_expected": 231569, - "groups_present": 231569 - }, - { - "timestamp_ms": 1793590983277, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 249249, - "frames_sent": 126, - "bytes_recv": 74774700, - "groups_lost": 0, - "groups_expected": 241569, - "groups_present": 241569 - } - ] - }, - "source_inclusive_samples": { - "resume_learn_end": 10, - "publisher_forward_drop": 284, - "track_state_on_stack": 254, - "resume_reader_on_stack": 224, - "waiter_list_on_stack": 129 - }, - "artifact_sha256": { - "load.jsonl": "00f4df2d4f298559a956bc73f8b9ad30e4f158e9650cccb1d5089fad690815c7", - "stat.json": "6f8a89d9bb2ed54a37bbc751ec68481c1f405e09922d3d48d011d47db868e348", - "window.json": "e9908747a036dde3c8025313281c7a13afb3daeb19bb05e8fdc64ea3abbbd5c9", - "perf.data": "fce504ebb39c8daa3939b72a573d8bb11d87b126467de40bbcc3ca4fbd873072", - "instructions.folded": "6517dfd50294a7831f5439cbe87a141430265a06175c3c992c9bfae31cef2e91" - } - } - ], - "summary": { - "base": { - "completed_cells": 3, - "median_instructions_per_frame": 142725.61793239173, - "min_instructions_per_frame": 138533.06771666187, - "max_instructions_per_frame": 143945.89612732406 - }, - "current": { - "completed_cells": 3, - "median_instructions_per_frame": 154269.3918295209, - "min_instructions_per_frame": 153035.81770559127, - "max_instructions_per_frame": 155433.22787603416 - } - }, - "pull_request_url": "https://github.com/moq-dev/moq/pull/6171" - }, - { - "artifact": "run.FlzdYt", - "description": "Immediately before per-reader ranges", - "base_revision": "dcd7d2cbb891dc1cf72c68295ba70fbd17a4ec15", - "comparison_status": "exploratory comparable-delivery capture", - "harness_head": "9c19f0a6cda900407ee7202b23e8633e95f87857", - "machine": { - "os": "Linux-7.0.0-42-generic-x86_64-with-glibc2.44", - "machine": "x86_64", - "rustc": "rustc 1.98.1 (48a229cea 2026-09-01)\nbinary: rustc\ncommit-hash: 48a229ceaefd4985c50990b14116b6d856af0985\ncommit-date: 2026-09-01\nhost: x86_64-unknown-linux-gnu\nrelease: 1.98.1\nLLVM version: 22.1.8", - "cargo": "cargo 1.98.1 (797e8a9bc 2026-08-05)", - "affinity": [ - 0, - 1, - 2, - 3, - 4, - 5, - 6, - 7 - ] - }, - "harness_dirty_patch_sha256": "d06a5759b069b0f7905e2c48b2b0a66c62495be6edd89e6d222a1ba7363be5ee", - "cells": [ - { - "variant": "base", - "round": 1, - "capture": { - "version": 1, - "relay_cpus": [ - 4, - 5, - 6, - 7 - ], - "generator_cpus": [ - 0, - 1, - 2, - 3 - ], - "period": 10000019, - "start_ms": [ - 1793591429532.2722, - 1793591429532.3203 - ], - "end_ms": [ - 1793591434532.479, - 1793591434532.7402 - ], - "threads": { - "2602269": "moq-relay", - "2602272": "tokio-rt-worker", - "2602273": "notify-rs inoti", - "2602274": "moq-uring-0", - "2602275": "moq-uring-1", - "2602276": "moq-uring-2", - "2602277": "moq-uring-3" - }, - "events": [ - "cpu_core/instructions/", - "cpu_core/cycles/" - ], - "sample_rate_limit": 40000, - "window_start_ms": 1793591429532.2964, - "window_end_ms": 1793591434532.6096, - "boundary_uncertainty_ms": [ - 0.048095703125, - 0.26123046875 - ], - "delivered_frames": 100006.2646484375, - "recv_fps": 20000.0, - "counters": { - "cpu_core/instructions/": 10514596796.0, - "cpu_core/cycles/": 19722345269.0, - "task-clock": 3737.32059 - }, - "instructions_per_frame": 105139.38134738922, - "cycles_per_frame": 197211.0981080238, - "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", - "instruction_samples": 1044 - }, - "delivery_boundary_reports": { - "window_start_ms": [ - { - "timestamp_ms": 1793591429332, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 139979, - "frames_sent": 71, - "bytes_recv": 41993700, - "groups_lost": 0, - "groups_expected": 133243, - "groups_present": 133243 - }, - { - "timestamp_ms": 1793591429832, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 149979, - "frames_sent": 75, - "bytes_recv": 44993700, - "groups_lost": 0, - "groups_expected": 143243, - "groups_present": 143243 - } - ], - "window_end_ms": [ - { - "timestamp_ms": 1793591434332, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 239979, - "frames_sent": 120, - "bytes_recv": 71993700, - "groups_lost": 0, - "groups_expected": 233243, - "groups_present": 233243 - }, - { - "timestamp_ms": 1793591434832, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 249979, - "frames_sent": 125, - "bytes_recv": 74993700, - "groups_lost": 0, - "groups_expected": 243243, - "groups_present": 243243 - } - ] - }, - "source_inclusive_samples": { - "resume_learn_end": 0, - "publisher_forward_drop": 65, - "track_state_on_stack": 103, - "resume_reader_on_stack": 115, - "waiter_list_on_stack": 117 - }, - "artifact_sha256": { - "load.jsonl": "18900ab161b6b103a6838c59a1ab282f4d0ed95a4506bdebde572fb8d430640d", - "stat.json": "79a43e7a88d738d8f67eef81c74179fa506b9fc434b8892513e8a25ad87b1e4e", - "window.json": "8032ba2ed5355ea28bcc20dff7a4beaab3d05453600bc99d24d82fc4dbd5b614", - "perf.data": "bc4bd163880aeb6c287f1973c232655b2f31b2654582fecc3a00e75a0fe9ae90", - "instructions.folded": "7ae8ac62057da779e04cbf2dbbe2770de2fc593e4897f8e431418811c41639a1" - } - }, - { - "variant": "base", - "round": 2, - "capture": { - "version": 1, - "relay_cpus": [ - 4, - 5, - 6, - 7 - ], - "generator_cpus": [ - 0, - 1, - 2, - 3 - ], - "period": 10000019, - "start_ms": [ - 1793591599197.893, - 1793591599198.4832 - ], - "end_ms": [ - 1793591604198.6462, - 1793591604199.917 - ], - "threads": { - "2803976": "moq-relay", - "2803981": "tokio-rt-worker", - "2803982": "notify-rs inoti", - "2803983": "moq-uring-0", - "2803984": "moq-uring-1", - "2803985": "moq-uring-2", - "2803986": "moq-uring-3" - }, - "events": [ - "cpu_core/instructions/", - "cpu_core/cycles/" - ], - "sample_rate_limit": 40000, - "window_start_ms": 1793591599198.188, - "window_end_ms": 1793591604199.2817, - "boundary_uncertainty_ms": [ - 0.590087890625, - 1.270751953125 - ], - "delivered_frames": 100009.62176305737, - "recv_fps": 19997.549888573347, - "counters": { - "cpu_core/instructions/": 10578214542.0, - "cpu_core/cycles/": 20004162999.0, - "task-clock": 3805.836228 - }, - "instructions_per_frame": 105771.96829182985, - "cycles_per_frame": 200022.38431012, - "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", - "instruction_samples": 1051 - }, - "delivery_boundary_reports": { - "window_start_ms": [ - { - "timestamp_ms": 1793591599005, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 139284, - "frames_sent": 70, - "bytes_recv": 41785200, - "groups_lost": 0, - "groups_expected": 130545, - "groups_present": 130545 - }, - { - "timestamp_ms": 1793591599505, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 149284, - "frames_sent": 75, - "bytes_recv": 44785200, - "groups_lost": 0, - "groups_expected": 140545, - "groups_present": 140545 - } - ], - "window_end_ms": [ - { - "timestamp_ms": 1793591604006, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 239284, - "frames_sent": 121, - "bytes_recv": 71785200, - "groups_lost": 0, - "groups_expected": 230545, - "groups_present": 230545 - }, - { - "timestamp_ms": 1793591604505, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 249284, - "frames_sent": 125, - "bytes_recv": 74785200, - "groups_lost": 0, - "groups_expected": 240545, - "groups_present": 240545 - } - ] - }, - "source_inclusive_samples": { - "resume_learn_end": 0, - "publisher_forward_drop": 65, - "track_state_on_stack": 73, - "resume_reader_on_stack": 127, - "waiter_list_on_stack": 124 - }, - "artifact_sha256": { - "load.jsonl": "b6a9e9fea814022133474d2c335ed406fe8f1d0e131ff4607a896d9bb4b22d91", - "stat.json": "8caff7a9e8ad0c632ffe28872d69d2341f3f2781be4c941f1973d8411ea298be", - "window.json": "c3324e3a4c2770dba5081272cc953c5d10b0364b2935574e7fa3e44ce50c1d5c", - "perf.data": "00763bf2de5d0c9a836baa7ffda3b649196f60dec428979f51637df88ffd1946", - "instructions.folded": "f4f6b20f2484c48922e21f8f36fcfc3c5cd869af8954c298d8b29728487bace7" - } - }, - { - "variant": "base", - "round": 3, - "capture": { - "version": 1, - "relay_cpus": [ - 4, - 5, - 6, - 7 - ], - "generator_cpus": [ - 0, - 1, - 2, - 3 - ], - "period": 10000019, - "start_ms": [ - 1793591639413.9666, - 1793591639414.2297 - ], - "end_ms": [ - 1793591644414.3, - 1793591644415.768 - ], - "threads": { - "2826068": "moq-relay", - "2826073": "tokio-rt-worker", - "2826074": "notify-rs inoti", - "2826076": "moq-uring-0", - "2826077": "moq-uring-1", - "2826078": "moq-uring-2", - "2826079": "moq-uring-3" - }, - "events": [ - "cpu_core/instructions/", - "cpu_core/cycles/" - ], - "sample_rate_limit": 40000, - "window_start_ms": 1793591639414.0981, - "window_end_ms": 1793591644415.0342, - "boundary_uncertainty_ms": [ - 0.26318359375, - 1.468017578125 - ], - "delivered_frames": 100031.09458816741, - "recv_fps": 20002.474313799543, - "counters": { - "cpu_core/instructions/": 10918608415.0, - "cpu_core/cycles/": 19942807466.0, - "task-clock": 3816.712397 - }, - "instructions_per_frame": 109152.14374042802, - "cycles_per_frame": 199366.08259767076, - "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", - "instruction_samples": 1086 - }, - "delivery_boundary_reports": { - "window_start_ms": [ - { - "timestamp_ms": 1793591639225, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 139984, - "frames_sent": 71, - "bytes_recv": 41995200, - "groups_lost": 0, - "groups_expected": 132784, - "groups_present": 132784 - }, - { - "timestamp_ms": 1793591639725, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 149984, - "frames_sent": 76, - "bytes_recv": 44995200, - "groups_lost": 0, - "groups_expected": 142784, - "groups_present": 142784 - } - ], - "window_end_ms": [ - { - "timestamp_ms": 1793591644224, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 239984, - "frames_sent": 121, - "bytes_recv": 71995200, - "groups_lost": 0, - "groups_expected": 232784, - "groups_present": 232784 - }, - { - "timestamp_ms": 1793591644725, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 249984, - "frames_sent": 126, - "bytes_recv": 74995200, - "groups_lost": 0, - "groups_expected": 242784, - "groups_present": 242784 - } - ] - }, - "source_inclusive_samples": { - "resume_learn_end": 0, - "publisher_forward_drop": 61, - "track_state_on_stack": 76, - "resume_reader_on_stack": 122, - "waiter_list_on_stack": 107 - }, - "artifact_sha256": { - "load.jsonl": "472fe13808cba4aefb9e0d312b1a74968118656a0ec2a97ff9bddad24ff88b7b", - "stat.json": "73fa91b9a99d0fed5074fa8aca44d9afaea4bea38bcca59d22941c141dc526f5", - "window.json": "0d12ad459a77b1b71ccb69d91e48c0ec0f76625e9c20c11499a305041e0040ae", - "perf.data": "01e1d99f7660e2e290c18f21e7c9c219cdaf7bfc659579b74bd80220cbd6c85d", - "instructions.folded": "aa738de3508117eae48aa0b5a95d0359d3170cb3d80c1195a7a7c3dc5f2c8da4" - } - }, - { - "variant": "current", - "round": 1, - "capture": { - "version": 1, - "relay_cpus": [ - 4, - 5, - 6, - 7 - ], - "generator_cpus": [ - 0, - 1, - 2, - 3 - ], - "period": 10000019, - "start_ms": [ - 1793591468080.8552, - 1793591468080.9146 - ], - "end_ms": [ - 1793591473080.9873, - 1793591473081.6592 - ], - "threads": { - "2614759": "moq-relay", - "2614770": "tokio-rt-worker", - "2614774": "notify-rs inoti", - "2614776": "moq-uring-0", - "2614777": "moq-uring-1", - "2614779": "moq-uring-2", - "2614780": "moq-uring-3" - }, - "events": [ - "cpu_core/instructions/", - "cpu_core/cycles/" - ], - "sample_rate_limit": 40000, - "window_start_ms": 1793591468080.8848, - "window_end_ms": 1793591473081.3232, - "boundary_uncertainty_ms": [ - 0.059326171875, - 0.671875 - ], - "delivered_frames": 100008.76953125, - "recv_fps": 20000.0, - "counters": { - "cpu_core/instructions/": 15745630432.0, - "cpu_core/cycles/": 29240252466.0, - "task-clock": 5579.799099 - }, - "instructions_per_frame": 157442.49735099403, - "cycles_per_frame": 292376.88457773917, - "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", - "instruction_samples": 1569 - }, - "delivery_boundary_reports": { - "window_start_ms": [ - { - "timestamp_ms": 1793591467895, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 139973, - "frames_sent": 70, - "bytes_recv": 41991900, - "groups_lost": 0, - "groups_expected": 135570, - "groups_present": 135570 - }, - { - "timestamp_ms": 1793591468395, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 149973, - "frames_sent": 75, - "bytes_recv": 44991900, - "groups_lost": 0, - "groups_expected": 145570, - "groups_present": 145570 - } - ], - "window_end_ms": [ - { - "timestamp_ms": 1793591472895, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 239973, - "frames_sent": 121, - "bytes_recv": 71991900, - "groups_lost": 0, - "groups_expected": 235570, - "groups_present": 235570 - }, - { - "timestamp_ms": 1793591473395, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 249973, - "frames_sent": 125, - "bytes_recv": 74991900, - "groups_lost": 0, - "groups_expected": 245570, - "groups_present": 245570 - } - ] - }, - "source_inclusive_samples": { - "resume_learn_end": 17, - "publisher_forward_drop": 321, - "track_state_on_stack": 256, - "resume_reader_on_stack": 234, - "waiter_list_on_stack": 155 - }, - "artifact_sha256": { - "load.jsonl": "92e49d1c7d0a81ffebe9c28d86ecfb80917afef109a3847907d2bd487cc4e23e", - "stat.json": "34efb2855ff09efbb27ed589ecc25359ff827b64dd9cd2cd97f837fff75a7cb5", - "window.json": "ab234eb58e797c67fa6e89fd5774eb3b0ef08aed1397dd49f6b78c226c173f69", - "perf.data": "66e5eebf77a530a074faf7be3e7605c6035cebb76a98bd79aa44bf65f9613060", - "instructions.folded": "80769aa1222e4fc20cea327ba70f98bd73bf03fc81f098e2358bcfed3ee47dd6" - } - }, - { - "variant": "current", - "round": 2, - "capture": { - "version": 1, - "relay_cpus": [ - 4, - 5, - 6, - 7 - ], - "generator_cpus": [ - 0, - 1, - 2, - 3 - ], - "period": 10000019, - "start_ms": [ - 1793591532508.8257, - 1793591532508.883 - ], - "end_ms": [ - 1793591537508.9658, - 1793591537510.3801 - ], - "threads": { - "2714794": "moq-relay", - "2714801": "tokio-rt-worker", - "2714806": "notify-rs inoti", - "2714807": "moq-uring-0", - "2714808": "moq-uring-1", - "2714810": "moq-uring-2", - "2714812": "moq-uring-3" - }, - "events": [ - "cpu_core/instructions/", - "cpu_core/cycles/" - ], - "sample_rate_limit": 40000, - "window_start_ms": 1793591532508.8545, - "window_end_ms": 1793591537509.6729, - "boundary_uncertainty_ms": [ - 0.057373046875, - 1.414306640625 - ], - "delivered_frames": 100016.39998747496, - "recv_fps": 20000.006558921483, - "counters": { - "cpu_core/instructions/": 16096769995.0, - "cpu_core/cycles/": 32341038169.0, - "task-clock": 6182.029905 - }, - "instructions_per_frame": 160941.3055960402, - "cycles_per_frame": 323357.35112491617, - "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", - "instruction_samples": 1603 - }, - "delivery_boundary_reports": { - "window_start_ms": [ - { - "timestamp_ms": 1793591532328, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 139962, - "frames_sent": 70, - "bytes_recv": 41988600, - "groups_lost": 0, - "groups_expected": 135340, - "groups_present": 135340 - }, - { - "timestamp_ms": 1793591532827, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 149962, - "frames_sent": 75, - "bytes_recv": 44988600, - "groups_lost": 0, - "groups_expected": 145340, - "groups_present": 145340 - } - ], - "window_end_ms": [ - { - "timestamp_ms": 1793591537328, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 239962, - "frames_sent": 120, - "bytes_recv": 71988600, - "groups_lost": 0, - "groups_expected": 235340, - "groups_present": 235340 - }, - { - "timestamp_ms": 1793591537827, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 249962, - "frames_sent": 125, - "bytes_recv": 74988600, - "groups_lost": 0, - "groups_expected": 245340, - "groups_present": 245340 - } - ] - }, - "source_inclusive_samples": { - "resume_learn_end": 12, - "publisher_forward_drop": 355, - "track_state_on_stack": 280, - "resume_reader_on_stack": 242, - "waiter_list_on_stack": 145 - }, - "artifact_sha256": { - "load.jsonl": "9bcd7ca7519beeb948685fc6b70604fb379e76d2a8f829cac879baf9456041a4", - "stat.json": "d8b9216a860adb4284d21b95dbfc1d2087a7567c4177344228c4810a04e02ce5", - "window.json": "a2722113b86f554bc3f8647f5de5f08c53a26388d76b0aad8134eedf6122e2db", - "perf.data": "5b46ef6d2c8733f7b497e059d3b821f55a9c81ee097ac4afa778eb3cf690641c", - "instructions.folded": "e9cafe61fd5a4c5d1d94221be4627140e114e009bf7ff7eaa17b6170953d193b" - } - }, - { - "variant": "current", - "round": 3, - "capture": { - "version": 1, - "relay_cpus": [ - 4, - 5, - 6, - 7 - ], - "generator_cpus": [ - 0, - 1, - 2, - 3 - ], - "period": 10000019, - "start_ms": [ - 1793591678816.9011, - 1793591678816.9712 - ], - "end_ms": [ - 1793591683817.052, - 1793591683817.2087 - ], - "threads": { - "2834707": "moq-relay", - "2834710": "tokio-rt-worker", - "2834711": "notify-rs inoti", - "2834712": "moq-uring-0", - "2834713": "moq-uring-1", - "2834714": "moq-uring-2", - "2834715": "moq-uring-3" - }, - "events": [ - "cpu_core/instructions/", - "cpu_core/cycles/" - ], - "sample_rate_limit": 40000, - "window_start_ms": 1793591678816.936, - "window_end_ms": 1793591683817.1304, - "boundary_uncertainty_ms": [ - 0.070068359375, - 0.15673828125 - ], - "delivered_frames": 99126.18189792291, - "recv_fps": 19824.465858352978, - "counters": { - "cpu_core/instructions/": 15546877923.0, - "cpu_core/cycles/": 31677863143.0, - "task-clock": 5994.939972 - }, - "instructions_per_frame": 156839.26915504216, - "cycles_per_frame": 319571.10156447755, - "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", - "instruction_samples": 1546 - }, - "delivery_boundary_reports": { - "window_start_ms": [ - { - "timestamp_ms": 1793591678634, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 139983, - "frames_sent": 70, - "bytes_recv": 41994900, - "groups_lost": 0, - "groups_expected": 135950, - "groups_present": 135950 - }, - { - "timestamp_ms": 1793591679134, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 149983, - "frames_sent": 75, - "bytes_recv": 44994900, - "groups_lost": 0, - "groups_expected": 145950, - "groups_present": 145950 - } - ], - "window_end_ms": [ - { - "timestamp_ms": 1793591683634, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 238943, - "frames_sent": 120, - "bytes_recv": 71682900, - "groups_lost": 0, - "groups_expected": 234910, - "groups_present": 234910 - }, - { - "timestamp_ms": 1793591684135, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 249407, - "frames_sent": 125, - "bytes_recv": 74822100, - "groups_lost": 0, - "groups_expected": 245374, - "groups_present": 245374 - } - ] - }, - "source_inclusive_samples": { - "resume_learn_end": 14, - "publisher_forward_drop": 273, - "track_state_on_stack": 245, - "resume_reader_on_stack": 181, - "waiter_list_on_stack": 157 - }, - "artifact_sha256": { - "load.jsonl": "8f7fc5db17c0fe9791dd998525a2197963ece459f8a0a368ca70bff5deaa342a", - "stat.json": "623cb58ccf32bbf612a62b333f636024e78843e5393eef394f97c63070f16820", - "window.json": "6537835011e0e20d9b2d0f8734b3f1e6f2a6961a7d98dd16f3a0180dc693e30d", - "perf.data": "f7c49a286fc4c6f8090b7a9de2670fa2736d4c2e1d7fd40aae4664e95efc33ad", - "instructions.folded": "8ba42bd224b9579db2b12345fe8d77683bd2e200e7b8a80500fb3ba79a2271d9" - } - } - ], - "summary": { - "base": { - "completed_cells": 3, - "median_instructions_per_frame": 105771.96829182985, - "min_instructions_per_frame": 105139.38134738922, - "max_instructions_per_frame": 109152.14374042802 - }, - "current": { - "completed_cells": 3, - "median_instructions_per_frame": 157442.49735099403, - "min_instructions_per_frame": 156839.26915504216, - "max_instructions_per_frame": 160941.3055960402 - } - }, - "pull_request_url": "https://github.com/moq-dev/moq/pull/5929" - }, - { - "artifact": "run.2bOgus", - "description": "Per-reader ranges", - "base_revision": "fe722cec9f8432e0e46cb633cc5e79e59e7724cb", - "comparison_status": "exploratory comparable-delivery capture", - "harness_head": "9c19f0a6cda900407ee7202b23e8633e95f87857", - "machine": { - "os": "Linux-7.0.0-42-generic-x86_64-with-glibc2.44", - "machine": "x86_64", - "rustc": "rustc 1.98.1 (48a229cea 2026-09-01)\nbinary: rustc\ncommit-hash: 48a229ceaefd4985c50990b14116b6d856af0985\ncommit-date: 2026-09-01\nhost: x86_64-unknown-linux-gnu\nrelease: 1.98.1\nLLVM version: 22.1.8", - "cargo": "cargo 1.98.1 (797e8a9bc 2026-08-05)", - "affinity": [ - 0, - 1, - 2, - 3, - 4, - 5, - 6, - 7 - ] - }, - "harness_dirty_patch_sha256": "1fb2250488672e6005088a1008ec311575d40c0a8ef4546a592f4ca2acea68a0", - "cells": [ - { - "variant": "base", - "round": 1, - "capture": { - "version": 1, - "relay_cpus": [ - 4, - 5, - 6, - 7 - ], - "generator_cpus": [ - 0, - 1, - 2, - 3 - ], - "period": 10000019, - "start_ms": [ - 1793592726175.0137, - 1793592726175.0774 - ], - "end_ms": [ - 1793592731175.1494, - 1793592731176.1135 - ], - "threads": { - "3403256": "moq-relay", - "3403259": "tokio-rt-worker", - "3403260": "notify-rs inoti", - "3403261": "moq-uring-0", - "3403262": "moq-uring-1", - "3403263": "moq-uring-2", - "3403264": "moq-uring-3" - }, - "events": [ - "cpu_core/instructions/", - "cpu_core/cycles/" - ], - "sample_rate_limit": 28000, - "window_start_ms": 1793592726175.0454, - "window_end_ms": 1793592731175.6313, - "boundary_uncertainty_ms": [ - 0.063720703125, - 0.964111328125 - ], - "delivered_frames": 99996.17166437415, - "recv_fps": 19996.890947216954, - "counters": { - "cpu_core/instructions/": 11314632417.0, - "cpu_core/cycles/": 21542189727.0, - "task-clock": 4131.846034 - }, - "instructions_per_frame": 113150.65595687287, - "cycles_per_frame": 215430.1446589768, - "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", - "instruction_samples": 1123 - }, - "delivery_boundary_reports": { - "window_start_ms": [ - { - "timestamp_ms": 1793592725981, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 139982, - "frames_sent": 70, - "bytes_recv": 41994600, - "groups_lost": 0, - "groups_expected": 133966, - "groups_present": 133966 - }, - { - "timestamp_ms": 1793592726480, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 149982, - "frames_sent": 75, - "bytes_recv": 44994600, - "groups_lost": 0, - "groups_expected": 143966, - "groups_present": 143966 - } - ], - "window_end_ms": [ - { - "timestamp_ms": 1793592730981, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 239982, - "frames_sent": 121, - "bytes_recv": 71994600, - "groups_lost": 0, - "groups_expected": 233966, - "groups_present": 233966 - }, - { - "timestamp_ms": 1793592731482, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 249982, - "frames_sent": 126, - "bytes_recv": 74994600, - "groups_lost": 0, - "groups_expected": 243966, - "groups_present": 243966 - } - ] - }, - "source_inclusive_samples": { - "resume_learn_end": 0, - "publisher_forward_drop": 75, - "track_state_on_stack": 118, - "resume_reader_on_stack": 137, - "waiter_list_on_stack": 111 - }, - "artifact_sha256": { - "load.jsonl": "dd5f67f709e6c35dbcfb529d1f45523d14a87983cf086221e937185e7d523227", - "stat.json": "7159bb7b77f8233904824d9dc4c2f2e276b718306c1ed2be6f7c23130c3d39eb", - "window.json": "2e417fc62091c4383c0b9ff0e68798c044c60445fd9c56fc96f44a25baa99909", - "perf.data": "8d3014e68d9f71df3b60c037ad04c6e321f63fabfd1ae6071baa440d45e77e81", - "instructions.folded": "9eade372a947195f6deb137fab1ea7f6d42667d0cca7ce819290baa9237dd91e" - } - }, - { - "variant": "base", - "round": 2, - "capture": { - "version": 1, - "relay_cpus": [ - 4, - 5, - 6, - 7 - ], - "generator_cpus": [ - 0, - 1, - 2, - 3 - ], - "period": 10000019, - "start_ms": [ - 1793592897069.7192, - 1793592897069.8096 - ], - "end_ms": [ - 1793592902069.882, - 1793592902070.324 - ], - "threads": { - "3458033": "moq-relay", - "3458036": "tokio-rt-worker", - "3458037": "notify-rs inoti", - "3458038": "moq-uring-0", - "3458039": "moq-uring-1", - "3458040": "moq-uring-2", - "3458041": "moq-uring-3" - }, - "events": [ - "cpu_core/instructions/", - "cpu_core/cycles/" - ], - "sample_rate_limit": 28000, - "window_start_ms": 1793592897069.7644, - "window_end_ms": 1793592902070.103, - "boundary_uncertainty_ms": [ - 0.09033203125, - 0.44189453125 - ], - "delivered_frames": 100006.7724609375, - "recv_fps": 20000.0, - "counters": { - "cpu_core/instructions/": 11211395662.0, - "cpu_core/cycles/": 21548382019.0, - "task-clock": 4110.358241 - }, - "instructions_per_frame": 112106.36426027202, - "cycles_per_frame": 215469.22762072706, - "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", - "instruction_samples": 1116 - }, - "delivery_boundary_reports": { - "window_start_ms": [ - { - "timestamp_ms": 1793592896879, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 139986, - "frames_sent": 70, - "bytes_recv": 41995800, - "groups_lost": 0, - "groups_expected": 132685, - "groups_present": 132685 - }, - { - "timestamp_ms": 1793592897379, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 149986, - "frames_sent": 75, - "bytes_recv": 44995800, - "groups_lost": 0, - "groups_expected": 142685, - "groups_present": 142685 - } - ], - "window_end_ms": [ - { - "timestamp_ms": 1793592901879, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 239986, - "frames_sent": 120, - "bytes_recv": 71995800, - "groups_lost": 0, - "groups_expected": 232685, - "groups_present": 232685 - }, - { - "timestamp_ms": 1793592902379, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 249986, - "frames_sent": 125, - "bytes_recv": 74995800, - "groups_lost": 0, - "groups_expected": 242685, - "groups_present": 242685 - } - ] - }, - "source_inclusive_samples": { - "resume_learn_end": 0, - "publisher_forward_drop": 84, - "track_state_on_stack": 110, - "resume_reader_on_stack": 148, - "waiter_list_on_stack": 115 - }, - "artifact_sha256": { - "load.jsonl": "636e399eabec69a817967b8c175519d31170de69b1a14092a2a97ccf259dbc8d", - "stat.json": "5f83462da4826283b2207df484c53cc0f07ba4ffb4169d59bd6eaffb84872431", - "window.json": "ac0e663e767e2c28da162cc5b969d468facc4f9c7126ccc5fde6d34ea2e8f0b0", - "perf.data": "12e271b74d443587a22b2c1e90fb262bd0c5d16e78b2658a1697da01cafa8eac", - "instructions.folded": "38cd2b29e52051621eed50956c71081e9fbbd13a7f7f3cf2bd25f79a8f47432c" - } - }, - { - "variant": "base", - "round": 3, - "capture": { - "version": 1, - "relay_cpus": [ - 4, - 5, - 6, - 7 - ], - "generator_cpus": [ - 0, - 1, - 2, - 3 - ], - "period": 10000019, - "start_ms": [ - 1793592937340.704, - 1793592937340.7566 - ], - "end_ms": [ - 1793592942340.8152, - 1793592942340.9216 - ], - "threads": { - "3463175": "moq-relay", - "3463178": "tokio-rt-worker", - "3463179": "notify-rs inoti", - "3463180": "moq-uring-0", - "3463181": "moq-uring-1", - "3463182": "moq-uring-2", - "3463183": "moq-uring-3" - }, - "events": [ - "cpu_core/instructions/", - "cpu_core/cycles/" - ], - "sample_rate_limit": 28000, - "window_start_ms": 1793592937340.7305, - "window_end_ms": 1793592942340.8684, - "boundary_uncertainty_ms": [ - 0.052490234375, - 0.1064453125 - ], - "delivered_frames": 99990.28858498245, - "recv_fps": 19997.50602798742, - "counters": { - "cpu_core/instructions/": 11190626305.0, - "cpu_core/cycles/": 21607081135.0, - "task-clock": 4097.69505 - }, - "instructions_per_frame": 111917.13178714359, - "cycles_per_frame": 216091.7969212179, - "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", - "instruction_samples": 1113 - }, - "delivery_boundary_reports": { - "window_start_ms": [ - { - "timestamp_ms": 1793592937152, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 140000, - "frames_sent": 70, - "bytes_recv": 42000000, - "groups_lost": 0, - "groups_expected": 132879, - "groups_present": 132879 - }, - { - "timestamp_ms": 1793592937652, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 150000, - "frames_sent": 75, - "bytes_recv": 45000000, - "groups_lost": 0, - "groups_expected": 142879, - "groups_present": 142879 - } - ], - "window_end_ms": [ - { - "timestamp_ms": 1793592942153, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 240000, - "frames_sent": 120, - "bytes_recv": 72000000, - "groups_lost": 0, - "groups_expected": 232879, - "groups_present": 232879 - }, - { - "timestamp_ms": 1793592942652, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 250000, - "frames_sent": 125, - "bytes_recv": 75000000, - "groups_lost": 0, - "groups_expected": 242879, - "groups_present": 242879 - } - ] - }, - "source_inclusive_samples": { - "resume_learn_end": 0, - "publisher_forward_drop": 78, - "track_state_on_stack": 97, - "resume_reader_on_stack": 140, - "waiter_list_on_stack": 87 - }, - "artifact_sha256": { - "load.jsonl": "7a6d9e0130d91ce0e8441bded0444a0e4e1db1878d5162c60a4c61f8ef905c63", - "stat.json": "07fa293123ba4198fa9377c53ddb3937d81aa354ea56e3c209b92883f6d584b1", - "window.json": "f421bf6b4a1e522ed5c2dc3bd93235f9179e7a4fac7a9d0ec96c45caed841d33", - "perf.data": "55985e651bb8ffff4a72ed340112fc373fefc5674f4fd78780cecd47ebb0554a", - "instructions.folded": "ab14407a6cdd7b41f6fbd07f206e74ba217fda1ab2460dbfc0b7e74b3d456211" - } - }, - { - "variant": "current", - "round": 1, - "capture": { - "version": 1, - "relay_cpus": [ - 4, - 5, - 6, - 7 - ], - "generator_cpus": [ - 0, - 1, - 2, - 3 - ], - "period": 10000019, - "start_ms": [ - 1793592766105.59, - 1793592766105.6628 - ], - "end_ms": [ - 1793592771105.8254, - 1793592771107.3223 - ], - "threads": { - "3415357": "moq-relay", - "3415362": "tokio-rt-worker", - "3415363": "notify-rs inoti", - "3415365": "moq-uring-0", - "3415366": "moq-uring-1", - "3415367": "moq-uring-2", - "3415368": "moq-uring-3" - }, - "events": [ - "cpu_core/instructions/", - "cpu_core/cycles/" - ], - "sample_rate_limit": 28000, - "window_start_ms": 1793592766105.6265, - "window_end_ms": 1793592771106.5737, - "boundary_uncertainty_ms": [ - 0.07275390625, - 1.496826171875 - ], - "delivered_frames": 100018.9453125, - "recv_fps": 20000.0, - "counters": { - "cpu_core/instructions/": 15711169720.0, - "cpu_core/cycles/": 28031733524.0, - "task-clock": 5358.969932 - }, - "instructions_per_frame": 157081.93753605275, - "cycles_per_frame": 280264.23830422753, - "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", - "instruction_samples": 1564 - }, - "delivery_boundary_reports": { - "window_start_ms": [ - { - "timestamp_ms": 1793592765920, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 139980, - "frames_sent": 71, - "bytes_recv": 41994000, - "groups_lost": 0, - "groups_expected": 136160, - "groups_present": 136160 - }, - { - "timestamp_ms": 1793592766420, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 149980, - "frames_sent": 75, - "bytes_recv": 44994000, - "groups_lost": 0, - "groups_expected": 146160, - "groups_present": 146160 - } - ], - "window_end_ms": [ - { - "timestamp_ms": 1793592770920, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 239980, - "frames_sent": 120, - "bytes_recv": 71994000, - "groups_lost": 0, - "groups_expected": 236160, - "groups_present": 236160 - }, - { - "timestamp_ms": 1793592771420, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 249980, - "frames_sent": 125, - "bytes_recv": 74994000, - "groups_lost": 0, - "groups_expected": 246160, - "groups_present": 246160 - } - ] - }, - "source_inclusive_samples": { - "resume_learn_end": 11, - "publisher_forward_drop": 318, - "track_state_on_stack": 260, - "resume_reader_on_stack": 222, - "waiter_list_on_stack": 171 - }, - "artifact_sha256": { - "load.jsonl": "bcbfc7727bc9bd01bb59d33ef9002e8db4e518b04b59d627e4c2344d6853b993", - "stat.json": "1cef97e8ac64b5763d8f1ce1029dc682d03ef7b043c02eaeb90bf6d42a94bb08", - "window.json": "1864085828f6ddcff3f0da2960d72f739059d2e8340d8cb45c1c48822db6ff07", - "perf.data": "270b786d8279bca3f2d63bb7f1adbbc7eba0f476051965bf1713d4caad7be6ac", - "instructions.folded": "1af64c071fcb65c83f2747e683c8d66dfa416b4ef72bf66817b7778df0efd0c2" - } - }, - { - "variant": "current", - "round": 2, - "capture": { - "version": 1, - "relay_cpus": [ - 4, - 5, - 6, - 7 - ], - "generator_cpus": [ - 0, - 1, - 2, - 3 - ], - "period": 10000019, - "start_ms": [ - 1793592831708.72, - 1793592831708.791 - ], - "end_ms": [ - 1793592836708.8792, - 1793592836710.3916 - ], - "threads": { - "3447207": "moq-relay", - "3447213": "tokio-rt-worker", - "3447215": "notify-rs inoti", - "3447217": "moq-uring-0", - "3447218": "moq-uring-1", - "3447219": "moq-uring-2", - "3447220": "moq-uring-3" - }, - "events": [ - "cpu_core/instructions/", - "cpu_core/cycles/" - ], - "sample_rate_limit": 28000, - "window_start_ms": 1793592831708.7554, - "window_end_ms": 1793592836709.6353, - "boundary_uncertainty_ms": [ - 0.071044921875, - 1.512451171875 - ], - "delivered_frames": 100017.59765625, - "recv_fps": 20000.0, - "counters": { - "cpu_core/instructions/": 16477354135.0, - "cpu_core/cycles/": 31166122602.0, - "task-clock": 6008.284902 - }, - "instructions_per_frame": 164744.5501703704, - "cycles_per_frame": 311606.3905985294, - "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", - "instruction_samples": 1637 - }, - "delivery_boundary_reports": { - "window_start_ms": [ - { - "timestamp_ms": 1793592831520, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 139971, - "frames_sent": 70, - "bytes_recv": 41991300, - "groups_lost": 0, - "groups_expected": 136123, - "groups_present": 136123 - }, - { - "timestamp_ms": 1793592832020, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 149971, - "frames_sent": 75, - "bytes_recv": 44991300, - "groups_lost": 0, - "groups_expected": 146123, - "groups_present": 146123 - } - ], - "window_end_ms": [ - { - "timestamp_ms": 1793592836520, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 239971, - "frames_sent": 120, - "bytes_recv": 71991300, - "groups_lost": 0, - "groups_expected": 236123, - "groups_present": 236123 - }, - { - "timestamp_ms": 1793592837020, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 249971, - "frames_sent": 125, - "bytes_recv": 74991300, - "groups_lost": 0, - "groups_expected": 246123, - "groups_present": 246123 - } - ] - }, - "source_inclusive_samples": { - "resume_learn_end": 18, - "publisher_forward_drop": 318, - "track_state_on_stack": 253, - "resume_reader_on_stack": 248, - "waiter_list_on_stack": 150 - }, - "artifact_sha256": { - "load.jsonl": "d976e8be43a5aea576bd68852a7ec3ba2c9962e00ac40a7facf35e9a00533713", - "stat.json": "c64bac5a7448524b4c6c13779b5a4ec0ac63077d025c0f3ba69e6a56c2508685", - "window.json": "970554cc01826cc3d020ab305155874062652f84b8860f9edd18702e2c89a6e1", - "perf.data": "d4fd6439d364fc468c07937048b55b5ceafe5732b9d2b97fc194a5e5e3b30797", - "instructions.folded": "c8afed451d97b180fcf9b579eb6d7a5e993c6716882ff209469ce7d5fe8ef586" - } - }, - { - "variant": "current", - "round": 3, - "capture": { - "version": 1, - "relay_cpus": [ - 4, - 5, - 6, - 7 - ], - "generator_cpus": [ - 0, - 1, - 2, - 3 - ], - "period": 10000019, - "start_ms": [ - 1793592977142.9905, - 1793592977143.1973 - ], - "end_ms": [ - 1793592982143.2788, - 1793592982143.3743 - ], - "threads": { - "3470232": "moq-relay", - "3470235": "tokio-rt-worker", - "3470236": "notify-rs inoti", - "3470237": "moq-uring-0", - "3470238": "moq-uring-1", - "3470239": "moq-uring-2", - "3470240": "moq-uring-3" - }, - "events": [ - "cpu_core/instructions/", - "cpu_core/cycles/" - ], - "sample_rate_limit": 28000, - "window_start_ms": 1793592977143.0938, - "window_end_ms": 1793592982143.3267, - "boundary_uncertainty_ms": [ - 0.206787109375, - 0.095458984375 - ], - "delivered_frames": 99442.91801408317, - "recv_fps": 19887.657195347674, - "counters": { - "cpu_core/instructions/": 15483800976.0, - "cpu_core/cycles/": 31019643084.0, - "task-clock": 5846.34583 - }, - "instructions_per_frame": 155705.41658690237, - "cycles_per_frame": 311934.15985245904, - "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", - "instruction_samples": 1543 - }, - "delivery_boundary_reports": { - "window_start_ms": [ - { - "timestamp_ms": 1793592976959, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 139959, - "frames_sent": 71, - "bytes_recv": 41987700, - "groups_lost": 0, - "groups_expected": 135047, - "groups_present": 135047 - }, - { - "timestamp_ms": 1793592977460, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 149348, - "frames_sent": 75, - "bytes_recv": 44804400, - "groups_lost": 0, - "groups_expected": 144436, - "groups_present": 144436 - } - ], - "window_end_ms": [ - { - "timestamp_ms": 1793592981959, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 238702, - "frames_sent": 120, - "bytes_recv": 71610600, - "groups_lost": 0, - "groups_expected": 233790, - "groups_present": 233790 - }, - { - "timestamp_ms": 1793592982459, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 249959, - "frames_sent": 126, - "bytes_recv": 74987700, - "groups_lost": 0, - "groups_expected": 245047, - "groups_present": 245047 - } - ] - }, - "source_inclusive_samples": { - "resume_learn_end": 4, - "publisher_forward_drop": 288, - "track_state_on_stack": 254, - "resume_reader_on_stack": 238, - "waiter_list_on_stack": 144 - }, - "artifact_sha256": { - "load.jsonl": "3915b34852bc9c694f2ca177bd52fcea6f0dffecad5b070030731ad7a26bd1d9", - "stat.json": "9f51ef443296da6fec4d138bdc430f37920947f7b73241054cc1f908ad9d7fc1", - "window.json": "aeb0d534b8bad83ba4c5b0c73bd01ac48fd34b95b06eb455f5d4f24642d8fe5b", - "perf.data": "db28232f4f94f8b8d33315c6dc6a6c7344a109af9efc2ebea73cb38743d7b8bb", - "instructions.folded": "211cbe80cbe847e693fc16edd69565cefd5435a7c5b512ceb7bd502b96530be7" - } - } - ], - "summary": { - "base": { - "completed_cells": 3, - "median_instructions_per_frame": 112106.36426027202, - "min_instructions_per_frame": 111917.13178714359, - "max_instructions_per_frame": 113150.65595687287 - }, - "current": { - "completed_cells": 3, - "median_instructions_per_frame": 157081.93753605275, - "min_instructions_per_frame": 155705.41658690237, - "max_instructions_per_frame": 164744.5501703704 - } - }, - "pull_request_url": "https://github.com/moq-dev/moq/pull/5929" - }, - { - "artifact": "run.iFmfVR", - "description": "Immediately before resume end tracking", - "base_revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", - "comparison_status": "exploratory comparable-delivery capture", - "harness_head": "9c19f0a6cda900407ee7202b23e8633e95f87857", - "machine": { - "os": "Linux-7.0.0-42-generic-x86_64-with-glibc2.44", - "machine": "x86_64", - "rustc": "rustc 1.98.1 (48a229cea 2026-09-01)\nbinary: rustc\ncommit-hash: 48a229ceaefd4985c50990b14116b6d856af0985\ncommit-date: 2026-09-01\nhost: x86_64-unknown-linux-gnu\nrelease: 1.98.1\nLLVM version: 22.1.8", - "cargo": "cargo 1.98.1 (797e8a9bc 2026-08-05)", - "affinity": [ - 0, - 1, - 2, - 3, - 4, - 5, - 6, - 7 - ] - }, - "harness_dirty_patch_sha256": "1fb2250488672e6005088a1008ec311575d40c0a8ef4546a592f4ca2acea68a0", - "cells": [ - { - "variant": "base", - "round": 1, - "capture": { - "version": 1, - "relay_cpus": [ - 4, - 5, - 6, - 7 - ], - "generator_cpus": [ - 0, - 1, - 2, - 3 - ], - "period": 10000019, - "start_ms": [ - 1793593639871.8943, - 1793593639871.9539 - ], - "end_ms": [ - 1793593644872.0403, - 1793593644872.16 - ], - "threads": { - "3853743": "moq-relay", - "3853746": "tokio-rt-worker", - "3853747": "notify-rs inoti", - "3853748": "moq-uring-0", - "3853749": "moq-uring-1", - "3853750": "moq-uring-2", - "3853751": "moq-uring-3" - }, - "events": [ - "cpu_core/instructions/", - "cpu_core/cycles/" - ], - "sample_rate_limit": 34000, - "window_start_ms": 1793593639871.924, - "window_end_ms": 1793593644872.1, - "boundary_uncertainty_ms": [ - 0.0595703125, - 0.11962890625 - ], - "delivered_frames": 100011.54142886103, - "recv_fps": 20001.604127736264, - "counters": { - "cpu_core/instructions/": 12769024351.0, - "cpu_core/cycles/": 26050400250.0, - "task-clock": 4972.217921 - }, - "instructions_per_frame": 127675.50793207905, - "cycles_per_frame": 260473.9400855035, - "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", - "instruction_samples": 1271 - }, - "delivery_boundary_reports": { - "window_start_ms": [ - { - "timestamp_ms": 1793593639671, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 139982, - "frames_sent": 70, - "bytes_recv": 41994600, - "groups_lost": 0, - "groups_expected": 133374, - "groups_present": 133374 - }, - { - "timestamp_ms": 1793593640172, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 149982, - "frames_sent": 75, - "bytes_recv": 44994600, - "groups_lost": 0, - "groups_expected": 143374, - "groups_present": 143374 - } - ], - "window_end_ms": [ - { - "timestamp_ms": 1793593644671, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 239982, - "frames_sent": 120, - "bytes_recv": 71994600, - "groups_lost": 0, - "groups_expected": 233374, - "groups_present": 233374 - }, - { - "timestamp_ms": 1793593645171, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 249982, - "frames_sent": 125, - "bytes_recv": 74994600, - "groups_lost": 0, - "groups_expected": 243374, - "groups_present": 243374 - } - ] - }, - "source_inclusive_samples": { - "resume_learn_end": 0, - "publisher_forward_drop": 76, - "track_state_on_stack": 121, - "resume_reader_on_stack": 236, - "waiter_list_on_stack": 123 - }, - "artifact_sha256": { - "load.jsonl": "1596865acaad0a890e1f47e22a42e43bcdc3f524fa58f935aedd9306fbe2b805", - "stat.json": "429263049c8d76b81daba010f9ea863ee60de0c85193ad08fd24eaf0938b3618", - "window.json": "c0266e3c8571a67fc691b8edb7595fa325fe3b6c543801e1cff7c53bc9447923", - "perf.data": "00ec413ee9809d3b968545996a4dc1021cb0e7b5fd7956d4c7b0e369c57f98aa", - "instructions.folded": "2b020f70432710eb4a264d1e7ae8fb244097f9f2ee79c066eef8b1a3ed85472c" - } - }, - { - "variant": "base", - "round": 2, - "capture": { - "version": 1, - "relay_cpus": [ - 4, - 5, - 6, - 7 - ], - "generator_cpus": [ - 0, - 1, - 2, - 3 - ], - "period": 10000019, - "start_ms": [ - 1793593814922.583, - 1793593814922.6758 - ], - "end_ms": [ - 1793593819922.7593, - 1793593819923.0583 - ], - "threads": { - "3882011": "moq-relay", - "3882014": "tokio-rt-worker", - "3882015": "notify-rs inoti", - "3882016": "moq-uring-0", - "3882017": "moq-uring-1", - "3882018": "moq-uring-2", - "3882019": "moq-uring-3" - }, - "events": [ - "cpu_core/instructions/", - "cpu_core/cycles/" - ], - "sample_rate_limit": 34000, - "window_start_ms": 1793593814922.6294, - "window_end_ms": 1793593819922.9087, - "boundary_uncertainty_ms": [ - 0.0927734375, - 0.299072265625 - ], - "delivered_frames": 100017.52571029816, - "recv_fps": 20002.38782117743, - "counters": { - "cpu_core/instructions/": 12654788263.0, - "cpu_core/cycles/": 29591227591.0, - "task-clock": 5609.588089 - }, - "instructions_per_frame": 126525.70810094554, - "cycles_per_frame": 295860.4242691557, - "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", - "instruction_samples": 1261 - }, - "delivery_boundary_reports": { - "window_start_ms": [ - { - "timestamp_ms": 1793593814722, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 139223, - "frames_sent": 71, - "bytes_recv": 41766900, - "groups_lost": 0, - "groups_expected": 129070, - "groups_present": 129070 - }, - { - "timestamp_ms": 1793593815222, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 149223, - "frames_sent": 76, - "bytes_recv": 44766900, - "groups_lost": 0, - "groups_expected": 139070, - "groups_present": 139070 - } - ], - "window_end_ms": [ - { - "timestamp_ms": 1793593819721, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 239223, - "frames_sent": 121, - "bytes_recv": 71766900, - "groups_lost": 0, - "groups_expected": 229070, - "groups_present": 229070 - }, - { - "timestamp_ms": 1793593820222, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 249223, - "frames_sent": 126, - "bytes_recv": 74766900, - "groups_lost": 0, - "groups_expected": 239070, - "groups_present": 239070 - } - ] - }, - "source_inclusive_samples": { - "resume_learn_end": 0, - "publisher_forward_drop": 89, - "track_state_on_stack": 102, - "resume_reader_on_stack": 240, - "waiter_list_on_stack": 126 - }, - "artifact_sha256": { - "load.jsonl": "01b91704669d71b4dcd5180ff722f58500f77e344cc0509e0922988d9a0cd4fd", - "stat.json": "b1aa02b423fa575ae62fc88b1f924c06573e582cf4bd7a2544df793de4605137", - "window.json": "3d9db44b9640e7dd7a821310e3af250e0ae3ccf8b6e7bd5e176791c8f9144abf", - "perf.data": "b6fc6e6db4a76da34bcc68d49b2df449ac3d9cf4b8a84b508f75c0b929a82e4c", - "instructions.folded": "b9d2bf3318a06d22175f8502f8e4638fc51aadc765b973f05ecf95cb931670bd" - } - }, - { - "variant": "base", - "round": 3, - "capture": { - "version": 1, - "relay_cpus": [ - 4, - 5, - 6, - 7 - ], - "generator_cpus": [ - 0, - 1, - 2, - 3 - ], - "period": 10000019, - "start_ms": [ - 1793593860507.2256, - 1793593860507.2815 - ], - "end_ms": [ - 1793593865507.3655, - 1793593865507.817 - ], - "threads": { - "3933959": "moq-relay", - "3933966": "tokio-rt-worker", - "3933968": "notify-rs inoti", - "3933972": "moq-uring-0", - "3933973": "moq-uring-1", - "3933974": "moq-uring-2", - "3933976": "moq-uring-3" - }, - "events": [ - "cpu_core/instructions/", - "cpu_core/cycles/" - ], - "sample_rate_limit": 34000, - "window_start_ms": 1793593860507.2534, - "window_end_ms": 1793593865507.5913, - "boundary_uncertainty_ms": [ - 0.055908203125, - 0.451416015625 - ], - "delivered_frames": 100006.7578125, - "recv_fps": 20000.0, - "counters": { - "cpu_core/instructions/": 13416194801.0, - "cpu_core/cycles/": 31466803079.0, - "task-clock": 6035.26755 - }, - "instructions_per_frame": 134152.8822097569, - "cycles_per_frame": 314646.7675514116, - "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", - "instruction_samples": 1333 - }, - "delivery_boundary_reports": { - "window_start_ms": [ - { - "timestamp_ms": 1793593860324, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 139979, - "frames_sent": 70, - "bytes_recv": 41993700, - "groups_lost": 0, - "groups_expected": 133884, - "groups_present": 133884 - }, - { - "timestamp_ms": 1793593860824, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 149979, - "frames_sent": 75, - "bytes_recv": 44993700, - "groups_lost": 0, - "groups_expected": 143884, - "groups_present": 143884 - } - ], - "window_end_ms": [ - { - "timestamp_ms": 1793593865324, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 239979, - "frames_sent": 120, - "bytes_recv": 71993700, - "groups_lost": 0, - "groups_expected": 233884, - "groups_present": 233884 - }, - { - "timestamp_ms": 1793593865824, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 249979, - "frames_sent": 125, - "bytes_recv": 74993700, - "groups_lost": 0, - "groups_expected": 243884, - "groups_present": 243884 - } - ] - }, - "source_inclusive_samples": { - "resume_learn_end": 0, - "publisher_forward_drop": 104, - "track_state_on_stack": 137, - "resume_reader_on_stack": 260, - "waiter_list_on_stack": 145 - }, - "artifact_sha256": { - "load.jsonl": "510731dc8d872b37d5948235c19fe62a1bdd31d59945edb0a607fb72379b05d6", - "stat.json": "32617342cd466bfad5e8beaa7ed6e5fadbb2a43063ceece96158ba268aac7057", - "window.json": "8f7168be3faf6104825296b8aee38773c3f75b5b01a619127dc27a2c2c5b6726", - "perf.data": "f4fc58b07e321395fca58b222877b31bf7fc95793365baa30ea28f9f60be3ac5", - "instructions.folded": "26a1fcda3d14edddd7937d7c1d037c922f0095b5a3ab1367e8e1c22dae90699a" - } - }, - { - "variant": "current", - "round": 1, - "capture": { - "version": 1, - "relay_cpus": [ - 4, - 5, - 6, - 7 - ], - "generator_cpus": [ - 0, - 1, - 2, - 3 - ], - "period": 10000019, - "start_ms": [ - 1793593685029.5342, - 1793593685029.6401 - ], - "end_ms": [ - 1793593690029.722, - 1793593690030.6287 - ], - "threads": { - "3860718": "moq-relay", - "3860729": "tokio-rt-worker", - "3860730": "notify-rs inoti", - "3860731": "moq-uring-0", - "3860732": "moq-uring-1", - "3860733": "moq-uring-2", - "3860734": "moq-uring-3" - }, - "events": [ - "cpu_core/instructions/", - "cpu_core/cycles/" - ], - "sample_rate_limit": 34000, - "window_start_ms": 1793593685029.5872, - "window_end_ms": 1793593690030.1753, - "boundary_uncertainty_ms": [ - 0.10595703125, - 0.90673828125 - ], - "delivered_frames": 100011.7626953125, - "recv_fps": 20000.0, - "counters": { - "cpu_core/instructions/": 15530949172.0, - "cpu_core/cycles/": 31052115126.0, - "task-clock": 5863.975766 - }, - "instructions_per_frame": 155291.22528632253, - "cycles_per_frame": 310484.62989899283, - "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", - "instruction_samples": 1552 - }, - "delivery_boundary_reports": { - "window_start_ms": [ - { - "timestamp_ms": 1793593684820, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 139114, - "frames_sent": 70, - "bytes_recv": 41734200, - "groups_lost": 0, - "groups_expected": 130750, - "groups_present": 130750 - }, - { - "timestamp_ms": 1793593685320, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 149114, - "frames_sent": 75, - "bytes_recv": 44734200, - "groups_lost": 0, - "groups_expected": 140750, - "groups_present": 140750 - } - ], - "window_end_ms": [ - { - "timestamp_ms": 1793593689820, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 239114, - "frames_sent": 120, - "bytes_recv": 71734200, - "groups_lost": 0, - "groups_expected": 230750, - "groups_present": 230750 - }, - { - "timestamp_ms": 1793593690320, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 249114, - "frames_sent": 125, - "bytes_recv": 74734200, - "groups_lost": 0, - "groups_expected": 240750, - "groups_present": 240750 - } - ] - }, - "source_inclusive_samples": { - "resume_learn_end": 17, - "publisher_forward_drop": 268, - "track_state_on_stack": 233, - "resume_reader_on_stack": 222, - "waiter_list_on_stack": 179 - }, - "artifact_sha256": { - "load.jsonl": "a1104b94382832194337330502439389218c7f0792d07f36ef73dd6299865fa3", - "stat.json": "31b9de14451c433738e4a04007ed552e17f6a92d975adffad721e76f2adc7030", - "window.json": "7f5b1e10790a269afa2890fac0cda062f77da7cad2e8f982fd269a281af875ca", - "perf.data": "a9b95e6a7017ed2f431fabac649b2cdea220f7fc445bc933a603166ccb6791dc", - "instructions.folded": "56e0437cf2b89c0c781234942cfd38ae2e6a9cca630b1d5c72748b0a03d83783" - } - }, - { - "variant": "current", - "round": 2, - "capture": { - "version": 1, - "relay_cpus": [ - 4, - 5, - 6, - 7 - ], - "generator_cpus": [ - 0, - 1, - 2, - 3 - ], - "period": 10000019, - "start_ms": [ - 1793593748793.664, - 1793593748793.9766 - ], - "end_ms": [ - 1793593753794.0137, - 1793593753795.1243 - ], - "threads": { - "3872892": "moq-relay", - "3872895": "tokio-rt-worker", - "3872896": "notify-rs inoti", - "3872897": "moq-uring-0", - "3872898": "moq-uring-1", - "3872899": "moq-uring-2", - "3872900": "moq-uring-3" - }, - "events": [ - "cpu_core/instructions/", - "cpu_core/cycles/" - ], - "sample_rate_limit": 34000, - "window_start_ms": 1793593748793.8203, - "window_end_ms": 1793593753794.5688, - "boundary_uncertainty_ms": [ - 0.3125, - 1.110595703125 - ], - "delivered_frames": 100007.08372317866, - "recv_fps": 19998.422840122654, - "counters": { - "cpu_core/instructions/": 15643159353.0, - "cpu_core/cycles/": 29421932916.0, - "task-clock": 5613.358136 - }, - "instructions_per_frame": 156420.51313385495, - "cycles_per_frame": 294198.48895344674, - "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", - "instruction_samples": 1557 - }, - "delivery_boundary_reports": { - "window_start_ms": [ - { - "timestamp_ms": 1793593748597, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 139490, - "frames_sent": 71, - "bytes_recv": 41847000, - "groups_lost": 0, - "groups_expected": 132312, - "groups_present": 132312 - }, - { - "timestamp_ms": 1793593749097, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 149490, - "frames_sent": 76, - "bytes_recv": 44847000, - "groups_lost": 0, - "groups_expected": 142312, - "groups_present": 142312 - } - ], - "window_end_ms": [ - { - "timestamp_ms": 1793593753597, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 239490, - "frames_sent": 121, - "bytes_recv": 71847000, - "groups_lost": 0, - "groups_expected": 232312, - "groups_present": 232312 - }, - { - "timestamp_ms": 1793593754098, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 249490, - "frames_sent": 126, - "bytes_recv": 74847000, - "groups_lost": 0, - "groups_expected": 242312, - "groups_present": 242312 - } - ] - }, - "source_inclusive_samples": { - "resume_learn_end": 17, - "publisher_forward_drop": 270, - "track_state_on_stack": 240, - "resume_reader_on_stack": 223, - "waiter_list_on_stack": 151 - }, - "artifact_sha256": { - "load.jsonl": "878cd6efd67ce31d406c8e8868aa5e09e3a00d35e2e531339493e0054adfc472", - "stat.json": "8dadbbc62284d5d8ad178ad4d5596a546426b7d5f75182cff25e153b5342c26a", - "window.json": "75a13755b3ed0b5c914358287bd2abf7134bc8f1a9d719d4494d2d8d6f99c79e", - "perf.data": "8cd1acaba3743f6e60d4162ec03b0a561c91a43d3a96ddfd19565a66a5d92edb", - "instructions.folded": "5f3adec772d21f451b0f1341c2314cc0d976de0648196ba666fa0acfd8bb9957" - } - }, - { - "variant": "current", - "round": 3, - "capture": { - "version": 1, - "relay_cpus": [ - 4, - 5, - 6, - 7 - ], - "generator_cpus": [ - 0, - 1, - 2, - 3 - ], - "period": 10000019, - "start_ms": [ - 1793593908674.517, - 1793593908674.5664 - ], - "end_ms": [ - 1793593913674.7961, - 1793593913675.5957 - ], - "threads": { - "3999994": "moq-relay", - "4000003": "tokio-rt-worker", - "4000005": "notify-rs inoti", - "4000009": "moq-uring-0", - "4000010": "moq-uring-1", - "4000011": "moq-uring-2", - "4000012": "moq-uring-3" - }, - "events": [ - "cpu_core/instructions/", - "cpu_core/cycles/" - ], - "sample_rate_limit": 34000, - "window_start_ms": 1793593908674.5417, - "window_end_ms": 1793593913675.1958, - "boundary_uncertainty_ms": [ - 0.04931640625, - 0.799560546875 - ], - "delivered_frames": 100005.44848779004, - "recv_fps": 19998.473686278438, - "counters": { - "cpu_core/instructions/": 15660840892.0, - "cpu_core/cycles/": 31134166368.0, - "task-clock": 5931.797298 - }, - "instructions_per_frame": 156599.8765948445, - "cycles_per_frame": 311324.7011916682, - "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", - "instruction_samples": 1559 - }, - "delivery_boundary_reports": { - "window_start_ms": [ - { - "timestamp_ms": 1793593908484, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 139977, - "frames_sent": 70, - "bytes_recv": 41993100, - "groups_lost": 0, - "groups_expected": 135246, - "groups_present": 135246 - }, - { - "timestamp_ms": 1793593908984, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 149977, - "frames_sent": 75, - "bytes_recv": 44993100, - "groups_lost": 0, - "groups_expected": 145246, - "groups_present": 145246 - } - ], - "window_end_ms": [ - { - "timestamp_ms": 1793593913484, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 239977, - "frames_sent": 120, - "bytes_recv": 71993100, - "groups_lost": 0, - "groups_expected": 235246, - "groups_present": 235246 - }, - { - "timestamp_ms": 1793593913985, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 249977, - "frames_sent": 126, - "bytes_recv": 74993100, - "groups_lost": 0, - "groups_expected": 245246, - "groups_present": 245246 - } - ] - }, - "source_inclusive_samples": { - "resume_learn_end": 11, - "publisher_forward_drop": 325, - "track_state_on_stack": 254, - "resume_reader_on_stack": 224, - "waiter_list_on_stack": 153 - }, - "artifact_sha256": { - "load.jsonl": "11c2cbc9dd3a74b7977886e1b4aa4e1dd0697d8cb20265598490e2d079803c84", - "stat.json": "7ca078952f185a2c00f77c48c4bc2b9f4e210f43e44015d989e449eb27e62758", - "window.json": "c7163d8126dcb546788bc51abb7142bc6d886f9c80c61514d82492b1992585ed", - "perf.data": "7408f1467e4bc481cda97f57026835c3adc0fdb51163548a4a209468334641e4", - "instructions.folded": "10abb2084945602304d0ec47bb25e78c7d658583fd75fb15f83549a487763c04" - } - } - ], - "summary": { - "base": { - "completed_cells": 3, - "median_instructions_per_frame": 127675.50793207905, - "min_instructions_per_frame": 126525.70810094554, - "max_instructions_per_frame": 134152.8822097569 - }, - "current": { - "completed_cells": 3, - "median_instructions_per_frame": 156420.51313385495, - "min_instructions_per_frame": 155291.22528632253, - "max_instructions_per_frame": 156599.8765948445 - } - }, - "pull_request_url": "https://github.com/moq-dev/moq/pull/6224" - }, - { - "artifact": "run.9iDBWq", - "description": "Resume end tracking", - "base_revision": "fd59eb18a50f6e0ac819e73d941a3832cbdac91f", - "comparison_status": "inconclusive: delivery changed materially and confirmed loss was reported; do not infer a regression from this run", - "harness_head": "9c19f0a6cda900407ee7202b23e8633e95f87857", - "machine": { - "os": "Linux-7.0.0-42-generic-x86_64-with-glibc2.44", - "machine": "x86_64", - "rustc": "rustc 1.98.1 (48a229cea 2026-09-01)\nbinary: rustc\ncommit-hash: 48a229ceaefd4985c50990b14116b6d856af0985\ncommit-date: 2026-09-01\nhost: x86_64-unknown-linux-gnu\nrelease: 1.98.1\nLLVM version: 22.1.8", - "cargo": "cargo 1.98.1 (797e8a9bc 2026-08-05)", - "affinity": [ - 0, - 1, - 2, - 3, - 4, - 5, - 6, - 7 - ] - }, - "harness_dirty_patch_sha256": "1fb2250488672e6005088a1008ec311575d40c0a8ef4546a592f4ca2acea68a0", - "cells": [ - { - "variant": "base", - "round": 1, - "capture": { - "version": 1, - "relay_cpus": [ - 4, - 5, - 6, - 7 - ], - "generator_cpus": [ - 0, - 1, - 2, - 3 - ], - "period": 10000019, - "start_ms": [ - 1793594381594.4658, - 1793594381594.559 - ], - "end_ms": [ - 1793594386594.65, - 1793594386594.7551 - ], - "threads": { - "331621": "moq-relay", - "331624": "tokio-rt-worker", - "331625": "notify-rs inoti", - "331627": "moq-uring-0", - "331628": "moq-uring-1", - "331629": "moq-uring-2", - "331630": "moq-uring-3" - }, - "events": [ - "cpu_core/instructions/", - "cpu_core/cycles/" - ], - "sample_rate_limit": 25000, - "window_start_ms": 1793594381594.5125, - "window_end_ms": 1793594386594.7026, - "boundary_uncertainty_ms": [ - 0.09326171875, - 0.105224609375 - ], - "delivered_frames": 100095.05315240734, - "recv_fps": 20018.249194147375, - "counters": { - "cpu_core/instructions/": 14399429625.0, - "cpu_core/cycles/": 39461160782.0, - "task-clock": 7432.597766 - }, - "instructions_per_frame": 143857.55510889288, - "cycles_per_frame": 394236.8732440294, - "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", - "instruction_samples": 1433 - }, - "delivery_boundary_reports": { - "window_start_ms": [ - { - "timestamp_ms": 1793594381226, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 136838, - "frames_sent": 70, - "bytes_recv": 41051400, - "groups_lost": 0, - "groups_expected": 125638, - "groups_present": 125638 - }, - { - "timestamp_ms": 1793594381726, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 147205, - "frames_sent": 76, - "bytes_recv": 44161500, - "groups_lost": 0, - "groups_expected": 136005, - "groups_present": 136005 - } - ], - "window_end_ms": [ - { - "timestamp_ms": 1793594386227, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 237205, - "frames_sent": 121, - "bytes_recv": 71161500, - "groups_lost": 0, - "groups_expected": 226005, - "groups_present": 226005 - }, - { - "timestamp_ms": 1793594386726, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 247205, - "frames_sent": 126, - "bytes_recv": 74161500, - "groups_lost": 0, - "groups_expected": 236005, - "groups_present": 236005 - } - ] - }, - "source_inclusive_samples": { - "resume_learn_end": 149, - "publisher_forward_drop": 135, - "track_state_on_stack": 253, - "resume_reader_on_stack": 427, - "waiter_list_on_stack": 220 - }, - "artifact_sha256": { - "load.jsonl": "c6635a2341f6e5d82258b8cc8cca1eb770d5cdf9bd21383dcb145a30ce943a0c", - "stat.json": "762933d9c68eea7e5fcfc50d4e1537654b50155b14c917dca7ae775fc38cf455", - "window.json": "e44d573b80528c963c803a2079ec5913919a5d6838587bf82b49d1c16911cf40", - "perf.data": "16c0b3a22610ccaf5d67903a405db41a339c052c7872200629244951e825fcb2", - "instructions.folded": "3f92bd25686b357f470abb8cfbced60fca3e7ccd4125380befcdef48d430f850" - } - }, - { - "variant": "base", - "round": 2, - "capture": { - "version": 1, - "relay_cpus": [ - 4, - 5, - 6, - 7 - ], - "generator_cpus": [ - 0, - 1, - 2, - 3 - ], - "period": 10000019, - "start_ms": [ - 1793594579283.6018, - 1793594579285.9014 - ], - "end_ms": [ - 1793594584285.9883, - 1793594584288.9255 - ], - "threads": { - "542687": "moq-relay", - "542709": "tokio-rt-worker", - "542713": "notify-rs inoti", - "542719": "moq-uring-0", - "542720": "moq-uring-1", - "542721": "moq-uring-2", - "542725": "moq-uring-3" - }, - "events": [ - "cpu_core/instructions/", - "cpu_core/cycles/" - ], - "sample_rate_limit": 25000, - "window_start_ms": 1793594579284.7515, - "window_end_ms": 1793594584287.457, - "boundary_uncertainty_ms": [ - 2.299560546875, - 2.937255859375 - ], - "delivered_frames": 111136.91455377889, - "recv_fps": 22215.3618833929, - "counters": { - "cpu_core/instructions/": 14496953653.0, - "cpu_core/cycles/": 38139777320.0, - "task-clock": 7159.595114 - }, - "instructions_per_frame": 130442.29013560529, - "cycles_per_frame": 343178.29924587527, - "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", - "instruction_samples": 1445 - }, - "delivery_boundary_reports": { - "window_start_ms": [ - { - "timestamp_ms": 1793594578941, - "ready": false, - "connections": { - "target": 2001, - "established": 1976 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 1916, - "established": 1841 - }, - "frames_recv": 103716, - "frames_sent": 71, - "bytes_recv": 31114800, - "groups_lost": 8698, - "groups_expected": 93167, - "groups_present": 93057 - }, - { - "timestamp_ms": 1793594579444, - "ready": false, - "connections": { - "target": 2001, - "established": 1990 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 1946, - "established": 1901 - }, - "frames_recv": 111655, - "frames_sent": 76, - "bytes_recv": 33496500, - "groups_lost": 8699, - "groups_expected": 100671, - "groups_present": 100561 - } - ], - "window_end_ms": [ - { - "timestamp_ms": 1793594583940, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 214150, - "frames_sent": 120, - "bytes_recv": 64245000, - "groups_lost": 10872, - "groups_expected": 201838, - "groups_present": 201724 - }, - { - "timestamp_ms": 1793594584440, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 222969, - "frames_sent": 125, - "bytes_recv": 66890700, - "groups_lost": 10872, - "groups_expected": 210657, - "groups_present": 210543 - } - ] - }, - "source_inclusive_samples": { - "resume_learn_end": 149, - "publisher_forward_drop": 109, - "track_state_on_stack": 259, - "resume_reader_on_stack": 388, - "waiter_list_on_stack": 214 - }, - "artifact_sha256": { - "load.jsonl": "8461651165eab4a26d5eed290af87252c60158767a987c41752e43408ecc32fe", - "stat.json": "c6038808d665bdbfbe91dfee959b9fc95f2c2e0631c2b97d6e6c58f1fcd82b67", - "window.json": "218faaba4fde6cb55a5e5d694be51726d30a1d55381bb3490b59a127bf88c79a", - "perf.data": "984910e6ec332d86f1386845c0f865d65d8c75e46b95c6c6f6a1309111a3c609", - "instructions.folded": "5005c77d20bd8631f6c96d205e7db6f234620136a8f820cebade588065d2d0b7" - } - }, - { - "variant": "base", - "round": 3, - "capture": { - "version": 1, - "relay_cpus": [ - 4, - 5, - 6, - 7 - ], - "generator_cpus": [ - 0, - 1, - 2, - 3 - ], - "period": 10000019, - "start_ms": [ - 1793594647413.7844, - 1793594647413.8716 - ], - "end_ms": [ - 1793594652413.9639, - 1793594652414.0789 - ], - "threads": { - "659791": "moq-relay", - "659808": "tokio-rt-worker", - "659812": "notify-rs inoti", - "659842": "moq-uring-0", - "659844": "moq-uring-1", - "659845": "moq-uring-2", - "659847": "moq-uring-3" - }, - "events": [ - "cpu_core/instructions/", - "cpu_core/cycles/" - ], - "sample_rate_limit": 25000, - "window_start_ms": 1793594647413.8281, - "window_end_ms": 1793594652414.0215, - "boundary_uncertainty_ms": [ - 0.087158203125, - 0.114990234375 - ], - "delivered_frames": 100245.75456898703, - "recv_fps": 20048.375605522036, - "counters": { - "cpu_core/instructions/": 14290822931.0, - "cpu_core/cycles/": 31924206244.0, - "task-clock": 6028.561508 - }, - "instructions_per_frame": 142557.8867897628, - "cycles_per_frame": 318459.4338309901, - "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", - "instruction_samples": 1423 - }, - "delivery_boundary_reports": { - "window_start_ms": [ - { - "timestamp_ms": 1793594647006, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 138869, - "frames_sent": 70, - "bytes_recv": 41660700, - "groups_lost": 0, - "groups_expected": 126705, - "groups_present": 126705 - }, - { - "timestamp_ms": 1793594647507, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 148840, - "frames_sent": 75, - "bytes_recv": 44652000, - "groups_lost": 0, - "groups_expected": 136676, - "groups_present": 136676 - } - ], - "window_end_ms": [ - { - "timestamp_ms": 1793594652006, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 239071, - "frames_sent": 120, - "bytes_recv": 71721300, - "groups_lost": 0, - "groups_expected": 226907, - "groups_present": 226907 - }, - { - "timestamp_ms": 1793594652506, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 249071, - "frames_sent": 125, - "bytes_recv": 74721300, - "groups_lost": 0, - "groups_expected": 236907, - "groups_present": 236907 - } - ] - }, - "source_inclusive_samples": { - "resume_learn_end": 163, - "publisher_forward_drop": 121, - "track_state_on_stack": 251, - "resume_reader_on_stack": 399, - "waiter_list_on_stack": 218 - }, - "artifact_sha256": { - "load.jsonl": "1874080e711ba7b934680da35586a5c48fbbabfa0fb155ef435a9bb037d598eb", - "stat.json": "150b5289d32f0a5570d6ade4c3b1d196397336acbfb54c3d91a2f24e695a37f6", - "window.json": "745d17d3675f91db7d481f4de11a8bb1354e062e9ec707c1e631b8b9b512c2d4", - "perf.data": "cb849a6cfdfc5e4a28ac731b6c5d5283ec90ee68c273fbeed6eb542128fe2300", - "instructions.folded": "bc70c79e82e28992f5bdaf996e6c96d1de787ef94618b52c653e8a5f249d966f" - } - }, - { - "variant": "current", - "round": 1, - "capture": { - "version": 1, - "relay_cpus": [ - 4, - 5, - 6, - 7 - ], - "generator_cpus": [ - 0, - 1, - 2, - 3 - ], - "period": 10000019, - "start_ms": [ - 1793594430209.9126, - 1793594430209.982 - ], - "end_ms": [ - 1793594435210.0576, - 1793594435210.8723 - ], - "threads": { - "379928": "moq-relay", - "379931": "tokio-rt-worker", - "379932": "notify-rs inoti", - "379933": "moq-uring-0", - "379934": "moq-uring-1", - "379935": "moq-uring-2", - "379936": "moq-uring-3" - }, - "events": [ - "cpu_core/instructions/", - "cpu_core/cycles/" - ], - "sample_rate_limit": 25000, - "window_start_ms": 1793594430209.9473, - "window_end_ms": 1793594435210.4648, - "boundary_uncertainty_ms": [ - 0.0693359375, - 0.814697265625 - ], - "delivered_frames": 101235.58537766157, - "recv_fps": 20245.021399489007, - "counters": { - "cpu_core/instructions/": 15529380560.0, - "cpu_core/cycles/": 32771189013.0, - "task-clock": 6142.396911 - }, - "instructions_per_frame": 153398.43694356392, - "cycles_per_frame": 323712.150137191, - "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", - "instruction_samples": 1552 - }, - "delivery_boundary_reports": { - "window_start_ms": [ - { - "timestamp_ms": 1793594429990, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 136127, - "frames_sent": 70, - "bytes_recv": 40838100, - "groups_lost": 0, - "groups_expected": 125410, - "groups_present": 125410 - }, - { - "timestamp_ms": 1793594430490, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 144795, - "frames_sent": 75, - "bytes_recv": 43438500, - "groups_lost": 0, - "groups_expected": 134078, - "groups_present": 134078 - } - ], - "window_end_ms": [ - { - "timestamp_ms": 1793594434990, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 236909, - "frames_sent": 121, - "bytes_recv": 71072700, - "groups_lost": 0, - "groups_expected": 226192, - "groups_present": 226192 - }, - { - "timestamp_ms": 1793594435489, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 246566, - "frames_sent": 125, - "bytes_recv": 73969800, - "groups_lost": 0, - "groups_expected": 235849, - "groups_present": 235849 - } - ] - }, - "source_inclusive_samples": { - "resume_learn_end": 6, - "publisher_forward_drop": 267, - "track_state_on_stack": 272, - "resume_reader_on_stack": 229, - "waiter_list_on_stack": 156 - }, - "artifact_sha256": { - "load.jsonl": "1018c7927375b1d99d229b29e1e602e1b9f14b0c8b245bbd8d1242def6a0faf1", - "stat.json": "6cec61573391d19c9323ed1efbfe97089c40bdf77b2956051d9282d46a9a8b19", - "window.json": "8f7eeaef1ba10e088ca3ee39084bba4e45c1806a65735d9a71a86b079a9fdc8b", - "perf.data": "1d4832ca0788473db120d9d9ccd4ce9adf8bf798c76709cdc9b9cd1c8fcdbb66", - "instructions.folded": "9a81ea5fe9a63b1348b4f46beda24ef01f84fc9226c71e90980804d7105c5303" - } - }, - { - "variant": "current", - "round": 2, - "capture": { - "version": 1, - "relay_cpus": [ - 4, - 5, - 6, - 7 - ], - "generator_cpus": [ - 0, - 1, - 2, - 3 - ], - "period": 10000019, - "start_ms": [ - 1793594500464.6292, - 1793594500464.705 - ], - "end_ms": [ - 1793594505464.9849, - 1793594505465.2336 - ], - "threads": { - "413374": "moq-relay", - "413378": "tokio-rt-worker", - "413381": "notify-rs inoti", - "413382": "moq-uring-0", - "413383": "moq-uring-1", - "413384": "moq-uring-2", - "413386": "moq-uring-3" - }, - "events": [ - "cpu_core/instructions/", - "cpu_core/cycles/" - ], - "sample_rate_limit": 25000, - "window_start_ms": 1793594500464.667, - "window_end_ms": 1793594505465.1094, - "boundary_uncertainty_ms": [ - 0.075927734375, - 0.248779296875 - ], - "delivered_frames": 99433.38824336545, - "recv_fps": 19884.91829945636, - "counters": { - "cpu_core/instructions/": 15418054843.0, - "cpu_core/cycles/": 30441872584.0, - "task-clock": 5712.46671 - }, - "instructions_per_frame": 155059.13169994735, - "cycles_per_frame": 306153.42715158046, - "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", - "instruction_samples": 1540 - }, - "delivery_boundary_reports": { - "window_start_ms": [ - { - "timestamp_ms": 1793594500244, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 139053, - "frames_sent": 70, - "bytes_recv": 41715900, - "groups_lost": 0, - "groups_expected": 131563, - "groups_present": 131563 - }, - { - "timestamp_ms": 1793594500745, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 149053, - "frames_sent": 76, - "bytes_recv": 44715900, - "groups_lost": 0, - "groups_expected": 141563, - "groups_present": 141563 - } - ], - "window_end_ms": [ - { - "timestamp_ms": 1793594505246, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 238067, - "frames_sent": 120, - "bytes_recv": 71420100, - "groups_lost": 0, - "groups_expected": 230577, - "groups_present": 230577 - }, - { - "timestamp_ms": 1793594505745, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 249053, - "frames_sent": 125, - "bytes_recv": 74715900, - "groups_lost": 0, - "groups_expected": 241563, - "groups_present": 241563 - } - ] - }, - "source_inclusive_samples": { - "resume_learn_end": 11, - "publisher_forward_drop": 258, - "track_state_on_stack": 226, - "resume_reader_on_stack": 233, - "waiter_list_on_stack": 168 - }, - "artifact_sha256": { - "load.jsonl": "f040c49aa1ae11940e2f9e9b16a0cbd9e07150e0ff481f79621d19aae4088b00", - "stat.json": "80c531a0880a9f1cae9d79ac5bf71b8ad85871d96ecd7b87cfefcd7d3cec888d", - "window.json": "f3ba3629de9b2d9b55dcf8584ac892e1b2f7ef003462bad971701bf3b1902334", - "perf.data": "87d1822b7515516f4cd55aa40f94abd8283b86c1701060566aa3c05a8609822d", - "instructions.folded": "441062103cb7eb8bc213981bbc82155494e86aba26caefd57f5f5d914d1ae3e2" - } - }, - { - "variant": "current", - "round": 3, - "capture": { - "version": 1, - "relay_cpus": [ - 4, - 5, - 6, - 7 - ], - "generator_cpus": [ - 0, - 1, - 2, - 3 - ], - "period": 10000019, - "start_ms": [ - 1793594694838.9155, - 1793594694839.8674 - ], - "end_ms": [ - 1793594699840.0994, - 1793594699840.213 - ], - "threads": { - "778541": "moq-relay", - "778554": "tokio-rt-worker", - "778558": "notify-rs inoti", - "778561": "moq-uring-0", - "778563": "moq-uring-1", - "778564": "moq-uring-2", - "778566": "moq-uring-3" - }, - "events": [ - "cpu_core/instructions/", - "cpu_core/cycles/" - ], - "sample_rate_limit": 25000, - "window_start_ms": 1793594694839.3916, - "window_end_ms": 1793594699840.1562, - "boundary_uncertainty_ms": [ - 0.951904296875, - 0.113525390625 - ], - "delivered_frames": 91833.47595923272, - "recv_fps": 18363.886808375653, - "counters": { - "cpu_core/instructions/": 14142104401.0, - "cpu_core/cycles/": 31545899811.0, - "task-clock": 5912.347715 - }, - "instructions_per_frame": 153997.26791652807, - "cycles_per_frame": 343511.98712116753, - "scope": "all relay threads, user and kernel; delivery interpolated at acknowledged counter boundaries", - "instruction_samples": 1411 - }, - "delivery_boundary_reports": { - "window_start_ms": [ - { - "timestamp_ms": 1793594694631, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 137055, - "frames_sent": 70, - "bytes_recv": 41116500, - "groups_lost": 68, - "groups_expected": 127560, - "groups_present": 127560 - }, - { - "timestamp_ms": 1793594695121, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 147270, - "frames_sent": 75, - "bytes_recv": 44181000, - "groups_lost": 68, - "groups_expected": 137775, - "groups_present": 137775 - } - ], - "window_end_ms": [ - { - "timestamp_ms": 1793594699622, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 226762, - "frames_sent": 120, - "bytes_recv": 68028600, - "groups_lost": 2959, - "groups_expected": 217263, - "groups_present": 217263 - }, - { - "timestamp_ms": 1793594700121, - "ready": true, - "connections": { - "target": 2001, - "established": 2001 - }, - "broadcasts": { - "target": 1, - "active": 1, - "established": 1 - }, - "subscriptions": { - "target": 2000, - "active": 2000, - "established": 2000 - }, - "frames_recv": 241563, - "frames_sent": 125, - "bytes_recv": 72468900, - "groups_lost": 4201, - "groups_expected": 232072, - "groups_present": 232068 - } - ] - }, - "source_inclusive_samples": { - "resume_learn_end": 15, - "publisher_forward_drop": 229, - "track_state_on_stack": 235, - "resume_reader_on_stack": 171, - "waiter_list_on_stack": 158 - }, - "artifact_sha256": { - "load.jsonl": "3077124b8edc46b3e23990e7bc30de6f33e4de9905677bdbf8d8736963d5a950", - "stat.json": "13a6eeee58035eaeb02c55853f5eeb2aa722aac2ff58e8531758762385e1a973", - "window.json": "d22e332093899df276d460548df681e5cea963fecaa56a865c000825dc260483", - "perf.data": "33f1b24821c432eab674a46976db0ea43a46b9fe1cb05f2ae269e7582c3926ff", - "instructions.folded": "0a430966175cc3889e469ce255213762a887cf789d2e4ba59320ad8564bdec3d" - } - } - ], - "summary": { - "base": { - "completed_cells": 3, - "median_instructions_per_frame": 142557.8867897628, - "min_instructions_per_frame": 130442.29013560529, - "max_instructions_per_frame": 143857.55510889288 - }, - "current": { - "completed_cells": 3, - "median_instructions_per_frame": 153997.26791652807, - "min_instructions_per_frame": 153398.43694356392, - "max_instructions_per_frame": 155059.13169994735 - } - }, - "pull_request_url": "https://github.com/moq-dev/moq/pull/6224" - } - ], - "excluded_attempts": [ - { - "artifact": "run.RBmaZ8", - "reason": "Current-tree samples throttled at period 1000003; predates the throttle validator. Any existing derived JSON is excluded." - }, - { - "artifact": "run.JuOoVh", - "reason": "Historical capture throttled at period 10000019 (957 samples and 957 THROTTLE records); command failed." - }, - { - "artifact": "run.hDJ7BQ", - "reason": "Nested Cargo alias configuration conflict; fixed by explicit isolated configuration." - }, - { - "artifact": "run.T2mHXL", - "reason": "New parser incorrectly rejected identical cumulative snapshots during teardown; fixed and the whole comparison repeated. This partial attempt is excluded." - } - ], - "production_control_url": "https://github.com/moq-dev/moq/commit/21d6da9fc108b48373a961f34e43c3fcd38b0146" + { + "artifact": "run.2DucdO", + "harness_revision": "82194441a92bb89fbc69f3759ef336d676fad643", + "workload": "chat", + "runtime": "io-uring-workers", + "workers": 4, + "generator_revision": "82194441a92bb89fbc69f3759ef336d676fad643", + "instruction_period": 10000019, + "sampling": "concurrent with the counter window", + "relay_cpus": "4-7", + "generator_cpus": "0-3", + "cells": [ + {"revision": "2d73eea67d083ecf18938f4fc7c4842118eb87cf", "label": "base-2d73eea67d08", "round": 1, "kinstr_per_frame": 123.1, "kcycles_per_frame": 235.4, "recv_fps": 20001, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 3.56, "samples": 1221, "markers": {"learn_end": 0.0, "poll_forward_drop": 7.9, "cursor_probe": 0.0, "syscall_leaf": 22.4, "mutex_leaf": 16.7}}, + {"revision": "2d73eea67d083ecf18938f4fc7c4842118eb87cf", "label": "base-2d73eea67d08", "round": 2, "kinstr_per_frame": 125.3, "kcycles_per_frame": 247.9, "recv_fps": 19999, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 3.19, "samples": 1248, "markers": {"learn_end": 0.0, "poll_forward_drop": 8.4, "cursor_probe": 0.0, "syscall_leaf": 24.8, "mutex_leaf": 15.3}}, + {"revision": "2d73eea67d083ecf18938f4fc7c4842118eb87cf", "label": "base-2d73eea67d08", "round": 3, "kinstr_per_frame": 114.9, "kcycles_per_frame": 297.8, "recv_fps": 19920, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 9.31, "samples": 1138, "markers": {"learn_end": 0.0, "poll_forward_drop": 6.4, "cursor_probe": 0.0, "syscall_leaf": 16.5, "mutex_leaf": 17.9}}, + {"revision": "41cff87192983aae80db976d0058661525daa8ed", "label": "base-41cff8719298", "round": 1, "kinstr_per_frame": 115.5, "kcycles_per_frame": 221.3, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 4.57, "samples": 1148, "markers": {"learn_end": 0.0, "poll_forward_drop": 6.4, "cursor_probe": 0.0, "syscall_leaf": 19.7, "mutex_leaf": 16.6}}, + {"revision": "41cff87192983aae80db976d0058661525daa8ed", "label": "base-41cff8719298", "round": 2, "kinstr_per_frame": 123.6, "kcycles_per_frame": 225.9, "recv_fps": 20003, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 2.99, "samples": 1228, "markers": {"learn_end": 0.0, "poll_forward_drop": 6.8, "cursor_probe": 0.0, "syscall_leaf": 23.7, "mutex_leaf": 17.5}}, + {"revision": "41cff87192983aae80db976d0058661525daa8ed", "label": "base-41cff8719298", "round": 3, "kinstr_per_frame": 142.6, "kcycles_per_frame": 412.5, "recv_fps": 12490, "groups_lost": 25854, "host_load_1m": null, "host_load_1m_logged": 30.78, "samples": 888, "markers": {"learn_end": 0.0, "poll_forward_drop": 4.3, "cursor_probe": 0.0, "syscall_leaf": 16.1, "mutex_leaf": 17.3}}, + {"revision": "72259ab3cc3d3ed53a9f3dad8ff37caeac310b83", "label": "base-72259ab3cc3d", "round": 1, "kinstr_per_frame": 126.5, "kcycles_per_frame": 244.6, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 3.19, "samples": 1259, "markers": {"learn_end": 0.0, "poll_forward_drop": 8.7, "cursor_probe": 0.0, "syscall_leaf": 24.7, "mutex_leaf": 15.3}}, + {"revision": "72259ab3cc3d3ed53a9f3dad8ff37caeac310b83", "label": "base-72259ab3cc3d", "round": 2, "kinstr_per_frame": 127.0, "kcycles_per_frame": 233.8, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 3.9, "samples": 1262, "markers": {"learn_end": 0.0, "poll_forward_drop": 9.0, "cursor_probe": 0.0, "syscall_leaf": 24.6, "mutex_leaf": 15.1}}, + {"revision": "72259ab3cc3d3ed53a9f3dad8ff37caeac310b83", "label": "base-72259ab3cc3d", "round": 3, "kinstr_per_frame": 120.2, "kcycles_per_frame": 238.3, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 9.69, "samples": 1193, "markers": {"learn_end": 0.0, "poll_forward_drop": 8.4, "cursor_probe": 0.0, "syscall_leaf": 21.8, "mutex_leaf": 16.7}}, + {"revision": "756637200cb10943871f6c51171f33a340addb6e", "label": "base-756637200cb1", "round": 1, "kinstr_per_frame": 138.4, "kcycles_per_frame": 267.2, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 3.97, "samples": 1378, "markers": {"learn_end": 0.0, "poll_forward_drop": 7.3, "cursor_probe": 0.0, "syscall_leaf": 21.9, "mutex_leaf": 15.1}}, + {"revision": "756637200cb10943871f6c51171f33a340addb6e", "label": "base-756637200cb1", "round": 2, "kinstr_per_frame": 133.9, "kcycles_per_frame": 256.6, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 3.41, "samples": 1330, "markers": {"learn_end": 0.0, "poll_forward_drop": 6.4, "cursor_probe": 0.0, "syscall_leaf": 18.4, "mutex_leaf": 16.9}}, + {"revision": "756637200cb10943871f6c51171f33a340addb6e", "label": "base-756637200cb1", "round": 3, "kinstr_per_frame": 133.7, "kcycles_per_frame": 266.5, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 15.73, "samples": 1331, "markers": {"learn_end": 0.0, "poll_forward_drop": 7.2, "cursor_probe": 0.0, "syscall_leaf": 21.5, "mutex_leaf": 15.2}}, + {"revision": "7648ce03908df33c168f7471787f3572e87eeddb", "label": "base-7648ce03908d", "round": 1, "kinstr_per_frame": 121.2, "kcycles_per_frame": 231.2, "recv_fps": 19998, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 4.22, "samples": 1204, "markers": {"learn_end": 0.0, "poll_forward_drop": 7.1, "cursor_probe": 0.0, "syscall_leaf": 22.8, "mutex_leaf": 17.4}}, + {"revision": "7648ce03908df33c168f7471787f3572e87eeddb", "label": "base-7648ce03908d", "round": 2, "kinstr_per_frame": 126.1, "kcycles_per_frame": 230.6, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 2.31, "samples": 1255, "markers": {"learn_end": 0.0, "poll_forward_drop": 8.0, "cursor_probe": 0.0, "syscall_leaf": 25.7, "mutex_leaf": 15.9}}, + {"revision": "7648ce03908df33c168f7471787f3572e87eeddb", "label": "base-7648ce03908d", "round": 3, "kinstr_per_frame": 156.9, "kcycles_per_frame": 384.1, "recv_fps": 13977, "groups_lost": 35553, "host_load_1m": null, "host_load_1m_logged": 55.74, "samples": 1087, "markers": {"learn_end": 0.0, "poll_forward_drop": 4.5, "cursor_probe": 0.0, "syscall_leaf": 20.8, "mutex_leaf": 17.7}}, + {"revision": "a9a85580a4488a3ad3739d8d130ebb0f66e80510", "label": "base-a9a85580a448", "round": 1, "kinstr_per_frame": 132.3, "kcycles_per_frame": 268.1, "recv_fps": 20001, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 4.9, "samples": 1316, "markers": {"learn_end": 0.0, "poll_forward_drop": 7.6, "cursor_probe": 0.0, "syscall_leaf": 22.1, "mutex_leaf": 17.3}}, + {"revision": "a9a85580a4488a3ad3739d8d130ebb0f66e80510", "label": "base-a9a85580a448", "round": 2, "kinstr_per_frame": 133.6, "kcycles_per_frame": 262.5, "recv_fps": 19999, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 3.31, "samples": 1330, "markers": {"learn_end": 0.0, "poll_forward_drop": 8.3, "cursor_probe": 0.0, "syscall_leaf": 22.3, "mutex_leaf": 16.7}}, + {"revision": "a9a85580a4488a3ad3739d8d130ebb0f66e80510", "label": "base-a9a85580a448", "round": 3, "kinstr_per_frame": 124.0, "kcycles_per_frame": 283.8, "recv_fps": 19998, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 18.89, "samples": 1237, "markers": {"learn_end": 0.0, "poll_forward_drop": 7.7, "cursor_probe": 0.0, "syscall_leaf": 17.3, "mutex_leaf": 17.8}}, + {"revision": "b4dfafac8879ffc4cb661c0d47200aeccea7a2ba", "label": "base-b4dfafac8879", "round": 1, "kinstr_per_frame": 130.1, "kcycles_per_frame": 243.7, "recv_fps": 20004, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 4.07, "samples": 1290, "markers": {"learn_end": 0.0, "poll_forward_drop": 9.5, "cursor_probe": 0.0, "syscall_leaf": 22.7, "mutex_leaf": 15.4}}, + {"revision": "b4dfafac8879ffc4cb661c0d47200aeccea7a2ba", "label": "base-b4dfafac8879", "round": 2, "kinstr_per_frame": 128.5, "kcycles_per_frame": 229.6, "recv_fps": 20001, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 3.77, "samples": 1278, "markers": {"learn_end": 0.0, "poll_forward_drop": 7.5, "cursor_probe": 0.0, "syscall_leaf": 24.2, "mutex_leaf": 14.9}}, + {"revision": "b4dfafac8879ffc4cb661c0d47200aeccea7a2ba", "label": "base-b4dfafac8879", "round": 3, "kinstr_per_frame": 117.7, "kcycles_per_frame": 282.4, "recv_fps": 20001, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 11.64, "samples": 1170, "markers": {"learn_end": 0.0, "poll_forward_drop": 5.9, "cursor_probe": 0.0, "syscall_leaf": 18.4, "mutex_leaf": 18.3}}, + {"revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", "label": "base-e84ffb3de313", "round": 1, "kinstr_per_frame": 136.9, "kcycles_per_frame": 260.8, "recv_fps": 20003, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 3.43, "samples": 1363, "markers": {"learn_end": 0.0, "poll_forward_drop": 9.6, "cursor_probe": 0.0, "syscall_leaf": 22.5, "mutex_leaf": 16.7}}, + {"revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", "label": "base-e84ffb3de313", "round": 2, "kinstr_per_frame": 134.9, "kcycles_per_frame": 262.6, "recv_fps": 20001, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 3.26, "samples": 1341, "markers": {"learn_end": 0.0, "poll_forward_drop": 7.8, "cursor_probe": 0.0, "syscall_leaf": 23.3, "mutex_leaf": 15.5}}, + {"revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", "label": "base-e84ffb3de313", "round": 3, "kinstr_per_frame": 134.9, "kcycles_per_frame": 267.8, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 11.14, "samples": 1342, "markers": {"learn_end": 0.0, "poll_forward_drop": 8.5, "cursor_probe": 0.0, "syscall_leaf": 20.1, "mutex_leaf": 17.2}}, + {"revision": "82194441a92bb89fbc69f3759ef336d676fad643", "label": "current", "round": 1, "kinstr_per_frame": 147.4, "kcycles_per_frame": 264.3, "recv_fps": 20001, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 3.09, "samples": 1466, "markers": {"learn_end": 0.4, "poll_forward_drop": 15.8, "cursor_probe": 8.0, "syscall_leaf": 17.9, "mutex_leaf": 17.9}}, + {"revision": "82194441a92bb89fbc69f3759ef336d676fad643", "label": "current", "round": 2, "kinstr_per_frame": 148.0, "kcycles_per_frame": 261.4, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 3.13, "samples": 1474, "markers": {"learn_end": 0.1, "poll_forward_drop": 14.1, "cursor_probe": 7.7, "syscall_leaf": 18.7, "mutex_leaf": 19.3}}, + {"revision": "82194441a92bb89fbc69f3759ef336d676fad643", "label": "current", "round": 3, "kinstr_per_frame": 148.0, "kcycles_per_frame": 266.1, "recv_fps": 19997, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 7.96, "samples": 1472, "markers": {"learn_end": 0.6, "poll_forward_drop": 15.4, "cursor_probe": 8.4, "syscall_leaf": 16.6, "mutex_leaf": 18.9}} + ] + }, + { + "artifact": "run.BIL6LP", + "harness_revision": "d935d13e8dba7d36e06f836a04984d940dbf0a57", + "workload": "chat", + "runtime": "io-uring-workers", + "workers": 4, + "generator_revision": "d935d13e8dba7d36e06f836a04984d940dbf0a57", + "instruction_period": 2000003, + "sampling": "concurrent with the counter window", + "relay_cpus": "4-7", + "generator_cpus": "0-3", + "cells": [ + {"revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", "label": "base-0c2a798de5c2", "round": 1, "kinstr_per_frame": 160.0, "kcycles_per_frame": 331.8, "recv_fps": 19997, "groups_lost": 0, "host_load_1m": 7.69, "host_load_1m_logged": 8.5, "samples": 7997, "markers": {"learn_end": 14.7, "poll_forward_drop": 9.7, "cursor_probe": 0.0, "syscall_leaf": 24.1, "mutex_leaf": 16.7}}, + {"revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", "label": "base-0c2a798de5c2", "round": 2, "kinstr_per_frame": 166.2, "kcycles_per_frame": 323.7, "recv_fps": 20003, "groups_lost": 0, "host_load_1m": 2.31, "host_load_1m_logged": 2.29, "samples": 8320, "markers": {"learn_end": 15.5, "poll_forward_drop": 11.3, "cursor_probe": 0.0, "syscall_leaf": 27.1, "mutex_leaf": 16.2}}, + {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 1, "kinstr_per_frame": 173.9, "kcycles_per_frame": 310.8, "recv_fps": 19997, "groups_lost": 0, "host_load_1m": 3.01, "host_load_1m_logged": 3.01, "samples": 8696, "markers": {"learn_end": 0.9, "poll_forward_drop": 20.9, "cursor_probe": 6.9, "syscall_leaf": 21.9, "mutex_leaf": 21.4}}, + {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 2, "kinstr_per_frame": 178.0, "kcycles_per_frame": 316.8, "recv_fps": 19997, "groups_lost": 0, "host_load_1m": 2.89, "host_load_1m_logged": 2.74, "samples": 8898, "markers": {"learn_end": 1.0, "poll_forward_drop": 20.9, "cursor_probe": 7.4, "syscall_leaf": 23.1, "mutex_leaf": 21.8}}, + {"revision": "2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3", "label": "base-2fff7ea7ae4e", "round": 1, "kinstr_per_frame": 209.5, "kcycles_per_frame": 415.5, "recv_fps": 19997, "groups_lost": 0, "host_load_1m": 2.45, "host_load_1m_logged": 2.43, "samples": 10473, "markers": {"learn_end": 16.6, "poll_forward_drop": 23.4, "cursor_probe": 7.3, "syscall_leaf": 31.6, "mutex_leaf": 17.2}}, + {"revision": "2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3", "label": "base-2fff7ea7ae4e", "round": 2, "kinstr_per_frame": 213.5, "kcycles_per_frame": 430.9, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 2.55, "host_load_1m_logged": 2.51, "samples": 10671, "markers": {"learn_end": 16.6, "poll_forward_drop": 23.4, "cursor_probe": 7.7, "syscall_leaf": 32.4, "mutex_leaf": 17.4}}, + {"revision": "49b5ec9b5630c54f2c5e99657b114535087b7da2", "label": "base-49b5ec9b5630", "round": 1, "kinstr_per_frame": 211.8, "kcycles_per_frame": 415.6, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 4.47, "host_load_1m_logged": 4.47, "samples": 10583, "markers": {"learn_end": 13.1, "poll_forward_drop": 19.1, "cursor_probe": 8.2, "syscall_leaf": 31.8, "mutex_leaf": 17.5}}, + {"revision": "49b5ec9b5630c54f2c5e99657b114535087b7da2", "label": "base-49b5ec9b5630", "round": 2, "kinstr_per_frame": 211.3, "kcycles_per_frame": 395.3, "recv_fps": 20003, "groups_lost": 0, "host_load_1m": 1.87, "host_load_1m_logged": 2.03, "samples": 10560, "markers": {"learn_end": 13.2, "poll_forward_drop": 19.3, "cursor_probe": 8.0, "syscall_leaf": 31.2, "mutex_leaf": 18.0}}, + {"revision": "55f6ed229250d17fc99389237659906d011d8a94", "label": "base-55f6ed229250", "round": 1, "kinstr_per_frame": 161.1, "kcycles_per_frame": 313.9, "recv_fps": 19998, "groups_lost": 0, "host_load_1m": 5.46, "host_load_1m_logged": 5.32, "samples": 8050, "markers": {"learn_end": 15.6, "poll_forward_drop": 10.9, "cursor_probe": 0.0, "syscall_leaf": 26.5, "mutex_leaf": 18.6}}, + {"revision": "55f6ed229250d17fc99389237659906d011d8a94", "label": "base-55f6ed229250", "round": 2, "kinstr_per_frame": 165.5, "kcycles_per_frame": 320.9, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 3.07, "host_load_1m_logged": 2.99, "samples": 8282, "markers": {"learn_end": 15.9, "poll_forward_drop": 11.0, "cursor_probe": 0.0, "syscall_leaf": 28.6, "mutex_leaf": 16.7}}, + {"revision": "991572d636d264c6189a3477849d2969e689f55c", "label": "base-991572d636d2", "round": 1, "kinstr_per_frame": 206.0, "kcycles_per_frame": 402.2, "recv_fps": 20003, "groups_lost": 0, "host_load_1m": 3.6, "host_load_1m_logged": 3.6, "samples": 10295, "markers": {"learn_end": 15.4, "poll_forward_drop": 19.7, "cursor_probe": 8.0, "syscall_leaf": 36.2, "mutex_leaf": 16.3}}, + {"revision": "991572d636d264c6189a3477849d2969e689f55c", "label": "base-991572d636d2", "round": 2, "kinstr_per_frame": 207.0, "kcycles_per_frame": 402.5, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 2.61, "host_load_1m_logged": 2.51, "samples": 10344, "markers": {"learn_end": 16.0, "poll_forward_drop": 19.6, "cursor_probe": 7.0, "syscall_leaf": 35.0, "mutex_leaf": 16.5}}, + {"revision": "e45064e3ccb43a535adb949b7db9a5a0aeb32413", "label": "base-e45064e3ccb4", "round": 1, "kinstr_per_frame": 158.5, "kcycles_per_frame": 316.3, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 5.21, "host_load_1m_logged": 5.97, "samples": 7921, "markers": {"learn_end": 14.5, "poll_forward_drop": 10.0, "cursor_probe": 0.0, "syscall_leaf": 24.8, "mutex_leaf": 17.0}}, + {"revision": "e45064e3ccb43a535adb949b7db9a5a0aeb32413", "label": "base-e45064e3ccb4", "round": 2, "kinstr_per_frame": 168.4, "kcycles_per_frame": 329.1, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 2.27, "host_load_1m_logged": 2.27, "samples": 8424, "markers": {"learn_end": 16.3, "poll_forward_drop": 11.4, "cursor_probe": 0.0, "syscall_leaf": 27.9, "mutex_leaf": 16.2}}, + {"revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", "label": "base-e84ffb3de313", "round": 1, "kinstr_per_frame": 136.2, "kcycles_per_frame": 277.1, "recv_fps": 19998, "groups_lost": 0, "host_load_1m": 6.92, "host_load_1m_logged": 7.17, "samples": 6802, "markers": {"learn_end": 0.0, "poll_forward_drop": 7.5, "cursor_probe": 0.0, "syscall_leaf": 21.4, "mutex_leaf": 16.4}}, + {"revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", "label": "base-e84ffb3de313", "round": 2, "kinstr_per_frame": 133.8, "kcycles_per_frame": 258.5, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 3.14, "host_load_1m_logged": 3.21, "samples": 6691, "markers": {"learn_end": 0.0, "poll_forward_drop": 7.5, "cursor_probe": 0.0, "syscall_leaf": 21.1, "mutex_leaf": 16.6}}, + {"revision": "f206893047dc2f1db7b3422622dbb505470d0c70", "label": "base-f206893047dc", "round": 1, "kinstr_per_frame": 168.5, "kcycles_per_frame": 338.2, "recv_fps": 20004, "groups_lost": 0, "host_load_1m": 9.21, "host_load_1m_logged": 9.23, "samples": 8424, "markers": {"learn_end": 16.6, "poll_forward_drop": 10.8, "cursor_probe": 0.0, "syscall_leaf": 29.4, "mutex_leaf": 16.6}}, + {"revision": "f206893047dc2f1db7b3422622dbb505470d0c70", "label": "base-f206893047dc", "round": 2, "kinstr_per_frame": 165.9, "kcycles_per_frame": 325.0, "recv_fps": 19997, "groups_lost": 0, "host_load_1m": 2.56, "host_load_1m_logged": 2.4, "samples": 8293, "markers": {"learn_end": 16.4, "poll_forward_drop": 11.2, "cursor_probe": 0.0, "syscall_leaf": 27.2, "mutex_leaf": 16.5}}, + {"revision": "d935d13e8dba7d36e06f836a04984d940dbf0a57", "label": "current", "round": 1, "kinstr_per_frame": 147.6, "kcycles_per_frame": 260.9, "recv_fps": 19997, "groups_lost": 0, "host_load_1m": 2.72, "host_load_1m_logged": 2.83, "samples": 7372, "markers": {"learn_end": 0.4, "poll_forward_drop": 14.5, "cursor_probe": 7.3, "syscall_leaf": 17.8, "mutex_leaf": 19.0}}, + {"revision": "d935d13e8dba7d36e06f836a04984d940dbf0a57", "label": "current", "round": 2, "kinstr_per_frame": 147.0, "kcycles_per_frame": 251.2, "recv_fps": 19999, "groups_lost": 0, "host_load_1m": 2.99, "host_load_1m_logged": 3.16, "samples": 7347, "markers": {"learn_end": 0.5, "poll_forward_drop": 14.5, "cursor_probe": 6.9, "syscall_leaf": 17.5, "mutex_leaf": 18.9}} + ] + }, + { + "artifact": "run.1KRu4S", + "harness_revision": "a6cd6f91b6a0c79a81f2a012a2683bc77bd6c7d7", + "workload": "chat", + "runtime": "io-uring-workers", + "workers": 4, + "generator_revision": "21d6da9fc1", + "instruction_period": 2000003, + "sampling": "concurrent with the counter window", + "relay_cpus": "4-7", + "generator_cpus": "0-3", + "cells": [ + {"revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", "label": "base-0c2a798de5c2", "round": 1, "kinstr_per_frame": 165.1, "kcycles_per_frame": 318.0, "recv_fps": 19999, "groups_lost": 0, "host_load_1m": 2.58, "host_load_1m_logged": 2.58, "samples": 8253, "markers": {"learn_end": 15.7, "poll_forward_drop": 10.8, "cursor_probe": 0.0, "syscall_leaf": 26.7, "mutex_leaf": 16.7}}, + {"revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", "label": "base-0c2a798de5c2", "round": 2, "kinstr_per_frame": 166.1, "kcycles_per_frame": 322.2, "recv_fps": 20001, "groups_lost": 0, "host_load_1m": 4.23, "host_load_1m_logged": 3.99, "samples": 8304, "markers": {"learn_end": 15.0, "poll_forward_drop": 10.7, "cursor_probe": 0.0, "syscall_leaf": 27.0, "mutex_leaf": 17.0}}, + {"revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", "label": "base-0c2a798de5c2", "round": 3, "kinstr_per_frame": 162.1, "kcycles_per_frame": 327.9, "recv_fps": 19997, "groups_lost": 0, "host_load_1m": 7.69, "host_load_1m_logged": 7.69, "samples": 8100, "markers": {"learn_end": 14.9, "poll_forward_drop": 10.9, "cursor_probe": 0.0, "syscall_leaf": 26.4, "mutex_leaf": 17.0}}, + {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 1, "kinstr_per_frame": 175.2, "kcycles_per_frame": 327.1, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 3.17, "host_load_1m_logged": 2.99, "samples": 8769, "markers": {"learn_end": 1.3, "poll_forward_drop": 22.0, "cursor_probe": 7.4, "syscall_leaf": 22.8, "mutex_leaf": 20.7}}, + {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 2, "kinstr_per_frame": 173.5, "kcycles_per_frame": 322.5, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 3.03, "host_load_1m_logged": 3.04, "samples": 8677, "markers": {"learn_end": 1.2, "poll_forward_drop": 21.4, "cursor_probe": 7.7, "syscall_leaf": 21.9, "mutex_leaf": 21.8}}, + {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 3, "kinstr_per_frame": 125.6, "kcycles_per_frame": 279.7, "recv_fps": 20569, "groups_lost": 32897, "host_load_1m": 24.77, "host_load_1m_logged": 24.77, "samples": 6509, "markers": {"learn_end": 0.4, "poll_forward_drop": 14.9, "cursor_probe": 6.5, "syscall_leaf": 12.4, "mutex_leaf": 23.5}}, + {"revision": "2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3", "label": "base-2fff7ea7ae4e", "round": 1, "kinstr_per_frame": 207.1, "kcycles_per_frame": 412.6, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 2.9, "host_load_1m_logged": 2.98, "samples": 10354, "markers": {"learn_end": 16.7, "poll_forward_drop": 22.5, "cursor_probe": 7.0, "syscall_leaf": 29.5, "mutex_leaf": 17.7}}, + {"revision": "2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3", "label": "base-2fff7ea7ae4e", "round": 2, "kinstr_per_frame": 202.8, "kcycles_per_frame": 412.8, "recv_fps": 20001, "groups_lost": 0, "host_load_1m": 3.67, "host_load_1m_logged": 4.13, "samples": 10136, "markers": {"learn_end": 16.3, "poll_forward_drop": 22.0, "cursor_probe": 6.7, "syscall_leaf": 29.1, "mutex_leaf": 18.1}}, + {"revision": "2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3", "label": "base-2fff7ea7ae4e", "round": 3, "kinstr_per_frame": 189.9, "kcycles_per_frame": 401.7, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 11.36, "host_load_1m_logged": 11.56, "samples": 9511, "markers": {"learn_end": 15.0, "poll_forward_drop": 21.0, "cursor_probe": 7.0, "syscall_leaf": 24.8, "mutex_leaf": 19.0}}, + {"revision": "55f6ed229250d17fc99389237659906d011d8a94", "label": "base-55f6ed229250", "round": 1, "kinstr_per_frame": 171.1, "kcycles_per_frame": 332.6, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 2.19, "host_load_1m_logged": 2.06, "samples": 8555, "markers": {"learn_end": 17.3, "poll_forward_drop": 11.6, "cursor_probe": 0.0, "syscall_leaf": 30.6, "mutex_leaf": 16.7}}, + {"revision": "55f6ed229250d17fc99389237659906d011d8a94", "label": "base-55f6ed229250", "round": 2, "kinstr_per_frame": 167.3, "kcycles_per_frame": 337.6, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 3.78, "host_load_1m_logged": 3.72, "samples": 8366, "markers": {"learn_end": 16.0, "poll_forward_drop": 10.1, "cursor_probe": 0.0, "syscall_leaf": 28.8, "mutex_leaf": 16.7}}, + {"revision": "55f6ed229250d17fc99389237659906d011d8a94", "label": "base-55f6ed229250", "round": 3, "kinstr_per_frame": 169.2, "kcycles_per_frame": 337.9, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 5.82, "host_load_1m_logged": 6.05, "samples": 8458, "markers": {"learn_end": 16.8, "poll_forward_drop": 11.5, "cursor_probe": 0.0, "syscall_leaf": 30.4, "mutex_leaf": 16.4}}, + {"revision": "991572d636d264c6189a3477849d2969e689f55c", "label": "base-991572d636d2", "round": 1, "kinstr_per_frame": 202.7, "kcycles_per_frame": 392.8, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 2.06, "host_load_1m_logged": 2.15, "samples": 10130, "markers": {"learn_end": 15.3, "poll_forward_drop": 19.7, "cursor_probe": 7.3, "syscall_leaf": 33.7, "mutex_leaf": 16.7}}, + {"revision": "991572d636d264c6189a3477849d2969e689f55c", "label": "base-991572d636d2", "round": 2, "kinstr_per_frame": 195.1, "kcycles_per_frame": 379.0, "recv_fps": 20002, "groups_lost": 0, "host_load_1m": 4.73, "host_load_1m_logged": 4.91, "samples": 9752, "markers": {"learn_end": 15.2, "poll_forward_drop": 18.3, "cursor_probe": 6.9, "syscall_leaf": 32.3, "mutex_leaf": 16.6}}, + {"revision": "991572d636d264c6189a3477849d2969e689f55c", "label": "base-991572d636d2", "round": 3, "kinstr_per_frame": 184.4, "kcycles_per_frame": 365.8, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 6.87, "host_load_1m_logged": 7.05, "samples": 9215, "markers": {"learn_end": 14.6, "poll_forward_drop": 18.3, "cursor_probe": 7.1, "syscall_leaf": 27.8, "mutex_leaf": 18.1}}, + {"revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", "label": "base-cdff3c8f952b", "round": 1, "kinstr_per_frame": 118.7, "kcycles_per_frame": 212.2, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 2.22, "host_load_1m_logged": 2.44, "samples": 5942, "markers": {"learn_end": 0.0, "poll_forward_drop": 6.8, "cursor_probe": 0.0, "syscall_leaf": 25.2, "mutex_leaf": 16.1}}, + {"revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", "label": "base-cdff3c8f952b", "round": 2, "kinstr_per_frame": 111.5, "kcycles_per_frame": 222.3, "recv_fps": 19999, "groups_lost": 0, "host_load_1m": 4.8, "host_load_1m_logged": 5.2, "samples": 5579, "markers": {"learn_end": 0.0, "poll_forward_drop": 7.4, "cursor_probe": 0.0, "syscall_leaf": 22.4, "mutex_leaf": 18.1}}, + {"revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", "label": "base-cdff3c8f952b", "round": 3, "kinstr_per_frame": 109.8, "kcycles_per_frame": 235.5, "recv_fps": 20003, "groups_lost": 0, "host_load_1m": 21.62, "host_load_1m_logged": 24.0, "samples": 5484, "markers": {"learn_end": 0.0, "poll_forward_drop": 6.9, "cursor_probe": 0.0, "syscall_leaf": 21.3, "mutex_leaf": 17.3}}, + {"revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", "label": "base-e84ffb3de313", "round": 1, "kinstr_per_frame": 135.0, "kcycles_per_frame": 266.2, "recv_fps": 19997, "groups_lost": 0, "host_load_1m": 2.84, "host_load_1m_logged": 3.41, "samples": 6746, "markers": {"learn_end": 0.0, "poll_forward_drop": 7.8, "cursor_probe": 0.0, "syscall_leaf": 22.2, "mutex_leaf": 16.0}}, + {"revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", "label": "base-e84ffb3de313", "round": 2, "kinstr_per_frame": 136.9, "kcycles_per_frame": 267.9, "recv_fps": 19997, "groups_lost": 0, "host_load_1m": 4.93, "host_load_1m_logged": 5.19, "samples": 6846, "markers": {"learn_end": 0.0, "poll_forward_drop": 8.0, "cursor_probe": 0.0, "syscall_leaf": 22.6, "mutex_leaf": 16.1}}, + {"revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", "label": "base-e84ffb3de313", "round": 3, "kinstr_per_frame": 116.1, "kcycles_per_frame": 310.9, "recv_fps": 22018, "groups_lost": 31280, "host_load_1m": 27.7, "host_load_1m_logged": 27.7, "samples": 6404, "markers": {"learn_end": 0.0, "poll_forward_drop": 4.8, "cursor_probe": 0.0, "syscall_leaf": 16.0, "mutex_leaf": 14.5}}, + {"revision": "a6cd6f91b6a0c79a81f2a012a2683bc77bd6c7d7", "label": "current", "round": 1, "kinstr_per_frame": 145.6, "kcycles_per_frame": 272.3, "recv_fps": 19997, "groups_lost": 0, "host_load_1m": 9.76, "host_load_1m_logged": 11.0, "samples": 7274, "markers": {"learn_end": 0.5, "poll_forward_drop": 14.8, "cursor_probe": 6.8, "syscall_leaf": 17.3, "mutex_leaf": 19.5}}, + {"revision": "a6cd6f91b6a0c79a81f2a012a2683bc77bd6c7d7", "label": "current", "round": 2, "kinstr_per_frame": 146.5, "kcycles_per_frame": 281.0, "recv_fps": 19998, "groups_lost": 0, "host_load_1m": 3.64, "host_load_1m_logged": 3.63, "samples": 7315, "markers": {"learn_end": 0.5, "poll_forward_drop": 14.3, "cursor_probe": 6.9, "syscall_leaf": 16.8, "mutex_leaf": 19.6}}, + {"revision": "a6cd6f91b6a0c79a81f2a012a2683bc77bd6c7d7", "label": "current", "round": 3, "kinstr_per_frame": 143.1, "kcycles_per_frame": 313.0, "recv_fps": 19304, "groups_lost": 0, "host_load_1m": 32.43, "host_load_1m_logged": 32.92, "samples": 6910, "markers": {"learn_end": 0.4, "poll_forward_drop": 14.1, "cursor_probe": 7.4, "syscall_leaf": 14.0, "mutex_leaf": 19.5}} + ] + }, + { + "artifact": "run.qYPSii", + "harness_revision": "74f46e79b75bb2cbb4c5d4c4d75e390249ba22d5", + "workload": "chat", + "runtime": "io-uring-workers", + "workers": 4, + "generator_revision": "21d6da9fc1", + "instruction_period": 2000003, + "sampling": "after the counter window", + "relay_cpus": "4-7", + "generator_cpus": "0-3", + "cells": [ + {"revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", "label": "base-0c2a798de5c2", "round": 1, "kinstr_per_frame": 246.0, "kcycles_per_frame": 599.4, "recv_fps": 7525, "groups_lost": 16750, "host_load_1m": 45.62, "host_load_1m_logged": 54.66, "samples": 5208, "markers": {"learn_end": 8.2, "poll_forward_drop": 11.1, "cursor_probe": 0.0, "syscall_leaf": 14.2, "mutex_leaf": 15.9}}, + {"revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", "label": "base-0c2a798de5c2", "round": 2, "kinstr_per_frame": 149.9, "kcycles_per_frame": 304.2, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 19.15, "host_load_1m_logged": 19.77, "samples": 7163, "markers": {"learn_end": 12.1, "poll_forward_drop": 8.9, "cursor_probe": 0.0, "syscall_leaf": 18.0, "mutex_leaf": 20.2}}, + {"revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", "label": "base-0c2a798de5c2", "round": 3, "kinstr_per_frame": 169.8, "kcycles_per_frame": 325.1, "recv_fps": 19999, "groups_lost": 0, "host_load_1m": 4.38, "host_load_1m_logged": 4.75, "samples": 8337, "markers": {"learn_end": 15.4, "poll_forward_drop": 11.7, "cursor_probe": 0.0, "syscall_leaf": 27.5, "mutex_leaf": 16.8}}, + {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 1, "kinstr_per_frame": 160.3, "kcycles_per_frame": 305.2, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 17.94, "host_load_1m_logged": 18.93, "samples": 7895, "markers": {"learn_end": 0.9, "poll_forward_drop": 19.7, "cursor_probe": 7.6, "syscall_leaf": 15.5, "mutex_leaf": 23.8}}, + {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 2, "kinstr_per_frame": 167.9, "kcycles_per_frame": 337.0, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 9.63, "host_load_1m_logged": 9.43, "samples": 8307, "markers": {"learn_end": 1.1, "poll_forward_drop": 21.0, "cursor_probe": 7.2, "syscall_leaf": 19.2, "mutex_leaf": 22.7}}, + {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 3, "kinstr_per_frame": 176.2, "kcycles_per_frame": 328.6, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 3.1, "host_load_1m_logged": 3.64, "samples": 8758, "markers": {"learn_end": 1.3, "poll_forward_drop": 21.2, "cursor_probe": 7.3, "syscall_leaf": 22.3, "mutex_leaf": 20.7}}, + {"revision": "2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3", "label": "base-2fff7ea7ae4e", "round": 1, "kinstr_per_frame": 169.1, "kcycles_per_frame": 409.6, "recv_fps": 17369, "groups_lost": 5362, "host_load_1m": 24.92, "host_load_1m_logged": 23.96, "samples": 9126, "markers": {"learn_end": 13.8, "poll_forward_drop": 18.9, "cursor_probe": 6.2, "syscall_leaf": 17.4, "mutex_leaf": 21.8}}, + {"revision": "2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3", "label": "base-2fff7ea7ae4e", "round": 2, "kinstr_per_frame": 161.7, "kcycles_per_frame": 349.3, "recv_fps": 19820, "groups_lost": 0, "host_load_1m": 11.62, "host_load_1m_logged": 10.84, "samples": 8749, "markers": {"learn_end": 14.1, "poll_forward_drop": 18.8, "cursor_probe": 6.7, "syscall_leaf": 19.0, "mutex_leaf": 21.1}}, + {"revision": "2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3", "label": "base-2fff7ea7ae4e", "round": 3, "kinstr_per_frame": 214.0, "kcycles_per_frame": 415.4, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 2.35, "host_load_1m_logged": 2.56, "samples": 10709, "markers": {"learn_end": 17.0, "poll_forward_drop": 24.0, "cursor_probe": 7.6, "syscall_leaf": 32.7, "mutex_leaf": 16.9}}, + {"revision": "55f6ed229250d17fc99389237659906d011d8a94", "label": "base-55f6ed229250", "round": 1, "kinstr_per_frame": 245.2, "kcycles_per_frame": 655.0, "recv_fps": 7433, "groups_lost": 28655, "host_load_1m": 53.37, "host_load_1m_logged": 53.25, "samples": 5224, "markers": {"learn_end": 7.2, "poll_forward_drop": 8.3, "cursor_probe": 0.0, "syscall_leaf": 15.9, "mutex_leaf": 16.9}}, + {"revision": "55f6ed229250d17fc99389237659906d011d8a94", "label": "base-55f6ed229250", "round": 2, "kinstr_per_frame": 166.6, "kcycles_per_frame": 340.8, "recv_fps": 19997, "groups_lost": 0, "host_load_1m": 11.77, "host_load_1m_logged": 12.27, "samples": 8277, "markers": {"learn_end": 16.1, "poll_forward_drop": 10.8, "cursor_probe": 0.0, "syscall_leaf": 27.6, "mutex_leaf": 17.9}}, + {"revision": "55f6ed229250d17fc99389237659906d011d8a94", "label": "base-55f6ed229250", "round": 3, "kinstr_per_frame": 174.6, "kcycles_per_frame": 343.7, "recv_fps": 19999, "groups_lost": 0, "host_load_1m": 3.23, "host_load_1m_logged": 3.42, "samples": 8758, "markers": {"learn_end": 17.5, "poll_forward_drop": 11.7, "cursor_probe": 0.0, "syscall_leaf": 31.6, "mutex_leaf": 16.3}}, + {"revision": "991572d636d264c6189a3477849d2969e689f55c", "label": "base-991572d636d2", "round": 1, "kinstr_per_frame": 164.2, "kcycles_per_frame": 331.5, "recv_fps": 19825, "groups_lost": 0, "host_load_1m": 30.42, "host_load_1m_logged": 31.94, "samples": 8019, "markers": {"learn_end": 11.0, "poll_forward_drop": 15.1, "cursor_probe": 6.5, "syscall_leaf": 17.6, "mutex_leaf": 20.5}}, + {"revision": "991572d636d264c6189a3477849d2969e689f55c", "label": "base-991572d636d2", "round": 2, "kinstr_per_frame": 181.6, "kcycles_per_frame": 363.0, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 9.45, "host_load_1m_logged": 9.45, "samples": 9524, "markers": {"learn_end": 14.6, "poll_forward_drop": 19.1, "cursor_probe": 7.1, "syscall_leaf": 30.3, "mutex_leaf": 17.2}}, + {"revision": "991572d636d264c6189a3477849d2969e689f55c", "label": "base-991572d636d2", "round": 3, "kinstr_per_frame": 204.7, "kcycles_per_frame": 382.8, "recv_fps": 20001, "groups_lost": 0, "host_load_1m": 2.13, "host_load_1m_logged": 2.32, "samples": 10294, "markers": {"learn_end": 15.5, "poll_forward_drop": 20.1, "cursor_probe": 7.7, "syscall_leaf": 35.0, "mutex_leaf": 16.6}}, + {"revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", "label": "base-cdff3c8f952b", "round": 1, "kinstr_per_frame": 124.4, "kcycles_per_frame": 287.4, "recv_fps": 16559, "groups_lost": 8554, "host_load_1m": 21.13, "host_load_1m_logged": 22.93, "samples": 5763, "markers": {"learn_end": 0.0, "poll_forward_drop": 5.3, "cursor_probe": 0.0, "syscall_leaf": 19.8, "mutex_leaf": 16.7}}, + {"revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", "label": "base-cdff3c8f952b", "round": 2, "kinstr_per_frame": 115.3, "kcycles_per_frame": 219.5, "recv_fps": 20001, "groups_lost": 0, "host_load_1m": 4.79, "host_load_1m_logged": 5.05, "samples": 5711, "markers": {"learn_end": 0.0, "poll_forward_drop": 7.5, "cursor_probe": 0.0, "syscall_leaf": 23.9, "mutex_leaf": 16.1}}, + {"revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", "label": "base-cdff3c8f952b", "round": 3, "kinstr_per_frame": 117.7, "kcycles_per_frame": 214.6, "recv_fps": 19998, "groups_lost": 0, "host_load_1m": 2.38, "host_load_1m_logged": 2.7, "samples": 5823, "markers": {"learn_end": 0.0, "poll_forward_drop": 7.1, "cursor_probe": 0.0, "syscall_leaf": 24.5, "mutex_leaf": 16.0}}, + {"revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", "label": "base-e84ffb3de313", "round": 1, "kinstr_per_frame": 125.5, "kcycles_per_frame": 294.0, "recv_fps": 19998, "groups_lost": 0, "host_load_1m": 21.29, "host_load_1m_logged": 22.36, "samples": 6352, "markers": {"learn_end": 0.0, "poll_forward_drop": 6.3, "cursor_probe": 0.0, "syscall_leaf": 16.8, "mutex_leaf": 17.2}}, + {"revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", "label": "base-e84ffb3de313", "round": 2, "kinstr_per_frame": 128.2, "kcycles_per_frame": 355.6, "recv_fps": 19590, "groups_lost": 0, "host_load_1m": 24.24, "host_load_1m_logged": 27.66, "samples": 6356, "markers": {"learn_end": 0.0, "poll_forward_drop": 5.0, "cursor_probe": 0.0, "syscall_leaf": 17.9, "mutex_leaf": 15.0}}, + {"revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", "label": "base-e84ffb3de313", "round": 3, "kinstr_per_frame": 135.7, "kcycles_per_frame": 264.2, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 2.24, "host_load_1m_logged": 2.24, "samples": 6774, "markers": {"learn_end": 0.0, "poll_forward_drop": 7.6, "cursor_probe": 0.0, "syscall_leaf": 21.6, "mutex_leaf": 16.2}}, + {"revision": "74f46e79b75bb2cbb4c5d4c4d75e390249ba22d5", "label": "current", "round": 1, "kinstr_per_frame": 145.2, "kcycles_per_frame": 278.0, "recv_fps": 19999, "groups_lost": 0, "host_load_1m": 11.7, "host_load_1m_logged": 11.76, "samples": 7185, "markers": {"learn_end": 0.4, "poll_forward_drop": 14.3, "cursor_probe": 6.6, "syscall_leaf": 15.5, "mutex_leaf": 19.8}}, + {"revision": "74f46e79b75bb2cbb4c5d4c4d75e390249ba22d5", "label": "current", "round": 2, "kinstr_per_frame": 147.0, "kcycles_per_frame": 278.5, "recv_fps": 20003, "groups_lost": 0, "host_load_1m": 6.81, "host_load_1m_logged": 6.97, "samples": 7385, "markers": {"learn_end": 0.6, "poll_forward_drop": 15.5, "cursor_probe": 7.1, "syscall_leaf": 17.7, "mutex_leaf": 18.0}}, + {"revision": "74f46e79b75bb2cbb4c5d4c4d75e390249ba22d5", "label": "current", "round": 3, "kinstr_per_frame": 148.8, "kcycles_per_frame": 269.2, "recv_fps": 20001, "groups_lost": 0, "host_load_1m": 3.07, "host_load_1m_logged": 3.07, "samples": 7392, "markers": {"learn_end": 0.5, "poll_forward_drop": 14.4, "cursor_probe": 6.6, "syscall_leaf": 17.7, "mutex_leaf": 19.3}} + ] + }, + { + "artifact": "run.16AJUb", + "harness_revision": "bd812a762c149a4345430b53530a597d261144b8", + "workload": "chat", + "runtime": "io-uring-workers", + "workers": 4, + "generator_revision": "21d6da9fc1", + "instruction_period": 2000003, + "sampling": "after the counter window", + "relay_cpus": "4-7", + "generator_cpus": "0-3", + "cells": [ + {"revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", "label": "base-0c2a798de5c2", "round": 1, "kinstr_per_frame": 158.5, "kcycles_per_frame": 330.8, "recv_fps": 20001, "groups_lost": 0, "host_load_1m": 13.91, "host_load_1m_logged": 14.83, "samples": 7839, "markers": {"learn_end": 14.9, "poll_forward_drop": 10.7, "cursor_probe": 0.0, "syscall_leaf": 23.5, "mutex_leaf": 17.1}}, + {"revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", "label": "base-0c2a798de5c2", "round": 2, "kinstr_per_frame": 139.5, "kcycles_per_frame": 330.1, "recv_fps": 19753, "groups_lost": 0, "host_load_1m": 25.02, "host_load_1m_logged": 25.76, "samples": 6782, "markers": {"learn_end": 11.1, "poll_forward_drop": 8.5, "cursor_probe": 0.0, "syscall_leaf": 15.1, "mutex_leaf": 18.7}}, + {"revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", "label": "base-0c2a798de5c2", "round": 3, "kinstr_per_frame": 140.8, "kcycles_per_frame": 335.0, "recv_fps": 19906, "groups_lost": 0, "host_load_1m": 21.62, "host_load_1m_logged": 21.62, "samples": 7065, "markers": {"learn_end": 12.0, "poll_forward_drop": 9.0, "cursor_probe": 0.0, "syscall_leaf": 16.1, "mutex_leaf": 19.6}}, + {"revision": "1e05129753c720d0b2318cedbab345d745227d2b", "label": "base-1e05129753c7", "round": 1, "kinstr_per_frame": 216.4, "kcycles_per_frame": 513.7, "recv_fps": 9349, "groups_lost": 22438, "host_load_1m": 32.14, "host_load_1m_logged": 32.14, "samples": 5175, "markers": {"learn_end": 8.5, "poll_forward_drop": 6.8, "cursor_probe": 0.0, "syscall_leaf": 13.7, "mutex_leaf": 17.7}}, + {"revision": "1e05129753c720d0b2318cedbab345d745227d2b", "label": "base-1e05129753c7", "round": 2, "kinstr_per_frame": 139.6, "kcycles_per_frame": 328.5, "recv_fps": 19696, "groups_lost": 6849, "host_load_1m": 29.39, "host_load_1m_logged": 29.39, "samples": 7166, "markers": {"learn_end": 12.1, "poll_forward_drop": 8.4, "cursor_probe": 0.0, "syscall_leaf": 17.0, "mutex_leaf": 19.3}}, + {"revision": "1e05129753c720d0b2318cedbab345d745227d2b", "label": "base-1e05129753c7", "round": 3, "kinstr_per_frame": 150.4, "kcycles_per_frame": 299.4, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 7.22, "host_load_1m_logged": 7.12, "samples": 7901, "markers": {"learn_end": 14.5, "poll_forward_drop": 11.2, "cursor_probe": 0.0, "syscall_leaf": 24.7, "mutex_leaf": 17.7}}, + {"revision": "45007f6fb14fb44adc02823a8035f4f39de8ab94", "label": "base-45007f6fb14f", "round": 1, "kinstr_per_frame": 144.6, "kcycles_per_frame": 353.6, "recv_fps": 16712, "groups_lost": 13701, "host_load_1m": 23.05, "host_load_1m_logged": 28.3, "samples": 5098, "markers": {"learn_end": 9.1, "poll_forward_drop": 6.8, "cursor_probe": 0.0, "syscall_leaf": 14.3, "mutex_leaf": 18.0}}, + {"revision": "45007f6fb14fb44adc02823a8035f4f39de8ab94", "label": "base-45007f6fb14f", "round": 2, "kinstr_per_frame": 156.9, "kcycles_per_frame": 330.2, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 20.44, "host_load_1m_logged": 22.17, "samples": 7645, "markers": {"learn_end": 13.6, "poll_forward_drop": 9.5, "cursor_probe": 0.0, "syscall_leaf": 21.8, "mutex_leaf": 19.1}}, + {"revision": "45007f6fb14fb44adc02823a8035f4f39de8ab94", "label": "base-45007f6fb14f", "round": 3, "kinstr_per_frame": 160.8, "kcycles_per_frame": 329.1, "recv_fps": 19998, "groups_lost": 0, "host_load_1m": 252.79, "host_load_1m_logged": 254.35, "samples": 7942, "markers": {"learn_end": 15.2, "poll_forward_drop": 11.2, "cursor_probe": 0.0, "syscall_leaf": 25.0, "mutex_leaf": 17.2}}, + {"revision": "87515b48c693457b7541db12c66836fd6fc6b1e8", "label": "base-87515b48c693", "round": 1, "kinstr_per_frame": 141.7, "kcycles_per_frame": 332.4, "recv_fps": 19983, "groups_lost": 0, "host_load_1m": 31.75, "host_load_1m_logged": 32.37, "samples": 6666, "markers": {"learn_end": 11.0, "poll_forward_drop": 8.6, "cursor_probe": 0.0, "syscall_leaf": 15.3, "mutex_leaf": 20.4}}, + {"revision": "87515b48c693457b7541db12c66836fd6fc6b1e8", "label": "base-87515b48c693", "round": 2, "kinstr_per_frame": 141.6, "kcycles_per_frame": 303.1, "recv_fps": 19765, "groups_lost": 0, "host_load_1m": 15.34, "host_load_1m_logged": 16.9, "samples": 6887, "markers": {"learn_end": 11.0, "poll_forward_drop": 8.4, "cursor_probe": 0.0, "syscall_leaf": 14.7, "mutex_leaf": 20.3}}, + {"revision": "87515b48c693457b7541db12c66836fd6fc6b1e8", "label": "base-87515b48c693", "round": 3, "kinstr_per_frame": 169.8, "kcycles_per_frame": 340.9, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 96.24, "host_load_1m_logged": 112.61, "samples": 8412, "markers": {"learn_end": 14.8, "poll_forward_drop": 11.1, "cursor_probe": 0.0, "syscall_leaf": 28.2, "mutex_leaf": 16.3}}, + {"revision": "8ee3c80894186483b3c0d8f7e84b37ab0f525390", "label": "base-8ee3c8089418", "round": 1, "kinstr_per_frame": 141.9, "kcycles_per_frame": 356.1, "recv_fps": 15002, "groups_lost": 38842, "host_load_1m": 30.58, "host_load_1m_logged": 31.49, "samples": 6099, "markers": {"learn_end": 10.8, "poll_forward_drop": 8.8, "cursor_probe": 0.0, "syscall_leaf": 12.7, "mutex_leaf": 18.4}}, + {"revision": "8ee3c80894186483b3c0d8f7e84b37ab0f525390", "label": "base-8ee3c8089418", "round": 2, "kinstr_per_frame": 152.6, "kcycles_per_frame": 349.8, "recv_fps": 20012, "groups_lost": 0, "host_load_1m": 29.14, "host_load_1m_logged": 29.13, "samples": 8262, "markers": {"learn_end": 15.4, "poll_forward_drop": 11.5, "cursor_probe": 0.0, "syscall_leaf": 27.4, "mutex_leaf": 16.7}}, + {"revision": "8ee3c80894186483b3c0d8f7e84b37ab0f525390", "label": "base-8ee3c8089418", "round": 3, "kinstr_per_frame": 150.4, "kcycles_per_frame": 303.5, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 10.23, "host_load_1m_logged": 10.69, "samples": 7586, "markers": {"learn_end": 13.8, "poll_forward_drop": 9.4, "cursor_probe": 0.0, "syscall_leaf": 20.6, "mutex_leaf": 17.5}}, + {"revision": "c3c88aec81350cece608599cef5c33322547d146", "label": "base-c3c88aec8135", "round": 1, "kinstr_per_frame": 150.7, "kcycles_per_frame": 322.3, "recv_fps": 19997, "groups_lost": 0, "host_load_1m": 20.63, "host_load_1m_logged": 22.23, "samples": 7644, "markers": {"learn_end": 14.9, "poll_forward_drop": 10.1, "cursor_probe": 0.0, "syscall_leaf": 22.2, "mutex_leaf": 17.7}}, + {"revision": "c3c88aec81350cece608599cef5c33322547d146", "label": "base-c3c88aec8135", "round": 2, "kinstr_per_frame": 123.9, "kcycles_per_frame": 272.4, "recv_fps": 22352, "groups_lost": 10324, "host_load_1m": 23.92, "host_load_1m_logged": 24.14, "samples": 6730, "markers": {"learn_end": 11.5, "poll_forward_drop": 8.4, "cursor_probe": 0.0, "syscall_leaf": 15.8, "mutex_leaf": 19.8}}, + {"revision": "c3c88aec81350cece608599cef5c33322547d146", "label": "base-c3c88aec8135", "round": 3, "kinstr_per_frame": 160.4, "kcycles_per_frame": 329.0, "recv_fps": 20003, "groups_lost": 0, "host_load_1m": 36.05, "host_load_1m_logged": 44.87, "samples": 8046, "markers": {"learn_end": 15.6, "poll_forward_drop": 10.2, "cursor_probe": 0.0, "syscall_leaf": 25.8, "mutex_leaf": 16.7}}, + {"revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", "label": "base-e84ffb3de313", "round": 1, "kinstr_per_frame": 129.7, "kcycles_per_frame": 275.4, "recv_fps": 20002, "groups_lost": 0, "host_load_1m": 32.54, "host_load_1m_logged": 32.54, "samples": 6495, "markers": {"learn_end": 0.0, "poll_forward_drop": 6.6, "cursor_probe": 0.0, "syscall_leaf": 19.1, "mutex_leaf": 16.6}}, + {"revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", "label": "base-e84ffb3de313", "round": 2, "kinstr_per_frame": 131.0, "kcycles_per_frame": 259.4, "recv_fps": 19883, "groups_lost": 0, "host_load_1m": 15.61, "host_load_1m_logged": 15.61, "samples": 6344, "markers": {"learn_end": 0.0, "poll_forward_drop": 6.3, "cursor_probe": 0.0, "syscall_leaf": 16.6, "mutex_leaf": 17.1}}, + {"revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", "label": "base-e84ffb3de313", "round": 3, "kinstr_per_frame": 130.0, "kcycles_per_frame": 267.5, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 9.84, "host_load_1m_logged": 9.84, "samples": 6231, "markers": {"learn_end": 0.0, "poll_forward_drop": 5.7, "cursor_probe": 0.0, "syscall_leaf": 16.4, "mutex_leaf": 17.0}}, + {"revision": "fd59eb18a50f6e0ac819e73d941a3832cbdac91f", "label": "base-fd59eb18a50f", "round": 1, "kinstr_per_frame": 136.4, "kcycles_per_frame": 357.7, "recv_fps": 21611, "groups_lost": 3490, "host_load_1m": 20.4, "host_load_1m_logged": 20.99, "samples": 6836, "markers": {"learn_end": 10.5, "poll_forward_drop": 8.2, "cursor_probe": 0.0, "syscall_leaf": 15.4, "mutex_leaf": 18.7}}, + {"revision": "fd59eb18a50f6e0ac819e73d941a3832cbdac91f", "label": "base-fd59eb18a50f", "round": 2, "kinstr_per_frame": 146.0, "kcycles_per_frame": 364.8, "recv_fps": 20004, "groups_lost": 0, "host_load_1m": 37.85, "host_load_1m_logged": 37.14, "samples": 7347, "markers": {"learn_end": 12.6, "poll_forward_drop": 8.9, "cursor_probe": 0.0, "syscall_leaf": 17.4, "mutex_leaf": 17.7}}, + {"revision": "fd59eb18a50f6e0ac819e73d941a3832cbdac91f", "label": "base-fd59eb18a50f", "round": 3, "kinstr_per_frame": 159.4, "kcycles_per_frame": 311.3, "recv_fps": 20003, "groups_lost": 0, "host_load_1m": 7.45, "host_load_1m_logged": 7.49, "samples": 8229, "markers": {"learn_end": 14.9, "poll_forward_drop": 10.2, "cursor_probe": 0.0, "syscall_leaf": 26.3, "mutex_leaf": 16.3}}, + {"revision": "bd812a762c149a4345430b53530a597d261144b8", "label": "current", "round": 1, "kinstr_per_frame": 144.6, "kcycles_per_frame": 354.3, "recv_fps": 19955, "groups_lost": 0, "host_load_1m": 15.03, "host_load_1m_logged": 15.03, "samples": 6179, "markers": {"learn_end": 0.3, "poll_forward_drop": 12.2, "cursor_probe": 6.4, "syscall_leaf": 14.2, "mutex_leaf": 19.0}}, + {"revision": "bd812a762c149a4345430b53530a597d261144b8", "label": "current", "round": 2, "kinstr_per_frame": 141.5, "kcycles_per_frame": 298.8, "recv_fps": 20036, "groups_lost": 0, "host_load_1m": 29.44, "host_load_1m_logged": 28.52, "samples": 7167, "markers": {"learn_end": 0.3, "poll_forward_drop": 13.6, "cursor_probe": 7.4, "syscall_leaf": 14.2, "mutex_leaf": 20.2}}, + {"revision": "bd812a762c149a4345430b53530a597d261144b8", "label": "current", "round": 3, "kinstr_per_frame": 142.7, "kcycles_per_frame": 281.2, "recv_fps": 19996, "groups_lost": 0, "host_load_1m": 16.58, "host_load_1m_logged": 17.57, "samples": 7166, "markers": {"learn_end": 0.3, "poll_forward_drop": 14.8, "cursor_probe": 6.7, "syscall_leaf": 16.0, "mutex_leaf": 18.4}} + ] + }, + { + "artifact": "run.SkAD0k", + "harness_revision": "bd812a762c149a4345430b53530a597d261144b8", + "workload": "video", + "runtime": "io-uring-workers", + "workers": 4, + "generator_revision": "21d6da9fc1", + "instruction_period": 2000003, + "sampling": "after the counter window", + "relay_cpus": "4-7", + "generator_cpus": "0-3", + "cells": [ + {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 1, "kinstr_per_frame": 141.1, "kcycles_per_frame": 330.6, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 5.42, "host_load_1m_logged": 5.31, "samples": 331, "markers": {"learn_end": 0.6, "poll_forward_drop": 6.0, "cursor_probe": 1.8, "syscall_leaf": 35.6, "mutex_leaf": 9.7}}, + {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 2, "kinstr_per_frame": 139.5, "kcycles_per_frame": 336.0, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 7.27, "host_load_1m_logged": 7.17, "samples": 324, "markers": {"learn_end": 0.0, "poll_forward_drop": 5.6, "cursor_probe": 1.2, "syscall_leaf": 33.6, "mutex_leaf": 9.6}}, + {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 3, "kinstr_per_frame": 138.3, "kcycles_per_frame": 339.8, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 7.29, "host_load_1m_logged": 7.18, "samples": 320, "markers": {"learn_end": 0.3, "poll_forward_drop": 5.9, "cursor_probe": 1.6, "syscall_leaf": 32.8, "mutex_leaf": 8.4}}, + {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 4, "kinstr_per_frame": 146.8, "kcycles_per_frame": 455.4, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 26.0, "host_load_1m_logged": 27.34, "samples": 343, "markers": {"learn_end": 0.3, "poll_forward_drop": 4.7, "cursor_probe": 1.7, "syscall_leaf": 37.6, "mutex_leaf": 9.3}}, + {"revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", "label": "base-cdff3c8f952b", "round": 1, "kinstr_per_frame": 109.6, "kcycles_per_frame": 291.5, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 4.97, "host_load_1m_logged": 5.13, "samples": 260, "markers": {"learn_end": 0.0, "poll_forward_drop": 1.5, "cursor_probe": 0.0, "syscall_leaf": 43.8, "mutex_leaf": 9.6}}, + {"revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", "label": "base-cdff3c8f952b", "round": 2, "kinstr_per_frame": 105.6, "kcycles_per_frame": 273.9, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 6.76, "host_load_1m_logged": 6.91, "samples": 253, "markers": {"learn_end": 0.0, "poll_forward_drop": 2.8, "cursor_probe": 0.0, "syscall_leaf": 42.7, "mutex_leaf": 7.9}}, + {"revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", "label": "base-cdff3c8f952b", "round": 3, "kinstr_per_frame": 109.9, "kcycles_per_frame": 286.4, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 6.84, "host_load_1m_logged": 7.46, "samples": 256, "markers": {"learn_end": 0.0, "poll_forward_drop": 1.2, "cursor_probe": 0.0, "syscall_leaf": 41.0, "mutex_leaf": 7.4}}, + {"revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", "label": "base-cdff3c8f952b", "round": 4, "kinstr_per_frame": 119.9, "kcycles_per_frame": 437.1, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 12.82, "host_load_1m_logged": 18.92, "samples": 282, "markers": {"learn_end": 0.0, "poll_forward_drop": 1.4, "cursor_probe": 0.0, "syscall_leaf": 42.6, "mutex_leaf": 8.2}}, + {"revision": "bd812a762c149a4345430b53530a597d261144b8", "label": "current", "round": 1, "kinstr_per_frame": 120.5, "kcycles_per_frame": 278.0, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 7.16, "host_load_1m_logged": 7.16, "samples": 279, "markers": {"learn_end": 0.0, "poll_forward_drop": 0.4, "cursor_probe": 0.0, "syscall_leaf": 35.1, "mutex_leaf": 4.3}}, + {"revision": "bd812a762c149a4345430b53530a597d261144b8", "label": "current", "round": 2, "kinstr_per_frame": 124.2, "kcycles_per_frame": 302.8, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 7.5, "host_load_1m_logged": 7.63, "samples": 284, "markers": {"learn_end": 0.0, "poll_forward_drop": 0.4, "cursor_probe": 0.4, "syscall_leaf": 39.8, "mutex_leaf": 4.2}}, + {"revision": "bd812a762c149a4345430b53530a597d261144b8", "label": "current", "round": 3, "kinstr_per_frame": 125.6, "kcycles_per_frame": 285.4, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 7.8, "host_load_1m_logged": 7.8, "samples": 288, "markers": {"learn_end": 0.0, "poll_forward_drop": 0.3, "cursor_probe": 0.0, "syscall_leaf": 38.9, "mutex_leaf": 3.1}}, + {"revision": "bd812a762c149a4345430b53530a597d261144b8", "label": "current", "round": 4, "kinstr_per_frame": 138.3, "kcycles_per_frame": 364.1, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 39.8, "host_load_1m_logged": 40.13, "samples": 326, "markers": {"learn_end": 0.0, "poll_forward_drop": 0.3, "cursor_probe": 0.0, "syscall_leaf": 46.3, "mutex_leaf": 5.2}} + ] + }, + { + "artifact": "run.IV47ua", + "harness_revision": "bd812a762c149a4345430b53530a597d261144b8", + "workload": "chat", + "runtime": "tokio-workers", + "workers": 4, + "generator_revision": "21d6da9fc1", + "instruction_period": 2000003, + "sampling": "after the counter window", + "relay_cpus": "4-7", + "generator_cpus": "0-3", + "cells": [ + {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 1, "kinstr_per_frame": 185.1, "kcycles_per_frame": 366.0, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 13.99, "host_load_1m_logged": 13.99, "samples": 9276, "markers": {"learn_end": 0.9, "poll_forward_drop": 17.1, "cursor_probe": 6.4, "syscall_leaf": 4.3, "mutex_leaf": 24.8}}, + {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 2, "kinstr_per_frame": 189.6, "kcycles_per_frame": 360.9, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 6.01, "host_load_1m_logged": 6.21, "samples": 9513, "markers": {"learn_end": 1.1, "poll_forward_drop": 17.0, "cursor_probe": 6.1, "syscall_leaf": 6.5, "mutex_leaf": 22.7}}, + {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 3, "kinstr_per_frame": 184.3, "kcycles_per_frame": 361.2, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 7.76, "host_load_1m_logged": 7.98, "samples": 9265, "markers": {"learn_end": 1.0, "poll_forward_drop": 17.1, "cursor_probe": 6.2, "syscall_leaf": 4.0, "mutex_leaf": 24.1}}, + {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 4, "kinstr_per_frame": 187.1, "kcycles_per_frame": 354.2, "recv_fps": 19997, "groups_lost": 0, "host_load_1m": 6.29, "host_load_1m_logged": 5.95, "samples": 9465, "markers": {"learn_end": 0.8, "poll_forward_drop": 18.0, "cursor_probe": 6.8, "syscall_leaf": 6.2, "mutex_leaf": 23.3}}, + {"revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", "label": "base-cdff3c8f952b", "round": 1, "kinstr_per_frame": 123.6, "kcycles_per_frame": 273.2, "recv_fps": 19998, "groups_lost": 0, "host_load_1m": 22.76, "host_load_1m_logged": 22.7, "samples": 6351, "markers": {"learn_end": 0.0, "poll_forward_drop": 5.1, "cursor_probe": 0.0, "syscall_leaf": 0.3, "mutex_leaf": 19.9}}, + {"revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", "label": "base-cdff3c8f952b", "round": 2, "kinstr_per_frame": 131.1, "kcycles_per_frame": 242.1, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 6.37, "host_load_1m_logged": 7.05, "samples": 6537, "markers": {"learn_end": 0.0, "poll_forward_drop": 5.8, "cursor_probe": 0.0, "syscall_leaf": 2.8, "mutex_leaf": 19.0}}, + {"revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", "label": "base-cdff3c8f952b", "round": 3, "kinstr_per_frame": 131.1, "kcycles_per_frame": 240.6, "recv_fps": 20004, "groups_lost": 0, "host_load_1m": 5.3, "host_load_1m_logged": 5.36, "samples": 6642, "markers": {"learn_end": 0.0, "poll_forward_drop": 6.9, "cursor_probe": 0.0, "syscall_leaf": 3.2, "mutex_leaf": 20.7}}, + {"revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", "label": "base-cdff3c8f952b", "round": 4, "kinstr_per_frame": 131.5, "kcycles_per_frame": 241.5, "recv_fps": 20003, "groups_lost": 0, "host_load_1m": 6.86, "host_load_1m_logged": 7.27, "samples": 6565, "markers": {"learn_end": 0.0, "poll_forward_drop": 6.0, "cursor_probe": 0.0, "syscall_leaf": 2.6, "mutex_leaf": 20.0}}, + {"revision": "bd812a762c149a4345430b53530a597d261144b8", "label": "current", "round": 1, "kinstr_per_frame": 172.8, "kcycles_per_frame": 324.4, "recv_fps": 19996, "groups_lost": 0, "host_load_1m": 7.61, "host_load_1m_logged": 8.28, "samples": 8639, "markers": {"learn_end": 0.3, "poll_forward_drop": 12.2, "cursor_probe": 6.2, "syscall_leaf": 1.6, "mutex_leaf": 20.5}}, + {"revision": "bd812a762c149a4345430b53530a597d261144b8", "label": "current", "round": 2, "kinstr_per_frame": 171.0, "kcycles_per_frame": 362.7, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 11.35, "host_load_1m_logged": 11.47, "samples": 8545, "markers": {"learn_end": 0.3, "poll_forward_drop": 12.4, "cursor_probe": 6.3, "syscall_leaf": 1.0, "mutex_leaf": 20.1}}, + {"revision": "bd812a762c149a4345430b53530a597d261144b8", "label": "current", "round": 3, "kinstr_per_frame": 172.2, "kcycles_per_frame": 318.2, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 5.71, "host_load_1m_logged": 5.76, "samples": 8638, "markers": {"learn_end": 0.4, "poll_forward_drop": 11.6, "cursor_probe": 6.0, "syscall_leaf": 1.9, "mutex_leaf": 19.8}}, + {"revision": "bd812a762c149a4345430b53530a597d261144b8", "label": "current", "round": 4, "kinstr_per_frame": 167.8, "kcycles_per_frame": 370.0, "recv_fps": 19845, "groups_lost": 0, "host_load_1m": 10.76, "host_load_1m_logged": 10.88, "samples": 8671, "markers": {"learn_end": 0.2, "poll_forward_drop": 12.1, "cursor_probe": 6.1, "syscall_leaf": 1.3, "mutex_leaf": 20.4}} + ] + }, + { + "artifact": "run.Z04ENl", + "harness_revision": "bd812a762c149a4345430b53530a597d261144b8", + "workload": "video", + "runtime": "tokio-workers", + "workers": 4, + "generator_revision": "21d6da9fc1", + "instruction_period": 2000003, + "sampling": "after the counter window", + "relay_cpus": "4-7", + "generator_cpus": "0-3", + "cells": [ + {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 1, "kinstr_per_frame": 170.2, "kcycles_per_frame": 535.3, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 25.25, "host_load_1m_logged": 27.59, "samples": 370, "markers": {"learn_end": 0.5, "poll_forward_drop": 9.5, "cursor_probe": 2.4, "syscall_leaf": 0.0, "mutex_leaf": 15.4}}, + {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 2, "kinstr_per_frame": 175.3, "kcycles_per_frame": 483.3, "recv_fps": 959, "groups_lost": 0, "host_load_1m": 30.01, "host_load_1m_logged": 28.96, "samples": 426, "markers": {"learn_end": 0.5, "poll_forward_drop": 6.6, "cursor_probe": 1.6, "syscall_leaf": 0.0, "mutex_leaf": 14.3}}, + {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 3, "kinstr_per_frame": 163.2, "kcycles_per_frame": 519.2, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 33.02, "host_load_1m_logged": 33.58, "samples": 384, "markers": {"learn_end": 0.3, "poll_forward_drop": 9.1, "cursor_probe": 2.3, "syscall_leaf": 0.0, "mutex_leaf": 17.7}}, + {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 4, "kinstr_per_frame": 164.5, "kcycles_per_frame": 386.2, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 39.42, "host_load_1m_logged": 39.65, "samples": 385, "markers": {"learn_end": 0.3, "poll_forward_drop": 9.1, "cursor_probe": 2.3, "syscall_leaf": 0.3, "mutex_leaf": 14.3}}, + {"revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", "label": "base-cdff3c8f952b", "round": 1, "kinstr_per_frame": 148.2, "kcycles_per_frame": 378.2, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 13.51, "host_load_1m_logged": 17.53, "samples": 357, "markers": {"learn_end": 0.0, "poll_forward_drop": 2.5, "cursor_probe": 0.0, "syscall_leaf": 0.0, "mutex_leaf": 12.0}}, + {"revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", "label": "base-cdff3c8f952b", "round": 2, "kinstr_per_frame": 149.5, "kcycles_per_frame": 302.2, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 18.81, "host_load_1m_logged": 19.83, "samples": 354, "markers": {"learn_end": 0.0, "poll_forward_drop": 2.8, "cursor_probe": 0.0, "syscall_leaf": 0.0, "mutex_leaf": 13.0}}, + {"revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", "label": "base-cdff3c8f952b", "round": 3, "kinstr_per_frame": 163.9, "kcycles_per_frame": 506.6, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 23.81, "host_load_1m_logged": 26.39, "samples": 383, "markers": {"learn_end": 0.0, "poll_forward_drop": 3.1, "cursor_probe": 0.0, "syscall_leaf": 0.0, "mutex_leaf": 12.5}}, + {"revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", "label": "base-cdff3c8f952b", "round": 4, "kinstr_per_frame": 156.0, "kcycles_per_frame": 556.9, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 39.7, "host_load_1m_logged": 39.65, "samples": 368, "markers": {"learn_end": 0.0, "poll_forward_drop": 3.5, "cursor_probe": 0.0, "syscall_leaf": 0.0, "mutex_leaf": 14.4}}, + {"revision": "bd812a762c149a4345430b53530a597d261144b8", "label": "current", "round": 1, "kinstr_per_frame": 147.3, "kcycles_per_frame": 372.5, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 28.85, "host_load_1m_logged": 28.75, "samples": 348, "markers": {"learn_end": 0.3, "poll_forward_drop": 0.3, "cursor_probe": 0.3, "syscall_leaf": 0.0, "mutex_leaf": 9.8}}, + {"revision": "bd812a762c149a4345430b53530a597d261144b8", "label": "current", "round": 2, "kinstr_per_frame": 142.9, "kcycles_per_frame": 322.9, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 23.15, "host_load_1m_logged": 24.73, "samples": 335, "markers": {"learn_end": 0.0, "poll_forward_drop": 0.6, "cursor_probe": 0.0, "syscall_leaf": 0.0, "mutex_leaf": 8.7}}, + {"revision": "bd812a762c149a4345430b53530a597d261144b8", "label": "current", "round": 3, "kinstr_per_frame": 146.4, "kcycles_per_frame": 428.6, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 17.83, "host_load_1m_logged": 18.09, "samples": 347, "markers": {"learn_end": 0.0, "poll_forward_drop": 0.3, "cursor_probe": 0.3, "syscall_leaf": 0.0, "mutex_leaf": 11.0}}, + {"revision": "bd812a762c149a4345430b53530a597d261144b8", "label": "current", "round": 4, "kinstr_per_frame": 147.2, "kcycles_per_frame": 330.8, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 24.93, "host_load_1m_logged": 24.93, "samples": 345, "markers": {"learn_end": 0.3, "poll_forward_drop": 0.3, "cursor_probe": 0.0, "syscall_leaf": 0.0, "mutex_leaf": 9.6}} + ] + } + ] } diff --git a/.github/relay-cpu-bisect.md b/.github/relay-cpu-bisect.md index e6fad74f482..d283544920d 100644 --- a/.github/relay-cpu-bisect.md +++ b/.github/relay-cpu-bisect.md @@ -1,48 +1,70 @@ -# Relay CPU bisect checkpoint +# Relay CPU bisect -Stopped at the user request on 2026-11-01. The investigation is unfinished. No production optimization, complete culprit attribution, or feature-cost acceptance is claimed. - -The reusable driver is `just bench-profile instructions`. Exact per-cell counts, boundary reports, source sample markers, artifact hashes, and excluded attempts are in [the evidence file](relay-cpu-bisect.json). Raw captures remain in this checkout’s ignored `.tmp/bench/`. +Relay instructions per delivered frame between [cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95) ([moq-dev/moq#5926](https://github.com/moq-dev/moq/pull/5926) as merged) and `main` at [c68c931025](https://github.com/moq-dev/moq/commit/c68c931025). Per-cell counts, delivery, loss, host load, and source markers are in [the evidence](relay-cpu-bisect.json). The [first checkpoint](https://github.com/moq-dev/moq/blob/8c62afd3195a3307054120226b54e3309a5dbc1f/.github/relay-cpu-bisect.md) and its [per-cell evidence](https://github.com/moq-dev/moq/blob/8c62afd3195a3307054120226b54e3309a5dbc1f/.github/relay-cpu-bisect.json) are superseded. ## Method -Each historical relay is paired with the same production control, [21d6da9fc1](https://github.com/moq-dev/moq/commit/21d6da9fc108b48373a961f34e43c3fcd38b0146), driven by the current generator. Builds use the profiling profile, frame pointers, and `moq-relay/io-uring,moq-bench/uring`, with a target per revision. Four io\_uring workers use cores 4-7; the generator uses 0-3. The chat workload has one publisher, 2,000 viewers, 10 fps, and one 300-byte frame per group. +`just bench-profile instructions` counts the relay's user and kernel instructions over a five-second window after seven seconds of warmup, divided by the frames the generator's 500 ms reports interpolate over the same window. Builds use the profiling profile with frame pointers and `moq-relay/io-uring,moq-bench/uring`; the relay runs four workers on cores 4-7 and the generator on cores 0-3. Chat is one publisher, 2,000 viewers, 10 fps, one 300-byte frame per group; video is the `video` workload at 960 delivered frames/s. -Three alternating historical/current pairs count user and kernel instructions over acknowledged five-second PMU windows after seven seconds of warmup. Delivery is interpolated from the generator’s surrounding 500 ms reports at those same boundaries. This is sampled delivery accounting. The normal host summaries cover a different final window and are not substituted for these counts. +One run measures several revisions: every round counts each base and `main` back to back, rotating which goes first. Each revision builds once into its own target and checkout, so later runs reuse it. Every run passed through `quest gate`, and each cell records the host's highest one-minute load during its window. -Fixed-period instruction profiles use 10,000,019 instructions/sample. The host’s original 1,000 Hz sample limit throttled every interrupt. Each successful gated run temporarily requested a 100,000 Hz limit and restored the original on exit. The kernel sometimes lowered the active limit itself; captures record it and refuse lost or throttled samples. The committed driver changes no sysctls. +Three findings shaped the method during this attempt: -## Measurements +- **Host load batches instructions down.** A relay contending with other work handles more frames per wakeup, so each costs fewer instructions: in [run.2DucdO](relay-cpu-bisect.json)'s third round, four revisions read 6-12k lower at load 9-19 than in the two rounds at load 2-5, three read the same, and the two at load 31-56 lost groups. Absolute counts from different sessions are not comparable: the first checkpoint read [55f6ed2292](https://github.com/moq-dev/moq/commit/55f6ed229250d17fc99389237659906d011d8a94) at 142.7k against 169-175k here. Only cells from one run, in a quiet stretch, are compared below. The 24-CPU host's load rose to 20-250 during several runs, from other work. +- **The generator is pinned.** From run.1KRu4S on, `BENCH_GENERATOR=21d6da9fc1` drives every relay, so later client changes cannot move what an old relay spends. With main's generator (run.BIL6LP), five of the seven relays it shares with run.qYPSii's quiet round read within 3k of the pinned figures; [0c2a798de5](https://github.com/moq-dev/moq/commit/0c2a798de5c2e9abb29815b86f945b0a980e5345) and [55f6ed2292](https://github.com/moq-dev/moq/commit/55f6ed229250d17fc99389237659906d011d8a94) read 7k and 11k lower. +- **Counting runs with the sampler off.** From run.qYPSii on, fixed-period stacks (period 2,000,003) come from the five seconds after the counter window, so sampling interrupts stay out of the counts. No measurable bias was found: [55f6ed2292](https://github.com/moq-dev/moq/commit/55f6ed229250d17fc99389237659906d011d8a94) read 167-171k with concurrent sampling and 175k without. -Cells are medians of three, with min-max in brackets, in thousands of instructions per delivered frame. Each row has its own interleaved current control; this is not a directly interleaved pair of adjacent historical commits. Host load and scheduling still affect instruction counts. No confidence interval or arbitrary performance threshold is asserted. +These are sampled-delivery estimates on a shared host. No confidence interval is claimed. -| Historical revision | Historical kinstr/frame | Paired current kinstr/frame | Evidence | -| --- | ---: | ---: | --- | -| [cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95), [moq-dev/moq#5926](https://github.com/moq-dev/moq/pull/5926) merged | 108.7 \[108.0-118.3] | 156.8 \[156.0-157.7] | `run.IkXbDf` | -| [55f6ed2292](https://github.com/moq-dev/moq/commit/55f6ed229250d17fc99389237659906d011d8a94), Immediately before dynamic tracks ([moq-dev/moq#6171](https://github.com/moq-dev/moq/pull/6171)) | 142.7 \[138.5-143.9] | 154.3 \[153.0-155.4] | `run.vgmg4w` | -| [dcd7d2cbb8](https://github.com/moq-dev/moq/commit/dcd7d2cbb891dc1cf72c68295ba70fbd17a4ec15), Immediately before per-reader ranges ([moq-dev/moq#5929](https://github.com/moq-dev/moq/pull/5929)) | 105.8 \[105.1-109.2] | 157.4 \[156.8-160.9] | `run.FlzdYt` | -| [fe722cec9f](https://github.com/moq-dev/moq/commit/fe722cec9f8432e0e46cb633cc5e79e59e7724cb), Per-reader ranges ([moq-dev/moq#5929](https://github.com/moq-dev/moq/pull/5929)) | 112.1 \[111.9-113.2] | 157.1 \[155.7-164.7] | `run.2bOgus` | -| [e84ffb3de3](https://github.com/moq-dev/moq/commit/e84ffb3de3139f7245bed5697754fd6add9a022e), Immediately before resume end tracking ([moq-dev/moq#6224](https://github.com/moq-dev/moq/pull/6224)) | 127.7 \[126.5-134.2] | 156.4 \[155.3-156.6] | `run.iFmfVR` | +## Chat on io_uring + +The quiet round of run.qYPSii (one-minute load 2-4, every cell at 20,000 frames/s with no loss): + +| Revision | kinstr/frame | `learn_end` | `Cursor::probe` | forward drop | syscall leaf | +| --- | ---: | ---: | ---: | ---: | ---: | +| [cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95), old endpoint | 117.7 | 0% | 0% | 7.1% | 24.5% | +| [e84ffb3de3](https://github.com/moq-dev/moq/commit/e84ffb3de3139f7245bed5697754fd6add9a022e), before [moq-dev/moq#6224](https://github.com/moq-dev/moq/pull/6224) | 135.7 | 0% | 0% | 7.6% | 21.6% | +| [0c2a798de5](https://github.com/moq-dev/moq/commit/0c2a798de5c2e9abb29815b86f945b0a980e5345) | 169.8 | 15.4% | 0% | 11.7% | 27.5% | +| [55f6ed2292](https://github.com/moq-dev/moq/commit/55f6ed229250d17fc99389237659906d011d8a94), before [moq-dev/moq#6171](https://github.com/moq-dev/moq/pull/6171) | 174.6 | 17.5% | 0% | 11.7% | 31.6% | +| [991572d636](https://github.com/moq-dev/moq/commit/991572d636d264c6189a3477849d2969e689f55c) | 204.7 | 15.5% | 7.7% | 20.1% | 35.0% | +| [2fff7ea7ae](https://github.com/moq-dev/moq/commit/2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3) | 214.0 | 17.0% | 7.6% | 24.0% | 32.7% | +| [21d6da9fc1](https://github.com/moq-dev/moq/commit/21d6da9fc108b48373a961f34e43c3fcd38b0146), first checkpoint's control | 176.2 | 1.3% | 7.3% | 21.2% | 22.3% | +| `main` | 148.8 | 0.5% | 6.6% | 14.4% | 17.7% | + +Markers are inclusive shares of the fixed-period samples; they overlap and are not additive. The two larger rises account for most of the peak, and `main` has already removed both: -The five comparisons above delivered approximately 20,000 frames/s and their boundary reports recorded zero confirmed lost groups. The evidence retains exact delivery estimates, including variation; no outlier cells were trimmed. Failed attempts are excluded explicitly rather than counted as completed comparisons. The final completed command, `run.9iDBWq` at [fd59eb18a5](https://github.com/moq-dev/moq/commit/fd59eb18a50f6e0ac819e73d941a3832cbdac91f), is retained separately in the JSON as **inconclusive**: delivery estimates ranged from approximately 18,364 to 22,215 frames/s and boundary reports included confirmed loss. Its counters and stacks were captured, but those differing workloads cannot establish the instruction effect of the resume-end change. No repeat was started after the stop request. +- **[moq-dev/moq#6224](https://github.com/moq-dev/moq/pull/6224), 26k of a +34k step, accidental, fixed.** Its `Reader::learn_end` polled each subscriber's finish through the track lock on every forward-drop pass (`poll_forward_drop` → `Subscriber::poll_drop` → `Reader::poll_serving` → `sync` → `learn_end`). It takes 10-18% of all relay instructions from [fd59eb18a5](https://github.com/moq-dev/moq/commit/fd59eb18a50f6e0ac819e73d941a3832cbdac91f), the PR's merge, through [2fff7ea7ae](https://github.com/moq-dev/moq/commit/2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3), and none at its parent ([run.16AJUb](relay-cpu-bisect.json) has every point in the interval). [moq-dev/moq#7532](https://github.com/moq-dev/moq/pull/7532) brought it to 0.5% on `main`. +- **[moq-dev/moq#6171](https://github.com/moq-dev/moq/pull/6171) and the changes after it, +39k to [2fff7ea7ae](https://github.com/moq-dev/moq/commit/2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3), mostly fixed.** Dynamic tracks added `Cursor::probe`, 15.8k at [991572d636](https://github.com/moq-dev/moq/commit/991572d636d264c6189a3477849d2969e689f55c) and 9.8k on `main` (below). The rest is track lock contention: futex syscalls and the forward-drop path. [moq-dev/moq#7493](https://github.com/moq-dev/moq/pull/7493) took the fanout subscribers off the shared lock, and `main`'s syscall leaf is now below the old endpoint's (26.3k against 28.8k). +- **[cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95) to [e84ffb3de3](https://github.com/moq-dev/moq/commit/e84ffb3de3139f7245bed5697754fd6add9a022e), +18k, a staircase of feature steps.** The first checkpoint measured [moq-dev/moq#5929](https://github.com/moq-dev/moq/pull/5929)'s per-reader ranges and time anchors at +6k. Runs eVjz2w and 2DucdO (main's generator, load 2-5 in the rounds used) put about +6k across [moq-dev/moq#5928](https://github.com/moq-dev/moq/pull/5928) (compressed tracks) and [moq-dev/moq#6129](https://github.com/moq-dev/moq/pull/6129) (a subscriber reads only its ranges' frames), and +12k in steps of 2-4k from [2d73eea67d](https://github.com/moq-dev/moq/commit/2d73eea67d083ecf18938f4fc7c4842118eb87cf): qmux in tree ([moq-dev/moq#6231](https://github.com/moq-dev/moq/pull/6231)), signed priorities ([moq-dev/moq#6142](https://github.com/moq-dev/moq/pull/6142)), the m0 simplification audit and public ingest ([moq-dev/moq#6108](https://github.com/moq-dev/moq/pull/6108), [moq-dev/moq#6111](https://github.com/moq-dev/moq/pull/6111)), and receive leases ([moq-dev/moq#5878](https://github.com/moq-dev/moq/pull/5878)). Each step is within twice the spread between rounds, so these name the interval, not a single commit's cost. +- **[moq-dev/moq#7530](https://github.com/moq-dev/moq/pull/7530) moved io_uring submission, not cost.** `udp::send` doubled after [21d6da9fc1](https://github.com/moq-dev/moq/commit/21d6da9fc108b48373a961f34e43c3fcd38b0146) because each turn now submits its queued sends inside `Shared::push_inner`, while total syscall instructions fell from 39.2k to 26.3k per frame. -## Source attribution and ownership +## Main against the old endpoint -- The overall chat increase is reproduced, but the complete regression has not been bisected. -- The range-serving boundary [moq-dev/moq#5929](https://github.com/moq-dev/moq/pull/5929) shows a smaller historical increase. It does not explain the whole current gap. Its cost has not been accepted as a feature cost in the release-perf epic. -- [moq-dev/moq#6224](https://github.com/moq-dev/moq/pull/6224) introduced `Reader::learn_end`, called by synchronization, retention, and finish polling. Its attempted after-change comparison was inconclusive because delivery changed and losses were reported; the source introduction is verified, but its instruction effect has not been confirmed. -- The current control’s `poll_forward_drop` path accounts for roughly 17-21% of sampled instructions across these runs, versus about 6-7% at the old endpoint. This is inclusive stack attribution, not an isolated cost or a claim that all of the path is avoidable. -- The historical pre-dynamic capture puts about 12.5% of samples under `learn_end`, while current captures put roughly 0.6-1.1% there. The later [moq-dev/moq#7532](https://github.com/moq-dev/moq/pull/7532) watch changes already alter this path. Do not attribute the whole current increase to the historical end-polling step or remove its correctness behavior. -- [moq-dev/moq#7493](https://github.com/moq-dev/moq/pull/7493) owns the overlapping lite publisher and track/resume/cache polling fixes, including the end-declared flag and fewer cursor locks. Those production paths were not edited here. The build-cost audit [moq-dev/moq#7602](https://github.com/moq-dev/moq/pull/7602) and archive epic [moq-dev/moq#7604](https://github.com/moq-dev/moq/pull/7604) are also untouched. +Every runtime and workload, with the pinned generator: + +| Workload, runtime | Run | [cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95) | [21d6da9fc1](https://github.com/moq-dev/moq/commit/21d6da9fc108b48373a961f34e43c3fcd38b0146) | `main` | `main` / old | Highest load | +| --- | --- | ---: | ---: | ---: | ---: | ---: | +| chat, io_uring | qYPSii, quiet round | 117.7 | 176.2 | 148.8 | 1.26 | 3 | +| chat, tokio workers | IV47ua, 4 rounds | 131.1 [123.6-131.5] | 186.1 [184.3-189.6] | 171.6 [167.8-172.8] | 1.31 | 23 | +| video, io_uring | SkAD0k, 4 rounds | 109.7 [105.6-119.9] | 140.3 [138.3-146.8] | 124.9 [120.5-138.3] | 1.14 | 40 | +| video, tokio workers | Z04ENl, 4 rounds | 152.8 [148.2-163.9] | 167.3 [163.2-175.3] | 146.8 [142.9-147.3] | 0.96 | 40 | + +Medians of four rounds, min-max in brackets; every cell delivered its workload's full rate with no loss. The video rows ran at load up to 40, so treat their ratios as approximate. Chat rose on both runtimes, so the remaining cost is shared code. By the nearest named function, chat's growth on `main` falls in the same places on io_uring (run.qYPSii) and the tokio workers (run.IV47ua), in instructions per frame: + +| Growth on `main` | io_uring | tokio | Introduced by | +| --- | ---: | ---: | --- | +| `Cursor::probe` and `Misses::judge` for every peek | 8.6k | 8.7k | [moq-dev/moq#6171](https://github.com/moq-dev/moq/pull/6171), fills for misses | +| `Cursor` `sync`, `poll_drift`, `poll_peek`, `TrackState::live_edge` and `live` | 11.0k | 9.0k | [moq-dev/moq#5929](https://github.com/moq-dev/moq/pull/5929) and its successors, per-reader ranges and time anchors | +| kio `Tasks::poll_each`, `WaiterList::register` growth, `Waiters::release` | 10.7k | 11.5k | kio's task and waiter rework, including [moq-dev/moq#7340](https://github.com/moq-dev/moq/pull/7340) and [moq-dev/moq#7532](https://github.com/moq-dev/moq/pull/7532) | +| `Subscriber` `poll_drop`, `take`, `poll_peek_group` growth | 5.0k | 4.7k | forward drops and resume | +| `SubscribeServe::poll`, `Writer::poll_close`, transport `poll_closed` | 5.6k | 6.6k | lite publisher serving | -The source markers in the evidence are inclusive, can overlap, and are not additive. They identify where to continue reading, not a finished attribution of deliberate versus avoidable costs. +The rows are self cost under the named function, including the std and libc leaves below it; they add to more than the net growth because other paths shrank. `Cursor::probe` reads the subscription, takes the track's shared state lock to judge misses, clones the track's fills, and runs the ask and resolve loop on every peek, even for a live reader that took every group as it arrived and never misses one. That work pays for relay backfill of missed groups, but the chat workload never needs it. -## Remaining scope +## Decisions -The existing [quest](../quest/m1/release-perf/relay-cpu-bisect.md) retains the work. Obtain a comparable-delivery measurement of the resume-end boundary. Resolve the unbisected growth between per-reader ranges and the parent of that change, then the later interval from the pre-dynamic point to the pinned control. Account for intervening improvements as well as regressions. Coordinate overlapping fixes with the fanout-lock owner before changing those files. Confirm attributed steps on video and Tokio workers; implement any remaining avoidable costs with publisher/viewer benchmark sweeps. Record justified feature-cost acceptances in the release-perf epic, then complete checks and local other-vendor review. None of this work was started after the stop request. +No feature cost is accepted in the [release-perf epic](../quest/m1/release-perf/README.md) yet; that is the maintainer's call, recorded in [the PR](https://github.com/moq-dev/moq/pull/7605). ## Validation -- Focused profile, relay lifecycle, and summary tests pass, along with Ruff and shellcheck. The profile tests cover matching windows, unsupported counters, CPU sets, loss/throttling, and identical cumulative snapshots at one timestamp. -- Two full `quest gate just ci` attempts failed on the known mbx `E2BIG` defect ([moq-dev/moq#7594](https://github.com/moq-dev/moq/pull/7594)) and runner-image `/proc/self/uid_map` namespace denial, including downstream build and tooling-test failures. The first also failed an unchanged JS announcement-sync test; the second passed JS tests. These are not full-gate passes. -- No other-vendor review or ready-for-review transition has been completed. The PR remains a draft. Public API and wire: no changes. +Every count, delivery estimate, loss figure, and host load cited here is in [the evidence](relay-cpu-bisect.json), keyed by run artifact. Raw captures (`perf.data`, folded stacks, reports) stay in the measuring checkout's ignored `.tmp/bench/`. diff --git a/quest/m1/release-perf/relay-cpu-bisect.md b/quest/m1/release-perf/relay-cpu-bisect.md index 923c3020549..19d67c9f07d 100644 --- a/quest/m1/release-perf/relay-cpu-bisect.md +++ b/quest/m1/release-perf/relay-cpu-bisect.md @@ -6,31 +6,22 @@ The commits that raised the relay's instructions per delivered frame between [cd ## Plan -Found by [moq-dev/moq#7505](https://github.com/moq-dev/moq/pull/7505), which profiled the io_uring clock read. At `main` [`30f23c3f8a`](https://github.com/moq-dev/moq/commit/30f23c3f8a), relay instructions per delivered frame had risen 25-56% on io_uring since [cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95) and 18-38% on the tokio workers since `1dd9b83d1`, with the same delivery: - -| workload | runtime | before, kinstr/frame | `30f23c3f8a`, kinstr/frame | -|---|---|---|---| -| chat | io_uring | 86.3 [84.5-87.4] at [cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95) | 134.8 [128.5-151.3] | -| video | io_uring | 108.7 [106.6-116.2] at [cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95) | 136.1 [132.6-152.8] | -| chat | tokio workers | 104.0 [101.4-105.0] at `1dd9b83d1` | 143.1 [127.5-205.3] | -| video | tokio workers | 141.6 [139.3-148.3] at `1dd9b83d1` | 167.1 [164.9-173.6] | - -Medians of 5 interleaved reps, min-max in brackets. The tokio rows start at [`1dd9b83d1`](https://github.com/moq-dev/moq/commit/1dd9b83d19), [moq-dev/moq#5926](https://github.com/moq-dev/moq/pull/5926)'s base, since [moq-dev/moq#7505](https://github.com/moq-dev/moq/pull/7505) measured the tokio workers only there and at `main`. Both runtimes moved, so at least one cause is shared code (moq-net's model, kio, or the session layer). That does not rule out a further io_uring-only cost, such as the clock read [moq-dev/moq#7505](https://github.com/moq-dev/moq/pull/7505) profiled. The tokio range also spans [moq-dev/moq#5926](https://github.com/moq-dev/moq/pull/5926) itself, so the two rises can't be subtracted to find one. - -Decided 2026-11-01 (maintainer, follow-up planning after the 2026-11-01 /quest-afk run): m1, in this epic, since `main` must not reach the cut carrying accidental per-frame cost. - -- Measure as [moq-dev/moq#7505](https://github.com/moq-dev/moq/pull/7505) did: `--profile profiling` builds with frame pointers and `moq-relay/io-uring,moq-bench/uring`, one target directory per revision, `perf stat` `cpu_core` instructions (user and kernel) over the steady window, divided by the frames the generator's 500 ms reports count over the same window. 4 io_uring workers, not 1, so chat does not saturate a worker. Pin the relay and generator cores so a loaded host still reads instruction deltas. -- Bisect `cdff3c8f95..main` on chat over io_uring first (the largest rise and the tightest ranges), with a step size that keeps each build's cost bounded, then confirm the culprits on video and on the tokio workers. A rise that shows on io_uring but not on the tokio workers is attributed on its own, not folded into a shared culprit. -- Attribute each step with `perf record` at a fixed period, as [moq-dev/moq#7505](https://github.com/moq-dev/moq/pull/7505) did, before deciding fix or accept. Record accepted costs in the epic's Plan with the feature they pay for. -- Commit the driver this needs as a deterministic mode of `just bench-profile` or `just bench-runtime` (pinned cores, instructions per delivered frame, fixed-period samples) instead of another throwaway script; [moq-dev/moq#7505](https://github.com/moq-dev/moq/pull/7505)'s driver on `bench/relay.sh` was not committed. -- Checkpointed at the maintainer's stop request in [moq-dev/moq#7605](https://github.com/moq-dev/moq/pull/7605), without a production fix or completed attribution. The [measurement report](/.github/relay-cpu-bisect.md) and [per-cell evidence](/.github/relay-cpu-bisect.json) retain counts, matching delivery boundaries, source sample markers, artifact hashes, and excluded attempts. Raw captures remain in that checkout's `.tmp/bench/`. -- `just bench-profile instructions` now provides disjoint relay/generator CPU sets, acknowledged PMU windows, delivery interpolated from surrounding 500 ms reports, alternating revision pairs, isolated revision builds, and fixed-period instruction stacks. It refuses unavailable/multiplexed counters and lost/throttled samples. Historical profiling checkouts live under `.tmp/` and load only their own Cargo configuration. -- The host's original 1,000 Hz sample-rate limit throttled every sample. Gated measurements temporarily requested 100,000 Hz and restored the original on exit; the driver changes no sysctls. A parser defect discovered during collection is fixed: identical cumulative snapshots at one millisecond remain valid, while contradictory observations are refused. Failed attempts are excluded explicitly. -- Completed exploratory chat comparisons use the same [21d6da9fc1](https://github.com/moq-dev/moq/commit/21d6da9fc108b48373a961f34e43c3fcd38b0146) production control, four io_uring workers, cores 4-7 for the relay and 0-3 for the generator, three alternating pairs, and a fixed period of 10,000,019 instructions. The old endpoint's median is 108.7k instructions/frame [108.0-118.3k], versus its paired current control's 156.8k [156.0-157.7k], at approximately 20,000 frames/s with zero confirmed loss in the boundary reports. No outlier cells were trimmed. These are sampled delivery estimates on a shared host, not confidence intervals. -- Attribution remains partial. The range-serving boundary [moq-dev/moq#5929](https://github.com/moq-dev/moq/pull/5929) accounts for a smaller measured historical step. [moq-dev/moq#6224](https://github.com/moq-dev/moq/pull/6224) introduced repeated `Reader::learn_end` polling; its after-change capture is retained but inconclusive because delivery changed and confirmed losses were reported. The source introduction is verified; its instruction effect is not yet confirmed. Later [moq-dev/moq#7532](https://github.com/moq-dev/moq/pull/7532) watch changes already reduce that path on the current control. Current `poll_forward_drop` stacks remain prominent and overlap the separately owned [moq-dev/moq#7493](https://github.com/moq-dev/moq/pull/7493); those production files were not edited here. No feature cost has been accepted in the epic from these incomplete measurements. -- Remaining: obtain a comparable-delivery measurement of the resume-end boundary, then attribute the growth between [fe722cec9f](https://github.com/moq-dev/moq/commit/fe722cec9f8432e0e46cb633cc5e79e59e7724cb) and the parent of [fd59eb18a5](https://github.com/moq-dev/moq/commit/fd59eb18a50f6e0ac819e73d941a3832cbdac91f), then account for changes through the pre-dynamic point [55f6ed2292](https://github.com/moq-dev/moq/commit/55f6ed229250d17fc99389237659906d011d8a94) and the pinned current control [21d6da9fc1](https://github.com/moq-dev/moq/commit/21d6da9fc108b48373a961f34e43c3fcd38b0146), including improvements that offset rises. Coordinate track/resume/publisher changes with [moq-dev/moq#7493](https://github.com/moq-dev/moq/pull/7493). Confirm attributed steps on video and Tokio workers, fix remaining avoidable costs with publisher/viewer benchmark sweeps, record justified feature-cost acceptances in the epic, and complete local review. None of those remaining experiments or fixes was started after the stop request. -- Validation: focused measurement tests and lints pass, but two full local gates failed on the known mbx E2BIG and runner namespace denial; an unchanged JS test also failed on the first attempt and passed on the second. The PR remains a draft and other-vendor review has not run. Preserve those limits when resuming. -- [moq-dev/moq#7493](https://github.com/moq-dev/moq/pull/7493) fixed the bimodal futex cost on the fanout workload; don't fix it twice. A bisect step that lands on the same lock starts from that PR's lock counts. +Found by [moq-dev/moq#7505](https://github.com/moq-dev/moq/pull/7505). Decided 2026-11-01 (maintainer): m1, in this epic, since `main` must not reach the cut carrying accidental per-frame cost. + +Bisected in [moq-dev/moq#7605](https://github.com/moq-dev/moq/pull/7605), which committed the driver (`just bench-profile instructions`, several rotated bases per run, a pinned `BENCH_GENERATOR`, per-cell host load) and [the report](/.github/relay-cpu-bisect.md). On chat over io_uring, the relay peaked at 1.82x the old endpoint and `main` is at 1.26x: + +- [moq-dev/moq#6224](https://github.com/moq-dev/moq/pull/6224)'s `Reader::learn_end` polling (+26k instructions per frame) and the track lock contention after [moq-dev/moq#6171](https://github.com/moq-dev/moq/pull/6171) were accidental. [moq-dev/moq#7532](https://github.com/moq-dev/moq/pull/7532) and [moq-dev/moq#7493](https://github.com/moq-dev/moq/pull/7493) already removed both from `main`. +- `main` against the old endpoint: chat 1.26x on io_uring and 1.31x on the tokio workers, video 1.14x on io_uring and 0.96x on the tokio workers. Chat's growth is shared code, in the same functions on both runtimes. + +What remains is set by the maintainer's decisions in [moq-dev/moq#7605](https://github.com/moq-dev/moq/pull/7605): + +- `Cursor::probe` and `Misses::judge` ([moq-dev/moq#6171](https://github.com/moq-dev/moq/pull/6171)'s fills for misses) cost about 8.6k per frame on every peek, taking the shared track lock and cloning its fills even for a live reader that never misses a group. Either a fast path that skips them while nothing is missable, or acceptance as the cost of relay backfill. +- The cursor's window, drift, and live-edge reads (about 9-11k, from [moq-dev/moq#5929](https://github.com/moq-dev/moq/pull/5929)'s per-reader ranges and time anchors) and the 2-4k feature steps before [e84ffb3de3](https://github.com/moq-dev/moq/commit/e84ffb3de3139f7245bed5697754fd6add9a022e) (compressed tracks, ranged frame reads, qmux in tree, signed priorities, public ingest, receive leases) are feature costs to accept or not. +- kio's task and waiter growth (about 11k: `Tasks::poll_each`, `WaiterList::register`, `Waiters::release`) and the lite publisher's serving and close polling (about 5-7k) have no single feature behind them yet. + +Then record each accepted cost and its feature in the epic's Plan, re-pair `main` against [cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95) on chat and video over both runtimes, and delete this quest. + +Measure as the report does: `BENCH_GENERATOR=21d6da9fc1`, `BENCH_INSTRUCTION_PERIOD=2000003`, `kernel.perf_event_max_sample_rate` raised to 100,000 for the gated run and restored after, and compare only cells from one run at low host load. Instructions per frame fall by up to 20% when other host work makes the relay batch more frames per wakeup, so cross-session absolutes don't compare. Public API: none expected. Wire: none. From ae981248fc1c8c1f39a43d9706e4f9138390d680 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 21:16:38 -0800 Subject: [PATCH 14/36] docs(bench): reflow the instructions mode notes Co-Authored-By: Claude Opus 5.5 --- bench/README.md | 66 +++++++++++++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 30 deletions(-) diff --git a/bench/README.md b/bench/README.md index ecc2386e375..4c08389d78b 100644 --- a/bench/README.md +++ b/bench/README.md @@ -297,20 +297,24 @@ BENCH_RELAY_CPUS=4-7 BENCH_GENERATOR_CPUS=0-3 BENCH_GENERATOR=21d6da9fc1 \ Choose disjoint allowed CPU sets, with at least one relay CPU per worker. Hybrid CPUs require performance cores, and count their `cpu_core` events. The relay and generator keep these affinities for every revision. Instructions -mode runs alone. Each base keeps a detached worktree and target under +mode runs alone. + +Each base keeps a detached worktree and target under `target/bench/revisions/`, so later runs reuse its build; delete them with `git worktree remove`. The revisions build side by side on every allowed CPU, -since fat LTO links on one thread. Every round measures each base and the current tree back -to back, rotating which goes first, for `BENCH_RUNTIME_ROUNDS` rounds (default -3); one generator drives them all. That is the current tree's unless -`BENCH_GENERATOR` names a revision: hold it fixed across a bisect's runs, since -client changes (flow control, subscription options) move what an old relay -spends per frame. The revisions are each other's -control, so there is no unprofiled run per revision. The run ends with each -revision's median and range in `instructions-summary.json`, including the groups -the generator confirmed lost between the reports bracketing each counter window -and the host's highest one-minute load during it. Admission only waits for room -when the run starts, so judge cells by the load they actually ran under. +since fat LTO links on one thread. Every round measures each base and the +current tree back to back, rotating which goes first, for +`BENCH_RUNTIME_ROUNDS` rounds (default 3). The revisions are each other's +control, so there is no unprofiled run per revision. One generator drives them +all: the current tree's, unless `BENCH_GENERATOR` names a revision. Hold it +fixed across a bisect's runs, so client changes (flow control, subscription +options) can't move what an old relay spends per frame. + +The run ends with each revision's median and range in +`instructions-summary.json`, including the groups the generator confirmed lost +between the reports bracketing each counter window and the host's highest +one-minute load during it. Admission only waits for room when the run starts, +so judge cells by the load they actually ran under. After seven seconds of warmup, acknowledged perf enable/disable commands bound five seconds of relay counters. `instructions/instructions.json` records both @@ -325,24 +329,26 @@ On the profiling host, a 1,000 Hz limit throttled every sample even at a period of 10,000,019. Temporarily raising `kernel.perf_event_max_sample_rate` to 100,000 allowed complete captures. Save and restore the original setting around the whole gated run; the driver records the observed limit and changes no sysctls. -The host sampler lasts nineteen seconds so delivery reports bracket both edges -and the sampling window that follows. -The normal `summary.json` still describes its own final throughput window; -use `instructions.json` for instruction comparisons. - -`BENCH_INSTRUCTION_PERIOD` defaults to 1,000,003 retired instructions per sample. -The fixed-period recording covers the five seconds after the counter window, -not the window itself, so sampling interrupts, which run as the relay's kernel -instructions, stay out of the counts. Scale stacks by their share of samples. -`instructions.folded` -weights each stack by that period. `perf.data`, raw `stat.json`, `perf.log`, -thread names, CPU sets, and delivery reports remain with the evidence. Both -counters and samples cover only the relay. Compare equivalent delivery and loss. -Instructions reduce frequency noise, but not batching: a relay contending with -other host work handles more frames per wakeup, so each frame costs fewer -instructions. On the profiling host, cells measured under a one-minute load -above about 5 read up to 20% low, so compare cells from quiet stretches, within -one run. Pinning does not remove interference from other host work. +The host sampler lasts nineteen seconds, so delivery reports bracket both edges +of the counter window and cover the sampling window after it. The normal +`summary.json` still describes its own final throughput window; use +`instructions.json` for instruction comparisons. + +`BENCH_INSTRUCTION_PERIOD` defaults to 1,000,003 retired instructions per +sample. The fixed-period recording covers the five seconds after the counter +window, not the window itself, so sampling interrupts, which run as the relay's +kernel instructions, stay out of the counts. `instructions.folded` weights each +stack by that period; scale stacks to a frame by their share of samples. +`perf.data`, raw `stat.json`, `perf.log`, thread names, CPU sets, and delivery +reports remain with the evidence. Both counters and samples cover only the +relay. + +Compare equivalent delivery and loss. Instructions reduce frequency noise, but +not batching: a relay contending with other host work handles more frames per +wakeup, so each frame costs fewer instructions. On the profiling host, cells +measured under a one-minute load above about 9 read 5-20% low, so compare +cells from quiet stretches within one run. Pinning does not remove interference +from other host work. ### Heap From 864b710f382f4807e1c05cea76143411e59e57a3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 21:25:58 -0800 Subject: [PATCH 15/36] bench(relay): check instructions bases out for the run only A base checkout kept under target/ put a historical tree's tests where the repository's test filters find them. The compiler cache keys on remapped paths, so a fresh checkout still reuses each revision's target. Co-Authored-By: Claude Opus 5.5 --- .github/relay-cpu-bisect.md | 16 ++++++++-------- bench/README.md | 6 +++--- bench/run.sh | 33 ++++++++++++++++++--------------- 3 files changed, 29 insertions(+), 26 deletions(-) diff --git a/.github/relay-cpu-bisect.md b/.github/relay-cpu-bisect.md index d283544920d..da86b916f2f 100644 --- a/.github/relay-cpu-bisect.md +++ b/.github/relay-cpu-bisect.md @@ -16,7 +16,7 @@ Three findings shaped the method during this attempt: These are sampled-delivery estimates on a shared host. No confidence interval is claimed. -## Chat on io_uring +## Chat on io\_uring The quiet round of run.qYPSii (one-minute load 2-4, every cell at 20,000 frames/s with no loss): @@ -36,7 +36,7 @@ Markers are inclusive shares of the fixed-period samples; they overlap and are n - **[moq-dev/moq#6224](https://github.com/moq-dev/moq/pull/6224), 26k of a +34k step, accidental, fixed.** Its `Reader::learn_end` polled each subscriber's finish through the track lock on every forward-drop pass (`poll_forward_drop` → `Subscriber::poll_drop` → `Reader::poll_serving` → `sync` → `learn_end`). It takes 10-18% of all relay instructions from [fd59eb18a5](https://github.com/moq-dev/moq/commit/fd59eb18a50f6e0ac819e73d941a3832cbdac91f), the PR's merge, through [2fff7ea7ae](https://github.com/moq-dev/moq/commit/2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3), and none at its parent ([run.16AJUb](relay-cpu-bisect.json) has every point in the interval). [moq-dev/moq#7532](https://github.com/moq-dev/moq/pull/7532) brought it to 0.5% on `main`. - **[moq-dev/moq#6171](https://github.com/moq-dev/moq/pull/6171) and the changes after it, +39k to [2fff7ea7ae](https://github.com/moq-dev/moq/commit/2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3), mostly fixed.** Dynamic tracks added `Cursor::probe`, 15.8k at [991572d636](https://github.com/moq-dev/moq/commit/991572d636d264c6189a3477849d2969e689f55c) and 9.8k on `main` (below). The rest is track lock contention: futex syscalls and the forward-drop path. [moq-dev/moq#7493](https://github.com/moq-dev/moq/pull/7493) took the fanout subscribers off the shared lock, and `main`'s syscall leaf is now below the old endpoint's (26.3k against 28.8k). - **[cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95) to [e84ffb3de3](https://github.com/moq-dev/moq/commit/e84ffb3de3139f7245bed5697754fd6add9a022e), +18k, a staircase of feature steps.** The first checkpoint measured [moq-dev/moq#5929](https://github.com/moq-dev/moq/pull/5929)'s per-reader ranges and time anchors at +6k. Runs eVjz2w and 2DucdO (main's generator, load 2-5 in the rounds used) put about +6k across [moq-dev/moq#5928](https://github.com/moq-dev/moq/pull/5928) (compressed tracks) and [moq-dev/moq#6129](https://github.com/moq-dev/moq/pull/6129) (a subscriber reads only its ranges' frames), and +12k in steps of 2-4k from [2d73eea67d](https://github.com/moq-dev/moq/commit/2d73eea67d083ecf18938f4fc7c4842118eb87cf): qmux in tree ([moq-dev/moq#6231](https://github.com/moq-dev/moq/pull/6231)), signed priorities ([moq-dev/moq#6142](https://github.com/moq-dev/moq/pull/6142)), the m0 simplification audit and public ingest ([moq-dev/moq#6108](https://github.com/moq-dev/moq/pull/6108), [moq-dev/moq#6111](https://github.com/moq-dev/moq/pull/6111)), and receive leases ([moq-dev/moq#5878](https://github.com/moq-dev/moq/pull/5878)). Each step is within twice the spread between rounds, so these name the interval, not a single commit's cost. -- **[moq-dev/moq#7530](https://github.com/moq-dev/moq/pull/7530) moved io_uring submission, not cost.** `udp::send` doubled after [21d6da9fc1](https://github.com/moq-dev/moq/commit/21d6da9fc108b48373a961f34e43c3fcd38b0146) because each turn now submits its queued sends inside `Shared::push_inner`, while total syscall instructions fell from 39.2k to 26.3k per frame. +- **[moq-dev/moq#7530](https://github.com/moq-dev/moq/pull/7530) moved io\_uring submission, not cost.** `udp::send` doubled after [21d6da9fc1](https://github.com/moq-dev/moq/commit/21d6da9fc108b48373a961f34e43c3fcd38b0146) because each turn now submits its queued sends inside `Shared::push_inner`, while total syscall instructions fell from 39.2k to 26.3k per frame. ## Main against the old endpoint @@ -44,14 +44,14 @@ Every runtime and workload, with the pinned generator: | Workload, runtime | Run | [cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95) | [21d6da9fc1](https://github.com/moq-dev/moq/commit/21d6da9fc108b48373a961f34e43c3fcd38b0146) | `main` | `main` / old | Highest load | | --- | --- | ---: | ---: | ---: | ---: | ---: | -| chat, io_uring | qYPSii, quiet round | 117.7 | 176.2 | 148.8 | 1.26 | 3 | -| chat, tokio workers | IV47ua, 4 rounds | 131.1 [123.6-131.5] | 186.1 [184.3-189.6] | 171.6 [167.8-172.8] | 1.31 | 23 | -| video, io_uring | SkAD0k, 4 rounds | 109.7 [105.6-119.9] | 140.3 [138.3-146.8] | 124.9 [120.5-138.3] | 1.14 | 40 | -| video, tokio workers | Z04ENl, 4 rounds | 152.8 [148.2-163.9] | 167.3 [163.2-175.3] | 146.8 [142.9-147.3] | 0.96 | 40 | +| chat, io\_uring | qYPSii, quiet round | 117.7 | 176.2 | 148.8 | 1.26 | 3 | +| chat, tokio workers | IV47ua, 4 rounds | 131.1 \[123.6-131.5] | 186.1 \[184.3-189.6] | 171.6 \[167.8-172.8] | 1.31 | 23 | +| video, io\_uring | SkAD0k, 4 rounds | 109.7 \[105.6-119.9] | 140.3 \[138.3-146.8] | 124.9 \[120.5-138.3] | 1.14 | 40 | +| video, tokio workers | Z04ENl, 4 rounds | 152.8 \[148.2-163.9] | 167.3 \[163.2-175.3] | 146.8 \[142.9-147.3] | 0.96 | 40 | -Medians of four rounds, min-max in brackets; every cell delivered its workload's full rate with no loss. The video rows ran at load up to 40, so treat their ratios as approximate. Chat rose on both runtimes, so the remaining cost is shared code. By the nearest named function, chat's growth on `main` falls in the same places on io_uring (run.qYPSii) and the tokio workers (run.IV47ua), in instructions per frame: +Medians of four rounds, min-max in brackets; every cell delivered its workload's full rate with no loss. The video rows ran at load up to 40, so treat their ratios as approximate. Chat rose on both runtimes, so the remaining cost is shared code. By the nearest named function, chat's growth on `main` falls in the same places on io\_uring (run.qYPSii) and the tokio workers (run.IV47ua), in instructions per frame: -| Growth on `main` | io_uring | tokio | Introduced by | +| Growth on `main` | io\_uring | tokio | Introduced by | | --- | ---: | ---: | --- | | `Cursor::probe` and `Misses::judge` for every peek | 8.6k | 8.7k | [moq-dev/moq#6171](https://github.com/moq-dev/moq/pull/6171), fills for misses | | `Cursor` `sync`, `poll_drift`, `poll_peek`, `TrackState::live_edge` and `live` | 11.0k | 9.0k | [moq-dev/moq#5929](https://github.com/moq-dev/moq/pull/5929) and its successors, per-reader ranges and time anchors | diff --git a/bench/README.md b/bench/README.md index 4c08389d78b..5056a217cde 100644 --- a/bench/README.md +++ b/bench/README.md @@ -299,9 +299,9 @@ Hybrid CPUs require performance cores, and count their `cpu_core` events. The relay and generator keep these affinities for every revision. Instructions mode runs alone. -Each base keeps a detached worktree and target under -`target/bench/revisions/`, so later runs reuse its build; delete them with -`git worktree remove`. The revisions build side by side on every allowed CPU, +Each base builds into its own target under `target/bench/revisions/`, +from a detached worktree in `.tmp/` that the run removes when it exits, so a +later run reuses that build. The revisions build side by side on every allowed CPU, since fat LTO links on one thread. Every round measures each base and the current tree back to back, rotating which goes first, for `BENCH_RUNTIME_ROUNDS` rounds (default 3). The revisions are each other's diff --git a/bench/run.sh b/bench/run.sh index c9a0c5f0574..eb2220101ab 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -186,6 +186,7 @@ fi RUN=$(mktemp -d "$ARTIFACTS/run.XXXXXX") printf 'Benchmark evidence: %s\n' "$RUN" WORKTREE= +WORKTREES=() RELAY_PID= HOST_PID= LOAD_PID= @@ -211,10 +212,11 @@ cleanup() { kill "$RELAY_PID" 2>/dev/null || true wait "$RELAY_PID" 2>/dev/null || true fi - if [[ -n $WORKTREE ]]; then - git -C "$ROOT" worktree remove --force "$WORKTREE" >/dev/null 2>&1 || true - rmdir "$WORKTREE" 2>/dev/null || true - fi + local worktree + for worktree in ${WORKTREE:+"$WORKTREE"} ${WORKTREES[@]+"${WORKTREES[@]}"}; do + git -C "$ROOT" worktree remove --force "$worktree" >/dev/null 2>&1 || true + rmdir "$worktree" 2>/dev/null || true + done # Close both pipe writers before waiting for the owned log copiers. if [[ -n $STDOUT_PID ]]; then exec 1>&3 2>&4 @@ -903,17 +905,18 @@ run_profiles() { done } -# Check out one revision where it stays put, so Cargo's fingerprints survive and a -# later bisect step reuses its build. It loads only its own Cargo configuration. +# Check out one revision for this run, in the checkout's ignored scratch space, +# where it loads only its own Cargo configuration. Its target persists, keyed by +# revision; the compiler cache keys on remapped paths, so a fresh checkout reuses it. revision_checkout() { - local sha=$1 label=$2 checkout=$TARGET/revisions/$1/checkout - if [[ $(git -C "$checkout" rev-parse HEAD 2>/dev/null) != "$sha" ]]; then - rm -rf "$checkout" - git worktree prune - git worktree add --detach "$checkout" "$sha" >/dev/null - fi + local sha=$1 label=$2 worktree + mkdir -p "$ROOT/.tmp" + worktree=$(mktemp -d "$ROOT/.tmp/moq-bench-base.XXXXXX") + WORKTREES+=("$worktree") + git worktree add --detach "$worktree" "$sha" >/dev/null mkdir -p "$RUN/base-context/$label" - (cd "$checkout" && python3 "$ROOT/bench/metadata.py" "$RUN/base-context/$label" "$sha" --profile) + (cd "$worktree" && python3 "$ROOT/bench/metadata.py" "$RUN/base-context/$label" "$sha" --profile) + REVISION_CHECKOUT=$worktree } # Count every base and the current tree in rotated order, so each round measures @@ -929,7 +932,7 @@ run_instructions() { if ((${#BASES[@]} > 1)); then label=base-${sha:0:12}; fi revision_checkout "$sha" "$label" labels+=("$label") - checkouts+=("$TARGET/revisions/$sha/checkout") + checkouts+=("$REVISION_CHECKOUT") targets+=("$TARGET/revisions/$sha/profile") done labels+=(current) @@ -943,7 +946,7 @@ run_instructions() { if [[ " ${BASES[*]} " != *" $GENERATOR "* ]]; then revision_checkout "$GENERATOR" generator build_labels+=(generator) - build_checkouts+=("$TARGET/revisions/$GENERATOR/checkout") + build_checkouts+=("$REVISION_CHECKOUT") build_targets+=("$generator") fi fi From 827924a3d107dedfdfb03ade134ec315b7d6bee3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 22:09:35 -0800 Subject: [PATCH 16/36] bench(relay): address review of the instructions driver - Discover the allowed CPUs only on Linux, where sched_getaffinity exists and instructions mode runs; other hosts' bench modes no longer abort. - Summarize each worker count separately, so one configuration's regression cannot hide in another's range. - Run each revision build as its own process group, and stop the outstanding ones before cleanup removes the worktrees they read. Co-Authored-By: Claude Opus 5.5 --- bench/README.md | 2 +- bench/instructions.py | 39 ++++++++++++++++++++++----------------- bench/profile.test.py | 19 +++++++++++++------ bench/run.sh | 30 +++++++++++++++++++++++------- 4 files changed, 59 insertions(+), 31 deletions(-) diff --git a/bench/README.md b/bench/README.md index 5056a217cde..eaefd9a47ba 100644 --- a/bench/README.md +++ b/bench/README.md @@ -310,7 +310,7 @@ all: the current tree's, unless `BENCH_GENERATOR` names a revision. Hold it fixed across a bisect's runs, so client changes (flow control, subscription options) can't move what an old relay spends per frame. -The run ends with each revision's median and range in +The run ends with each revision's median and range at each worker count in `instructions-summary.json`, including the groups the generator confirmed lost between the reports bracketing each counter window and the host's highest one-minute load during it. Admission only waits for room when the run starts, diff --git a/bench/instructions.py b/bench/instructions.py index aafddfa83c0..82ce8eaaaed 100644 --- a/bench/instructions.py +++ b/bench/instructions.py @@ -290,16 +290,18 @@ def lost(rows, start, end): def summarize(run, labels): - """Print each revision's median and range across rounds, and keep them beside the evidence.""" + """Print each revision's median and range across rounds at each worker count, and keep them.""" from statistics import median result = {} for label in labels: - cells = [] + configurations = {} for path in sorted((run / "relay" / label).glob("round-*/instructions-*/*/instructions/instructions.json")): + # Worker counts are separate configurations, never one revision-wide range. + workers = path.relative_to(run / "relay" / label).parts[1].removeprefix("instructions-") cell = json.loads(path.read_text()) rows = [json.loads(line) for line in (path.parent.parent / "load.jsonl").read_text().splitlines()] - cells.append( + configurations.setdefault(workers, dict(cells=[]))["cells"].append( dict( cell=str(path.relative_to(run)), instructions_per_frame=cell["instructions_per_frame"], @@ -309,24 +311,27 @@ def summarize(run, labels): host_load_1m=max(cell.get("host_load_1m", [math.nan])), ) ) - if not cells: + if not configurations: raise ValueError(f"no instruction cells for {label}") - result[label] = dict(cells=cells) - for key in ("instructions_per_frame", "cycles_per_frame", "recv_fps", "groups_lost", "host_load_1m"): - values = [cell[key] for cell in cells] - result[label][key] = dict(median=median(values), min=min(values), max=max(values)) + for configuration in configurations.values(): + cells = configuration["cells"] + for key in ("instructions_per_frame", "cycles_per_frame", "recv_fps", "groups_lost", "host_load_1m"): + values = [cell[key] for cell in cells] + configuration[key] = dict(median=median(values), min=min(values), max=max(values)) + result[label] = dict(sorted(configurations.items(), key=lambda item: int(item[0]))) (run / "instructions-summary.json").write_text(json.dumps(result, indent=2) + "\n") print( - f"\n{'revision':<20} {'kinstr/frame [min-max]':>26} {'kcycles/frame':>14} {'fps [min-max]':>24} " - f"{'lost':>6} {'load':>5}" + f"\n{'revision':<20} {'workers':>7} {'kinstr/frame [min-max]':>26} {'kcycles/frame':>14} " + f"{'fps [min-max]':>24} {'lost':>6} {'load':>5}" ) - for label, row in result.items(): - i, c, f = row["instructions_per_frame"], row["cycles_per_frame"], row["recv_fps"] - print( - f"{label:<20} {i['median'] / 1e3:>9.1f} [{i['min'] / 1e3:.1f}-{i['max'] / 1e3:.1f}] " - f"{c['median'] / 1e3:>14.1f} {f['median']:>9.0f} [{f['min']:.0f}-{f['max']:.0f}] " - f"{row['groups_lost']['max']:>6} {row['host_load_1m']['max']:>5.1f}" - ) + for label, configurations in result.items(): + for workers, row in configurations.items(): + i, c, f = row["instructions_per_frame"], row["cycles_per_frame"], row["recv_fps"] + print( + f"{label:<20} {workers:>7} {i['median'] / 1e3:>9.1f} [{i['min'] / 1e3:.1f}-{i['max'] / 1e3:.1f}] " + f"{c['median'] / 1e3:>14.1f} {f['median']:>9.0f} [{f['min']:.0f}-{f['max']:.0f}] " + f"{row['groups_lost']['max']:>6} {row['host_load_1m']['max']:>5.1f}" + ) def validate_data(path): diff --git a/bench/profile.test.py b/bench/profile.test.py index 63333f14896..59723561df3 100644 --- a/bench/profile.test.py +++ b/bench/profile.test.py @@ -75,9 +75,13 @@ def test_cpu_sets_cannot_overlap_or_escape_the_allowed_set(self): def test_summary_reports_each_revision_across_rounds(self): with tempfile.TemporaryDirectory() as directory: run = Path(directory) - for label, values in [("base-0123", [12, 10, 11]), ("current", [20, 21])]: + for label, workers, values in [ + ("base-0123", 4, [12, 10, 11]), + ("base-0123", 16, [40]), + ("current", 4, [20, 21]), + ]: for round, value in enumerate(values, 1): - cell = run / "relay" / label / f"round-{round}" / "instructions-4" / "chat" + cell = run / "relay" / label / f"round-{round}" / f"instructions-{workers}" / "chat" (cell / "instructions").mkdir(parents=True) (cell / "instructions" / "instructions.json").write_text( json.dumps( @@ -96,11 +100,14 @@ def test_summary_reports_each_revision_across_rounds(self): with patch("builtins.print"): instructions.summarize(run, ["base-0123", "current"]) summary = json.loads((run / "instructions-summary.json").read_text()) - self.assertEqual(summary["base-0123"]["instructions_per_frame"], dict(median=11, min=10, max=12)) - self.assertEqual(summary["current"]["instructions_per_frame"]["median"], 20.5) + # Each worker count is its own configuration, not part of one revision-wide range. + self.assertEqual(list(summary["base-0123"]), ["4", "16"]) + self.assertEqual(summary["base-0123"]["4"]["instructions_per_frame"], dict(median=11, min=10, max=12)) + self.assertEqual(summary["base-0123"]["16"]["instructions_per_frame"], dict(median=40, min=40, max=40)) + self.assertEqual(summary["current"]["4"]["instructions_per_frame"]["median"], 20.5) # Loss counts between the reports bracketing the window, not over the lifetime. - self.assertEqual([cell["groups_lost"] for cell in summary["base-0123"]["cells"]], [0, 1, 2]) - self.assertEqual(summary["base-0123"]["host_load_1m"], dict(median=2, min=1, max=3)) + self.assertEqual([cell["groups_lost"] for cell in summary["base-0123"]["4"]["cells"]], [0, 1, 2]) + self.assertEqual(summary["base-0123"]["4"]["host_load_1m"], dict(median=2, min=1, max=3)) with self.assertRaisesRegex(ValueError, "no instruction cells"): instructions.summarize(run, ["missing"]) diff --git a/bench/run.sh b/bench/run.sh index eb2220101ab..455b8f11664 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -24,8 +24,12 @@ ROUNDS=$((10#$ROUNDS)) # the performance cores it may already use. The kernel only lists them on Intel # hybrid parts. PINNED= -# Builds don't affect what a benchmark measures, so instructions mode compiles on every allowed CPU. -ALLOWED_CPUS=$(python3 -c 'import os; print(",".join(map(str, sorted(os.sched_getaffinity(0)))))') +# Builds don't affect what a benchmark measures, so instructions mode compiles on +# every CPU this shell may use before it pins itself. Linux only, like that mode. +ALLOWED_CPUS= +if [[ $(uname -s) == Linux ]]; then + ALLOWED_CPUS=$(python3 -c 'import os; print(",".join(map(str, sorted(os.sched_getaffinity(0)))))') +fi if [[ -r /sys/devices/cpu_core/cpus ]]; then PINNED=$( python3 - "$$" <<'EOF' @@ -187,6 +191,7 @@ RUN=$(mktemp -d "$ARTIFACTS/run.XXXXXX") printf 'Benchmark evidence: %s\n' "$RUN" WORKTREE= WORKTREES=() +BUILD_PIDS=() RELAY_PID= HOST_PID= LOAD_PID= @@ -212,7 +217,12 @@ cleanup() { kill "$RELAY_PID" 2>/dev/null || true wait "$RELAY_PID" 2>/dev/null || true fi - local worktree + # Builds read the worktrees below, so stop them first. Each leads its own group. + local build worktree + for build in ${BUILD_PIDS[@]+"${BUILD_PIDS[@]}"}; do + kill -- -"$build" 2>/dev/null || true + wait "$build" 2>/dev/null || true + done for worktree in ${WORKTREE:+"$WORKTREE"} ${WORKTREES[@]+"${WORKTREES[@]}"}; do git -C "$ROOT" worktree remove --force "$worktree" >/dev/null 2>&1 || true rmdir "$worktree" 2>/dev/null || true @@ -926,7 +936,7 @@ revision_checkout() { run_instructions() { local label sha round index next generator built # Measured revisions, then what gets built: those plus a generator measured nowhere. - local -a labels=() targets=() checkouts=() builds=() + local -a labels=() targets=() checkouts=() for sha in ${BASES[@]+"${BASES[@]}"}; do label=base if ((${#BASES[@]} > 1)); then label=base-${sha:0:12}; fi @@ -955,14 +965,20 @@ run_instructions() { # Fat LTO links on one thread, so revisions build side by side on every allowed CPU. mkdir -p "$RUN/build" + # setsid makes each build a group leader without forking, so cleanup stops its rustc too. for ((index = 0; index < ${#build_labels[@]}; index++)); do - taskset -c "$ALLOWED_CPUS" bash -c "$(declare -f build_workers); build_workers \"\$@\"" _ \ + setsid taskset -c "$ALLOWED_CPUS" bash -c "$(declare -f build_workers); build_workers \"\$@\"" _ \ "${build_checkouts[$index]}" "${build_targets[$index]}" profiling instructions \ >"$RUN/build/${build_labels[$index]}.log" 2>&1 & - builds+=($!) + BUILD_PIDS+=($!) done + local status for ((index = 0; index < ${#build_labels[@]}; index++)); do - if ! wait "${builds[$index]}"; then + status=0 + wait "${BUILD_PIDS[$index]}" || status=$? + # Reaped, so cleanup must not signal a group that ID may now name. + unset "BUILD_PIDS[$index]" + if ((status != 0)); then printf 'build failed for %s:\n' "${build_labels[$index]}" >&2 tail -n 40 "$RUN/build/${build_labels[$index]}.log" >&2 return 1 From 0db3f6c91b5482429b3bf3b6d40313c3c923ae98 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 10:26:00 -0800 Subject: [PATCH 17/36] perf(moq-net): a live reader's probe skips the fills while it misses nothing Every peek probes the reader's misses: it judges what arrived since the last probe for holes the live feed won't fill, then asks and resolves fills. A live reader that takes each group as it arrives never misses one, yet paid for all of it on every group. Now: - A probe with nothing to ask, no join, and no deferred gap skips the subscription read, the fills clone, and the ask/resolve loop (a priority change still reaches any joins the reader holds). - When only the feed's newest group moved, with nothing deferred or refused, a group the arrival walk serves is judged without the hole, backfill, and ledger passes, which would find nothing. - A reader's eviction inbox starts above what it handed out every group below: an evicted group it took is never a miss, but its landing counted as an eviction and forced a full judgement on every reader for every group the cache aged out. Instructions per group per subscriber in moq-net's track_fanout_group shape (one finished group, every subscriber takes it and parks), counted exactly: subscribers before after 512 13,669 10,780 (-21.1%) 64 13,888 10,981 (-20.9%) 1 27,287 24,500 (-10.2%) Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/track.rs | 68 ++++++++++++------ rs/moq-net/src/model/track/miss.rs | 111 ++++++++++++++++++++++++++++- 2 files changed, 158 insertions(+), 21 deletions(-) diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index e7a598b0dba..c46380c5f31 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -6276,6 +6276,17 @@ impl Admitted { } /// `runs`, lowest first, below the exclusive `cap`. +/// What a reader asks its fills with: read only once it asks, since every probe that asks +/// nothing would otherwise take the subscription's lock for it. +fn asking(subscription: &Preferences) -> miss::Asking { + let subscription = subscription.read(); + miss::Asking { + priority: subscription.priority, + max_delay: subscription.delay.max, + order: subscription.order, + } +} + fn below(runs: impl Iterator>, cap: Option) -> RangeInclusiveSet { let mut below = RangeInclusiveSet::new(); let last = match cap { @@ -7509,11 +7520,12 @@ impl Cursor { /// Look up each time start the window can't resolve from the cache, once, and take in /// the answers, parking `waiter` on the lookups still out. - fn look_up(&mut self, asking: &miss::Asking, waiter: &kio::Waiter) { + fn look_up(&mut self, waiter: &kio::Waiter) { let lookups: Vec<(Timestamp, u64)> = self.filter.window.lookups().collect(); if lookups.is_empty() && self.filter.lookups.iter().all(|lookup| lookup.join.is_none()) { return; } + let asking = &asking(&self.subscription); let fills = self.state.read().fills.clone(); for (pts, reach) in lookups { if self.filter.lookups.iter().any(|lookup| lookup.pts == pts) { @@ -7577,16 +7589,12 @@ impl Cursor { /// Ask fills for what this cursor misses from `from`, and take in what they answered, /// parking `waiter` on the fills still out. Whatever it gives up goes to the drop cursor. /// `taken` is what its subscriber handed out, which an eviction never makes a miss. + /// + /// A live reader that takes each group as it arrives never misses one, so most probes + /// find nothing to ask and hold no fill: those skip the fills, the subscription, and + /// the answers entirely, paying only for the judgement that the new groups arrived. fn probe(&mut self, from: u64, drift: &Drift, taken: &Delivered, waiter: &kio::Waiter) { - let asking = { - let subscription = self.subscription.read(); - miss::Asking { - priority: subscription.priority, - max_delay: subscription.delay.max, - order: subscription.order, - } - }; - self.look_up(&asking, waiter); + self.look_up(waiter); let (window, cap, revision) = (&self.filter.window, self.end_sequence, self.filter.revision); let misses = self.misses.get_or_insert_with(Box::default); let mut given_up = Vec::new(); @@ -7619,15 +7627,26 @@ impl Cursor { &mut given_up, waiter, ); - (asks, state.fills.clone()) + // Only an ask needs the fills. + let fills = (!asks.is_empty()).then(|| state.fills.clone()); + (asks, fills) }; - misses.ask(&self.state, &fills, &asks, &asking, &mut given_up); - loop { - let retry = misses.resolve(&self.state, asking.max_delay, &mut given_up, waiter); - if retry.is_empty() { - break; + // Nothing to ask, no fill to hear from, and no gap to time: no answer can arrive. + if fills.is_some() || !misses.is_idle() { + let asking = asking(&self.subscription); + match &fills { + Some(fills) => misses.ask(&self.state, fills, &asks, &asking, &mut given_up), + // What the reader asks with still reaches the fills it joined. + None => misses.update(&asking), + } + loop { + let retry = misses.resolve(&self.state, asking.max_delay, &mut given_up, waiter); + if retry.is_empty() { + break; + } + let fills = self.state.read().fills.clone(); + misses.ask(&self.state, &fills, &retry, &asking, &mut given_up); } - misses.ask(&self.state, &fills, &retry, &asking, &mut given_up); } // Only what a fill still owes can expire. if misses.is_waiting() { @@ -21150,7 +21169,8 @@ mod test { /// An eviction waits only for a reader that judged the evicted group cached: a reader /// that judged once and stalled holds no more than what was cached then, however many - /// groups the feed caches and evicts after, late lower ones included. + /// groups the feed caches and evicts after, late lower ones included. A reader that + /// took every group up to it holds none, since a group it took is never a miss again. #[test] #[cfg(feature = "async")] fn an_eviction_waits_only_for_readers_that_judged_it_cached() { @@ -21177,9 +21197,17 @@ mod test { assert_eq!(held(), [vec![], vec![(198, 199)]], "the stalled reader judged none"); drain_groups(&mut active); evict(&producer, 1); - assert_eq!(held(), [vec![(1, 2)], vec![(1, 2)]], "both judged group 1 cached"); + assert_eq!( + held(), + [vec![(1, 2)], vec![]], + "the active reader judged after taking every group up to 2; the stalled one judged before it took group 1" + ); drain_groups(&mut active); - assert_eq!(held(), [vec![(1, 2)], vec![]], "the active reader took it in"); + assert_eq!( + held(), + [vec![(1, 2)], vec![]], + "the active reader has nothing to take in" + ); drop(stalled); assert_eq!(held(), [vec![]], "a dropped reader holds nothing"); write_group(&producer, 3); diff --git a/rs/moq-net/src/model/track/miss.rs b/rs/moq-net/src/model/track/miss.rs index f26910b129a..b51f6548291 100644 --- a/rs/moq-net/src/model/track/miss.rs +++ b/rs/moq-net/src/model/track/miss.rs @@ -264,6 +264,12 @@ impl Misses { self.release(lost); } + /// Whether no fill can answer this reader and no gap is timed: nothing joined, nothing + /// deferred. Its probe then has nothing to resolve. + pub fn is_idle(&self) -> bool { + self.joins.is_empty() && self.deferred.runs.is_empty() + } + /// Whether a fill still owes this reader a group. pub fn is_waiting(&self) -> bool { !self.wanted.is_empty() @@ -632,6 +638,10 @@ impl Misses { return asks; } let grace = state.miss_grace(); + if self.judge_arrivals(state, key, at, view, &mut target, taken, drift, grace, ended) { + self.ledger_revision = ledger.revision(); + return asks; + } let unchanged = unchanged && self.grace == Some(grace); let deadline = self.deferred.deadline(); if unchanged && deadline.is_none() { @@ -784,7 +794,7 @@ impl Misses { .register(inbox.clone()); inbox }); - let evicted = inbox.judge(state.cached, low, top); + let evicted = inbox.judge(state.cached, inbox_floor(low, taken), top); self.evictions = state.evictions; let mut refills = RangeSet::new(); let mut again = RangeSet::new(); @@ -928,6 +938,97 @@ impl Misses { asks } + /// Judge what changed when only the feed's newest group moved up, and the reader's floor + /// with it, returning whether that was all. Each group arriving live changes nothing else, + /// so a reader that takes them as they come judges every one here. + /// + /// With nothing deferred or refused, the arrival ledger's changes touch nothing this + /// reader holds, and with no eviction counted, its inbox holds none. A new group the + /// arrival walk serves is then neither a hole nor backfill: judged, with nothing to ask. + /// Anything else, a group not walked included, is left to [`Self::judge`] whole. + #[expect( + clippy::too_many_arguments, + reason = "the judgement's inputs, as Self::judge takes them" + )] + fn judge_arrivals( + &mut self, + state: &TrackState, + key: (u64, u64, Option), + at: JudgedAt, + view: Option<&Layered>, + target: &mut impl FnMut(RangeInclusive) -> RangeInclusiveSet, + taken: &Delivered, + drift: &Drift, + grace: Duration, + ended: bool, + ) -> bool { + let Some(previous) = self.at else { + return false; + }; + let (Some(judged_to), Some(newest)) = (previous.3, at.3) else { + return false; + }; + let only_arrivals = previous.0 <= at.0 + && judged_to <= newest + && (previous.1, previous.2, previous.4, previous.5, previous.6) == (at.1, at.2, at.4, at.5, at.6); + if !only_arrivals + || !self.deferred.runs.is_empty() + || !self.refused.is_empty() + || self.budget != Some(drift.budget) + || self.grace != Some(grace) + || self.ended != ended + || self.drops_revision != state.drops_changes.revision() + || self.evictions != state.evictions + || self.view.as_ref() != view + { + return false; + } + let Some(inbox) = &self.inbox else { + return false; + }; + let low = state.live_floor.unwrap_or(0).max(key.0); + let top = newest.min(u64::MAX - 1); + let first = judged_to.saturating_add(1).max(low); + let mut fresh = RangeSet::new(); + if first <= top { + for run in target(first..=top).iter() { + for gap in self.judged.gaps(&(*run.start()..*run.end() + 1)) { + fresh.insert(gap); + } + } + } + let walked = fresh.iter().all(|run| { + let mut next = run.start; + for (&sequence, slot) in state.lookup.range(run.clone()) { + if sequence != next || !slot.is_walked() { + return false; + } + next = sequence + 1; + } + next == run.end + }); + if !walked { + return false; + } + if low > 0 { + for set in [ + &mut self.judged, + &mut self.refused, + &mut self.given_up, + &mut self.refilled, + ] { + set.remove(0..low); + } + } + let evicted = inbox.judge(state.cached, inbox_floor(low, taken), top); + debug_assert!(evicted.is_empty(), "an eviction landed without counting"); + for run in fresh.iter() { + self.judged.insert(run.clone()); + } + self.at = Some(at); + true + } + /// Give up what a fill still owes this reader once it is stale under the reader's budget, /// as the reader skips a stale cached group, parking `waiter` on each run going stale. /// A fill answers on its handler's time, so an ordered reader would otherwise wait out @@ -1265,6 +1366,14 @@ impl TrackState { } /// The runs of `run` the cache holds as backfill: servable, but out of the arrival walk. +/// Where a reader's inbox starts taking evictions: no lower than what it handed out every +/// group below, since an evicted group it took is never a miss again. A live reader that +/// reads by arrival never raises its floor, so without this every old group the cache +/// evicts would land with it and count as an eviction to judge again. +fn inbox_floor(low: u64, taken: &Delivered) -> u64 { + low.max(taken.floor().unwrap_or(u64::MAX)) +} + fn backfill(state: &TrackState, run: ops::Range) -> RangeSet { state .lookup From 5132fc4a455bf6a063112d1d5185c6e3b8a2a4f5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 10:31:59 -0800 Subject: [PATCH 18/36] perf(moq-net): read a window with one set of runs without growing across two A window's runs are its fixed ranges and the layer its time ends walk on. Every read of the union grew each run across both sets, though most windows have no layer, and one set's runs are already merged and whole. A peek reads the window seven times per group, so a set alone now answers directly. Instructions per group per subscriber in moq-net's track_fanout_group shape, counted exactly, on top of the previous commit: subscribers before after 512 10,780 10,425 (-3.3%) 64 10,981 10,638 (-3.1%) 1 24,500 24,169 (-1.4%) Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/subscription/runs.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/rs/moq-net/src/model/subscription/runs.rs b/rs/moq-net/src/model/subscription/runs.rs index cbce7e8f218..e60e6f43bb3 100644 --- a/rs/moq-net/src/model/subscription/runs.rs +++ b/rs/moq-net/src/model/subscription/runs.rs @@ -133,8 +133,22 @@ impl Layered { self.fixed.contains(sequence) || self.layer.contains(sequence) } + /// The one set holding runs, when the other holds none: its runs are the union's, each + /// already whole, so a read needs no growing across the two. + fn only(&self) -> Option<&Runs> { + match (self.fixed.0.is_empty(), self.layer.0.is_empty()) { + (_, true) => Some(&self.fixed), + (true, false) => Some(&self.layer), + (false, false) => None, + } + } + /// The lowest whole run of the union reaching into `lo..=hi`, its start clipped to `lo`. pub fn first(&self, lo: u64, hi: u64) -> Option> { + // Most windows have no layer: their fixed runs are already the union's. + if let Some(only) = self.only() { + return only.first(lo, hi).map(|run| (*run.start()).max(lo)..=*run.end()); + } let starts = [self.fixed.first(lo, hi), self.layer.first(lo, hi)]; let start = starts.into_iter().flatten().map(|run| (*run.start()).max(lo)).min()?; // Grow the end through whatever either set holds next to it: each step crosses into @@ -159,6 +173,9 @@ impl Layered { /// The highest whole run of the union reaching into `lo..=hi`, its end clipped to `hi`. pub fn last(&self, lo: u64, hi: u64) -> Option> { + if let Some(only) = self.only() { + return only.last(lo, hi).map(|run| *run.start()..=(*run.end()).min(hi)); + } let ends = [self.fixed.last(lo, hi), self.layer.last(lo, hi)]; let end = ends.into_iter().flatten().map(|run| (*run.end()).min(hi)).max()?; let mut start = end; From 87e0fcdaedfff244ec61ff710d08aae5c2abf340 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 10:42:10 -0800 Subject: [PATCH 19/36] quest: record the maintainer's 2026-11-03 decisions on the relay CPU growth Co-Authored-By: Claude Opus 5.5 --- quest/m1/release-perf/relay-cpu-bisect.md | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/quest/m1/release-perf/relay-cpu-bisect.md b/quest/m1/release-perf/relay-cpu-bisect.md index 19d67c9f07d..5bd51123340 100644 --- a/quest/m1/release-perf/relay-cpu-bisect.md +++ b/quest/m1/release-perf/relay-cpu-bisect.md @@ -13,13 +13,23 @@ Bisected in [moq-dev/moq#7605](https://github.com/moq-dev/moq/pull/7605), which - [moq-dev/moq#6224](https://github.com/moq-dev/moq/pull/6224)'s `Reader::learn_end` polling (+26k instructions per frame) and the track lock contention after [moq-dev/moq#6171](https://github.com/moq-dev/moq/pull/6171) were accidental. [moq-dev/moq#7532](https://github.com/moq-dev/moq/pull/7532) and [moq-dev/moq#7493](https://github.com/moq-dev/moq/pull/7493) already removed both from `main`. - `main` against the old endpoint: chat 1.26x on io_uring and 1.31x on the tokio workers, video 1.14x on io_uring and 0.96x on the tokio workers. Chat's growth is shared code, in the same functions on both runtimes. -What remains is set by the maintainer's decisions in [moq-dev/moq#7605](https://github.com/moq-dev/moq/pull/7605): +Decided 2026-11-03 (maintainer, on [moq-dev/moq#7605](https://github.com/moq-dev/moq/pull/7605)): -- `Cursor::probe` and `Misses::judge` ([moq-dev/moq#6171](https://github.com/moq-dev/moq/pull/6171)'s fills for misses) cost about 8.6k per frame on every peek, taking the shared track lock and cloning its fills even for a live reader that never misses a group. Either a fast path that skips them while nothing is missable, or acceptance as the cost of relay backfill. -- The cursor's window, drift, and live-edge reads (about 9-11k, from [moq-dev/moq#5929](https://github.com/moq-dev/moq/pull/5929)'s per-reader ranges and time anchors) and the 2-4k feature steps before [e84ffb3de3](https://github.com/moq-dev/moq/commit/e84ffb3de3139f7245bed5697754fd6add9a022e) (compressed tracks, ranged frame reads, qmux in tree, signed priorities, public ingest, receive leases) are feature costs to accept or not. -- kio's task and waiter growth (about 11k: `Tasks::poll_each`, `WaiterList::register`, `Waiters::release`) and the lite publisher's serving and close polling (about 5-7k) have no single feature behind them yet. +1. `Cursor::probe` on every peek ([moq-dev/moq#6171](https://github.com/moq-dev/moq/pull/6171), about 8.6k instructions per frame): + - ✅ Add a fast path in this quest that skips the work while nothing is missable. + - Accept it as the cost of relay backfill. +2. The cursor's window, drift, and live-edge reads ([moq-dev/moq#5929](https://github.com/moq-dev/moq/pull/5929), about 9-11k) and the 2-4k steps from compressed tracks, ranged reads, in-tree qmux, signed priorities, public ingest, and receive leases: + - Accept them as feature costs. + - ✅ Investigate each: profile each step's added instructions, fix what can be cut without changing behavior, and attribute what can't with the evidence. +3. kio's task and waiter growth (about 11k) and the lite publisher's serving and close polling (about 6k): + - ✅ Profile them in this quest, then fix or attribute them before closing. + - Hand them to the [performance audit](/quest/m2/audit-perf.md). -Then record each accepted cost and its feature in the epic's Plan, re-pair `main` against [cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95) on chat and video over both runtimes, and delete this quest. +`Cursor::probe` is the relay's check for groups a reader has missed. Each time a subscriber asks for its next group, the probe looks at what arrived on the track since its last look and decides whether any group the reader still wants has left the cache without the live feed delivering it (evicted, skipped, or from before the feed started), and asks upstream to fill those. A live reader that takes each group as it arrives never has such a gap, yet it paid for the whole check on every group: locking the subscription and the track's shared state, cloning the track's fills, walking the arrival ledger, checking each new group for holes, and running the ask-and-answer loop. On top of that, each time the cache aged out an old group, the eviction landed with every reader, all of which had already taken it, and forced each one to re-run the full check. + +Fixes measure each change exactly with a deterministic replay of moq-net's `track_fanout_group` bench (one finished group, every subscriber takes it and parks), counting user instructions per group per subscriber. The same harness builds against historical revisions, so the model's share of the growth is bisected exactly; the session and QUIC layers are attributed from relay cells. + +Remaining: finish the per-step attribution of decision 2 and the profiling of decision 3, land what can be cut, record each attributed cost and its feature in the epic's Plan, re-pair `main` against [cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95) on chat and video over both runtimes, and delete this quest. Measure as the report does: `BENCH_GENERATOR=21d6da9fc1`, `BENCH_INSTRUCTION_PERIOD=2000003`, `kernel.perf_event_max_sample_rate` raised to 100,000 for the gated run and restored after, and compare only cells from one run at low host load. Instructions per frame fall by up to 20% when other host work makes the relay batch more frames per wakeup, so cross-session absolutes don't compare. From 3c3c33026a925b20336fe516f5f0b3c987967c53 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 11:11:57 -0800 Subject: [PATCH 20/36] bench(relay): count the chat fan-out per group independent of host load Relay instruction counts move 5-20% with host load, which hid the 2-4k steps the bisect was after. `just bench-fanout` builds two one-thread harnesses against each base and the current tree, and counts their user instructions with perf on one pinned CPU, less a run of zero cycles: - model: a finished group a cycle through the track model to every subscriber, building back to cdff3c8f95. - session: a publisher session into a relay origin and a viewer session out of it per subscriber, over the in-memory transport. Repeated counts agree within a few instructions per group. A revision whose session API predates the harness reports it unsupported, and callgrind outputs are kept on request for per-function attribution. Co-Authored-By: Claude Opus 5.5 --- .github/justfile | 1 + bench/README.md | 35 ++++++ bench/fanout.py | 193 +++++++++++++++++++++++++++++++ bench/fanout.test.py | 54 +++++++++ bench/fanout/model/Cargo.toml | 19 +++ bench/fanout/model/src/main.rs | 41 +++++++ bench/fanout/session/Cargo.toml | 21 ++++ bench/fanout/session/src/main.rs | 88 ++++++++++++++ bench/run.sh | 107 +++++++++++++++++ justfile | 6 + 10 files changed, 565 insertions(+) create mode 100644 bench/fanout.py create mode 100644 bench/fanout.test.py create mode 100644 bench/fanout/model/Cargo.toml create mode 100644 bench/fanout/model/src/main.rs create mode 100644 bench/fanout/session/Cargo.toml create mode 100644 bench/fanout/session/src/main.rs diff --git a/.github/justfile b/.github/justfile index 73d61a8e986..0bd5133c648 100644 --- a/.github/justfile +++ b/.github/justfile @@ -44,6 +44,7 @@ test: python3 bench/compare.test.py python3 bench/summary.test.py python3 bench/profile.test.py + python3 bench/fanout.test.py python3 sh/gh/gst-formula.test.py python3 sh/changed.test.py python3 sh/large-files.test.py diff --git a/bench/README.md b/bench/README.md index d74b2058f03..a23578e429c 100644 --- a/bench/README.md +++ b/bench/README.md @@ -355,6 +355,41 @@ measured under a one-minute load above about 9 read 5-20% low, so compare cells from quiet stretches within one run. Pinning does not remove interference from other host work. +### Fan-out + +Relay instruction counts move with host load (see Instructions above), which +hides steps of a few percent. `just bench-fanout` counts the chat fan-out +without a relay process or sockets, so the counts are the code's alone: + +```bash +nix develop --command just bench-fanout "cdff3c8f95 21d6da9fc1" +nix develop --command just bench-fanout "" "model" "1 512" +``` + +Two scenarios run every subscriber count on one thread: + +- `model`: one finished group a cycle through the track model, which every + subscriber takes before parking again, as moq-net's `track_fanout_group` + bench does. It builds against every revision since + [cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95). +- `session`: one publisher session into a relay origin and one viewer session + out of it per subscriber, over `transport::memory`, on one current-thread + runtime. It counts the publisher, relay, and viewer sessions together, but no + QUIC: in-tree qmux, receive leases, and crypto only show in relay counts. A + revision whose session API predates this harness reports it unsupported. + +Each `bench/fanout/` crate builds against each base's and the current +tree's own crates and lock, side by side, into `target/bench/fanout/`. Each +count pins the harness to one CPU (`BENCH_FANOUT_CPU`, by default the last +allowed one) and takes its user instructions from `perf stat` for a run of +zero cycles and one of many; the difference over the groups delivered is +`fanout-summary.json`. Repeated counts agree to within a few instructions per +group. `BENCH_FANOUT_CALLGRIND=1`, with `valgrind` on `PATH`, also keeps +callgrind outputs of both runs under `fanout///callgrind/`; +`python3 bench/fanout.py functions ZERO FULL UNITS` ranks functions by +instructions per group. `BENCH_FANOUT_VERSION` picks the session's protocol +version (default `moq-lite-06`). + ### Heap The heap mode builds the relay with `moq-relay/jemalloc` (the other modes keep diff --git a/bench/fanout.py b/bench/fanout.py new file mode 100644 index 00000000000..3cb6f62f34b --- /dev/null +++ b/bench/fanout.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +"""Count chat fan-out instructions per delivered group, independent of host load. + +`bench/run.sh --fanout` builds each `bench/fanout/` harness against every +revision. Each harness runs its whole fan-out on one thread, so the user instructions it +retires depend only on the code: this pins it to one CPU, counts a run of zero cycles and +one of many, and divides the difference by the groups delivered. +""" + +import argparse +import json +import math +import os +import re +import subprocess +from pathlib import Path + +# Group deliveries per measurement: subscribers times cycles, so every subscriber count +# runs about as long. The session harness costs about ten times the model's per group. +UNITS = {"model": 1_000_000, "session": 100_000} + + +def cycles(scenario, subscribers): + return max(1, math.ceil(UNITS[scenario] / subscribers)) + + +def parse_stat(text): + """Sum the user instructions perf stat's CSV reports, one line per PMU on a hybrid CPU.""" + total, counted = 0, False + for line in text.splitlines(): + fields = line.split(",") + if len(fields) < 5 or "instructions" not in fields[2]: + continue + if fields[0] in ("", ""): + continue + if float(fields[4] or 0) != 100: + raise ValueError(f"instructions were multiplexed and undercounted: {line}") + total += int(fields[0]) + counted = True + if not counted: + raise ValueError(f"perf counted no instructions:\n{text}") + return total + + +def count(binary, args, cpu): + """User instructions `binary args` retires pinned to `cpu`.""" + run = subprocess.run( + ["taskset", "-c", str(cpu), "perf", "stat", "-x,", "-e", "instructions:u", binary, *map(str, args)], + capture_output=True, + text=True, + env=dict(os.environ, LC_ALL="C"), + ) + if run.returncode != 0: + raise ValueError(f"{binary} {args} failed:\n{run.stderr}") + return parse_stat(run.stderr) + + +def per_group(scenario, binary, subscribers, cpu, extra=()): + n = cycles(scenario, subscribers) + base = count(binary, [subscribers, 0, *extra], cpu) + full = count(binary, [subscribers, n, *extra], cpu) + return (full - base) / (n * subscribers) + + +def callgrind(scenario, binary, subscribers, out, extra=()): + """Exact per-function attribution: both runs under callgrind, kept for callgrind_annotate.""" + n = max(1, cycles(scenario, subscribers) // 10) + out.mkdir(parents=True, exist_ok=True) + for label, run in (("zero", 0), ("full", n)): + subprocess.run( + [ + "valgrind", + "--tool=callgrind", + f"--callgrind-out-file={out / f'{subscribers}.{label}.out'}", + binary, + str(subscribers), + str(run), + *extra, + ], + check=True, + capture_output=True, + ) + (out / f"{subscribers}.units").write_text(f"{n * subscribers}\n") + + +def functions(zero, full, units, inclusive=False): + """Instructions per group by function, from two callgrind outputs of the same binary.""" + + def load(path): + text = subprocess.run( + [ + "callgrind_annotate", + f"--inclusive={'yes' if inclusive else 'no'}", + "--threshold=100", + "--auto=no", + "--show-percs=no", + str(path), + ], + capture_output=True, + text=True, + check=True, + ).stdout + totals = {} + for line in text.splitlines(): + match = re.match(r"\s*([\d,]+)\s+(\S.*)$", line) + if not match: + continue + name = re.sub(r"\s+\[[^\]]*\]$", "", match.group(2)) + name = re.sub(r"^(/[^:]*|\?\?\?):", "", name) + totals[name] = totals.get(name, 0) + int(match.group(1).replace(",", "")) + return totals + + before, after = load(zero), load(full) + names = set(before) | set(after) + return {name: (after.get(name, 0) - before.get(name, 0)) / units for name in names} + + +def summarize(run): + """Print and keep each scenario's instructions per group, by subscribers and revision.""" + rows = [json.loads(line) for line in (run / "fanout.jsonl").read_text().splitlines()] + (run / "fanout-summary.json").write_text(json.dumps(rows, indent=2) + "\n") + labels = list(dict.fromkeys(row["label"] for row in rows)) + for scenario in dict.fromkeys(row["scenario"] for row in rows): + print(f"\n{scenario}: instructions per delivered group") + print(f"{'subscribers':>11}" + "".join(f"{label[:16]:>18}" for label in labels)) + for subscribers in sorted({row["subscribers"] for row in rows if row["scenario"] == scenario}): + cells = { + row["label"]: row.get("per_group") + for row in rows + if row["scenario"] == scenario and row["subscribers"] == subscribers + } + print( + f"{subscribers:>11}" + + "".join( + f"{cells[label]:>18.1f}" if cells.get(label) is not None else f"{'unsupported':>18}" + for label in labels + ) + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + measure = sub.add_parser("measure") + measure.add_argument("run", type=Path) + measure.add_argument("label") + measure.add_argument("revision") + measure.add_argument("scenario", choices=sorted(UNITS)) + measure.add_argument("binary") + measure.add_argument("cpu") + measure.add_argument("subscribers", type=int, nargs="+") + measure.add_argument("--version", default=None) + measure.add_argument("--callgrind", action="store_true") + summary = sub.add_parser("summarize") + summary.add_argument("run", type=Path) + top = sub.add_parser("functions") + top.add_argument("zero", type=Path) + top.add_argument("full", type=Path) + top.add_argument("units", type=int) + top.add_argument("--inclusive", action="store_true") + top.add_argument("--top", type=int, default=40) + args = parser.parse_args() + try: + if args.command == "measure": + extra = [args.version] if args.version and args.scenario == "session" else [] + with (args.run / "fanout.jsonl").open("a") as out: + for subscribers in args.subscribers: + row = dict( + label=args.label, + revision=args.revision, + scenario=args.scenario, + subscribers=subscribers, + cycles=cycles(args.scenario, subscribers), + per_group=None, + ) + if args.binary != "-": + row["per_group"] = per_group(args.scenario, args.binary, subscribers, args.cpu, extra) + if args.callgrind: + target = args.run / "fanout" / args.label / args.scenario / "callgrind" + callgrind(args.scenario, args.binary, subscribers, target, extra) + out.write(json.dumps(row) + "\n") + elif args.command == "summarize": + summarize(args.run) + else: + ranked = functions(args.zero, args.full, args.units, args.inclusive) + for name, value in sorted(ranked.items(), key=lambda item: -item[1])[: args.top]: + print(f"{value:10.1f} {name[:180]}") + except (ValueError, OSError, subprocess.CalledProcessError) as error: + parser.exit(1, f"fanout: {error}\n") + + +if __name__ == "__main__": + main() diff --git a/bench/fanout.test.py b/bench/fanout.test.py new file mode 100644 index 00000000000..fa590415cd3 --- /dev/null +++ b/bench/fanout.test.py @@ -0,0 +1,54 @@ +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location("fanout", Path(__file__).with_name("fanout.py")) +fanout = importlib.util.module_from_spec(spec) +spec.loader.exec_module(fanout) + + +class Fanout(unittest.TestCase): + def test_hybrid_counts_sum_each_pmu_that_ran(self): + stat = "\n".join( + [ + "1200,,cpu_atom/instructions/u,100,100.00,,", + ",,cpu_core/instructions/u,0,0.00,,", + "34,,cpu_atom/instructions/u,100,100.00,,", + ] + ) + self.assertEqual(fanout.parse_stat(stat), 1234) + + def test_multiplexed_or_absent_counts_are_refused(self): + with self.assertRaisesRegex(ValueError, "multiplexed"): + fanout.parse_stat("1200,,instructions/u,100,50.00,,") + with self.assertRaisesRegex(ValueError, "no instructions"): + fanout.parse_stat(",,cpu_core/instructions/u,0,0.00,,") + + def test_each_count_divides_the_cycles_it_added_by_the_groups_delivered(self): + counts = {0: 1_000, fanout.cycles("model", 4): 1_000 + 7 * fanout.cycles("model", 4) * 4} + with patch.object(fanout, "count", side_effect=lambda _, args, __: counts[args[1]]): + self.assertEqual(fanout.per_group("model", "bin", 4, "0"), 7) + self.assertEqual(fanout.cycles("session", 512), 196) + self.assertEqual(fanout.cycles("model", 2_000_000), 1) + + def test_summary_names_unsupported_cells(self): + with tempfile.TemporaryDirectory() as directory: + run = Path(directory) + rows = [ + dict(label="base", scenario="session", subscribers=8, per_group=None), + dict(label="current", scenario="session", subscribers=8, per_group=12.5), + ] + (run / "fanout.jsonl").write_text("".join(json.dumps(row) + "\n" for row in rows)) + with patch("builtins.print") as printed: + fanout.summarize(run) + table = "\n".join(str(call.args[0]) for call in printed.call_args_list) + self.assertIn("unsupported", table) + self.assertIn("12.5", table) + self.assertEqual(json.loads((run / "fanout-summary.json").read_text()), rows) + + +if __name__ == "__main__": + unittest.main() diff --git a/bench/fanout/model/Cargo.toml b/bench/fanout/model/Cargo.toml new file mode 100644 index 00000000000..dc7272be082 --- /dev/null +++ b/bench/fanout/model/Cargo.toml @@ -0,0 +1,19 @@ +# Built by `bench/run.sh --fanout` against each revision's moq-net, outside the workspace, +# so a historical checkout measures the same scenario. Its own lock is that revision's. +[package] +name = "fanout-model" +version = "0.0.0" +edition = "2024" +publish = false + +[workspace] + +[dependencies] +bytes = "1" +kio = { path = "../../../rs/kio" } +moq-net = { path = "../../../rs/moq-net", features = ["async"] } + +[profile.release] +codegen-units = 1 +debug = "line-tables-only" +lto = "fat" diff --git a/bench/fanout/model/src/main.rs b/bench/fanout/model/src/main.rs new file mode 100644 index 00000000000..6bc52556714 --- /dev/null +++ b/bench/fanout/model/src/main.rs @@ -0,0 +1,41 @@ +//! Chat fan-out through the model alone: one finished group a cycle, which every +//! subscriber takes before parking again, as moq-net's `track_fanout_group` bench does. +//! Usage: `fanout-model SUBSCRIBERS CYCLES`. The work is deterministic, so its retired +//! instructions, less a run of zero cycles, are the per-group cost whatever the host does. +use std::{hint::black_box, task::Poll}; + +use bytes::Bytes; +use moq_net::{Timestamp, broadcast, cache, track}; + +fn main() { + let mut args = std::env::args().skip(1).map(|a| a.parse::().unwrap()); + let (subscribers, cycles) = (args.next().unwrap(), args.next().unwrap()); + let mut info = broadcast::Info::default(); + info.pool = cache::Pool::new(cache::Config::default().with_capacity(64 * 1024).with_expiry(cache::DEFAULT_EXPIRY)); + let broadcast = broadcast::Producer::new(info); + let track = broadcast + .create_track("bench", track::Info::default().with_timescale(moq_net::Timescale::MILLI)) + .unwrap(); + let mut subs: Vec<_> = (0..subscribers).map(|_| track.subscribe(None)).collect(); + let waiters: Vec<_> = (0..subscribers).map(|_| kio::Waiter::noop()).collect(); + for (s, w) in subs.iter_mut().zip(&waiters) { + assert!(matches!(s.poll_recv_group(w), Poll::Pending)); + } + let payload = Bytes::from(vec![0; 64]); + // Warm up past the first groups' one-time work, so both runs share it. + for _ in 0..(16 + cycles) { + let mut group = track.append_group().unwrap(); + group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); + group.finish().unwrap(); + for (s, w) in subs.iter_mut().zip(&waiters) { + match s.poll_recv_group(w) { + Poll::Ready(Ok(Some(g))) => { + black_box(g); + } + _ => unreachable!(), + } + assert!(matches!(s.poll_recv_group(w), Poll::Pending)); + } + } + black_box(&broadcast); +} diff --git a/bench/fanout/session/Cargo.toml b/bench/fanout/session/Cargo.toml new file mode 100644 index 00000000000..76eaa646abc --- /dev/null +++ b/bench/fanout/session/Cargo.toml @@ -0,0 +1,21 @@ +# Built by `bench/run.sh --fanout` against each revision's moq-net, outside the workspace, +# so a historical checkout measures the same scenario. Its own lock is that revision's. +[package] +name = "fanout-session" +version = "0.0.0" +edition = "2024" +publish = false + +[workspace] + +[dependencies] +bytes = "1" +futures = "0.3" +tokio = { version = "1", features = ["rt", "time"] } +kio = { path = "../../../rs/kio" } +moq-net = { path = "../../../rs/moq-net", features = ["async"] } + +[profile.release] +codegen-units = 1 +debug = "line-tables-only" +lto = "fat" diff --git a/bench/fanout/session/src/main.rs b/bench/fanout/session/src/main.rs new file mode 100644 index 00000000000..daa3b4ee077 --- /dev/null +++ b/bench/fanout/session/src/main.rs @@ -0,0 +1,88 @@ +//! Chat fan-out through a relay over the in-memory transport: one publisher session into +//! a relay origin and one viewer session out of it per subscriber, all on one thread. Each +//! cycle writes one 64-byte group that every viewer receives and reads. +//! +//! Usage: `fanout-session VIEWERS CYCLES [VERSION]`. Every session runs on this thread's +//! runtime and the memory transport has no sockets or clock waits, so the retired +//! instructions, less a run of zero cycles, are the per-group cost whatever the host does. +use std::hint::black_box; + +use bytes::Bytes; +use moq_net::{Client, Hop, Server, Timestamp, Version, origin, transport::memory}; + +fn origin(hop: u64) -> origin::Producer { + let (producer, driver) = origin::Producer::new(origin::Config::new(Hop::new(hop).unwrap())); + tokio::spawn(moq_net::time::run(driver)); + producer +} + +/// Connect `client` to `server` over one in-memory pair, returning both sessions. +async fn link(version: Version, client: Client, server: Server) -> (moq_net::Session, moq_net::Session) { + let mut config = memory::Config::default(); + config.protocol = Some(version.alpn().to_string()); + let (near, far) = memory::pair(config); + // Each driver runs as soon as its handshake resolves: a newer draft's accept waits on + // the client's SETUP, which only its driver sends. + futures::join!( + async { + let (session, driver) = client.connect(kio::clock::now(), near).await.expect("client handshake"); + tokio::spawn(moq_net::time::run(driver)); + session + }, + async { + let (session, driver) = server.accept(kio::clock::now(), far).await.expect("server handshake"); + tokio::spawn(moq_net::time::run(driver)); + session + } + ) +} + +fn main() { + let mut args = std::env::args().skip(1); + let viewers: u64 = args.next().unwrap().parse().unwrap(); + let cycles: usize = args.next().unwrap().parse().unwrap(); + let version: Version = args.next().as_deref().unwrap_or("moq-lite-06").parse().unwrap(); + let runtime = tokio::runtime::Builder::new_current_thread().enable_time().build().unwrap(); + runtime.block_on(async move { + let relay = origin(1); + let publisher = origin(2); + let ingest = link( + version, + Client::new().with_versions(version.into()).with_publisher(publisher.consume()), + Server::new().with_versions(version.into()).with_subscriber(relay.clone()), + ) + .await; + let broadcast = publisher.create_broadcast("room").unwrap(); + let info = moq_net::track::Info::default().with_timescale(moq_net::Timescale::MILLI); + let mut track = broadcast.create_track("chat", info).unwrap(); + broadcast.announce(Default::default()).unwrap(); + + let mut audience = Vec::new(); + for viewer in 0..viewers { + let local = origin(3 + viewer); + let sessions = link( + version, + Client::new().with_versions(version.into()).with_subscriber(local.clone()), + Server::new().with_versions(version.into()).with_publisher(relay.consume()), + ) + .await; + let remote = local.consume().follow("room", None).unwrap().available().await.expect("broadcast resolves"); + let subscriber = remote.track("chat").unwrap().subscribe(None).await.expect("subscribe"); + audience.push((sessions, local, subscriber)); + } + while track.subscription_changed().await.expect("track open").is_none() {} + + let payload = Bytes::from(vec![0; 64]); + // Past the first groups' one-time work, so the zero-cycle run shares it. + for _ in 0..(16 + cycles) { + let mut group = track.append_group().unwrap(); + group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); + group.finish().unwrap(); + for (_, _, subscriber) in &mut audience { + let mut group = subscriber.recv_group().await.unwrap().expect("a group"); + black_box(group.read_frame().await.unwrap().expect("a frame")); + } + } + black_box((ingest, audience, broadcast)); + }); +} diff --git a/bench/run.sh b/bench/run.sh index 00f7eddd072..5bba284f9f4 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -10,6 +10,8 @@ set -euo pipefail # `--profile` captures relay profiles: std mutex waits by stack (lock), CPU stacks # (cpu), or jemalloc heap snapshots and allocation churn (heap). # `--shard` sweeps the QUIC shard count against the connection count. +# `--fanout` counts the chat fan-out's instructions per delivered group on one thread, +# for each BENCH_BASE revision and the current tree, independent of host load. MODE=${1:-} ROUNDS=${BENCH_RUNTIME_ROUNDS:-3} @@ -130,6 +132,32 @@ case $MODE in # and CPU stacks unwind by frame pointer alone. export RUSTFLAGS="${RUSTFLAGS:+$RUSTFLAGS }-C force-frame-pointers=yes" ;; + --fanout) + # Several bases, each counted beside the current tree; see bench/fanout.py. + BASES=() + for value in ${BENCH_BASE:-}; do + BASES+=("$(git rev-parse --verify "$value^{commit}")") + done + BASE=${BASES[*]} + FANOUT_SCENARIOS=${BENCH_FANOUT_SCENARIOS:-model session} + FANOUT_SUBSCRIBERS=${BENCH_FANOUT_SUBSCRIBERS:-1 64 512} + for value in $FANOUT_SCENARIOS; do + if [[ ! $value =~ ^(model|session)$ ]]; then + printf 'unknown fan-out scenario %q; expected model or session\n' "$value" >&2 + exit 1 + fi + done + for value in $FANOUT_SUBSCRIBERS; do + if [[ ! $value =~ ^[1-9][0-9]{0,4}$ ]]; then + printf 'fan-out subscribers must be integers from 1 to 99999, got %q\n' "$value" >&2 + exit 1 + fi + done + if [[ -n ${BENCH_FANOUT_CALLGRIND:-} ]] && ! command -v valgrind >/dev/null; then + printf 'BENCH_FANOUT_CALLGRIND needs valgrind on PATH\n' >&2 + exit 1 + fi + ;; --shard) BASE= SHARD_COUNTS=${BENCH_SHARD_COUNTS:-0 1 4 16} @@ -1002,6 +1030,82 @@ run_instructions() { python3 "$ROOT/bench/instructions.py" summarize "$RUN" "${labels[@]}" } +# Count the chat fan-out per delivered group on one thread, for every base and the current +# tree: each `bench/fanout/` harness builds against that revision's own crates and +# lock, so a count is the code's alone, whatever else the host runs. A scenario whose API a +# revision predates is reported unsupported rather than failing the run. +run_fanout() { + if [[ $(uname -s) != Linux ]] || ! command -v perf >/dev/null; then + printf 'fan-out counts need Linux perf\n' >&2 + return 1 + fi + local cpu=${BENCH_FANOUT_CPU:-${ALLOWED_CPUS##*,}} + local label sha index scenario dir key binary status + local -a labels=() revisions=() checkouts=() keys=() build_dirs=() extra=() + for sha in ${BASES[@]+"${BASES[@]}"}; do + label=base + if ((${#BASES[@]} > 1)); then label=base-${sha:0:12}; fi + revision_checkout "$sha" "$label" + labels+=("$label") + revisions+=("$sha") + checkouts+=("$REVISION_CHECKOUT") + keys+=("$sha") + done + labels+=(current) + revisions+=("$(git rev-parse HEAD)") + checkouts+=("$ROOT") + # The working tree changes under one target; Cargo's fingerprints track it. + keys+=(current) + + # Fat LTO links on one thread, so the harnesses build side by side on every allowed CPU. + for ((index = 0; index < ${#labels[@]}; index++)); do + for scenario in $FANOUT_SCENARIOS; do + dir=$RUN/fanout/${labels[$index]}/$scenario + mkdir -p "$dir" + cp -R "$ROOT/bench/fanout/$scenario/." "$dir/" + # This revision's crates, resolved by its own lock. + sed -i "s|path = \"../../../rs/|path = \"${checkouts[$index]}/rs/|" "$dir/Cargo.toml" + cp "${checkouts[$index]}/Cargo.lock" "$dir/Cargo.lock" + # shellcheck disable=SC2016 # the inner shell expands its own arguments + setsid taskset -c "$ALLOWED_CPUS" bash -c 'cd "$1" && CARGO_TARGET_DIR=$2 cargo build --release' _ \ + "$dir" "$TARGET/fanout/${keys[$index]}/$scenario" >"$dir/build.log" 2>&1 & + BUILD_PIDS+=($!) + build_dirs+=("$dir") + done + done + for ((index = 0; index < ${#build_dirs[@]}; index++)); do + status=0 + wait "${BUILD_PIDS[$index]}" || status=$? + # Reaped, so cleanup must not signal a group that ID may now name. + unset "BUILD_PIDS[$index]" + if ((status == 0)); then + continue + fi + if [[ ${build_dirs[$index]} == "$RUN/fanout/current/"* ]]; then + printf 'the current tree'"'"'s fan-out harness failed to build:\n' >&2 + tail -n 40 "${build_dirs[$index]}/build.log" >&2 + return 1 + fi + printf 'fan-out harness %s does not build; reported unsupported\n' "${build_dirs[$index]#"$RUN"/fanout/}" + touch "${build_dirs[$index]}/unsupported" + done + + if [[ -n ${BENCH_FANOUT_VERSION:-} ]]; then extra+=(--version "$BENCH_FANOUT_VERSION"); fi + if [[ -n ${BENCH_FANOUT_CALLGRIND:-} ]]; then extra+=(--callgrind); fi + printf 'Counting user instructions on CPU %s\n' "$cpu" + for scenario in $FANOUT_SCENARIOS; do + for ((index = 0; index < ${#labels[@]}; index++)); do + dir=$RUN/fanout/${labels[$index]}/$scenario + binary=$TARGET/fanout/${keys[$index]}/$scenario/release/fanout-$scenario + if [[ -f $dir/unsupported ]]; then binary=-; fi + # shellcheck disable=SC2086 # one argument per subscriber count + python3 "$ROOT/bench/fanout.py" measure "$RUN" "${labels[$index]}" "${revisions[$index]}" \ + "$scenario" "$binary" "$cpu" $FANOUT_SUBSCRIBERS ${extra[@]+"${extra[@]}"} + done + done + python3 "$ROOT/bench/fanout.py" summarize "$RUN" +} + # Each profiled run follows an unprofiled control on the same binary, so the # throughput and CPU difference shows the profiler's overhead. run_profile() { @@ -1115,6 +1219,9 @@ elif [[ $MODE == --profile ]]; then elif [[ $MODE == --shard ]]; then run_shard_sweep exit 0 +elif [[ $MODE == --fanout ]]; then + run_fanout + exit 0 elif [[ -n $BASE ]]; then BASE_COMMIT=$(git rev-parse --verify "$BASE^{commit}") checkout_base "$BASE_COMMIT" diff --git a/justfile b/justfile index 4135a67c5c6..f3a9d59a0be 100644 --- a/justfile +++ b/justfile @@ -76,6 +76,12 @@ bench-shard $SHARDS="0 1 4 16" $CONNECTIONS="16 64 256": bench-profile $MODES $WORKLOAD="fanout" $WORKERS="1 4 16" $RUNTIME="tokio-workers" $BASE="" $TRANSPORT="quic": BENCH_PROFILE_MODES="$MODES" BENCH_PROFILE_WORKLOAD="$WORKLOAD" BENCH_PROFILE_WORKERS="$WORKERS" BENCH_PROFILE_RUNTIME="$RUNTIME" BENCH_BASE="$BASE" BENCH_PROFILE_TRANSPORT="$TRANSPORT" bench/run.sh --profile +# Count the chat fan-out's instructions per delivered group on one thread, through the model +# alone and through a relay over the in-memory transport, for each revision in BASE and the +# current tree. The counts don't move with host load: see the Fan-out section of bench/README.md. +bench-fanout $BASE="" $SCENARIOS="model session" $SUBSCRIBERS="1 64 512": + BENCH_BASE="$BASE" BENCH_FANOUT_SCENARIOS="$SCENARIOS" BENCH_FANOUT_SUBSCRIBERS="$SUBSCRIBERS" bench/run.sh --fanout + # Real-browser transport, CMAF, and publish/watch costs, optionally against a base: # `just bench-browser origin/main`. `--smoke` is the bounded correctness lane. [positional-arguments] From 44c0dc494ff3262de6f8bad38230f8ff09e6ebf1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 11:12:47 -0800 Subject: [PATCH 21/36] bench(relay): build the fan-out harnesses a few at a time Co-Authored-By: Claude Opus 5.5 --- bench/run.sh | 48 +++++++++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/bench/run.sh b/bench/run.sh index 5bba284f9f4..e4f89cca04e 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -1057,7 +1057,9 @@ run_fanout() { # The working tree changes under one target; Cargo's fingerprints track it. keys+=(current) - # Fat LTO links on one thread, so the harnesses build side by side on every allowed CPU. + # Fat LTO links on one thread, so the harnesses build side by side on every allowed CPU, + # BENCH_FANOUT_JOBS at a time. + local jobs=${BENCH_FANOUT_JOBS:-4} first launched for ((index = 0; index < ${#labels[@]}; index++)); do for scenario in $FANOUT_SCENARIOS; do dir=$RUN/fanout/${labels[$index]}/$scenario @@ -1066,28 +1068,36 @@ run_fanout() { # This revision's crates, resolved by its own lock. sed -i "s|path = \"../../../rs/|path = \"${checkouts[$index]}/rs/|" "$dir/Cargo.toml" cp "${checkouts[$index]}/Cargo.lock" "$dir/Cargo.lock" + printf '%s\n' "$TARGET/fanout/${keys[$index]}/$scenario" >"$dir/target" + build_dirs+=("$dir") + done + done + for ((first = 0; first < ${#build_dirs[@]}; first += jobs)); do + BUILD_PIDS=() + for dir in "${build_dirs[@]:first:jobs}"; do # shellcheck disable=SC2016 # the inner shell expands its own arguments setsid taskset -c "$ALLOWED_CPUS" bash -c 'cd "$1" && CARGO_TARGET_DIR=$2 cargo build --release' _ \ - "$dir" "$TARGET/fanout/${keys[$index]}/$scenario" >"$dir/build.log" 2>&1 & + "$dir" "$(cat "$dir/target")" >"$dir/build.log" 2>&1 & BUILD_PIDS+=($!) - build_dirs+=("$dir") done - done - for ((index = 0; index < ${#build_dirs[@]}; index++)); do - status=0 - wait "${BUILD_PIDS[$index]}" || status=$? - # Reaped, so cleanup must not signal a group that ID may now name. - unset "BUILD_PIDS[$index]" - if ((status == 0)); then - continue - fi - if [[ ${build_dirs[$index]} == "$RUN/fanout/current/"* ]]; then - printf 'the current tree'"'"'s fan-out harness failed to build:\n' >&2 - tail -n 40 "${build_dirs[$index]}/build.log" >&2 - return 1 - fi - printf 'fan-out harness %s does not build; reported unsupported\n' "${build_dirs[$index]#"$RUN"/fanout/}" - touch "${build_dirs[$index]}/unsupported" + launched=${#BUILD_PIDS[@]} + for ((index = 0; index < launched; index++)); do + dir=${build_dirs[$((first + index))]} + status=0 + wait "${BUILD_PIDS[$index]}" || status=$? + # Reaped, so cleanup must not signal a group that ID may now name. + unset "BUILD_PIDS[$index]" + if ((status == 0)); then + continue + fi + if [[ $dir == "$RUN/fanout/current/"* ]]; then + printf 'the current tree'"'"'s fan-out harness failed to build:\n' >&2 + tail -n 40 "$dir/build.log" >&2 + return 1 + fi + printf 'fan-out harness %s does not build; reported unsupported\n' "${dir#"$RUN"/fanout/}" + touch "$dir/unsupported" + done done if [[ -n ${BENCH_FANOUT_VERSION:-} ]]; then extra+=(--version "$BENCH_FANOUT_VERSION"); fi From 101ed5a8066d4c17ec85ac3cc2d5ec40686c4962 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 11:38:42 -0800 Subject: [PATCH 22/36] perf(kio): a waiter remembers 16 lists, not 8 A waiter records the lists it is parked on, so registering again on one that still holds it is free. A relay session's control and serving tasks park on more than 8 (stream accepts, deadlines, the auth link, expiry landings, channel closes), so every list past the eighth took a duplicate entry on every poll and grew until a sweep. Instructions per delivered group, counted exactly (bench/fanout): scenario subscribers 8 slots 16 slots session (lite-06) 64 115,000 110,524 (-3.9%) session (lite-06) 512 113,200 108,747 (-3.9%) model 512 10,425 10,185 (-2.3%) 12 slots measured 112,354 and 32 slots 111,692 at 64 viewers: each registration searches every slot, so more costs more than it saves. Co-Authored-By: Claude Opus 5.5 --- rs/kio/src/waiter.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/rs/kio/src/waiter.rs b/rs/kio/src/waiter.rs index e2dc3bc5404..707db955cf6 100644 --- a/rs/kio/src/waiter.rs +++ b/rs/kio/src/waiter.rs @@ -47,8 +47,13 @@ pub struct Waiter { noop: bool, } -/// Lists a waiter remembers being parked on. A relay's busiest tasks park on up to 8. -const PARKED: usize = 8; +/// Lists a waiter remembers being parked on. A relay session's control and serving tasks +/// park on more than 8 (stream accepts, deadlines, the auth link, expiry landings, channel +/// closes), and each list past the last remembered stacks a duplicate on every poll. 16 +/// holds them: in the chat fan-out over the memory transport, 8 slots cost 3.9% more +/// instructions per delivered group and 12 or 32 cost 1.1-1.7% more, since every +/// registration searches the slots. +const PARKED: usize = 16; impl Waiter { /// Create a new waiter from an async [`Waker`]. From a4650108f6c5e5dbec92253331a62bfd932c309f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 11:39:01 -0800 Subject: [PATCH 23/36] bench(relay): recreate an evicted target directory before building A build cache may evict the directory a target symlink points to, which failed every revision build in that run through the dangling link. Co-Authored-By: Claude Opus 5.5 --- bench/run.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bench/run.sh b/bench/run.sh index e4f89cca04e..8b79f608696 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -202,6 +202,9 @@ TARGET=${CARGO_TARGET_DIR:-$ROOT/target/bench} if [[ $TARGET != /* ]]; then TARGET=$ROOT/$TARGET fi +# A build cache may evict the directory a `target` symlink points to: recreate it rather +# than fail every build through a dangling link. +mkdir -p "$(readlink -m "$TARGET")" BASE_TARGET=$TARGET/base CURRENT_TARGET=$TARGET/current From b243c4046f472d0a547e401deb71036f1a5c94c3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 11:49:56 -0800 Subject: [PATCH 24/36] bench(relay): let a fan-out sweep build outside an evictable target Co-Authored-By: Claude Opus 5.5 --- bench/README.md | 3 ++- bench/run.sh | 9 ++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/bench/README.md b/bench/README.md index a23578e429c..383b2d85fe1 100644 --- a/bench/README.md +++ b/bench/README.md @@ -379,7 +379,8 @@ Two scenarios run every subscriber count on one thread: revision whose session API predates this harness reports it unsupported. Each `bench/fanout/` crate builds against each base's and the current -tree's own crates and lock, side by side, into `target/bench/fanout/`. Each +tree's own crates and lock, side by side, into `target/bench/fanout/`, or +`BENCH_FANOUT_TARGET` where a build cache might evict that during a long sweep. Each count pins the harness to one CPU (`BENCH_FANOUT_CPU`, by default the last allowed one) and takes its user instructions from `perf stat` for a run of zero cycles and one of many; the difference over the groups delivered is diff --git a/bench/run.sh b/bench/run.sh index 8b79f608696..d55ae80db68 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -1063,6 +1063,9 @@ run_fanout() { # Fat LTO links on one thread, so the harnesses build side by side on every allowed CPU, # BENCH_FANOUT_JOBS at a time. local jobs=${BENCH_FANOUT_JOBS:-4} first launched + # A build cache that evicts the `target` directory under a long sweep fails every later + # build; BENCH_FANOUT_TARGET names a directory it doesn't manage. + local targets=${BENCH_FANOUT_TARGET:-$TARGET/fanout} for ((index = 0; index < ${#labels[@]}; index++)); do for scenario in $FANOUT_SCENARIOS; do dir=$RUN/fanout/${labels[$index]}/$scenario @@ -1071,7 +1074,7 @@ run_fanout() { # This revision's crates, resolved by its own lock. sed -i "s|path = \"../../../rs/|path = \"${checkouts[$index]}/rs/|" "$dir/Cargo.toml" cp "${checkouts[$index]}/Cargo.lock" "$dir/Cargo.lock" - printf '%s\n' "$TARGET/fanout/${keys[$index]}/$scenario" >"$dir/target" + printf '%s\n' "$targets/${keys[$index]}/$scenario" >"$dir/target" build_dirs+=("$dir") done done @@ -1079,7 +1082,7 @@ run_fanout() { BUILD_PIDS=() for dir in "${build_dirs[@]:first:jobs}"; do # shellcheck disable=SC2016 # the inner shell expands its own arguments - setsid taskset -c "$ALLOWED_CPUS" bash -c 'cd "$1" && CARGO_TARGET_DIR=$2 cargo build --release' _ \ + setsid taskset -c "$ALLOWED_CPUS" bash -c 'mkdir -p "$(readlink -m "$2")" && cd "$1" && CARGO_TARGET_DIR=$2 cargo build --release' _ \ "$dir" "$(cat "$dir/target")" >"$dir/build.log" 2>&1 & BUILD_PIDS+=($!) done @@ -1109,7 +1112,7 @@ run_fanout() { for scenario in $FANOUT_SCENARIOS; do for ((index = 0; index < ${#labels[@]}; index++)); do dir=$RUN/fanout/${labels[$index]}/$scenario - binary=$TARGET/fanout/${keys[$index]}/$scenario/release/fanout-$scenario + binary=$targets/${keys[$index]}/$scenario/release/fanout-$scenario if [[ -f $dir/unsupported ]]; then binary=-; fi # shellcheck disable=SC2086 # one argument per subscriber count python3 "$ROOT/bench/fanout.py" measure "$RUN" "${labels[$index]}" "${revisions[$index]}" \ From 983bd3693f8546602157225a47a5ad74aab5c8d2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 12:10:46 -0800 Subject: [PATCH 25/36] bench(relay): keep teardown and growing caches out of the fan-out counts A run's teardown grows with the groups it delivered and each viewer's origin cached every group it received, so neither cancelled against the zero-cycle run. Every origin now caches 64 KiB, as the model scenario's track does, and the harnesses exit without tearing down. Co-Authored-By: Claude Opus 5.5 --- bench/fanout/model/src/main.rs | 4 +++- bench/fanout/session/src/main.rs | 13 ++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/bench/fanout/model/src/main.rs b/bench/fanout/model/src/main.rs index 6bc52556714..93ef1b310ae 100644 --- a/bench/fanout/model/src/main.rs +++ b/bench/fanout/model/src/main.rs @@ -37,5 +37,7 @@ fn main() { assert!(matches!(s.poll_recv_group(w), Poll::Pending)); } } - black_box(&broadcast); + // Left standing, so teardown stays out of the count. + std::mem::forget((broadcast, track, subs)); + std::process::exit(0); } diff --git a/bench/fanout/session/src/main.rs b/bench/fanout/session/src/main.rs index daa3b4ee077..d692315eae2 100644 --- a/bench/fanout/session/src/main.rs +++ b/bench/fanout/session/src/main.rs @@ -8,10 +8,14 @@ use std::hint::black_box; use bytes::Bytes; -use moq_net::{Client, Hop, Server, Timestamp, Version, origin, transport::memory}; +use moq_net::{Client, Hop, Server, Timestamp, Version, cache, origin, transport::memory}; +/// An origin whose cache holds what the model scenario's does, so neither the relay's nor a +/// viewer's grows with the groups a run delivers. fn origin(hop: u64) -> origin::Producer { - let (producer, driver) = origin::Producer::new(origin::Config::new(Hop::new(hop).unwrap())); + let mut config = origin::Config::new(Hop::new(hop).unwrap()); + config.pool = cache::Pool::new(cache::Config::default().with_capacity(64 * 1024)); + let (producer, driver) = origin::Producer::new(config); tokio::spawn(moq_net::time::run(driver)); producer } @@ -83,6 +87,9 @@ fn main() { black_box(group.read_frame().await.unwrap().expect("a frame")); } } - black_box((ingest, audience, broadcast)); + // Left standing: their teardown grows with what they delivered, which the zero-cycle + // run would not subtract. + std::mem::forget((ingest, audience, broadcast, relay, publisher)); }); + std::process::exit(0); } From df74d8dbe07f760b2ff5eb6da9dda57d9ee8006d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 12:23:33 -0800 Subject: [PATCH 26/36] bench(relay): compare two revisions' fan-out profiles by function Callgrind names carry each revision's checkout path, so names are matched without it. Co-Authored-By: Claude Opus 5.5 --- bench/README.md | 3 ++- bench/fanout.py | 34 +++++++++++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/bench/README.md b/bench/README.md index 383b2d85fe1..332bac280f2 100644 --- a/bench/README.md +++ b/bench/README.md @@ -388,7 +388,8 @@ zero cycles and one of many; the difference over the groups delivered is group. `BENCH_FANOUT_CALLGRIND=1`, with `valgrind` on `PATH`, also keeps callgrind outputs of both runs under `fanout///callgrind/`; `python3 bench/fanout.py functions ZERO FULL UNITS` ranks functions by -instructions per group. `BENCH_FANOUT_VERSION` picks the session's protocol +instructions per group, and `python3 bench/fanout.py compare BASE CURRENT +SUBSCRIBERS` ranks what moved between two revisions' `callgrind/` directories. `BENCH_FANOUT_VERSION` picks the session's protocol version (default `moq-lite-06`). ### Heap diff --git a/bench/fanout.py b/bench/fanout.py index 3cb6f62f34b..0702f6c1285 100644 --- a/bench/fanout.py +++ b/bench/fanout.py @@ -106,7 +106,7 @@ def load(path): if not match: continue name = re.sub(r"\s+\[[^\]]*\]$", "", match.group(2)) - name = re.sub(r"^(/[^:]*|\?\?\?):", "", name) + name = re.sub(r"^([^ <]*\.rs|\?\?\?):", "", name) totals[name] = totals.get(name, 0) + int(match.group(1).replace(",", "")) return totals @@ -115,6 +115,29 @@ def load(path): return {name: (after.get(name, 0) - before.get(name, 0)) / units for name in names} +def differences(base, current, subscribers, inclusive, top): + """What each function costs per group in `current` against `base`: the largest moves + either way, from the callgrind outputs `--callgrind` kept for each.""" + + def per_group(directory): + units = int((directory / f"{subscribers}.units").read_text()) + zero, full = directory / f"{subscribers}.zero.out", directory / f"{subscribers}.full.out" + return functions(zero, full, units, inclusive) + + before, after = per_group(base), per_group(current) + moves = sorted( + ( + (after.get(name, 0) - before.get(name, 0), before.get(name, 0), after.get(name, 0), name) + for name in set(before) | set(after) + ), + reverse=True, + ) + shown = moves[:top] + [None] + moves[-top // 3 :] + return [ + " ..." if row is None else f"{row[0]:+10.1f} {row[1]:10.1f} {row[2]:10.1f} {row[3][:160]}" for row in shown + ] + + def summarize(run): """Print and keep each scenario's instructions per group, by subscribers and revision.""" rows = [json.loads(line) for line in (run / "fanout.jsonl").read_text().splitlines()] @@ -153,6 +176,12 @@ def main(): measure.add_argument("--callgrind", action="store_true") summary = sub.add_parser("summarize") summary.add_argument("run", type=Path) + compare = sub.add_parser("compare") + compare.add_argument("base", type=Path, help="callgrind/ directory of the base revision") + compare.add_argument("current", type=Path, help="callgrind/ directory of the revision compared") + compare.add_argument("subscribers", type=int) + compare.add_argument("--inclusive", action="store_true") + compare.add_argument("--top", type=int, default=30) top = sub.add_parser("functions") top.add_argument("zero", type=Path) top.add_argument("full", type=Path) @@ -181,6 +210,9 @@ def main(): out.write(json.dumps(row) + "\n") elif args.command == "summarize": summarize(args.run) + elif args.command == "compare": + for line in differences(args.base, args.current, args.subscribers, args.inclusive, args.top): + print(line) else: ranked = functions(args.zero, args.full, args.units, args.inclusive) for name, value in sorted(ranked.items(), key=lambda item: -item[1])[: args.top]: From 1a5ceeca84c243a79853fdca8323d724b2c5dbfd Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 12:35:06 -0800 Subject: [PATCH 27/36] refactor(net): box the pending subscriptions a larger waiter outgrew A pending answer holds a task waiter, which now remembers 16 lists, so the copy's and the stats reader's pending states dwarfed their siblings. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/resume.rs | 9 +++++---- rs/moq-stats/src/aggregate.rs | 8 ++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 10e31cc604c..09955ee2c28 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -510,9 +510,10 @@ struct Copy { feeds: bool, } -// Boxed: a subscriber is twice a pending answer, and a copy holds one for its whole life. +// Both boxed: a copy holds one for its whole life, a subscriber is twice a pending answer, +// and a pending answer holds a task waiter with the lists it remembers. enum Sub { - Pending(kio::Pending), + Pending(Box>), Ready(Box), } @@ -1274,7 +1275,7 @@ impl Reader { let wire = self.mirrored.clamped(floor, None, &resolved); self.copies.push(Copy { generation, - sub: Sub::Pending(track.subscribe(wire)), + sub: Sub::Pending(Box::new(track.subscribe(wire))), track, floor, from: self.mirrored.ranges.clone(), @@ -1988,7 +1989,7 @@ impl Reader { let wire = self.mirrored.clamped(floor, Some(lowered.end), &resolved); let copy = Copy { generation, - sub: Sub::Pending(track.subscribe(wire)), + sub: Sub::Pending(Box::new(track.subscribe(wire))), floor, from: self.mirrored.ranges.clone(), resolved, diff --git a/rs/moq-stats/src/aggregate.rs b/rs/moq-stats/src/aggregate.rs index 290c807b91b..52d94668a38 100644 --- a/rs/moq-stats/src/aggregate.rs +++ b/rs/moq-stats/src/aggregate.rs @@ -697,8 +697,8 @@ enum Reader { Resolving(Box>), /// Waiting for the node's listing to offer the track. Waiting, - /// Awaiting the subscription handshake. - Subscribing(Pending), + /// Awaiting the subscription handshake. Boxed: its task waiter dwarfs the other states. + Subscribing(Box>), /// Reading frames. Boxed: the snapshot consumer dwarfs the other variants, /// and one lives per node in a map. Active(Box>), @@ -1001,7 +1001,7 @@ fn advance( } }, Poll::Ready(Ok(broadcast)) => match broadcast.track(name) { - Ok(track) => node.reader = Reader::Subscribing(track.subscribe(None)), + Ok(track) => node.reader = Reader::Subscribing(Box::new(track.subscribe(None))), Err(err) => { tracing::debug!(?err, name, "stats: node missing track"); return changed | node.depart(); @@ -1030,7 +1030,7 @@ fn advance( } node.offered = false; match listing.broadcast().track(name) { - Ok(track) => node.reader = Reader::Subscribing(track.subscribe(None)), + Ok(track) => node.reader = Reader::Subscribing(Box::new(track.subscribe(None))), Err(err) => { tracing::debug!(?err, name, "stats: node missing track"); return changed | node.depart(); From 018254b7d63b1b04590b111b0359433b3cafce70 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 12:40:19 -0800 Subject: [PATCH 28/36] bench(relay): bound parallel builds, and count each fan-out as a median Every revision built at once ran a 15 GiB host out of memory in its fat-LTO links; both instructions and fan-out modes now build BENCH_BUILD_JOBS at a time through one runner. A fan-out count moves up to 0.5% with each process's hash seeds, so each is the median of three. Co-Authored-By: Claude Opus 5.5 --- bench/README.md | 13 +++--- bench/fanout.py | 16 ++++++-- bench/fanout.test.py | 3 +- bench/run.sh | 95 ++++++++++++++++++++++++-------------------- 4 files changed, 74 insertions(+), 53 deletions(-) diff --git a/bench/README.md b/bench/README.md index 332bac280f2..0b647f5b2f5 100644 --- a/bench/README.md +++ b/bench/README.md @@ -306,8 +306,10 @@ mode runs alone. Each base builds into its own target under `target/bench/revisions/`, from a detached worktree in `.tmp/` that the run removes when it exits, so a -later run reuses that build. The revisions build side by side on every allowed CPU, -since fat LTO links on one thread. Every round measures each base and the +later run reuses that build. The revisions build side by side on every allowed +CPU, since fat LTO links on one thread, `BENCH_BUILD_JOBS` (default 4) at a +time: each link takes gigabytes, so lower it on a host with little memory. +Every round measures each base and the current tree back to back, rotating which goes first, for `BENCH_RUNTIME_ROUNDS` rounds (default 3). The revisions are each other's control, so there is no unprofiled run per revision. One generator drives them @@ -379,13 +381,14 @@ Two scenarios run every subscriber count on one thread: revision whose session API predates this harness reports it unsupported. Each `bench/fanout/` crate builds against each base's and the current -tree's own crates and lock, side by side, into `target/bench/fanout/`, or +tree's own crates and lock, `BENCH_BUILD_JOBS` at a time, into +`target/bench/fanout/`, or `BENCH_FANOUT_TARGET` where a build cache might evict that during a long sweep. Each count pins the harness to one CPU (`BENCH_FANOUT_CPU`, by default the last allowed one) and takes its user instructions from `perf stat` for a run of zero cycles and one of many; the difference over the groups delivered is -`fanout-summary.json`. Repeated counts agree to within a few instructions per -group. `BENCH_FANOUT_CALLGRIND=1`, with `valgrind` on `PATH`, also keeps +`fanout-summary.json`. Each process draws its own hash seeds, which move a +count by up to about 0.5%, so each is the median of three. `BENCH_FANOUT_CALLGRIND=1`, with `valgrind` on `PATH`, also keeps callgrind outputs of both runs under `fanout///callgrind/`; `python3 bench/fanout.py functions ZERO FULL UNITS` ranks functions by instructions per group, and `python3 bench/fanout.py compare BASE CURRENT diff --git a/bench/fanout.py b/bench/fanout.py index 0702f6c1285..5e7c04840f2 100644 --- a/bench/fanout.py +++ b/bench/fanout.py @@ -55,11 +55,21 @@ def count(binary, args, cpu): return parse_stat(run.stderr) +# Each process draws its own hash seeds, which move a count by up to about 0.5%, so each +# count is the median of a few. +REPEATS = 3 + + def per_group(scenario, binary, subscribers, cpu, extra=()): + from statistics import median + n = cycles(scenario, subscribers) - base = count(binary, [subscribers, 0, *extra], cpu) - full = count(binary, [subscribers, n, *extra], cpu) - return (full - base) / (n * subscribers) + counts = [] + for _ in range(REPEATS): + base = count(binary, [subscribers, 0, *extra], cpu) + full = count(binary, [subscribers, n, *extra], cpu) + counts.append((full - base) / (n * subscribers)) + return median(counts) def callgrind(scenario, binary, subscribers, out, extra=()): diff --git a/bench/fanout.test.py b/bench/fanout.test.py index fa590415cd3..33f2d738a1b 100644 --- a/bench/fanout.test.py +++ b/bench/fanout.test.py @@ -29,8 +29,9 @@ def test_multiplexed_or_absent_counts_are_refused(self): def test_each_count_divides_the_cycles_it_added_by_the_groups_delivered(self): counts = {0: 1_000, fanout.cycles("model", 4): 1_000 + 7 * fanout.cycles("model", 4) * 4} - with patch.object(fanout, "count", side_effect=lambda _, args, __: counts[args[1]]): + with patch.object(fanout, "count", side_effect=lambda _, args, __: counts[args[1]]) as counted: self.assertEqual(fanout.per_group("model", "bin", 4, "0"), 7) + self.assertEqual(counted.call_count, 2 * fanout.REPEATS) self.assertEqual(fanout.cycles("session", 512), 196) self.assertEqual(fanout.cycles("model", 2_000_000), 1) diff --git a/bench/run.sh b/bench/run.sh index d55ae80db68..356c7282b8a 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -949,6 +949,30 @@ run_profiles() { done } +# Run each QUEUED_BUILDS shell command with its output in the matching QUEUED_LOGS file, on +# every allowed CPU and BENCH_BUILD_JOBS at a time: fat LTO links on one thread, but each +# link takes gigabytes. Each runs as its own process group, so cleanup stops its rustc too. +# Sets BUILD_STATUS to each command's exit status. +run_queued_builds() { + local jobs=${BENCH_BUILD_JOBS:-4} first index launched + BUILD_STATUS=() + for ((first = 0; first < ${#QUEUED_BUILDS[@]}; first += jobs)); do + BUILD_PIDS=() + for ((index = first; index < first + jobs && index < ${#QUEUED_BUILDS[@]}; index++)); do + # setsid makes each build a group leader without forking. + setsid taskset -c "$ALLOWED_CPUS" bash -c "${QUEUED_BUILDS[$index]}" >"${QUEUED_LOGS[$index]}" 2>&1 & + BUILD_PIDS+=($!) + done + launched=${#BUILD_PIDS[@]} + for ((index = 0; index < launched; index++)); do + BUILD_STATUS[first + index]=0 + wait "${BUILD_PIDS[$index]}" || BUILD_STATUS[first + index]=$? + # Reaped, so cleanup must not signal a group that ID may now name. + unset "BUILD_PIDS[$index]" + done + done +} + # Check out one revision for this run, in the checkout's ignored scratch space, # where it loads only its own Cargo configuration. Its target persists, keyed by # revision; the compiler cache keys on remapped paths, so a fresh checkout reuses it. @@ -997,22 +1021,16 @@ run_instructions() { LOAD_BIN=$generator/profiling/moq-bench HOST_BIN=$generator/profiling/moq-bench-host - # Fat LTO links on one thread, so revisions build side by side on every allowed CPU. mkdir -p "$RUN/build" - # setsid makes each build a group leader without forking, so cleanup stops its rustc too. + QUEUED_BUILDS=() QUEUED_LOGS=() for ((index = 0; index < ${#build_labels[@]}; index++)); do - setsid taskset -c "$ALLOWED_CPUS" bash -c "$(declare -f build_workers); build_workers \"\$@\"" _ \ - "${build_checkouts[$index]}" "${build_targets[$index]}" profiling instructions \ - >"$RUN/build/${build_labels[$index]}.log" 2>&1 & - BUILD_PIDS+=($!) + QUEUED_BUILDS+=("$(declare -f build_workers); build_workers $(printf '%q ' \ + "${build_checkouts[$index]}" "${build_targets[$index]}" profiling instructions)") + QUEUED_LOGS+=("$RUN/build/${build_labels[$index]}.log") done - local status + run_queued_builds for ((index = 0; index < ${#build_labels[@]}; index++)); do - status=0 - wait "${BUILD_PIDS[$index]}" || status=$? - # Reaped, so cleanup must not signal a group that ID may now name. - unset "BUILD_PIDS[$index]" - if ((status != 0)); then + if ((BUILD_STATUS[index] != 0)); then printf 'build failed for %s:\n' "${build_labels[$index]}" >&2 tail -n 40 "$RUN/build/${build_labels[$index]}.log" >&2 return 1 @@ -1060,12 +1078,10 @@ run_fanout() { # The working tree changes under one target; Cargo's fingerprints track it. keys+=(current) - # Fat LTO links on one thread, so the harnesses build side by side on every allowed CPU, - # BENCH_FANOUT_JOBS at a time. - local jobs=${BENCH_FANOUT_JOBS:-4} first launched # A build cache that evicts the `target` directory under a long sweep fails every later # build; BENCH_FANOUT_TARGET names a directory it doesn't manage. - local targets=${BENCH_FANOUT_TARGET:-$TARGET/fanout} + local harnesses=${BENCH_FANOUT_TARGET:-$TARGET/fanout} target + QUEUED_BUILDS=() QUEUED_LOGS=() for ((index = 0; index < ${#labels[@]}; index++)); do for scenario in $FANOUT_SCENARIOS; do dir=$RUN/fanout/${labels[$index]}/$scenario @@ -1074,36 +1090,27 @@ run_fanout() { # This revision's crates, resolved by its own lock. sed -i "s|path = \"../../../rs/|path = \"${checkouts[$index]}/rs/|" "$dir/Cargo.toml" cp "${checkouts[$index]}/Cargo.lock" "$dir/Cargo.lock" - printf '%s\n' "$targets/${keys[$index]}/$scenario" >"$dir/target" + target=$harnesses/${keys[$index]}/$scenario + # shellcheck disable=SC2016 # expanded by the build's own shell + QUEUED_BUILDS+=("$(printf 'mkdir -p "$(readlink -m %q)" && cd %q && CARGO_TARGET_DIR=%q cargo build --release' \ + "$target" "$dir" "$target")") + QUEUED_LOGS+=("$dir/build.log") build_dirs+=("$dir") done done - for ((first = 0; first < ${#build_dirs[@]}; first += jobs)); do - BUILD_PIDS=() - for dir in "${build_dirs[@]:first:jobs}"; do - # shellcheck disable=SC2016 # the inner shell expands its own arguments - setsid taskset -c "$ALLOWED_CPUS" bash -c 'mkdir -p "$(readlink -m "$2")" && cd "$1" && CARGO_TARGET_DIR=$2 cargo build --release' _ \ - "$dir" "$(cat "$dir/target")" >"$dir/build.log" 2>&1 & - BUILD_PIDS+=($!) - done - launched=${#BUILD_PIDS[@]} - for ((index = 0; index < launched; index++)); do - dir=${build_dirs[$((first + index))]} - status=0 - wait "${BUILD_PIDS[$index]}" || status=$? - # Reaped, so cleanup must not signal a group that ID may now name. - unset "BUILD_PIDS[$index]" - if ((status == 0)); then - continue - fi - if [[ $dir == "$RUN/fanout/current/"* ]]; then - printf 'the current tree'"'"'s fan-out harness failed to build:\n' >&2 - tail -n 40 "$dir/build.log" >&2 - return 1 - fi - printf 'fan-out harness %s does not build; reported unsupported\n' "${dir#"$RUN"/fanout/}" - touch "$dir/unsupported" - done + run_queued_builds + for ((index = 0; index < ${#build_dirs[@]}; index++)); do + dir=${build_dirs[$index]} + if ((BUILD_STATUS[index] == 0)); then + continue + fi + if [[ $dir == "$RUN/fanout/current/"* ]]; then + printf 'the current tree'"'"'s fan-out harness failed to build:\n' >&2 + tail -n 40 "$dir/build.log" >&2 + return 1 + fi + printf 'fan-out harness %s does not build; reported unsupported\n' "${dir#"$RUN"/fanout/}" + touch "$dir/unsupported" done if [[ -n ${BENCH_FANOUT_VERSION:-} ]]; then extra+=(--version "$BENCH_FANOUT_VERSION"); fi @@ -1112,7 +1119,7 @@ run_fanout() { for scenario in $FANOUT_SCENARIOS; do for ((index = 0; index < ${#labels[@]}; index++)); do dir=$RUN/fanout/${labels[$index]}/$scenario - binary=$targets/${keys[$index]}/$scenario/release/fanout-$scenario + binary=$harnesses/${keys[$index]}/$scenario/release/fanout-$scenario if [[ -f $dir/unsupported ]]; then binary=-; fi # shellcheck disable=SC2086 # one argument per subscriber count python3 "$ROOT/bench/fanout.py" measure "$RUN" "${labels[$index]}" "${revisions[$index]}" \ From 8cfc3f7e416b67bbb4e1063161d5174c5621b81c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 12:42:59 -0800 Subject: [PATCH 29/36] docs(bench): exact per-commit attribution and this branch's fixes Co-Authored-By: Claude Opus 5.5 --- .github/relay-cpu-bisect.md | 51 ++++++++++++++++++++++- quest/m1/release-perf/relay-cpu-bisect.md | 10 +++-- 2 files changed, 56 insertions(+), 5 deletions(-) diff --git a/.github/relay-cpu-bisect.md b/.github/relay-cpu-bisect.md index da86b916f2f..e554007e9b9 100644 --- a/.github/relay-cpu-bisect.md +++ b/.github/relay-cpu-bisect.md @@ -61,9 +61,56 @@ Medians of four rounds, min-max in brackets; every cell delivered its workload's The rows are self cost under the named function, including the std and libc leaves below it; they add to more than the net growth because other paths shrank. `Cursor::probe` reads the subscription, takes the track's shared state lock to judge misses, clones the track's fills, and runs the ask and resolve loop on every peek, even for a live reader that took every group as it arrived and never misses one. That work pays for relay backfill of missed groups, but the chat workload never needs it. -## Decisions +## Exact counts by commit -No feature cost is accepted in the [release-perf epic](../quest/m1/release-perf/README.md) yet; that is the maintainer's call, recorded in [the PR](https://github.com/moq-dev/moq/pull/7605). +Relay counts move 5-20% with host load, so a step of a few thousand instructions can't be pinned to one commit from them. `just bench-fanout` (see `bench/README.md`) counts the chat fan-out on one pinned thread instead, against a run of zero cycles, so repeated counts agree within a few instructions per group: + +- `model`: one finished group a cycle through the track model, which every subscriber takes. It builds against every revision back to [cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95). +- `session`: a publisher session into a relay origin and a viewer session out of it per subscriber, over the in-memory transport. It counts publisher, relay, and viewers together, without QUIC, and builds back to [moq-dev/moq#7027](https://github.com/moq-dev/moq/pull/7027)'s `kio::clock` (2026-11-01). + +At 512 subscribers, in instructions per delivered group, the model path cost 4,285 at the old endpoint and 13,300 on `main`, 3.1x; its growth is about 9k of the relay's per-frame growth on chat. The features the maintainer asked about, each against its parent (model, built with frame pointers): + +| Feature | Parent | Merge | Change | +| --- | ---: | ---: | ---: | +| Per-reader ranges and time anchors, [moq-dev/moq#5929](https://github.com/moq-dev/moq/pull/5929) | 4,411 | 5,313 | +902 | +| Compressed tracks, [moq-dev/moq#5928](https://github.com/moq-dev/moq/pull/5928) | 5,579 | 5,612 | +33 | +| A subscriber reads only its ranges' frames, [moq-dev/moq#6129](https://github.com/moq-dev/moq/pull/6129) | 5,604 | 5,787 | +183 | +| Signed priorities, [moq-dev/moq#6142](https://github.com/moq-dev/moq/pull/6142) | 5,888 | 5,816 | -72 | +| Public ingest, [moq-dev/moq#6111](https://github.com/moq-dev/moq/pull/6111) | 5,816 | 6,016 | +200 | +| The m0 simplification audit, [moq-dev/moq#6108](https://github.com/moq-dev/moq/pull/6108) | 6,017 | 6,017 | 0 | + +In-tree qmux ([moq-dev/moq#6231](https://github.com/moq-dev/moq/pull/6231)) and receive leases ([moq-dev/moq#5878](https://github.com/moq-dev/moq/pull/5878)) are QUIC-level, which neither scenario reaches; their relay counts on the quiet test host are below. The model path barely moved before [e84ffb3de3](https://github.com/moq-dev/moq/commit/e84ffb3de3139f7245bed5697754fd6add9a022e) (5,840) and then grew in a few large steps, each against its parent (run.TmGdFy, one build profile throughout): + +| Commit | Model | Session | +| --- | ---: | ---: | +| Dynamic tracks serve fills for misses, [moq-dev/moq#6171](https://github.com/moq-dev/moq/pull/6171) | +4,264 | | +| Refill groups evicted before a reader took them, [moq-dev/moq#7339](https://github.com/moq-dev/moq/pull/7339) | +1,154 | | +| Lite-07 frame bounds, timed datagram admission, layered windows, [moq-dev/moq#7542](https://github.com/moq-dev/moq/pull/7542) | +876 | +3,054 | +| Fill linearly in range count and span, [moq-dev/moq#6995](https://github.com/moq-dev/moq/pull/6995) | +558 | +3,290 | +| Close the relay demand gaps, [moq-dev/moq#7052](https://github.com/moq-dev/moq/pull/7052) | +365 | | +| Bounded reads end through routes, [c68c931025](https://github.com/moq-dev/moq/commit/c68c931025) | +275 | +450 | +| An idle track read parks without walking the read path, [3c0dd8a2af](https://github.com/moq-dev/moq/commit/3c0dd8a2af) | +213 | -3,142 | +| Take a reserved group without asking its cursor again, [68fb455bac](https://github.com/moq-dev/moq/commit/68fb455bac) | +204 | -455 | + +The session column starts at the 2026-11-01 harness; blank cells predate it. [moq-dev/moq#6995](https://github.com/moq-dev/moq/pull/6995)'s session cost is the probe's walk of the live groups (`TrackState::live`, +4.0k inclusive) on both the reader's cursor and its forward-drop cursor. + +## Fixes in this PR + +Each is its own commit, counted in one run (run.s6IjbN in the evidence) at 512 subscribers per delivered group, each row on top of the one above: + +| Change | Model | Session | +| --- | ---: | ---: | +| `main` | 13,300 | 120,948 | +| A live reader's probe skips the fills while it misses nothing | 10,477 (-2,823) | 114,139 (-6,809) | +| A window with one set of runs reads without growing across two | 10,126 (-351) | 113,133 (-1,007) | +| A kio waiter remembers 16 lists, not 8 | 10,189 (+63) | 108,709 (-4,424) | +| This branch | 10,189 (-23.4%) | 108,709 (-10.1%) | + +`Cursor::probe` is the relay's check for groups a reader has missed. Each time a subscriber asks for its next group, the probe looks at what arrived on the track since its last look and decides whether any group the reader still wants has left the cache without the live feed delivering it (evicted, skipped, or from before the feed started), and asks upstream to fill those. A live reader that takes each group as it arrives never has such a gap, yet it paid for the whole check on every group: locking the subscription and the track's shared state, cloning the track's fills, walking the arrival ledger, checking each new group for holes, and running the ask-and-answer loop. Each time the cache aged out an old group, the eviction also landed with every reader, all of which had already taken it, and forced each one to re-run the full check. The fast path judges a group the arrival walk serves without the hole, backfill, and ledger passes, skips the subscription, fills, and answers when nothing is asked or owed, and starts each reader's eviction inbox above what it handed out. It recovers most of [moq-dev/moq#6171](https://github.com/moq-dev/moq/pull/6171)'s and [moq-dev/moq#7339](https://github.com/moq-dev/moq/pull/7339)'s cost. + +kio's `WaiterList::register` was the largest self cost in the session path: a relay session's control and serving tasks park on more than the 8 lists a waiter remembered, so every list past the eighth took a duplicate entry on every poll. 16 measured best; 12 and 32 cost 1.1-1.7% more. Its commit message also claims 2.3% on the model, measured with frame pointers forced; in the build measured here the model moves +0.6%, and the change pays off in the session. + +On the quiet test host (run.we4umM, one-minute load at most 4.7), chat over io\_uring went from 154.5k \[154.1-155.0] instructions per frame on `main` to 148.9k \[148.3-150.8] with the probe and window changes (-3.6%), four rounds each, every cell at 20,000 frames/s with no loss. The relay's QUIC, crypto, and syscalls are most of its cost, so the model's -23% shows as a few percent there. ## Validation diff --git a/quest/m1/release-perf/relay-cpu-bisect.md b/quest/m1/release-perf/relay-cpu-bisect.md index 5bd51123340..00be1cc5739 100644 --- a/quest/m1/release-perf/relay-cpu-bisect.md +++ b/quest/m1/release-perf/relay-cpu-bisect.md @@ -27,11 +27,15 @@ Decided 2026-11-03 (maintainer, on [moq-dev/moq#7605](https://github.com/moq-dev `Cursor::probe` is the relay's check for groups a reader has missed. Each time a subscriber asks for its next group, the probe looks at what arrived on the track since its last look and decides whether any group the reader still wants has left the cache without the live feed delivering it (evicted, skipped, or from before the feed started), and asks upstream to fill those. A live reader that takes each group as it arrives never has such a gap, yet it paid for the whole check on every group: locking the subscription and the track's shared state, cloning the track's fills, walking the arrival ledger, checking each new group for holes, and running the ask-and-answer loop. On top of that, each time the cache aged out an old group, the eviction landed with every reader, all of which had already taken it, and forced each one to re-run the full check. -Fixes measure each change exactly with a deterministic replay of moq-net's `track_fanout_group` bench (one finished group, every subscriber takes it and parks), counting user instructions per group per subscriber. The same harness builds against historical revisions, so the model's share of the growth is bisected exactly; the session and QUIC layers are attributed from relay cells. +Follow-up planning on 2026-11-03 (maintainer) added a load-independent mode to the driver: `just bench-fanout` counts the chat fan-out per delivered group on one pinned thread, through the model alone and through a relay over the in-memory transport, for any list of revisions. [The report](/.github/relay-cpu-bisect.md) attributes each step with it, and the fixes so far take `main`'s model path from 13,300 to 10,189 instructions per group (-23.4%) and the in-process session from 120,948 to 108,709 (-10.1%), at 512 subscribers. -Remaining: finish the per-step attribution of decision 2 and the profiling of decision 3, land what can be cut, record each attributed cost and its feature in the epic's Plan, re-pair `main` against [cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95) on chat and video over both runtimes, and delete this quest. +Remaining: -Measure as the report does: `BENCH_GENERATOR=21d6da9fc1`, `BENCH_INSTRUCTION_PERIOD=2000003`, `kernel.perf_event_max_sample_rate` raised to 100,000 for the gated run and restored after, and compare only cells from one run at low host load. Instructions per frame fall by up to 20% when other host work makes the relay batch more frames per wakeup, so cross-session absolutes don't compare. +- Settle the attributed costs nothing here cut without changing behavior (the report's tables: the window, drift, and live-group walks under [moq-dev/moq#5929](https://github.com/moq-dev/moq/pull/5929), [moq-dev/moq#6995](https://github.com/moq-dev/moq/pull/6995), and [moq-dev/moq#7542](https://github.com/moq-dev/moq/pull/7542); the small flat steps of compressed tracks, ranged reads, and public ingest; the lite publisher's per-group stream open and close), and record each accepted one and its feature in the epic's Plan. +- The session path's remaining largest self costs: moves of large values (about 9.4k per group, `group::Consumer` and `Peeked` by value through `poll_peek_group` and `take`), and waiter registration (about 4.0k). +- Re-pair `main` against [cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95) on chat and video over both runtimes on a quiet host, then delete this quest. + +Measure as the report does. For a model or session change, `just bench-fanout` against the base, which host load doesn't move. For the relay, `BENCH_GENERATOR=21d6da9fc1`, `BENCH_INSTRUCTION_PERIOD=2000003`, `kernel.perf_event_max_sample_rate` raised to 100,000 and restored after, on a quiet host such as `nixos2` (6 cores: relay on 0-3, generator on 4,5,10,11, `BENCH_BUILD_JOBS=2` for its 15 GiB), comparing only cells from one run; instructions per frame fall by up to 20% when other host work makes the relay batch more frames per wakeup. Public API: none expected. Wire: none. From e2e2dcad4b16aca6c60a56e72d20b86f11586850 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 12:53:44 -0800 Subject: [PATCH 30/36] Revert "refactor(net): box the pending subscriptions a larger waiter outgrew" This reverts commit 1a5ceeca84c243a79853fdca8323d724b2c5dbfd. --- rs/moq-net/src/model/resume.rs | 9 ++++----- rs/moq-stats/src/aggregate.rs | 8 ++++---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 09955ee2c28..10e31cc604c 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -510,10 +510,9 @@ struct Copy { feeds: bool, } -// Both boxed: a copy holds one for its whole life, a subscriber is twice a pending answer, -// and a pending answer holds a task waiter with the lists it remembers. +// Boxed: a subscriber is twice a pending answer, and a copy holds one for its whole life. enum Sub { - Pending(Box>), + Pending(kio::Pending), Ready(Box), } @@ -1275,7 +1274,7 @@ impl Reader { let wire = self.mirrored.clamped(floor, None, &resolved); self.copies.push(Copy { generation, - sub: Sub::Pending(Box::new(track.subscribe(wire))), + sub: Sub::Pending(track.subscribe(wire)), track, floor, from: self.mirrored.ranges.clone(), @@ -1989,7 +1988,7 @@ impl Reader { let wire = self.mirrored.clamped(floor, Some(lowered.end), &resolved); let copy = Copy { generation, - sub: Sub::Pending(Box::new(track.subscribe(wire))), + sub: Sub::Pending(track.subscribe(wire)), floor, from: self.mirrored.ranges.clone(), resolved, diff --git a/rs/moq-stats/src/aggregate.rs b/rs/moq-stats/src/aggregate.rs index 52d94668a38..290c807b91b 100644 --- a/rs/moq-stats/src/aggregate.rs +++ b/rs/moq-stats/src/aggregate.rs @@ -697,8 +697,8 @@ enum Reader { Resolving(Box>), /// Waiting for the node's listing to offer the track. Waiting, - /// Awaiting the subscription handshake. Boxed: its task waiter dwarfs the other states. - Subscribing(Box>), + /// Awaiting the subscription handshake. + Subscribing(Pending), /// Reading frames. Boxed: the snapshot consumer dwarfs the other variants, /// and one lives per node in a map. Active(Box>), @@ -1001,7 +1001,7 @@ fn advance( } }, Poll::Ready(Ok(broadcast)) => match broadcast.track(name) { - Ok(track) => node.reader = Reader::Subscribing(Box::new(track.subscribe(None))), + Ok(track) => node.reader = Reader::Subscribing(track.subscribe(None)), Err(err) => { tracing::debug!(?err, name, "stats: node missing track"); return changed | node.depart(); @@ -1030,7 +1030,7 @@ fn advance( } node.offered = false; match listing.broadcast().track(name) { - Ok(track) => node.reader = Reader::Subscribing(Box::new(track.subscribe(None))), + Ok(track) => node.reader = Reader::Subscribing(track.subscribe(None)), Err(err) => { tracing::debug!(?err, name, "stats: node missing track"); return changed | node.depart(); From d532154d8dcefe05d96572ed85cc4af5d96cde3d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 12:53:44 -0800 Subject: [PATCH 31/36] Revert "perf(kio): a waiter remembers 16 lists, not 8" This reverts commit 101ed5a806 and, before it, the enum boxing it needed. With 16 records, two moq-net integration tests fail (errored_end::an_errored_end_keeps_its_history_across_a_hop and quiet_catalog::a_quiet_catalog_reaches_an_unfloored_relay_reader_when_a_peer_resumes_past_it) and pass with 8. A read whose waiter parks on more lists than it records is retired and walks again; with more records, those reads stay parked, and something they depend on doesn't wake them. That needs its own investigation before the waiter can remember more. Co-Authored-By: Claude Opus 5.5 --- rs/kio/src/waiter.rs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/rs/kio/src/waiter.rs b/rs/kio/src/waiter.rs index 707db955cf6..e2dc3bc5404 100644 --- a/rs/kio/src/waiter.rs +++ b/rs/kio/src/waiter.rs @@ -47,13 +47,8 @@ pub struct Waiter { noop: bool, } -/// Lists a waiter remembers being parked on. A relay session's control and serving tasks -/// park on more than 8 (stream accepts, deadlines, the auth link, expiry landings, channel -/// closes), and each list past the last remembered stacks a duplicate on every poll. 16 -/// holds them: in the chat fan-out over the memory transport, 8 slots cost 3.9% more -/// instructions per delivered group and 12 or 32 cost 1.1-1.7% more, since every -/// registration searches the slots. -const PARKED: usize = 16; +/// Lists a waiter remembers being parked on. A relay's busiest tasks park on up to 8. +const PARKED: usize = 8; impl Waiter { /// Create a new waiter from an async [`Waker`]. From af83ff87fcb3cb0e2ff3969ed1f96fcebee62043 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 12:56:26 -0800 Subject: [PATCH 32/36] docs(bench): the waiter change is reverted; record why Co-Authored-By: Claude Opus 5.5 --- .github/relay-cpu-bisect.md | 5 ++--- quest/m1/release-perf/relay-cpu-bisect.md | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/relay-cpu-bisect.md b/.github/relay-cpu-bisect.md index e554007e9b9..a12a8594f03 100644 --- a/.github/relay-cpu-bisect.md +++ b/.github/relay-cpu-bisect.md @@ -103,12 +103,11 @@ Each is its own commit, counted in one run (run.s6IjbN in the evidence) at 512 s | `main` | 13,300 | 120,948 | | A live reader's probe skips the fills while it misses nothing | 10,477 (-2,823) | 114,139 (-6,809) | | A window with one set of runs reads without growing across two | 10,126 (-351) | 113,133 (-1,007) | -| A kio waiter remembers 16 lists, not 8 | 10,189 (+63) | 108,709 (-4,424) | -| This branch | 10,189 (-23.4%) | 108,709 (-10.1%) | +| This branch | 10,126 (-23.9%) | 113,133 (-6.5%) | `Cursor::probe` is the relay's check for groups a reader has missed. Each time a subscriber asks for its next group, the probe looks at what arrived on the track since its last look and decides whether any group the reader still wants has left the cache without the live feed delivering it (evicted, skipped, or from before the feed started), and asks upstream to fill those. A live reader that takes each group as it arrives never has such a gap, yet it paid for the whole check on every group: locking the subscription and the track's shared state, cloning the track's fills, walking the arrival ledger, checking each new group for holes, and running the ask-and-answer loop. Each time the cache aged out an old group, the eviction also landed with every reader, all of which had already taken it, and forced each one to re-run the full check. The fast path judges a group the arrival walk serves without the hole, backfill, and ledger passes, skips the subscription, fills, and answers when nothing is asked or owed, and starts each reader's eviction inbox above what it handed out. It recovers most of [moq-dev/moq#6171](https://github.com/moq-dev/moq/pull/6171)'s and [moq-dev/moq#7339](https://github.com/moq-dev/moq/pull/7339)'s cost. -kio's `WaiterList::register` was the largest self cost in the session path: a relay session's control and serving tasks park on more than the 8 lists a waiter remembered, so every list past the eighth took a duplicate entry on every poll. 16 measured best; 12 and 32 cost 1.1-1.7% more. Its commit message also claims 2.3% on the model, measured with frame pointers forced; in the build measured here the model moves +0.6%, and the change pays off in the session. +kio's `WaiterList::register` is the session path's second-largest self cost: 7.2k per group. A waiter records 8 lists, and a relay session's control and serving tasks park on more, so every list past the eighth takes a duplicate entry on every poll. Remembering 16 cut the session by 4,424 (-3.9%) and was reverted: with it, two moq-net integration tests (`errored_end::an_errored_end_keeps_its_history_across_a_hop`, `quiet_catalog::a_quiet_catalog_reaches_an_unfloored_relay_reader_when_a_peer_resumes_past_it`) fail, and they pass with 8. A read whose waiter parks on more lists than it records is retired and walks again; with more records those reads stay parked, and something they depend on doesn't wake them. That needs its own investigation before the waiter can remember more. On the quiet test host (run.we4umM, one-minute load at most 4.7), chat over io\_uring went from 154.5k \[154.1-155.0] instructions per frame on `main` to 148.9k \[148.3-150.8] with the probe and window changes (-3.6%), four rounds each, every cell at 20,000 frames/s with no loss. The relay's QUIC, crypto, and syscalls are most of its cost, so the model's -23% shows as a few percent there. diff --git a/quest/m1/release-perf/relay-cpu-bisect.md b/quest/m1/release-perf/relay-cpu-bisect.md index 00be1cc5739..08c40414ab4 100644 --- a/quest/m1/release-perf/relay-cpu-bisect.md +++ b/quest/m1/release-perf/relay-cpu-bisect.md @@ -27,12 +27,12 @@ Decided 2026-11-03 (maintainer, on [moq-dev/moq#7605](https://github.com/moq-dev `Cursor::probe` is the relay's check for groups a reader has missed. Each time a subscriber asks for its next group, the probe looks at what arrived on the track since its last look and decides whether any group the reader still wants has left the cache without the live feed delivering it (evicted, skipped, or from before the feed started), and asks upstream to fill those. A live reader that takes each group as it arrives never has such a gap, yet it paid for the whole check on every group: locking the subscription and the track's shared state, cloning the track's fills, walking the arrival ledger, checking each new group for holes, and running the ask-and-answer loop. On top of that, each time the cache aged out an old group, the eviction landed with every reader, all of which had already taken it, and forced each one to re-run the full check. -Follow-up planning on 2026-11-03 (maintainer) added a load-independent mode to the driver: `just bench-fanout` counts the chat fan-out per delivered group on one pinned thread, through the model alone and through a relay over the in-memory transport, for any list of revisions. [The report](/.github/relay-cpu-bisect.md) attributes each step with it, and the fixes so far take `main`'s model path from 13,300 to 10,189 instructions per group (-23.4%) and the in-process session from 120,948 to 108,709 (-10.1%), at 512 subscribers. +Follow-up planning on 2026-11-03 (maintainer) added a load-independent mode to the driver: `just bench-fanout` counts the chat fan-out per delivered group on one pinned thread, through the model alone and through a relay over the in-memory transport, for any list of revisions. [The report](/.github/relay-cpu-bisect.md) attributes each step with it, and the fixes so far take `main`'s model path from 13,300 to 10,126 instructions per group (-23.9%) and the in-process session from 120,948 to 113,133 (-6.5%), at 512 subscribers. Remaining: - Settle the attributed costs nothing here cut without changing behavior (the report's tables: the window, drift, and live-group walks under [moq-dev/moq#5929](https://github.com/moq-dev/moq/pull/5929), [moq-dev/moq#6995](https://github.com/moq-dev/moq/pull/6995), and [moq-dev/moq#7542](https://github.com/moq-dev/moq/pull/7542); the small flat steps of compressed tracks, ranged reads, and public ingest; the lite publisher's per-group stream open and close), and record each accepted one and its feature in the epic's Plan. -- The session path's remaining largest self costs: moves of large values (about 9.4k per group, `group::Consumer` and `Peeked` by value through `poll_peek_group` and `take`), and waiter registration (about 4.0k). +- The session path's remaining largest self costs: moves of large values (about 9.4k per group: `group::Consumer` is 280 bytes, moved by value through `poll_peek_group`, `take`, and the viewer's serve loop), and waiter registration (about 7.2k). A waiter that remembered 16 lists instead of 8 cut the session by 3.9% but made two integration tests miss a wake, since reads it no longer retires stay parked; see the report. - Re-pair `main` against [cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95) on chat and video over both runtimes on a quiet host, then delete this quest. Measure as the report does. For a model or session change, `just bench-fanout` against the base, which host load doesn't move. For the relay, `BENCH_GENERATOR=21d6da9fc1`, `BENCH_INSTRUCTION_PERIOD=2000003`, `kernel.perf_event_max_sample_rate` raised to 100,000 and restored after, on a quiet host such as `nixos2` (6 cores: relay on 0-3, generator on 4,5,10,11, `BENCH_BUILD_JOBS=2` for its 15 GiB), comparing only cells from one run; instructions per frame fall by up to 20% when other host work makes the relay batch more frames per wakeup. From 0e9b7ae6bb4d22b50da244a3c1146acd9bb91f5a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 13:48:12 -0800 Subject: [PATCH 33/36] fix(moq-net): a live arrival's fast judgement keeps history below a resumed floor moq-dev/moq#7708 judges from the reader's floor rather than a resumed copy's live floor, which can sit above history an unlimited reader still reads. The fast path for a live arrival still took the higher floor: it forgot those groups and left them out of the eviction inbox, so one evicted before the reader took it was never asked again. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/track.rs | 47 ++++++++++++++++++++++++++++++ rs/moq-net/src/model/track/miss.rs | 4 ++- 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 6638314365a..bef9290abac 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -21434,6 +21434,53 @@ mod test { assert!(misses.refilled().is_empty(), "a refill handed out is forgotten"); } + /// A live arrival judged on its own keeps the history a resumed copy's live floor sits + /// above: a cached group the reader hasn't taken yet, evicted after, is still refilled. + #[test] + #[cfg(feature = "async")] + fn history_below_a_resumed_live_floor_is_refilled_after_a_live_arrival() { + let mut producer = track_producer("test", Info::default()); + let dynamic = producer.requests(); + producer.received(); + for sequence in 0..5 { + framed(&producer, sequence, 1).finish().unwrap(); + } + producer.set_idle(); + // The feed resumes at 8, above the cached 0..=4: its live floor is 5. + producer.set_live(Some(Position::group(8)), None); + framed(&producer, 8, 1).finish().unwrap(); + let mut reader = producer.subscribe( + Subscription::default() + .with_range(0..) + .with_order(Order::Ascending) + .with_delay(Delay::default().with_max(None)), + ); + let waiter = kio::Waiter::noop(); + let next = |reader: &mut Subscriber| match reader.poll_next_group(&waiter) { + Poll::Ready(Ok(Some(group))) => Some(group.sequence), + _ => None, + }; + assert_eq!(next(&mut reader), Some(0)); + // The gap the feed may still deliver arrives late, so nothing is deferred after. + for sequence in 5..8 { + framed(&producer, sequence, 1).finish().unwrap(); + } + assert_eq!(next(&mut reader), Some(1)); + // Only the newest group moves before this read: judged on its own. + framed(&producer, 9, 1).finish().unwrap(); + assert_eq!(next(&mut reader), Some(2)); + evict(&producer, 3); + let _ = next(&mut reader); + let mut asked = Vec::new(); + while let Poll::Ready(Ok(fill)) = dynamic.poll_requested_fill(&waiter) { + asked.push((fill.start().group, fill.end())); + } + assert!( + asked.iter().any(|&(start, end)| start <= 3 && 3 < end), + "group 3 was evicted before the reader took it: asked {asked:?}" + ); + } + /// An eviction waits only for a reader that judged the evicted group cached: a reader /// that judged once and stalled holds no more than what was cached then, however many /// groups the feed caches and evicts after, late lower ones included. A reader that diff --git a/rs/moq-net/src/model/track/miss.rs b/rs/moq-net/src/model/track/miss.rs index a27a7ba8b70..6c353c28355 100644 --- a/rs/moq-net/src/model/track/miss.rs +++ b/rs/moq-net/src/model/track/miss.rs @@ -997,7 +997,9 @@ impl Misses { let Some(inbox) = &self.inbox else { return false; }; - let low = state.live_floor.unwrap_or(0).max(key.0); + // The reader's floor, as the whole judgement takes it: a resumed copy's live floor + // can sit above history the reader still reads. + let low = key.0; let top = newest.min(u64::MAX - 1); let first = judged_to.saturating_add(1).max(low); let mut fresh = RangeSet::new(); From 8c4202592fa89d7d638e2d48b809ab0866c4b01e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 13:50:57 -0800 Subject: [PATCH 34/36] docs(bench): keep the fan-out and quiet-host evidence Co-Authored-By: Claude Opus 5.5 --- .github/relay-cpu-bisect.json | 5554 +++++++++++++++++++++++++++++++-- .github/relay-cpu-bisect.md | 4 +- 2 files changed, 5244 insertions(+), 314 deletions(-) diff --git a/.github/relay-cpu-bisect.json b/.github/relay-cpu-bisect.json index cdd1d3e742b..b97fc71bf25 100644 --- a/.github/relay-cpu-bisect.json +++ b/.github/relay-cpu-bisect.json @@ -1,315 +1,5243 @@ { - "version": 2, - "metric": "relay instructions (user and kernel, all threads) per delivered frame over a 5 s counter window after 7 s warmup", - "markers": { - "learn_end": "inclusive share of fixed-period instruction samples whose stack matches resume::Reader>::learn_end", - "poll_forward_drop": "inclusive share of fixed-period instruction samples whose stack matches poll_forward_drop", - "cursor_probe": "inclusive share of fixed-period instruction samples whose stack matches model::track::Cursor>::probe", - "syscall_leaf": "inclusive share of fixed-period instruction samples whose stack matches ;syscall$", - "mutex_leaf": "inclusive share of fixed-period instruction samples whose stack matches ;(lock|unlock)$" - }, - "runs": [ - { - "artifact": "run.eVjz2w", - "harness_revision": "82194441a92bb89fbc69f3759ef336d676fad643", - "workload": "chat", - "runtime": "io-uring-workers", - "workers": 4, - "generator_revision": "82194441a92bb89fbc69f3759ef336d676fad643", - "instruction_period": 10000019, - "sampling": "concurrent with the counter window", - "relay_cpus": "4-7", - "generator_cpus": "0-3", - "cells": [ - {"revision": "2d73eea67d083ecf18938f4fc7c4842118eb87cf", "label": "base-2d73eea67d08", "round": 1, "kinstr_per_frame": 121.8, "kcycles_per_frame": 224.8, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 4.81, "samples": 1209, "markers": {"learn_end": 0.0, "poll_forward_drop": 8.9, "cursor_probe": 0.0, "syscall_leaf": 23.2, "mutex_leaf": 16.3}}, - {"revision": "2d73eea67d083ecf18938f4fc7c4842118eb87cf", "label": "base-2d73eea67d08", "round": 2, "kinstr_per_frame": 117.8, "kcycles_per_frame": 232.6, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 5.87, "samples": 1167, "markers": {"learn_end": 0.0, "poll_forward_drop": 6.9, "cursor_probe": 0.0, "syscall_leaf": 20.1, "mutex_leaf": 16.5}}, - {"revision": "2d73eea67d083ecf18938f4fc7c4842118eb87cf", "label": "base-2d73eea67d08", "round": 3, "kinstr_per_frame": 122.6, "kcycles_per_frame": 235.2, "recv_fps": 19999, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 4.2, "samples": 1216, "markers": {"learn_end": 0.0, "poll_forward_drop": 6.6, "cursor_probe": 0.0, "syscall_leaf": 21.7, "mutex_leaf": 16.0}}, - {"revision": "32c1e9940e5519a2bdc7c853d84846f55195cb94", "label": "base-32c1e9940e55", "round": 1, "kinstr_per_frame": 123.4, "kcycles_per_frame": 221.0, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 5.81, "samples": 1229, "markers": {"learn_end": 0.0, "poll_forward_drop": 8.0, "cursor_probe": 0.0, "syscall_leaf": 23.6, "mutex_leaf": 17.8}}, - {"revision": "32c1e9940e5519a2bdc7c853d84846f55195cb94", "label": "base-32c1e9940e55", "round": 2, "kinstr_per_frame": 115.8, "kcycles_per_frame": 220.7, "recv_fps": 20003, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 3.96, "samples": 1150, "markers": {"learn_end": 0.0, "poll_forward_drop": 8.5, "cursor_probe": 0.0, "syscall_leaf": 22.8, "mutex_leaf": 19.7}}, - {"revision": "32c1e9940e5519a2bdc7c853d84846f55195cb94", "label": "base-32c1e9940e55", "round": 3, "kinstr_per_frame": 117.3, "kcycles_per_frame": 223.7, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 3.13, "samples": 1168, "markers": {"learn_end": 0.0, "poll_forward_drop": 8.0, "cursor_probe": 0.0, "syscall_leaf": 21.8, "mutex_leaf": 17.6}}, - {"revision": "cbab2a8922c53a4541f919b89a95c131628f0cb0", "label": "base-cbab2a8922c5", "round": 1, "kinstr_per_frame": 129.2, "kcycles_per_frame": 241.4, "recv_fps": 19997, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 5.81, "samples": 1282, "markers": {"learn_end": 0.0, "poll_forward_drop": 8.7, "cursor_probe": 0.0, "syscall_leaf": 24.1, "mutex_leaf": 16.3}}, - {"revision": "cbab2a8922c53a4541f919b89a95c131628f0cb0", "label": "base-cbab2a8922c5", "round": 2, "kinstr_per_frame": 121.7, "kcycles_per_frame": 233.1, "recv_fps": 19997, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 4.88, "samples": 1210, "markers": {"learn_end": 0.0, "poll_forward_drop": 8.8, "cursor_probe": 0.0, "syscall_leaf": 20.6, "mutex_leaf": 18.1}}, - {"revision": "cbab2a8922c53a4541f919b89a95c131628f0cb0", "label": "base-cbab2a8922c5", "round": 3, "kinstr_per_frame": 124.0, "kcycles_per_frame": 244.0, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 4.37, "samples": 1233, "markers": {"learn_end": 0.0, "poll_forward_drop": 9.3, "cursor_probe": 0.0, "syscall_leaf": 23.5, "mutex_leaf": 16.7}}, - {"revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", "label": "base-e84ffb3de313", "round": 1, "kinstr_per_frame": 133.4, "kcycles_per_frame": 270.7, "recv_fps": 20003, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 4.41, "samples": 1327, "markers": {"learn_end": 0.0, "poll_forward_drop": 6.9, "cursor_probe": 0.0, "syscall_leaf": 22.0, "mutex_leaf": 16.9}}, - {"revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", "label": "base-e84ffb3de313", "round": 2, "kinstr_per_frame": 132.9, "kcycles_per_frame": 282.7, "recv_fps": 20001, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 6.04, "samples": 1320, "markers": {"learn_end": 0.0, "poll_forward_drop": 7.6, "cursor_probe": 0.0, "syscall_leaf": 21.4, "mutex_leaf": 15.4}}, - {"revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", "label": "base-e84ffb3de313", "round": 3, "kinstr_per_frame": 135.4, "kcycles_per_frame": 269.0, "recv_fps": 20002, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 4.05, "samples": 1346, "markers": {"learn_end": 0.0, "poll_forward_drop": 7.6, "cursor_probe": 0.0, "syscall_leaf": 21.1, "mutex_leaf": 16.5}}, - {"revision": "fe722cec9f8432e0e46cb633cc5e79e59e7724cb", "label": "base-fe722cec9f84", "round": 1, "kinstr_per_frame": 117.5, "kcycles_per_frame": 230.1, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 6.52, "samples": 1166, "markers": {"learn_end": 0.0, "poll_forward_drop": 8.1, "cursor_probe": 0.0, "syscall_leaf": 22.4, "mutex_leaf": 16.8}}, - {"revision": "fe722cec9f8432e0e46cb633cc5e79e59e7724cb", "label": "base-fe722cec9f84", "round": 2, "kinstr_per_frame": 114.6, "kcycles_per_frame": 222.0, "recv_fps": 20003, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 4.66, "samples": 1139, "markers": {"learn_end": 0.0, "poll_forward_drop": 8.5, "cursor_probe": 0.0, "syscall_leaf": 20.9, "mutex_leaf": 18.1}}, - {"revision": "fe722cec9f8432e0e46cb633cc5e79e59e7724cb", "label": "base-fe722cec9f84", "round": 3, "kinstr_per_frame": 120.5, "kcycles_per_frame": 220.7, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 3.06, "samples": 1200, "markers": {"learn_end": 0.0, "poll_forward_drop": 9.0, "cursor_probe": 0.0, "syscall_leaf": 25.0, "mutex_leaf": 17.9}}, - {"revision": "82194441a92bb89fbc69f3759ef336d676fad643", "label": "current", "round": 1, "kinstr_per_frame": 146.9, "kcycles_per_frame": 248.6, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 3.8, "samples": 1461, "markers": {"learn_end": 0.5, "poll_forward_drop": 14.9, "cursor_probe": 6.4, "syscall_leaf": 17.2, "mutex_leaf": 17.5}}, - {"revision": "82194441a92bb89fbc69f3759ef336d676fad643", "label": "current", "round": 2, "kinstr_per_frame": 145.2, "kcycles_per_frame": 278.8, "recv_fps": 20004, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 5.49, "samples": 1447, "markers": {"learn_end": 0.5, "poll_forward_drop": 13.3, "cursor_probe": 6.7, "syscall_leaf": 16.6, "mutex_leaf": 18.9}}, - {"revision": "82194441a92bb89fbc69f3759ef336d676fad643", "label": "current", "round": 3, "kinstr_per_frame": 147.1, "kcycles_per_frame": 270.2, "recv_fps": 20001, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 3.47, "samples": 1465, "markers": {"learn_end": 0.5, "poll_forward_drop": 15.1, "cursor_probe": 6.8, "syscall_leaf": 18.3, "mutex_leaf": 17.2}} - ] - }, - { - "artifact": "run.2DucdO", - "harness_revision": "82194441a92bb89fbc69f3759ef336d676fad643", - "workload": "chat", - "runtime": "io-uring-workers", - "workers": 4, - "generator_revision": "82194441a92bb89fbc69f3759ef336d676fad643", - "instruction_period": 10000019, - "sampling": "concurrent with the counter window", - "relay_cpus": "4-7", - "generator_cpus": "0-3", - "cells": [ - {"revision": "2d73eea67d083ecf18938f4fc7c4842118eb87cf", "label": "base-2d73eea67d08", "round": 1, "kinstr_per_frame": 123.1, "kcycles_per_frame": 235.4, "recv_fps": 20001, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 3.56, "samples": 1221, "markers": {"learn_end": 0.0, "poll_forward_drop": 7.9, "cursor_probe": 0.0, "syscall_leaf": 22.4, "mutex_leaf": 16.7}}, - {"revision": "2d73eea67d083ecf18938f4fc7c4842118eb87cf", "label": "base-2d73eea67d08", "round": 2, "kinstr_per_frame": 125.3, "kcycles_per_frame": 247.9, "recv_fps": 19999, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 3.19, "samples": 1248, "markers": {"learn_end": 0.0, "poll_forward_drop": 8.4, "cursor_probe": 0.0, "syscall_leaf": 24.8, "mutex_leaf": 15.3}}, - {"revision": "2d73eea67d083ecf18938f4fc7c4842118eb87cf", "label": "base-2d73eea67d08", "round": 3, "kinstr_per_frame": 114.9, "kcycles_per_frame": 297.8, "recv_fps": 19920, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 9.31, "samples": 1138, "markers": {"learn_end": 0.0, "poll_forward_drop": 6.4, "cursor_probe": 0.0, "syscall_leaf": 16.5, "mutex_leaf": 17.9}}, - {"revision": "41cff87192983aae80db976d0058661525daa8ed", "label": "base-41cff8719298", "round": 1, "kinstr_per_frame": 115.5, "kcycles_per_frame": 221.3, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 4.57, "samples": 1148, "markers": {"learn_end": 0.0, "poll_forward_drop": 6.4, "cursor_probe": 0.0, "syscall_leaf": 19.7, "mutex_leaf": 16.6}}, - {"revision": "41cff87192983aae80db976d0058661525daa8ed", "label": "base-41cff8719298", "round": 2, "kinstr_per_frame": 123.6, "kcycles_per_frame": 225.9, "recv_fps": 20003, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 2.99, "samples": 1228, "markers": {"learn_end": 0.0, "poll_forward_drop": 6.8, "cursor_probe": 0.0, "syscall_leaf": 23.7, "mutex_leaf": 17.5}}, - {"revision": "41cff87192983aae80db976d0058661525daa8ed", "label": "base-41cff8719298", "round": 3, "kinstr_per_frame": 142.6, "kcycles_per_frame": 412.5, "recv_fps": 12490, "groups_lost": 25854, "host_load_1m": null, "host_load_1m_logged": 30.78, "samples": 888, "markers": {"learn_end": 0.0, "poll_forward_drop": 4.3, "cursor_probe": 0.0, "syscall_leaf": 16.1, "mutex_leaf": 17.3}}, - {"revision": "72259ab3cc3d3ed53a9f3dad8ff37caeac310b83", "label": "base-72259ab3cc3d", "round": 1, "kinstr_per_frame": 126.5, "kcycles_per_frame": 244.6, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 3.19, "samples": 1259, "markers": {"learn_end": 0.0, "poll_forward_drop": 8.7, "cursor_probe": 0.0, "syscall_leaf": 24.7, "mutex_leaf": 15.3}}, - {"revision": "72259ab3cc3d3ed53a9f3dad8ff37caeac310b83", "label": "base-72259ab3cc3d", "round": 2, "kinstr_per_frame": 127.0, "kcycles_per_frame": 233.8, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 3.9, "samples": 1262, "markers": {"learn_end": 0.0, "poll_forward_drop": 9.0, "cursor_probe": 0.0, "syscall_leaf": 24.6, "mutex_leaf": 15.1}}, - {"revision": "72259ab3cc3d3ed53a9f3dad8ff37caeac310b83", "label": "base-72259ab3cc3d", "round": 3, "kinstr_per_frame": 120.2, "kcycles_per_frame": 238.3, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 9.69, "samples": 1193, "markers": {"learn_end": 0.0, "poll_forward_drop": 8.4, "cursor_probe": 0.0, "syscall_leaf": 21.8, "mutex_leaf": 16.7}}, - {"revision": "756637200cb10943871f6c51171f33a340addb6e", "label": "base-756637200cb1", "round": 1, "kinstr_per_frame": 138.4, "kcycles_per_frame": 267.2, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 3.97, "samples": 1378, "markers": {"learn_end": 0.0, "poll_forward_drop": 7.3, "cursor_probe": 0.0, "syscall_leaf": 21.9, "mutex_leaf": 15.1}}, - {"revision": "756637200cb10943871f6c51171f33a340addb6e", "label": "base-756637200cb1", "round": 2, "kinstr_per_frame": 133.9, "kcycles_per_frame": 256.6, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 3.41, "samples": 1330, "markers": {"learn_end": 0.0, "poll_forward_drop": 6.4, "cursor_probe": 0.0, "syscall_leaf": 18.4, "mutex_leaf": 16.9}}, - {"revision": "756637200cb10943871f6c51171f33a340addb6e", "label": "base-756637200cb1", "round": 3, "kinstr_per_frame": 133.7, "kcycles_per_frame": 266.5, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 15.73, "samples": 1331, "markers": {"learn_end": 0.0, "poll_forward_drop": 7.2, "cursor_probe": 0.0, "syscall_leaf": 21.5, "mutex_leaf": 15.2}}, - {"revision": "7648ce03908df33c168f7471787f3572e87eeddb", "label": "base-7648ce03908d", "round": 1, "kinstr_per_frame": 121.2, "kcycles_per_frame": 231.2, "recv_fps": 19998, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 4.22, "samples": 1204, "markers": {"learn_end": 0.0, "poll_forward_drop": 7.1, "cursor_probe": 0.0, "syscall_leaf": 22.8, "mutex_leaf": 17.4}}, - {"revision": "7648ce03908df33c168f7471787f3572e87eeddb", "label": "base-7648ce03908d", "round": 2, "kinstr_per_frame": 126.1, "kcycles_per_frame": 230.6, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 2.31, "samples": 1255, "markers": {"learn_end": 0.0, "poll_forward_drop": 8.0, "cursor_probe": 0.0, "syscall_leaf": 25.7, "mutex_leaf": 15.9}}, - {"revision": "7648ce03908df33c168f7471787f3572e87eeddb", "label": "base-7648ce03908d", "round": 3, "kinstr_per_frame": 156.9, "kcycles_per_frame": 384.1, "recv_fps": 13977, "groups_lost": 35553, "host_load_1m": null, "host_load_1m_logged": 55.74, "samples": 1087, "markers": {"learn_end": 0.0, "poll_forward_drop": 4.5, "cursor_probe": 0.0, "syscall_leaf": 20.8, "mutex_leaf": 17.7}}, - {"revision": "a9a85580a4488a3ad3739d8d130ebb0f66e80510", "label": "base-a9a85580a448", "round": 1, "kinstr_per_frame": 132.3, "kcycles_per_frame": 268.1, "recv_fps": 20001, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 4.9, "samples": 1316, "markers": {"learn_end": 0.0, "poll_forward_drop": 7.6, "cursor_probe": 0.0, "syscall_leaf": 22.1, "mutex_leaf": 17.3}}, - {"revision": "a9a85580a4488a3ad3739d8d130ebb0f66e80510", "label": "base-a9a85580a448", "round": 2, "kinstr_per_frame": 133.6, "kcycles_per_frame": 262.5, "recv_fps": 19999, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 3.31, "samples": 1330, "markers": {"learn_end": 0.0, "poll_forward_drop": 8.3, "cursor_probe": 0.0, "syscall_leaf": 22.3, "mutex_leaf": 16.7}}, - {"revision": "a9a85580a4488a3ad3739d8d130ebb0f66e80510", "label": "base-a9a85580a448", "round": 3, "kinstr_per_frame": 124.0, "kcycles_per_frame": 283.8, "recv_fps": 19998, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 18.89, "samples": 1237, "markers": {"learn_end": 0.0, "poll_forward_drop": 7.7, "cursor_probe": 0.0, "syscall_leaf": 17.3, "mutex_leaf": 17.8}}, - {"revision": "b4dfafac8879ffc4cb661c0d47200aeccea7a2ba", "label": "base-b4dfafac8879", "round": 1, "kinstr_per_frame": 130.1, "kcycles_per_frame": 243.7, "recv_fps": 20004, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 4.07, "samples": 1290, "markers": {"learn_end": 0.0, "poll_forward_drop": 9.5, "cursor_probe": 0.0, "syscall_leaf": 22.7, "mutex_leaf": 15.4}}, - {"revision": "b4dfafac8879ffc4cb661c0d47200aeccea7a2ba", "label": "base-b4dfafac8879", "round": 2, "kinstr_per_frame": 128.5, "kcycles_per_frame": 229.6, "recv_fps": 20001, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 3.77, "samples": 1278, "markers": {"learn_end": 0.0, "poll_forward_drop": 7.5, "cursor_probe": 0.0, "syscall_leaf": 24.2, "mutex_leaf": 14.9}}, - {"revision": "b4dfafac8879ffc4cb661c0d47200aeccea7a2ba", "label": "base-b4dfafac8879", "round": 3, "kinstr_per_frame": 117.7, "kcycles_per_frame": 282.4, "recv_fps": 20001, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 11.64, "samples": 1170, "markers": {"learn_end": 0.0, "poll_forward_drop": 5.9, "cursor_probe": 0.0, "syscall_leaf": 18.4, "mutex_leaf": 18.3}}, - {"revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", "label": "base-e84ffb3de313", "round": 1, "kinstr_per_frame": 136.9, "kcycles_per_frame": 260.8, "recv_fps": 20003, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 3.43, "samples": 1363, "markers": {"learn_end": 0.0, "poll_forward_drop": 9.6, "cursor_probe": 0.0, "syscall_leaf": 22.5, "mutex_leaf": 16.7}}, - {"revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", "label": "base-e84ffb3de313", "round": 2, "kinstr_per_frame": 134.9, "kcycles_per_frame": 262.6, "recv_fps": 20001, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 3.26, "samples": 1341, "markers": {"learn_end": 0.0, "poll_forward_drop": 7.8, "cursor_probe": 0.0, "syscall_leaf": 23.3, "mutex_leaf": 15.5}}, - {"revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", "label": "base-e84ffb3de313", "round": 3, "kinstr_per_frame": 134.9, "kcycles_per_frame": 267.8, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 11.14, "samples": 1342, "markers": {"learn_end": 0.0, "poll_forward_drop": 8.5, "cursor_probe": 0.0, "syscall_leaf": 20.1, "mutex_leaf": 17.2}}, - {"revision": "82194441a92bb89fbc69f3759ef336d676fad643", "label": "current", "round": 1, "kinstr_per_frame": 147.4, "kcycles_per_frame": 264.3, "recv_fps": 20001, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 3.09, "samples": 1466, "markers": {"learn_end": 0.4, "poll_forward_drop": 15.8, "cursor_probe": 8.0, "syscall_leaf": 17.9, "mutex_leaf": 17.9}}, - {"revision": "82194441a92bb89fbc69f3759ef336d676fad643", "label": "current", "round": 2, "kinstr_per_frame": 148.0, "kcycles_per_frame": 261.4, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 3.13, "samples": 1474, "markers": {"learn_end": 0.1, "poll_forward_drop": 14.1, "cursor_probe": 7.7, "syscall_leaf": 18.7, "mutex_leaf": 19.3}}, - {"revision": "82194441a92bb89fbc69f3759ef336d676fad643", "label": "current", "round": 3, "kinstr_per_frame": 148.0, "kcycles_per_frame": 266.1, "recv_fps": 19997, "groups_lost": 0, "host_load_1m": null, "host_load_1m_logged": 7.96, "samples": 1472, "markers": {"learn_end": 0.6, "poll_forward_drop": 15.4, "cursor_probe": 8.4, "syscall_leaf": 16.6, "mutex_leaf": 18.9}} - ] - }, - { - "artifact": "run.BIL6LP", - "harness_revision": "d935d13e8dba7d36e06f836a04984d940dbf0a57", - "workload": "chat", - "runtime": "io-uring-workers", - "workers": 4, - "generator_revision": "d935d13e8dba7d36e06f836a04984d940dbf0a57", - "instruction_period": 2000003, - "sampling": "concurrent with the counter window", - "relay_cpus": "4-7", - "generator_cpus": "0-3", - "cells": [ - {"revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", "label": "base-0c2a798de5c2", "round": 1, "kinstr_per_frame": 160.0, "kcycles_per_frame": 331.8, "recv_fps": 19997, "groups_lost": 0, "host_load_1m": 7.69, "host_load_1m_logged": 8.5, "samples": 7997, "markers": {"learn_end": 14.7, "poll_forward_drop": 9.7, "cursor_probe": 0.0, "syscall_leaf": 24.1, "mutex_leaf": 16.7}}, - {"revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", "label": "base-0c2a798de5c2", "round": 2, "kinstr_per_frame": 166.2, "kcycles_per_frame": 323.7, "recv_fps": 20003, "groups_lost": 0, "host_load_1m": 2.31, "host_load_1m_logged": 2.29, "samples": 8320, "markers": {"learn_end": 15.5, "poll_forward_drop": 11.3, "cursor_probe": 0.0, "syscall_leaf": 27.1, "mutex_leaf": 16.2}}, - {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 1, "kinstr_per_frame": 173.9, "kcycles_per_frame": 310.8, "recv_fps": 19997, "groups_lost": 0, "host_load_1m": 3.01, "host_load_1m_logged": 3.01, "samples": 8696, "markers": {"learn_end": 0.9, "poll_forward_drop": 20.9, "cursor_probe": 6.9, "syscall_leaf": 21.9, "mutex_leaf": 21.4}}, - {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 2, "kinstr_per_frame": 178.0, "kcycles_per_frame": 316.8, "recv_fps": 19997, "groups_lost": 0, "host_load_1m": 2.89, "host_load_1m_logged": 2.74, "samples": 8898, "markers": {"learn_end": 1.0, "poll_forward_drop": 20.9, "cursor_probe": 7.4, "syscall_leaf": 23.1, "mutex_leaf": 21.8}}, - {"revision": "2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3", "label": "base-2fff7ea7ae4e", "round": 1, "kinstr_per_frame": 209.5, "kcycles_per_frame": 415.5, "recv_fps": 19997, "groups_lost": 0, "host_load_1m": 2.45, "host_load_1m_logged": 2.43, "samples": 10473, "markers": {"learn_end": 16.6, "poll_forward_drop": 23.4, "cursor_probe": 7.3, "syscall_leaf": 31.6, "mutex_leaf": 17.2}}, - {"revision": "2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3", "label": "base-2fff7ea7ae4e", "round": 2, "kinstr_per_frame": 213.5, "kcycles_per_frame": 430.9, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 2.55, "host_load_1m_logged": 2.51, "samples": 10671, "markers": {"learn_end": 16.6, "poll_forward_drop": 23.4, "cursor_probe": 7.7, "syscall_leaf": 32.4, "mutex_leaf": 17.4}}, - {"revision": "49b5ec9b5630c54f2c5e99657b114535087b7da2", "label": "base-49b5ec9b5630", "round": 1, "kinstr_per_frame": 211.8, "kcycles_per_frame": 415.6, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 4.47, "host_load_1m_logged": 4.47, "samples": 10583, "markers": {"learn_end": 13.1, "poll_forward_drop": 19.1, "cursor_probe": 8.2, "syscall_leaf": 31.8, "mutex_leaf": 17.5}}, - {"revision": "49b5ec9b5630c54f2c5e99657b114535087b7da2", "label": "base-49b5ec9b5630", "round": 2, "kinstr_per_frame": 211.3, "kcycles_per_frame": 395.3, "recv_fps": 20003, "groups_lost": 0, "host_load_1m": 1.87, "host_load_1m_logged": 2.03, "samples": 10560, "markers": {"learn_end": 13.2, "poll_forward_drop": 19.3, "cursor_probe": 8.0, "syscall_leaf": 31.2, "mutex_leaf": 18.0}}, - {"revision": "55f6ed229250d17fc99389237659906d011d8a94", "label": "base-55f6ed229250", "round": 1, "kinstr_per_frame": 161.1, "kcycles_per_frame": 313.9, "recv_fps": 19998, "groups_lost": 0, "host_load_1m": 5.46, "host_load_1m_logged": 5.32, "samples": 8050, "markers": {"learn_end": 15.6, "poll_forward_drop": 10.9, "cursor_probe": 0.0, "syscall_leaf": 26.5, "mutex_leaf": 18.6}}, - {"revision": "55f6ed229250d17fc99389237659906d011d8a94", "label": "base-55f6ed229250", "round": 2, "kinstr_per_frame": 165.5, "kcycles_per_frame": 320.9, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 3.07, "host_load_1m_logged": 2.99, "samples": 8282, "markers": {"learn_end": 15.9, "poll_forward_drop": 11.0, "cursor_probe": 0.0, "syscall_leaf": 28.6, "mutex_leaf": 16.7}}, - {"revision": "991572d636d264c6189a3477849d2969e689f55c", "label": "base-991572d636d2", "round": 1, "kinstr_per_frame": 206.0, "kcycles_per_frame": 402.2, "recv_fps": 20003, "groups_lost": 0, "host_load_1m": 3.6, "host_load_1m_logged": 3.6, "samples": 10295, "markers": {"learn_end": 15.4, "poll_forward_drop": 19.7, "cursor_probe": 8.0, "syscall_leaf": 36.2, "mutex_leaf": 16.3}}, - {"revision": "991572d636d264c6189a3477849d2969e689f55c", "label": "base-991572d636d2", "round": 2, "kinstr_per_frame": 207.0, "kcycles_per_frame": 402.5, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 2.61, "host_load_1m_logged": 2.51, "samples": 10344, "markers": {"learn_end": 16.0, "poll_forward_drop": 19.6, "cursor_probe": 7.0, "syscall_leaf": 35.0, "mutex_leaf": 16.5}}, - {"revision": "e45064e3ccb43a535adb949b7db9a5a0aeb32413", "label": "base-e45064e3ccb4", "round": 1, "kinstr_per_frame": 158.5, "kcycles_per_frame": 316.3, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 5.21, "host_load_1m_logged": 5.97, "samples": 7921, "markers": {"learn_end": 14.5, "poll_forward_drop": 10.0, "cursor_probe": 0.0, "syscall_leaf": 24.8, "mutex_leaf": 17.0}}, - {"revision": "e45064e3ccb43a535adb949b7db9a5a0aeb32413", "label": "base-e45064e3ccb4", "round": 2, "kinstr_per_frame": 168.4, "kcycles_per_frame": 329.1, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 2.27, "host_load_1m_logged": 2.27, "samples": 8424, "markers": {"learn_end": 16.3, "poll_forward_drop": 11.4, "cursor_probe": 0.0, "syscall_leaf": 27.9, "mutex_leaf": 16.2}}, - {"revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", "label": "base-e84ffb3de313", "round": 1, "kinstr_per_frame": 136.2, "kcycles_per_frame": 277.1, "recv_fps": 19998, "groups_lost": 0, "host_load_1m": 6.92, "host_load_1m_logged": 7.17, "samples": 6802, "markers": {"learn_end": 0.0, "poll_forward_drop": 7.5, "cursor_probe": 0.0, "syscall_leaf": 21.4, "mutex_leaf": 16.4}}, - {"revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", "label": "base-e84ffb3de313", "round": 2, "kinstr_per_frame": 133.8, "kcycles_per_frame": 258.5, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 3.14, "host_load_1m_logged": 3.21, "samples": 6691, "markers": {"learn_end": 0.0, "poll_forward_drop": 7.5, "cursor_probe": 0.0, "syscall_leaf": 21.1, "mutex_leaf": 16.6}}, - {"revision": "f206893047dc2f1db7b3422622dbb505470d0c70", "label": "base-f206893047dc", "round": 1, "kinstr_per_frame": 168.5, "kcycles_per_frame": 338.2, "recv_fps": 20004, "groups_lost": 0, "host_load_1m": 9.21, "host_load_1m_logged": 9.23, "samples": 8424, "markers": {"learn_end": 16.6, "poll_forward_drop": 10.8, "cursor_probe": 0.0, "syscall_leaf": 29.4, "mutex_leaf": 16.6}}, - {"revision": "f206893047dc2f1db7b3422622dbb505470d0c70", "label": "base-f206893047dc", "round": 2, "kinstr_per_frame": 165.9, "kcycles_per_frame": 325.0, "recv_fps": 19997, "groups_lost": 0, "host_load_1m": 2.56, "host_load_1m_logged": 2.4, "samples": 8293, "markers": {"learn_end": 16.4, "poll_forward_drop": 11.2, "cursor_probe": 0.0, "syscall_leaf": 27.2, "mutex_leaf": 16.5}}, - {"revision": "d935d13e8dba7d36e06f836a04984d940dbf0a57", "label": "current", "round": 1, "kinstr_per_frame": 147.6, "kcycles_per_frame": 260.9, "recv_fps": 19997, "groups_lost": 0, "host_load_1m": 2.72, "host_load_1m_logged": 2.83, "samples": 7372, "markers": {"learn_end": 0.4, "poll_forward_drop": 14.5, "cursor_probe": 7.3, "syscall_leaf": 17.8, "mutex_leaf": 19.0}}, - {"revision": "d935d13e8dba7d36e06f836a04984d940dbf0a57", "label": "current", "round": 2, "kinstr_per_frame": 147.0, "kcycles_per_frame": 251.2, "recv_fps": 19999, "groups_lost": 0, "host_load_1m": 2.99, "host_load_1m_logged": 3.16, "samples": 7347, "markers": {"learn_end": 0.5, "poll_forward_drop": 14.5, "cursor_probe": 6.9, "syscall_leaf": 17.5, "mutex_leaf": 18.9}} - ] - }, - { - "artifact": "run.1KRu4S", - "harness_revision": "a6cd6f91b6a0c79a81f2a012a2683bc77bd6c7d7", - "workload": "chat", - "runtime": "io-uring-workers", - "workers": 4, - "generator_revision": "21d6da9fc1", - "instruction_period": 2000003, - "sampling": "concurrent with the counter window", - "relay_cpus": "4-7", - "generator_cpus": "0-3", - "cells": [ - {"revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", "label": "base-0c2a798de5c2", "round": 1, "kinstr_per_frame": 165.1, "kcycles_per_frame": 318.0, "recv_fps": 19999, "groups_lost": 0, "host_load_1m": 2.58, "host_load_1m_logged": 2.58, "samples": 8253, "markers": {"learn_end": 15.7, "poll_forward_drop": 10.8, "cursor_probe": 0.0, "syscall_leaf": 26.7, "mutex_leaf": 16.7}}, - {"revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", "label": "base-0c2a798de5c2", "round": 2, "kinstr_per_frame": 166.1, "kcycles_per_frame": 322.2, "recv_fps": 20001, "groups_lost": 0, "host_load_1m": 4.23, "host_load_1m_logged": 3.99, "samples": 8304, "markers": {"learn_end": 15.0, "poll_forward_drop": 10.7, "cursor_probe": 0.0, "syscall_leaf": 27.0, "mutex_leaf": 17.0}}, - {"revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", "label": "base-0c2a798de5c2", "round": 3, "kinstr_per_frame": 162.1, "kcycles_per_frame": 327.9, "recv_fps": 19997, "groups_lost": 0, "host_load_1m": 7.69, "host_load_1m_logged": 7.69, "samples": 8100, "markers": {"learn_end": 14.9, "poll_forward_drop": 10.9, "cursor_probe": 0.0, "syscall_leaf": 26.4, "mutex_leaf": 17.0}}, - {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 1, "kinstr_per_frame": 175.2, "kcycles_per_frame": 327.1, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 3.17, "host_load_1m_logged": 2.99, "samples": 8769, "markers": {"learn_end": 1.3, "poll_forward_drop": 22.0, "cursor_probe": 7.4, "syscall_leaf": 22.8, "mutex_leaf": 20.7}}, - {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 2, "kinstr_per_frame": 173.5, "kcycles_per_frame": 322.5, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 3.03, "host_load_1m_logged": 3.04, "samples": 8677, "markers": {"learn_end": 1.2, "poll_forward_drop": 21.4, "cursor_probe": 7.7, "syscall_leaf": 21.9, "mutex_leaf": 21.8}}, - {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 3, "kinstr_per_frame": 125.6, "kcycles_per_frame": 279.7, "recv_fps": 20569, "groups_lost": 32897, "host_load_1m": 24.77, "host_load_1m_logged": 24.77, "samples": 6509, "markers": {"learn_end": 0.4, "poll_forward_drop": 14.9, "cursor_probe": 6.5, "syscall_leaf": 12.4, "mutex_leaf": 23.5}}, - {"revision": "2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3", "label": "base-2fff7ea7ae4e", "round": 1, "kinstr_per_frame": 207.1, "kcycles_per_frame": 412.6, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 2.9, "host_load_1m_logged": 2.98, "samples": 10354, "markers": {"learn_end": 16.7, "poll_forward_drop": 22.5, "cursor_probe": 7.0, "syscall_leaf": 29.5, "mutex_leaf": 17.7}}, - {"revision": "2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3", "label": "base-2fff7ea7ae4e", "round": 2, "kinstr_per_frame": 202.8, "kcycles_per_frame": 412.8, "recv_fps": 20001, "groups_lost": 0, "host_load_1m": 3.67, "host_load_1m_logged": 4.13, "samples": 10136, "markers": {"learn_end": 16.3, "poll_forward_drop": 22.0, "cursor_probe": 6.7, "syscall_leaf": 29.1, "mutex_leaf": 18.1}}, - {"revision": "2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3", "label": "base-2fff7ea7ae4e", "round": 3, "kinstr_per_frame": 189.9, "kcycles_per_frame": 401.7, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 11.36, "host_load_1m_logged": 11.56, "samples": 9511, "markers": {"learn_end": 15.0, "poll_forward_drop": 21.0, "cursor_probe": 7.0, "syscall_leaf": 24.8, "mutex_leaf": 19.0}}, - {"revision": "55f6ed229250d17fc99389237659906d011d8a94", "label": "base-55f6ed229250", "round": 1, "kinstr_per_frame": 171.1, "kcycles_per_frame": 332.6, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 2.19, "host_load_1m_logged": 2.06, "samples": 8555, "markers": {"learn_end": 17.3, "poll_forward_drop": 11.6, "cursor_probe": 0.0, "syscall_leaf": 30.6, "mutex_leaf": 16.7}}, - {"revision": "55f6ed229250d17fc99389237659906d011d8a94", "label": "base-55f6ed229250", "round": 2, "kinstr_per_frame": 167.3, "kcycles_per_frame": 337.6, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 3.78, "host_load_1m_logged": 3.72, "samples": 8366, "markers": {"learn_end": 16.0, "poll_forward_drop": 10.1, "cursor_probe": 0.0, "syscall_leaf": 28.8, "mutex_leaf": 16.7}}, - {"revision": "55f6ed229250d17fc99389237659906d011d8a94", "label": "base-55f6ed229250", "round": 3, "kinstr_per_frame": 169.2, "kcycles_per_frame": 337.9, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 5.82, "host_load_1m_logged": 6.05, "samples": 8458, "markers": {"learn_end": 16.8, "poll_forward_drop": 11.5, "cursor_probe": 0.0, "syscall_leaf": 30.4, "mutex_leaf": 16.4}}, - {"revision": "991572d636d264c6189a3477849d2969e689f55c", "label": "base-991572d636d2", "round": 1, "kinstr_per_frame": 202.7, "kcycles_per_frame": 392.8, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 2.06, "host_load_1m_logged": 2.15, "samples": 10130, "markers": {"learn_end": 15.3, "poll_forward_drop": 19.7, "cursor_probe": 7.3, "syscall_leaf": 33.7, "mutex_leaf": 16.7}}, - {"revision": "991572d636d264c6189a3477849d2969e689f55c", "label": "base-991572d636d2", "round": 2, "kinstr_per_frame": 195.1, "kcycles_per_frame": 379.0, "recv_fps": 20002, "groups_lost": 0, "host_load_1m": 4.73, "host_load_1m_logged": 4.91, "samples": 9752, "markers": {"learn_end": 15.2, "poll_forward_drop": 18.3, "cursor_probe": 6.9, "syscall_leaf": 32.3, "mutex_leaf": 16.6}}, - {"revision": "991572d636d264c6189a3477849d2969e689f55c", "label": "base-991572d636d2", "round": 3, "kinstr_per_frame": 184.4, "kcycles_per_frame": 365.8, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 6.87, "host_load_1m_logged": 7.05, "samples": 9215, "markers": {"learn_end": 14.6, "poll_forward_drop": 18.3, "cursor_probe": 7.1, "syscall_leaf": 27.8, "mutex_leaf": 18.1}}, - {"revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", "label": "base-cdff3c8f952b", "round": 1, "kinstr_per_frame": 118.7, "kcycles_per_frame": 212.2, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 2.22, "host_load_1m_logged": 2.44, "samples": 5942, "markers": {"learn_end": 0.0, "poll_forward_drop": 6.8, "cursor_probe": 0.0, "syscall_leaf": 25.2, "mutex_leaf": 16.1}}, - {"revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", "label": "base-cdff3c8f952b", "round": 2, "kinstr_per_frame": 111.5, "kcycles_per_frame": 222.3, "recv_fps": 19999, "groups_lost": 0, "host_load_1m": 4.8, "host_load_1m_logged": 5.2, "samples": 5579, "markers": {"learn_end": 0.0, "poll_forward_drop": 7.4, "cursor_probe": 0.0, "syscall_leaf": 22.4, "mutex_leaf": 18.1}}, - {"revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", "label": "base-cdff3c8f952b", "round": 3, "kinstr_per_frame": 109.8, "kcycles_per_frame": 235.5, "recv_fps": 20003, "groups_lost": 0, "host_load_1m": 21.62, "host_load_1m_logged": 24.0, "samples": 5484, "markers": {"learn_end": 0.0, "poll_forward_drop": 6.9, "cursor_probe": 0.0, "syscall_leaf": 21.3, "mutex_leaf": 17.3}}, - {"revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", "label": "base-e84ffb3de313", "round": 1, "kinstr_per_frame": 135.0, "kcycles_per_frame": 266.2, "recv_fps": 19997, "groups_lost": 0, "host_load_1m": 2.84, "host_load_1m_logged": 3.41, "samples": 6746, "markers": {"learn_end": 0.0, "poll_forward_drop": 7.8, "cursor_probe": 0.0, "syscall_leaf": 22.2, "mutex_leaf": 16.0}}, - {"revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", "label": "base-e84ffb3de313", "round": 2, "kinstr_per_frame": 136.9, "kcycles_per_frame": 267.9, "recv_fps": 19997, "groups_lost": 0, "host_load_1m": 4.93, "host_load_1m_logged": 5.19, "samples": 6846, "markers": {"learn_end": 0.0, "poll_forward_drop": 8.0, "cursor_probe": 0.0, "syscall_leaf": 22.6, "mutex_leaf": 16.1}}, - {"revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", "label": "base-e84ffb3de313", "round": 3, "kinstr_per_frame": 116.1, "kcycles_per_frame": 310.9, "recv_fps": 22018, "groups_lost": 31280, "host_load_1m": 27.7, "host_load_1m_logged": 27.7, "samples": 6404, "markers": {"learn_end": 0.0, "poll_forward_drop": 4.8, "cursor_probe": 0.0, "syscall_leaf": 16.0, "mutex_leaf": 14.5}}, - {"revision": "a6cd6f91b6a0c79a81f2a012a2683bc77bd6c7d7", "label": "current", "round": 1, "kinstr_per_frame": 145.6, "kcycles_per_frame": 272.3, "recv_fps": 19997, "groups_lost": 0, "host_load_1m": 9.76, "host_load_1m_logged": 11.0, "samples": 7274, "markers": {"learn_end": 0.5, "poll_forward_drop": 14.8, "cursor_probe": 6.8, "syscall_leaf": 17.3, "mutex_leaf": 19.5}}, - {"revision": "a6cd6f91b6a0c79a81f2a012a2683bc77bd6c7d7", "label": "current", "round": 2, "kinstr_per_frame": 146.5, "kcycles_per_frame": 281.0, "recv_fps": 19998, "groups_lost": 0, "host_load_1m": 3.64, "host_load_1m_logged": 3.63, "samples": 7315, "markers": {"learn_end": 0.5, "poll_forward_drop": 14.3, "cursor_probe": 6.9, "syscall_leaf": 16.8, "mutex_leaf": 19.6}}, - {"revision": "a6cd6f91b6a0c79a81f2a012a2683bc77bd6c7d7", "label": "current", "round": 3, "kinstr_per_frame": 143.1, "kcycles_per_frame": 313.0, "recv_fps": 19304, "groups_lost": 0, "host_load_1m": 32.43, "host_load_1m_logged": 32.92, "samples": 6910, "markers": {"learn_end": 0.4, "poll_forward_drop": 14.1, "cursor_probe": 7.4, "syscall_leaf": 14.0, "mutex_leaf": 19.5}} - ] - }, - { - "artifact": "run.qYPSii", - "harness_revision": "74f46e79b75bb2cbb4c5d4c4d75e390249ba22d5", - "workload": "chat", - "runtime": "io-uring-workers", - "workers": 4, - "generator_revision": "21d6da9fc1", - "instruction_period": 2000003, - "sampling": "after the counter window", - "relay_cpus": "4-7", - "generator_cpus": "0-3", - "cells": [ - {"revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", "label": "base-0c2a798de5c2", "round": 1, "kinstr_per_frame": 246.0, "kcycles_per_frame": 599.4, "recv_fps": 7525, "groups_lost": 16750, "host_load_1m": 45.62, "host_load_1m_logged": 54.66, "samples": 5208, "markers": {"learn_end": 8.2, "poll_forward_drop": 11.1, "cursor_probe": 0.0, "syscall_leaf": 14.2, "mutex_leaf": 15.9}}, - {"revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", "label": "base-0c2a798de5c2", "round": 2, "kinstr_per_frame": 149.9, "kcycles_per_frame": 304.2, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 19.15, "host_load_1m_logged": 19.77, "samples": 7163, "markers": {"learn_end": 12.1, "poll_forward_drop": 8.9, "cursor_probe": 0.0, "syscall_leaf": 18.0, "mutex_leaf": 20.2}}, - {"revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", "label": "base-0c2a798de5c2", "round": 3, "kinstr_per_frame": 169.8, "kcycles_per_frame": 325.1, "recv_fps": 19999, "groups_lost": 0, "host_load_1m": 4.38, "host_load_1m_logged": 4.75, "samples": 8337, "markers": {"learn_end": 15.4, "poll_forward_drop": 11.7, "cursor_probe": 0.0, "syscall_leaf": 27.5, "mutex_leaf": 16.8}}, - {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 1, "kinstr_per_frame": 160.3, "kcycles_per_frame": 305.2, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 17.94, "host_load_1m_logged": 18.93, "samples": 7895, "markers": {"learn_end": 0.9, "poll_forward_drop": 19.7, "cursor_probe": 7.6, "syscall_leaf": 15.5, "mutex_leaf": 23.8}}, - {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 2, "kinstr_per_frame": 167.9, "kcycles_per_frame": 337.0, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 9.63, "host_load_1m_logged": 9.43, "samples": 8307, "markers": {"learn_end": 1.1, "poll_forward_drop": 21.0, "cursor_probe": 7.2, "syscall_leaf": 19.2, "mutex_leaf": 22.7}}, - {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 3, "kinstr_per_frame": 176.2, "kcycles_per_frame": 328.6, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 3.1, "host_load_1m_logged": 3.64, "samples": 8758, "markers": {"learn_end": 1.3, "poll_forward_drop": 21.2, "cursor_probe": 7.3, "syscall_leaf": 22.3, "mutex_leaf": 20.7}}, - {"revision": "2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3", "label": "base-2fff7ea7ae4e", "round": 1, "kinstr_per_frame": 169.1, "kcycles_per_frame": 409.6, "recv_fps": 17369, "groups_lost": 5362, "host_load_1m": 24.92, "host_load_1m_logged": 23.96, "samples": 9126, "markers": {"learn_end": 13.8, "poll_forward_drop": 18.9, "cursor_probe": 6.2, "syscall_leaf": 17.4, "mutex_leaf": 21.8}}, - {"revision": "2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3", "label": "base-2fff7ea7ae4e", "round": 2, "kinstr_per_frame": 161.7, "kcycles_per_frame": 349.3, "recv_fps": 19820, "groups_lost": 0, "host_load_1m": 11.62, "host_load_1m_logged": 10.84, "samples": 8749, "markers": {"learn_end": 14.1, "poll_forward_drop": 18.8, "cursor_probe": 6.7, "syscall_leaf": 19.0, "mutex_leaf": 21.1}}, - {"revision": "2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3", "label": "base-2fff7ea7ae4e", "round": 3, "kinstr_per_frame": 214.0, "kcycles_per_frame": 415.4, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 2.35, "host_load_1m_logged": 2.56, "samples": 10709, "markers": {"learn_end": 17.0, "poll_forward_drop": 24.0, "cursor_probe": 7.6, "syscall_leaf": 32.7, "mutex_leaf": 16.9}}, - {"revision": "55f6ed229250d17fc99389237659906d011d8a94", "label": "base-55f6ed229250", "round": 1, "kinstr_per_frame": 245.2, "kcycles_per_frame": 655.0, "recv_fps": 7433, "groups_lost": 28655, "host_load_1m": 53.37, "host_load_1m_logged": 53.25, "samples": 5224, "markers": {"learn_end": 7.2, "poll_forward_drop": 8.3, "cursor_probe": 0.0, "syscall_leaf": 15.9, "mutex_leaf": 16.9}}, - {"revision": "55f6ed229250d17fc99389237659906d011d8a94", "label": "base-55f6ed229250", "round": 2, "kinstr_per_frame": 166.6, "kcycles_per_frame": 340.8, "recv_fps": 19997, "groups_lost": 0, "host_load_1m": 11.77, "host_load_1m_logged": 12.27, "samples": 8277, "markers": {"learn_end": 16.1, "poll_forward_drop": 10.8, "cursor_probe": 0.0, "syscall_leaf": 27.6, "mutex_leaf": 17.9}}, - {"revision": "55f6ed229250d17fc99389237659906d011d8a94", "label": "base-55f6ed229250", "round": 3, "kinstr_per_frame": 174.6, "kcycles_per_frame": 343.7, "recv_fps": 19999, "groups_lost": 0, "host_load_1m": 3.23, "host_load_1m_logged": 3.42, "samples": 8758, "markers": {"learn_end": 17.5, "poll_forward_drop": 11.7, "cursor_probe": 0.0, "syscall_leaf": 31.6, "mutex_leaf": 16.3}}, - {"revision": "991572d636d264c6189a3477849d2969e689f55c", "label": "base-991572d636d2", "round": 1, "kinstr_per_frame": 164.2, "kcycles_per_frame": 331.5, "recv_fps": 19825, "groups_lost": 0, "host_load_1m": 30.42, "host_load_1m_logged": 31.94, "samples": 8019, "markers": {"learn_end": 11.0, "poll_forward_drop": 15.1, "cursor_probe": 6.5, "syscall_leaf": 17.6, "mutex_leaf": 20.5}}, - {"revision": "991572d636d264c6189a3477849d2969e689f55c", "label": "base-991572d636d2", "round": 2, "kinstr_per_frame": 181.6, "kcycles_per_frame": 363.0, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 9.45, "host_load_1m_logged": 9.45, "samples": 9524, "markers": {"learn_end": 14.6, "poll_forward_drop": 19.1, "cursor_probe": 7.1, "syscall_leaf": 30.3, "mutex_leaf": 17.2}}, - {"revision": "991572d636d264c6189a3477849d2969e689f55c", "label": "base-991572d636d2", "round": 3, "kinstr_per_frame": 204.7, "kcycles_per_frame": 382.8, "recv_fps": 20001, "groups_lost": 0, "host_load_1m": 2.13, "host_load_1m_logged": 2.32, "samples": 10294, "markers": {"learn_end": 15.5, "poll_forward_drop": 20.1, "cursor_probe": 7.7, "syscall_leaf": 35.0, "mutex_leaf": 16.6}}, - {"revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", "label": "base-cdff3c8f952b", "round": 1, "kinstr_per_frame": 124.4, "kcycles_per_frame": 287.4, "recv_fps": 16559, "groups_lost": 8554, "host_load_1m": 21.13, "host_load_1m_logged": 22.93, "samples": 5763, "markers": {"learn_end": 0.0, "poll_forward_drop": 5.3, "cursor_probe": 0.0, "syscall_leaf": 19.8, "mutex_leaf": 16.7}}, - {"revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", "label": "base-cdff3c8f952b", "round": 2, "kinstr_per_frame": 115.3, "kcycles_per_frame": 219.5, "recv_fps": 20001, "groups_lost": 0, "host_load_1m": 4.79, "host_load_1m_logged": 5.05, "samples": 5711, "markers": {"learn_end": 0.0, "poll_forward_drop": 7.5, "cursor_probe": 0.0, "syscall_leaf": 23.9, "mutex_leaf": 16.1}}, - {"revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", "label": "base-cdff3c8f952b", "round": 3, "kinstr_per_frame": 117.7, "kcycles_per_frame": 214.6, "recv_fps": 19998, "groups_lost": 0, "host_load_1m": 2.38, "host_load_1m_logged": 2.7, "samples": 5823, "markers": {"learn_end": 0.0, "poll_forward_drop": 7.1, "cursor_probe": 0.0, "syscall_leaf": 24.5, "mutex_leaf": 16.0}}, - {"revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", "label": "base-e84ffb3de313", "round": 1, "kinstr_per_frame": 125.5, "kcycles_per_frame": 294.0, "recv_fps": 19998, "groups_lost": 0, "host_load_1m": 21.29, "host_load_1m_logged": 22.36, "samples": 6352, "markers": {"learn_end": 0.0, "poll_forward_drop": 6.3, "cursor_probe": 0.0, "syscall_leaf": 16.8, "mutex_leaf": 17.2}}, - {"revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", "label": "base-e84ffb3de313", "round": 2, "kinstr_per_frame": 128.2, "kcycles_per_frame": 355.6, "recv_fps": 19590, "groups_lost": 0, "host_load_1m": 24.24, "host_load_1m_logged": 27.66, "samples": 6356, "markers": {"learn_end": 0.0, "poll_forward_drop": 5.0, "cursor_probe": 0.0, "syscall_leaf": 17.9, "mutex_leaf": 15.0}}, - {"revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", "label": "base-e84ffb3de313", "round": 3, "kinstr_per_frame": 135.7, "kcycles_per_frame": 264.2, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 2.24, "host_load_1m_logged": 2.24, "samples": 6774, "markers": {"learn_end": 0.0, "poll_forward_drop": 7.6, "cursor_probe": 0.0, "syscall_leaf": 21.6, "mutex_leaf": 16.2}}, - {"revision": "74f46e79b75bb2cbb4c5d4c4d75e390249ba22d5", "label": "current", "round": 1, "kinstr_per_frame": 145.2, "kcycles_per_frame": 278.0, "recv_fps": 19999, "groups_lost": 0, "host_load_1m": 11.7, "host_load_1m_logged": 11.76, "samples": 7185, "markers": {"learn_end": 0.4, "poll_forward_drop": 14.3, "cursor_probe": 6.6, "syscall_leaf": 15.5, "mutex_leaf": 19.8}}, - {"revision": "74f46e79b75bb2cbb4c5d4c4d75e390249ba22d5", "label": "current", "round": 2, "kinstr_per_frame": 147.0, "kcycles_per_frame": 278.5, "recv_fps": 20003, "groups_lost": 0, "host_load_1m": 6.81, "host_load_1m_logged": 6.97, "samples": 7385, "markers": {"learn_end": 0.6, "poll_forward_drop": 15.5, "cursor_probe": 7.1, "syscall_leaf": 17.7, "mutex_leaf": 18.0}}, - {"revision": "74f46e79b75bb2cbb4c5d4c4d75e390249ba22d5", "label": "current", "round": 3, "kinstr_per_frame": 148.8, "kcycles_per_frame": 269.2, "recv_fps": 20001, "groups_lost": 0, "host_load_1m": 3.07, "host_load_1m_logged": 3.07, "samples": 7392, "markers": {"learn_end": 0.5, "poll_forward_drop": 14.4, "cursor_probe": 6.6, "syscall_leaf": 17.7, "mutex_leaf": 19.3}} - ] - }, - { - "artifact": "run.16AJUb", - "harness_revision": "bd812a762c149a4345430b53530a597d261144b8", - "workload": "chat", - "runtime": "io-uring-workers", - "workers": 4, - "generator_revision": "21d6da9fc1", - "instruction_period": 2000003, - "sampling": "after the counter window", - "relay_cpus": "4-7", - "generator_cpus": "0-3", - "cells": [ - {"revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", "label": "base-0c2a798de5c2", "round": 1, "kinstr_per_frame": 158.5, "kcycles_per_frame": 330.8, "recv_fps": 20001, "groups_lost": 0, "host_load_1m": 13.91, "host_load_1m_logged": 14.83, "samples": 7839, "markers": {"learn_end": 14.9, "poll_forward_drop": 10.7, "cursor_probe": 0.0, "syscall_leaf": 23.5, "mutex_leaf": 17.1}}, - {"revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", "label": "base-0c2a798de5c2", "round": 2, "kinstr_per_frame": 139.5, "kcycles_per_frame": 330.1, "recv_fps": 19753, "groups_lost": 0, "host_load_1m": 25.02, "host_load_1m_logged": 25.76, "samples": 6782, "markers": {"learn_end": 11.1, "poll_forward_drop": 8.5, "cursor_probe": 0.0, "syscall_leaf": 15.1, "mutex_leaf": 18.7}}, - {"revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", "label": "base-0c2a798de5c2", "round": 3, "kinstr_per_frame": 140.8, "kcycles_per_frame": 335.0, "recv_fps": 19906, "groups_lost": 0, "host_load_1m": 21.62, "host_load_1m_logged": 21.62, "samples": 7065, "markers": {"learn_end": 12.0, "poll_forward_drop": 9.0, "cursor_probe": 0.0, "syscall_leaf": 16.1, "mutex_leaf": 19.6}}, - {"revision": "1e05129753c720d0b2318cedbab345d745227d2b", "label": "base-1e05129753c7", "round": 1, "kinstr_per_frame": 216.4, "kcycles_per_frame": 513.7, "recv_fps": 9349, "groups_lost": 22438, "host_load_1m": 32.14, "host_load_1m_logged": 32.14, "samples": 5175, "markers": {"learn_end": 8.5, "poll_forward_drop": 6.8, "cursor_probe": 0.0, "syscall_leaf": 13.7, "mutex_leaf": 17.7}}, - {"revision": "1e05129753c720d0b2318cedbab345d745227d2b", "label": "base-1e05129753c7", "round": 2, "kinstr_per_frame": 139.6, "kcycles_per_frame": 328.5, "recv_fps": 19696, "groups_lost": 6849, "host_load_1m": 29.39, "host_load_1m_logged": 29.39, "samples": 7166, "markers": {"learn_end": 12.1, "poll_forward_drop": 8.4, "cursor_probe": 0.0, "syscall_leaf": 17.0, "mutex_leaf": 19.3}}, - {"revision": "1e05129753c720d0b2318cedbab345d745227d2b", "label": "base-1e05129753c7", "round": 3, "kinstr_per_frame": 150.4, "kcycles_per_frame": 299.4, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 7.22, "host_load_1m_logged": 7.12, "samples": 7901, "markers": {"learn_end": 14.5, "poll_forward_drop": 11.2, "cursor_probe": 0.0, "syscall_leaf": 24.7, "mutex_leaf": 17.7}}, - {"revision": "45007f6fb14fb44adc02823a8035f4f39de8ab94", "label": "base-45007f6fb14f", "round": 1, "kinstr_per_frame": 144.6, "kcycles_per_frame": 353.6, "recv_fps": 16712, "groups_lost": 13701, "host_load_1m": 23.05, "host_load_1m_logged": 28.3, "samples": 5098, "markers": {"learn_end": 9.1, "poll_forward_drop": 6.8, "cursor_probe": 0.0, "syscall_leaf": 14.3, "mutex_leaf": 18.0}}, - {"revision": "45007f6fb14fb44adc02823a8035f4f39de8ab94", "label": "base-45007f6fb14f", "round": 2, "kinstr_per_frame": 156.9, "kcycles_per_frame": 330.2, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 20.44, "host_load_1m_logged": 22.17, "samples": 7645, "markers": {"learn_end": 13.6, "poll_forward_drop": 9.5, "cursor_probe": 0.0, "syscall_leaf": 21.8, "mutex_leaf": 19.1}}, - {"revision": "45007f6fb14fb44adc02823a8035f4f39de8ab94", "label": "base-45007f6fb14f", "round": 3, "kinstr_per_frame": 160.8, "kcycles_per_frame": 329.1, "recv_fps": 19998, "groups_lost": 0, "host_load_1m": 252.79, "host_load_1m_logged": 254.35, "samples": 7942, "markers": {"learn_end": 15.2, "poll_forward_drop": 11.2, "cursor_probe": 0.0, "syscall_leaf": 25.0, "mutex_leaf": 17.2}}, - {"revision": "87515b48c693457b7541db12c66836fd6fc6b1e8", "label": "base-87515b48c693", "round": 1, "kinstr_per_frame": 141.7, "kcycles_per_frame": 332.4, "recv_fps": 19983, "groups_lost": 0, "host_load_1m": 31.75, "host_load_1m_logged": 32.37, "samples": 6666, "markers": {"learn_end": 11.0, "poll_forward_drop": 8.6, "cursor_probe": 0.0, "syscall_leaf": 15.3, "mutex_leaf": 20.4}}, - {"revision": "87515b48c693457b7541db12c66836fd6fc6b1e8", "label": "base-87515b48c693", "round": 2, "kinstr_per_frame": 141.6, "kcycles_per_frame": 303.1, "recv_fps": 19765, "groups_lost": 0, "host_load_1m": 15.34, "host_load_1m_logged": 16.9, "samples": 6887, "markers": {"learn_end": 11.0, "poll_forward_drop": 8.4, "cursor_probe": 0.0, "syscall_leaf": 14.7, "mutex_leaf": 20.3}}, - {"revision": "87515b48c693457b7541db12c66836fd6fc6b1e8", "label": "base-87515b48c693", "round": 3, "kinstr_per_frame": 169.8, "kcycles_per_frame": 340.9, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 96.24, "host_load_1m_logged": 112.61, "samples": 8412, "markers": {"learn_end": 14.8, "poll_forward_drop": 11.1, "cursor_probe": 0.0, "syscall_leaf": 28.2, "mutex_leaf": 16.3}}, - {"revision": "8ee3c80894186483b3c0d8f7e84b37ab0f525390", "label": "base-8ee3c8089418", "round": 1, "kinstr_per_frame": 141.9, "kcycles_per_frame": 356.1, "recv_fps": 15002, "groups_lost": 38842, "host_load_1m": 30.58, "host_load_1m_logged": 31.49, "samples": 6099, "markers": {"learn_end": 10.8, "poll_forward_drop": 8.8, "cursor_probe": 0.0, "syscall_leaf": 12.7, "mutex_leaf": 18.4}}, - {"revision": "8ee3c80894186483b3c0d8f7e84b37ab0f525390", "label": "base-8ee3c8089418", "round": 2, "kinstr_per_frame": 152.6, "kcycles_per_frame": 349.8, "recv_fps": 20012, "groups_lost": 0, "host_load_1m": 29.14, "host_load_1m_logged": 29.13, "samples": 8262, "markers": {"learn_end": 15.4, "poll_forward_drop": 11.5, "cursor_probe": 0.0, "syscall_leaf": 27.4, "mutex_leaf": 16.7}}, - {"revision": "8ee3c80894186483b3c0d8f7e84b37ab0f525390", "label": "base-8ee3c8089418", "round": 3, "kinstr_per_frame": 150.4, "kcycles_per_frame": 303.5, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 10.23, "host_load_1m_logged": 10.69, "samples": 7586, "markers": {"learn_end": 13.8, "poll_forward_drop": 9.4, "cursor_probe": 0.0, "syscall_leaf": 20.6, "mutex_leaf": 17.5}}, - {"revision": "c3c88aec81350cece608599cef5c33322547d146", "label": "base-c3c88aec8135", "round": 1, "kinstr_per_frame": 150.7, "kcycles_per_frame": 322.3, "recv_fps": 19997, "groups_lost": 0, "host_load_1m": 20.63, "host_load_1m_logged": 22.23, "samples": 7644, "markers": {"learn_end": 14.9, "poll_forward_drop": 10.1, "cursor_probe": 0.0, "syscall_leaf": 22.2, "mutex_leaf": 17.7}}, - {"revision": "c3c88aec81350cece608599cef5c33322547d146", "label": "base-c3c88aec8135", "round": 2, "kinstr_per_frame": 123.9, "kcycles_per_frame": 272.4, "recv_fps": 22352, "groups_lost": 10324, "host_load_1m": 23.92, "host_load_1m_logged": 24.14, "samples": 6730, "markers": {"learn_end": 11.5, "poll_forward_drop": 8.4, "cursor_probe": 0.0, "syscall_leaf": 15.8, "mutex_leaf": 19.8}}, - {"revision": "c3c88aec81350cece608599cef5c33322547d146", "label": "base-c3c88aec8135", "round": 3, "kinstr_per_frame": 160.4, "kcycles_per_frame": 329.0, "recv_fps": 20003, "groups_lost": 0, "host_load_1m": 36.05, "host_load_1m_logged": 44.87, "samples": 8046, "markers": {"learn_end": 15.6, "poll_forward_drop": 10.2, "cursor_probe": 0.0, "syscall_leaf": 25.8, "mutex_leaf": 16.7}}, - {"revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", "label": "base-e84ffb3de313", "round": 1, "kinstr_per_frame": 129.7, "kcycles_per_frame": 275.4, "recv_fps": 20002, "groups_lost": 0, "host_load_1m": 32.54, "host_load_1m_logged": 32.54, "samples": 6495, "markers": {"learn_end": 0.0, "poll_forward_drop": 6.6, "cursor_probe": 0.0, "syscall_leaf": 19.1, "mutex_leaf": 16.6}}, - {"revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", "label": "base-e84ffb3de313", "round": 2, "kinstr_per_frame": 131.0, "kcycles_per_frame": 259.4, "recv_fps": 19883, "groups_lost": 0, "host_load_1m": 15.61, "host_load_1m_logged": 15.61, "samples": 6344, "markers": {"learn_end": 0.0, "poll_forward_drop": 6.3, "cursor_probe": 0.0, "syscall_leaf": 16.6, "mutex_leaf": 17.1}}, - {"revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", "label": "base-e84ffb3de313", "round": 3, "kinstr_per_frame": 130.0, "kcycles_per_frame": 267.5, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 9.84, "host_load_1m_logged": 9.84, "samples": 6231, "markers": {"learn_end": 0.0, "poll_forward_drop": 5.7, "cursor_probe": 0.0, "syscall_leaf": 16.4, "mutex_leaf": 17.0}}, - {"revision": "fd59eb18a50f6e0ac819e73d941a3832cbdac91f", "label": "base-fd59eb18a50f", "round": 1, "kinstr_per_frame": 136.4, "kcycles_per_frame": 357.7, "recv_fps": 21611, "groups_lost": 3490, "host_load_1m": 20.4, "host_load_1m_logged": 20.99, "samples": 6836, "markers": {"learn_end": 10.5, "poll_forward_drop": 8.2, "cursor_probe": 0.0, "syscall_leaf": 15.4, "mutex_leaf": 18.7}}, - {"revision": "fd59eb18a50f6e0ac819e73d941a3832cbdac91f", "label": "base-fd59eb18a50f", "round": 2, "kinstr_per_frame": 146.0, "kcycles_per_frame": 364.8, "recv_fps": 20004, "groups_lost": 0, "host_load_1m": 37.85, "host_load_1m_logged": 37.14, "samples": 7347, "markers": {"learn_end": 12.6, "poll_forward_drop": 8.9, "cursor_probe": 0.0, "syscall_leaf": 17.4, "mutex_leaf": 17.7}}, - {"revision": "fd59eb18a50f6e0ac819e73d941a3832cbdac91f", "label": "base-fd59eb18a50f", "round": 3, "kinstr_per_frame": 159.4, "kcycles_per_frame": 311.3, "recv_fps": 20003, "groups_lost": 0, "host_load_1m": 7.45, "host_load_1m_logged": 7.49, "samples": 8229, "markers": {"learn_end": 14.9, "poll_forward_drop": 10.2, "cursor_probe": 0.0, "syscall_leaf": 26.3, "mutex_leaf": 16.3}}, - {"revision": "bd812a762c149a4345430b53530a597d261144b8", "label": "current", "round": 1, "kinstr_per_frame": 144.6, "kcycles_per_frame": 354.3, "recv_fps": 19955, "groups_lost": 0, "host_load_1m": 15.03, "host_load_1m_logged": 15.03, "samples": 6179, "markers": {"learn_end": 0.3, "poll_forward_drop": 12.2, "cursor_probe": 6.4, "syscall_leaf": 14.2, "mutex_leaf": 19.0}}, - {"revision": "bd812a762c149a4345430b53530a597d261144b8", "label": "current", "round": 2, "kinstr_per_frame": 141.5, "kcycles_per_frame": 298.8, "recv_fps": 20036, "groups_lost": 0, "host_load_1m": 29.44, "host_load_1m_logged": 28.52, "samples": 7167, "markers": {"learn_end": 0.3, "poll_forward_drop": 13.6, "cursor_probe": 7.4, "syscall_leaf": 14.2, "mutex_leaf": 20.2}}, - {"revision": "bd812a762c149a4345430b53530a597d261144b8", "label": "current", "round": 3, "kinstr_per_frame": 142.7, "kcycles_per_frame": 281.2, "recv_fps": 19996, "groups_lost": 0, "host_load_1m": 16.58, "host_load_1m_logged": 17.57, "samples": 7166, "markers": {"learn_end": 0.3, "poll_forward_drop": 14.8, "cursor_probe": 6.7, "syscall_leaf": 16.0, "mutex_leaf": 18.4}} - ] - }, - { - "artifact": "run.SkAD0k", - "harness_revision": "bd812a762c149a4345430b53530a597d261144b8", - "workload": "video", - "runtime": "io-uring-workers", - "workers": 4, - "generator_revision": "21d6da9fc1", - "instruction_period": 2000003, - "sampling": "after the counter window", - "relay_cpus": "4-7", - "generator_cpus": "0-3", - "cells": [ - {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 1, "kinstr_per_frame": 141.1, "kcycles_per_frame": 330.6, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 5.42, "host_load_1m_logged": 5.31, "samples": 331, "markers": {"learn_end": 0.6, "poll_forward_drop": 6.0, "cursor_probe": 1.8, "syscall_leaf": 35.6, "mutex_leaf": 9.7}}, - {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 2, "kinstr_per_frame": 139.5, "kcycles_per_frame": 336.0, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 7.27, "host_load_1m_logged": 7.17, "samples": 324, "markers": {"learn_end": 0.0, "poll_forward_drop": 5.6, "cursor_probe": 1.2, "syscall_leaf": 33.6, "mutex_leaf": 9.6}}, - {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 3, "kinstr_per_frame": 138.3, "kcycles_per_frame": 339.8, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 7.29, "host_load_1m_logged": 7.18, "samples": 320, "markers": {"learn_end": 0.3, "poll_forward_drop": 5.9, "cursor_probe": 1.6, "syscall_leaf": 32.8, "mutex_leaf": 8.4}}, - {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 4, "kinstr_per_frame": 146.8, "kcycles_per_frame": 455.4, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 26.0, "host_load_1m_logged": 27.34, "samples": 343, "markers": {"learn_end": 0.3, "poll_forward_drop": 4.7, "cursor_probe": 1.7, "syscall_leaf": 37.6, "mutex_leaf": 9.3}}, - {"revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", "label": "base-cdff3c8f952b", "round": 1, "kinstr_per_frame": 109.6, "kcycles_per_frame": 291.5, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 4.97, "host_load_1m_logged": 5.13, "samples": 260, "markers": {"learn_end": 0.0, "poll_forward_drop": 1.5, "cursor_probe": 0.0, "syscall_leaf": 43.8, "mutex_leaf": 9.6}}, - {"revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", "label": "base-cdff3c8f952b", "round": 2, "kinstr_per_frame": 105.6, "kcycles_per_frame": 273.9, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 6.76, "host_load_1m_logged": 6.91, "samples": 253, "markers": {"learn_end": 0.0, "poll_forward_drop": 2.8, "cursor_probe": 0.0, "syscall_leaf": 42.7, "mutex_leaf": 7.9}}, - {"revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", "label": "base-cdff3c8f952b", "round": 3, "kinstr_per_frame": 109.9, "kcycles_per_frame": 286.4, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 6.84, "host_load_1m_logged": 7.46, "samples": 256, "markers": {"learn_end": 0.0, "poll_forward_drop": 1.2, "cursor_probe": 0.0, "syscall_leaf": 41.0, "mutex_leaf": 7.4}}, - {"revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", "label": "base-cdff3c8f952b", "round": 4, "kinstr_per_frame": 119.9, "kcycles_per_frame": 437.1, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 12.82, "host_load_1m_logged": 18.92, "samples": 282, "markers": {"learn_end": 0.0, "poll_forward_drop": 1.4, "cursor_probe": 0.0, "syscall_leaf": 42.6, "mutex_leaf": 8.2}}, - {"revision": "bd812a762c149a4345430b53530a597d261144b8", "label": "current", "round": 1, "kinstr_per_frame": 120.5, "kcycles_per_frame": 278.0, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 7.16, "host_load_1m_logged": 7.16, "samples": 279, "markers": {"learn_end": 0.0, "poll_forward_drop": 0.4, "cursor_probe": 0.0, "syscall_leaf": 35.1, "mutex_leaf": 4.3}}, - {"revision": "bd812a762c149a4345430b53530a597d261144b8", "label": "current", "round": 2, "kinstr_per_frame": 124.2, "kcycles_per_frame": 302.8, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 7.5, "host_load_1m_logged": 7.63, "samples": 284, "markers": {"learn_end": 0.0, "poll_forward_drop": 0.4, "cursor_probe": 0.4, "syscall_leaf": 39.8, "mutex_leaf": 4.2}}, - {"revision": "bd812a762c149a4345430b53530a597d261144b8", "label": "current", "round": 3, "kinstr_per_frame": 125.6, "kcycles_per_frame": 285.4, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 7.8, "host_load_1m_logged": 7.8, "samples": 288, "markers": {"learn_end": 0.0, "poll_forward_drop": 0.3, "cursor_probe": 0.0, "syscall_leaf": 38.9, "mutex_leaf": 3.1}}, - {"revision": "bd812a762c149a4345430b53530a597d261144b8", "label": "current", "round": 4, "kinstr_per_frame": 138.3, "kcycles_per_frame": 364.1, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 39.8, "host_load_1m_logged": 40.13, "samples": 326, "markers": {"learn_end": 0.0, "poll_forward_drop": 0.3, "cursor_probe": 0.0, "syscall_leaf": 46.3, "mutex_leaf": 5.2}} - ] - }, - { - "artifact": "run.IV47ua", - "harness_revision": "bd812a762c149a4345430b53530a597d261144b8", - "workload": "chat", - "runtime": "tokio-workers", - "workers": 4, - "generator_revision": "21d6da9fc1", - "instruction_period": 2000003, - "sampling": "after the counter window", - "relay_cpus": "4-7", - "generator_cpus": "0-3", - "cells": [ - {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 1, "kinstr_per_frame": 185.1, "kcycles_per_frame": 366.0, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 13.99, "host_load_1m_logged": 13.99, "samples": 9276, "markers": {"learn_end": 0.9, "poll_forward_drop": 17.1, "cursor_probe": 6.4, "syscall_leaf": 4.3, "mutex_leaf": 24.8}}, - {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 2, "kinstr_per_frame": 189.6, "kcycles_per_frame": 360.9, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 6.01, "host_load_1m_logged": 6.21, "samples": 9513, "markers": {"learn_end": 1.1, "poll_forward_drop": 17.0, "cursor_probe": 6.1, "syscall_leaf": 6.5, "mutex_leaf": 22.7}}, - {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 3, "kinstr_per_frame": 184.3, "kcycles_per_frame": 361.2, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 7.76, "host_load_1m_logged": 7.98, "samples": 9265, "markers": {"learn_end": 1.0, "poll_forward_drop": 17.1, "cursor_probe": 6.2, "syscall_leaf": 4.0, "mutex_leaf": 24.1}}, - {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 4, "kinstr_per_frame": 187.1, "kcycles_per_frame": 354.2, "recv_fps": 19997, "groups_lost": 0, "host_load_1m": 6.29, "host_load_1m_logged": 5.95, "samples": 9465, "markers": {"learn_end": 0.8, "poll_forward_drop": 18.0, "cursor_probe": 6.8, "syscall_leaf": 6.2, "mutex_leaf": 23.3}}, - {"revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", "label": "base-cdff3c8f952b", "round": 1, "kinstr_per_frame": 123.6, "kcycles_per_frame": 273.2, "recv_fps": 19998, "groups_lost": 0, "host_load_1m": 22.76, "host_load_1m_logged": 22.7, "samples": 6351, "markers": {"learn_end": 0.0, "poll_forward_drop": 5.1, "cursor_probe": 0.0, "syscall_leaf": 0.3, "mutex_leaf": 19.9}}, - {"revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", "label": "base-cdff3c8f952b", "round": 2, "kinstr_per_frame": 131.1, "kcycles_per_frame": 242.1, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 6.37, "host_load_1m_logged": 7.05, "samples": 6537, "markers": {"learn_end": 0.0, "poll_forward_drop": 5.8, "cursor_probe": 0.0, "syscall_leaf": 2.8, "mutex_leaf": 19.0}}, - {"revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", "label": "base-cdff3c8f952b", "round": 3, "kinstr_per_frame": 131.1, "kcycles_per_frame": 240.6, "recv_fps": 20004, "groups_lost": 0, "host_load_1m": 5.3, "host_load_1m_logged": 5.36, "samples": 6642, "markers": {"learn_end": 0.0, "poll_forward_drop": 6.9, "cursor_probe": 0.0, "syscall_leaf": 3.2, "mutex_leaf": 20.7}}, - {"revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", "label": "base-cdff3c8f952b", "round": 4, "kinstr_per_frame": 131.5, "kcycles_per_frame": 241.5, "recv_fps": 20003, "groups_lost": 0, "host_load_1m": 6.86, "host_load_1m_logged": 7.27, "samples": 6565, "markers": {"learn_end": 0.0, "poll_forward_drop": 6.0, "cursor_probe": 0.0, "syscall_leaf": 2.6, "mutex_leaf": 20.0}}, - {"revision": "bd812a762c149a4345430b53530a597d261144b8", "label": "current", "round": 1, "kinstr_per_frame": 172.8, "kcycles_per_frame": 324.4, "recv_fps": 19996, "groups_lost": 0, "host_load_1m": 7.61, "host_load_1m_logged": 8.28, "samples": 8639, "markers": {"learn_end": 0.3, "poll_forward_drop": 12.2, "cursor_probe": 6.2, "syscall_leaf": 1.6, "mutex_leaf": 20.5}}, - {"revision": "bd812a762c149a4345430b53530a597d261144b8", "label": "current", "round": 2, "kinstr_per_frame": 171.0, "kcycles_per_frame": 362.7, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 11.35, "host_load_1m_logged": 11.47, "samples": 8545, "markers": {"learn_end": 0.3, "poll_forward_drop": 12.4, "cursor_probe": 6.3, "syscall_leaf": 1.0, "mutex_leaf": 20.1}}, - {"revision": "bd812a762c149a4345430b53530a597d261144b8", "label": "current", "round": 3, "kinstr_per_frame": 172.2, "kcycles_per_frame": 318.2, "recv_fps": 20000, "groups_lost": 0, "host_load_1m": 5.71, "host_load_1m_logged": 5.76, "samples": 8638, "markers": {"learn_end": 0.4, "poll_forward_drop": 11.6, "cursor_probe": 6.0, "syscall_leaf": 1.9, "mutex_leaf": 19.8}}, - {"revision": "bd812a762c149a4345430b53530a597d261144b8", "label": "current", "round": 4, "kinstr_per_frame": 167.8, "kcycles_per_frame": 370.0, "recv_fps": 19845, "groups_lost": 0, "host_load_1m": 10.76, "host_load_1m_logged": 10.88, "samples": 8671, "markers": {"learn_end": 0.2, "poll_forward_drop": 12.1, "cursor_probe": 6.1, "syscall_leaf": 1.3, "mutex_leaf": 20.4}} - ] - }, - { - "artifact": "run.Z04ENl", - "harness_revision": "bd812a762c149a4345430b53530a597d261144b8", - "workload": "video", - "runtime": "tokio-workers", - "workers": 4, - "generator_revision": "21d6da9fc1", - "instruction_period": 2000003, - "sampling": "after the counter window", - "relay_cpus": "4-7", - "generator_cpus": "0-3", - "cells": [ - {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 1, "kinstr_per_frame": 170.2, "kcycles_per_frame": 535.3, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 25.25, "host_load_1m_logged": 27.59, "samples": 370, "markers": {"learn_end": 0.5, "poll_forward_drop": 9.5, "cursor_probe": 2.4, "syscall_leaf": 0.0, "mutex_leaf": 15.4}}, - {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 2, "kinstr_per_frame": 175.3, "kcycles_per_frame": 483.3, "recv_fps": 959, "groups_lost": 0, "host_load_1m": 30.01, "host_load_1m_logged": 28.96, "samples": 426, "markers": {"learn_end": 0.5, "poll_forward_drop": 6.6, "cursor_probe": 1.6, "syscall_leaf": 0.0, "mutex_leaf": 14.3}}, - {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 3, "kinstr_per_frame": 163.2, "kcycles_per_frame": 519.2, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 33.02, "host_load_1m_logged": 33.58, "samples": 384, "markers": {"learn_end": 0.3, "poll_forward_drop": 9.1, "cursor_probe": 2.3, "syscall_leaf": 0.0, "mutex_leaf": 17.7}}, - {"revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", "label": "base-21d6da9fc108", "round": 4, "kinstr_per_frame": 164.5, "kcycles_per_frame": 386.2, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 39.42, "host_load_1m_logged": 39.65, "samples": 385, "markers": {"learn_end": 0.3, "poll_forward_drop": 9.1, "cursor_probe": 2.3, "syscall_leaf": 0.3, "mutex_leaf": 14.3}}, - {"revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", "label": "base-cdff3c8f952b", "round": 1, "kinstr_per_frame": 148.2, "kcycles_per_frame": 378.2, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 13.51, "host_load_1m_logged": 17.53, "samples": 357, "markers": {"learn_end": 0.0, "poll_forward_drop": 2.5, "cursor_probe": 0.0, "syscall_leaf": 0.0, "mutex_leaf": 12.0}}, - {"revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", "label": "base-cdff3c8f952b", "round": 2, "kinstr_per_frame": 149.5, "kcycles_per_frame": 302.2, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 18.81, "host_load_1m_logged": 19.83, "samples": 354, "markers": {"learn_end": 0.0, "poll_forward_drop": 2.8, "cursor_probe": 0.0, "syscall_leaf": 0.0, "mutex_leaf": 13.0}}, - {"revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", "label": "base-cdff3c8f952b", "round": 3, "kinstr_per_frame": 163.9, "kcycles_per_frame": 506.6, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 23.81, "host_load_1m_logged": 26.39, "samples": 383, "markers": {"learn_end": 0.0, "poll_forward_drop": 3.1, "cursor_probe": 0.0, "syscall_leaf": 0.0, "mutex_leaf": 12.5}}, - {"revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", "label": "base-cdff3c8f952b", "round": 4, "kinstr_per_frame": 156.0, "kcycles_per_frame": 556.9, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 39.7, "host_load_1m_logged": 39.65, "samples": 368, "markers": {"learn_end": 0.0, "poll_forward_drop": 3.5, "cursor_probe": 0.0, "syscall_leaf": 0.0, "mutex_leaf": 14.4}}, - {"revision": "bd812a762c149a4345430b53530a597d261144b8", "label": "current", "round": 1, "kinstr_per_frame": 147.3, "kcycles_per_frame": 372.5, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 28.85, "host_load_1m_logged": 28.75, "samples": 348, "markers": {"learn_end": 0.3, "poll_forward_drop": 0.3, "cursor_probe": 0.3, "syscall_leaf": 0.0, "mutex_leaf": 9.8}}, - {"revision": "bd812a762c149a4345430b53530a597d261144b8", "label": "current", "round": 2, "kinstr_per_frame": 142.9, "kcycles_per_frame": 322.9, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 23.15, "host_load_1m_logged": 24.73, "samples": 335, "markers": {"learn_end": 0.0, "poll_forward_drop": 0.6, "cursor_probe": 0.0, "syscall_leaf": 0.0, "mutex_leaf": 8.7}}, - {"revision": "bd812a762c149a4345430b53530a597d261144b8", "label": "current", "round": 3, "kinstr_per_frame": 146.4, "kcycles_per_frame": 428.6, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 17.83, "host_load_1m_logged": 18.09, "samples": 347, "markers": {"learn_end": 0.0, "poll_forward_drop": 0.3, "cursor_probe": 0.3, "syscall_leaf": 0.0, "mutex_leaf": 11.0}}, - {"revision": "bd812a762c149a4345430b53530a597d261144b8", "label": "current", "round": 4, "kinstr_per_frame": 147.2, "kcycles_per_frame": 330.8, "recv_fps": 960, "groups_lost": 0, "host_load_1m": 24.93, "host_load_1m_logged": 24.93, "samples": 345, "markers": {"learn_end": 0.3, "poll_forward_drop": 0.3, "cursor_probe": 0.0, "syscall_leaf": 0.0, "mutex_leaf": 9.6}} - ] - } - ] + "version": 2, + "metric": "relay instructions (user and kernel, all threads) per delivered frame over a 5 s counter window after 7 s warmup", + "markers": { + "learn_end": "inclusive share of fixed-period instruction samples whose stack matches resume::Reader>::learn_end", + "poll_forward_drop": "inclusive share of fixed-period instruction samples whose stack matches poll_forward_drop", + "cursor_probe": "inclusive share of fixed-period instruction samples whose stack matches model::track::Cursor>::probe", + "syscall_leaf": "inclusive share of fixed-period instruction samples whose stack matches ;syscall$", + "mutex_leaf": "inclusive share of fixed-period instruction samples whose stack matches ;(lock|unlock)$" + }, + "runs": [ + { + "artifact": "run.eVjz2w", + "harness_revision": "82194441a92bb89fbc69f3759ef336d676fad643", + "workload": "chat", + "runtime": "io-uring-workers", + "workers": 4, + "generator_revision": "82194441a92bb89fbc69f3759ef336d676fad643", + "instruction_period": 10000019, + "sampling": "concurrent with the counter window", + "relay_cpus": "4-7", + "generator_cpus": "0-3", + "cells": [ + { + "revision": "2d73eea67d083ecf18938f4fc7c4842118eb87cf", + "label": "base-2d73eea67d08", + "round": 1, + "kinstr_per_frame": 121.8, + "kcycles_per_frame": 224.8, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 4.81, + "samples": 1209, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 8.9, + "cursor_probe": 0.0, + "syscall_leaf": 23.2, + "mutex_leaf": 16.3 + } + }, + { + "revision": "2d73eea67d083ecf18938f4fc7c4842118eb87cf", + "label": "base-2d73eea67d08", + "round": 2, + "kinstr_per_frame": 117.8, + "kcycles_per_frame": 232.6, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 5.87, + "samples": 1167, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 6.9, + "cursor_probe": 0.0, + "syscall_leaf": 20.1, + "mutex_leaf": 16.5 + } + }, + { + "revision": "2d73eea67d083ecf18938f4fc7c4842118eb87cf", + "label": "base-2d73eea67d08", + "round": 3, + "kinstr_per_frame": 122.6, + "kcycles_per_frame": 235.2, + "recv_fps": 19999, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 4.2, + "samples": 1216, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 6.6, + "cursor_probe": 0.0, + "syscall_leaf": 21.7, + "mutex_leaf": 16.0 + } + }, + { + "revision": "32c1e9940e5519a2bdc7c853d84846f55195cb94", + "label": "base-32c1e9940e55", + "round": 1, + "kinstr_per_frame": 123.4, + "kcycles_per_frame": 221.0, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 5.81, + "samples": 1229, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 8.0, + "cursor_probe": 0.0, + "syscall_leaf": 23.6, + "mutex_leaf": 17.8 + } + }, + { + "revision": "32c1e9940e5519a2bdc7c853d84846f55195cb94", + "label": "base-32c1e9940e55", + "round": 2, + "kinstr_per_frame": 115.8, + "kcycles_per_frame": 220.7, + "recv_fps": 20003, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 3.96, + "samples": 1150, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 8.5, + "cursor_probe": 0.0, + "syscall_leaf": 22.8, + "mutex_leaf": 19.7 + } + }, + { + "revision": "32c1e9940e5519a2bdc7c853d84846f55195cb94", + "label": "base-32c1e9940e55", + "round": 3, + "kinstr_per_frame": 117.3, + "kcycles_per_frame": 223.7, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 3.13, + "samples": 1168, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 8.0, + "cursor_probe": 0.0, + "syscall_leaf": 21.8, + "mutex_leaf": 17.6 + } + }, + { + "revision": "cbab2a8922c53a4541f919b89a95c131628f0cb0", + "label": "base-cbab2a8922c5", + "round": 1, + "kinstr_per_frame": 129.2, + "kcycles_per_frame": 241.4, + "recv_fps": 19997, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 5.81, + "samples": 1282, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 8.7, + "cursor_probe": 0.0, + "syscall_leaf": 24.1, + "mutex_leaf": 16.3 + } + }, + { + "revision": "cbab2a8922c53a4541f919b89a95c131628f0cb0", + "label": "base-cbab2a8922c5", + "round": 2, + "kinstr_per_frame": 121.7, + "kcycles_per_frame": 233.1, + "recv_fps": 19997, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 4.88, + "samples": 1210, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 8.8, + "cursor_probe": 0.0, + "syscall_leaf": 20.6, + "mutex_leaf": 18.1 + } + }, + { + "revision": "cbab2a8922c53a4541f919b89a95c131628f0cb0", + "label": "base-cbab2a8922c5", + "round": 3, + "kinstr_per_frame": 124.0, + "kcycles_per_frame": 244.0, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 4.37, + "samples": 1233, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 9.3, + "cursor_probe": 0.0, + "syscall_leaf": 23.5, + "mutex_leaf": 16.7 + } + }, + { + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "label": "base-e84ffb3de313", + "round": 1, + "kinstr_per_frame": 133.4, + "kcycles_per_frame": 270.7, + "recv_fps": 20003, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 4.41, + "samples": 1327, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 6.9, + "cursor_probe": 0.0, + "syscall_leaf": 22.0, + "mutex_leaf": 16.9 + } + }, + { + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "label": "base-e84ffb3de313", + "round": 2, + "kinstr_per_frame": 132.9, + "kcycles_per_frame": 282.7, + "recv_fps": 20001, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 6.04, + "samples": 1320, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 7.6, + "cursor_probe": 0.0, + "syscall_leaf": 21.4, + "mutex_leaf": 15.4 + } + }, + { + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "label": "base-e84ffb3de313", + "round": 3, + "kinstr_per_frame": 135.4, + "kcycles_per_frame": 269.0, + "recv_fps": 20002, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 4.05, + "samples": 1346, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 7.6, + "cursor_probe": 0.0, + "syscall_leaf": 21.1, + "mutex_leaf": 16.5 + } + }, + { + "revision": "fe722cec9f8432e0e46cb633cc5e79e59e7724cb", + "label": "base-fe722cec9f84", + "round": 1, + "kinstr_per_frame": 117.5, + "kcycles_per_frame": 230.1, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 6.52, + "samples": 1166, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 8.1, + "cursor_probe": 0.0, + "syscall_leaf": 22.4, + "mutex_leaf": 16.8 + } + }, + { + "revision": "fe722cec9f8432e0e46cb633cc5e79e59e7724cb", + "label": "base-fe722cec9f84", + "round": 2, + "kinstr_per_frame": 114.6, + "kcycles_per_frame": 222.0, + "recv_fps": 20003, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 4.66, + "samples": 1139, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 8.5, + "cursor_probe": 0.0, + "syscall_leaf": 20.9, + "mutex_leaf": 18.1 + } + }, + { + "revision": "fe722cec9f8432e0e46cb633cc5e79e59e7724cb", + "label": "base-fe722cec9f84", + "round": 3, + "kinstr_per_frame": 120.5, + "kcycles_per_frame": 220.7, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 3.06, + "samples": 1200, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 9.0, + "cursor_probe": 0.0, + "syscall_leaf": 25.0, + "mutex_leaf": 17.9 + } + }, + { + "revision": "82194441a92bb89fbc69f3759ef336d676fad643", + "label": "current", + "round": 1, + "kinstr_per_frame": 146.9, + "kcycles_per_frame": 248.6, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 3.8, + "samples": 1461, + "markers": { + "learn_end": 0.5, + "poll_forward_drop": 14.9, + "cursor_probe": 6.4, + "syscall_leaf": 17.2, + "mutex_leaf": 17.5 + } + }, + { + "revision": "82194441a92bb89fbc69f3759ef336d676fad643", + "label": "current", + "round": 2, + "kinstr_per_frame": 145.2, + "kcycles_per_frame": 278.8, + "recv_fps": 20004, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 5.49, + "samples": 1447, + "markers": { + "learn_end": 0.5, + "poll_forward_drop": 13.3, + "cursor_probe": 6.7, + "syscall_leaf": 16.6, + "mutex_leaf": 18.9 + } + }, + { + "revision": "82194441a92bb89fbc69f3759ef336d676fad643", + "label": "current", + "round": 3, + "kinstr_per_frame": 147.1, + "kcycles_per_frame": 270.2, + "recv_fps": 20001, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 3.47, + "samples": 1465, + "markers": { + "learn_end": 0.5, + "poll_forward_drop": 15.1, + "cursor_probe": 6.8, + "syscall_leaf": 18.3, + "mutex_leaf": 17.2 + } + } + ] + }, + { + "artifact": "run.2DucdO", + "harness_revision": "82194441a92bb89fbc69f3759ef336d676fad643", + "workload": "chat", + "runtime": "io-uring-workers", + "workers": 4, + "generator_revision": "82194441a92bb89fbc69f3759ef336d676fad643", + "instruction_period": 10000019, + "sampling": "concurrent with the counter window", + "relay_cpus": "4-7", + "generator_cpus": "0-3", + "cells": [ + { + "revision": "2d73eea67d083ecf18938f4fc7c4842118eb87cf", + "label": "base-2d73eea67d08", + "round": 1, + "kinstr_per_frame": 123.1, + "kcycles_per_frame": 235.4, + "recv_fps": 20001, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 3.56, + "samples": 1221, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 7.9, + "cursor_probe": 0.0, + "syscall_leaf": 22.4, + "mutex_leaf": 16.7 + } + }, + { + "revision": "2d73eea67d083ecf18938f4fc7c4842118eb87cf", + "label": "base-2d73eea67d08", + "round": 2, + "kinstr_per_frame": 125.3, + "kcycles_per_frame": 247.9, + "recv_fps": 19999, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 3.19, + "samples": 1248, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 8.4, + "cursor_probe": 0.0, + "syscall_leaf": 24.8, + "mutex_leaf": 15.3 + } + }, + { + "revision": "2d73eea67d083ecf18938f4fc7c4842118eb87cf", + "label": "base-2d73eea67d08", + "round": 3, + "kinstr_per_frame": 114.9, + "kcycles_per_frame": 297.8, + "recv_fps": 19920, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 9.31, + "samples": 1138, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 6.4, + "cursor_probe": 0.0, + "syscall_leaf": 16.5, + "mutex_leaf": 17.9 + } + }, + { + "revision": "41cff87192983aae80db976d0058661525daa8ed", + "label": "base-41cff8719298", + "round": 1, + "kinstr_per_frame": 115.5, + "kcycles_per_frame": 221.3, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 4.57, + "samples": 1148, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 6.4, + "cursor_probe": 0.0, + "syscall_leaf": 19.7, + "mutex_leaf": 16.6 + } + }, + { + "revision": "41cff87192983aae80db976d0058661525daa8ed", + "label": "base-41cff8719298", + "round": 2, + "kinstr_per_frame": 123.6, + "kcycles_per_frame": 225.9, + "recv_fps": 20003, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 2.99, + "samples": 1228, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 6.8, + "cursor_probe": 0.0, + "syscall_leaf": 23.7, + "mutex_leaf": 17.5 + } + }, + { + "revision": "41cff87192983aae80db976d0058661525daa8ed", + "label": "base-41cff8719298", + "round": 3, + "kinstr_per_frame": 142.6, + "kcycles_per_frame": 412.5, + "recv_fps": 12490, + "groups_lost": 25854, + "host_load_1m": null, + "host_load_1m_logged": 30.78, + "samples": 888, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 4.3, + "cursor_probe": 0.0, + "syscall_leaf": 16.1, + "mutex_leaf": 17.3 + } + }, + { + "revision": "72259ab3cc3d3ed53a9f3dad8ff37caeac310b83", + "label": "base-72259ab3cc3d", + "round": 1, + "kinstr_per_frame": 126.5, + "kcycles_per_frame": 244.6, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 3.19, + "samples": 1259, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 8.7, + "cursor_probe": 0.0, + "syscall_leaf": 24.7, + "mutex_leaf": 15.3 + } + }, + { + "revision": "72259ab3cc3d3ed53a9f3dad8ff37caeac310b83", + "label": "base-72259ab3cc3d", + "round": 2, + "kinstr_per_frame": 127.0, + "kcycles_per_frame": 233.8, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 3.9, + "samples": 1262, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 9.0, + "cursor_probe": 0.0, + "syscall_leaf": 24.6, + "mutex_leaf": 15.1 + } + }, + { + "revision": "72259ab3cc3d3ed53a9f3dad8ff37caeac310b83", + "label": "base-72259ab3cc3d", + "round": 3, + "kinstr_per_frame": 120.2, + "kcycles_per_frame": 238.3, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 9.69, + "samples": 1193, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 8.4, + "cursor_probe": 0.0, + "syscall_leaf": 21.8, + "mutex_leaf": 16.7 + } + }, + { + "revision": "756637200cb10943871f6c51171f33a340addb6e", + "label": "base-756637200cb1", + "round": 1, + "kinstr_per_frame": 138.4, + "kcycles_per_frame": 267.2, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 3.97, + "samples": 1378, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 7.3, + "cursor_probe": 0.0, + "syscall_leaf": 21.9, + "mutex_leaf": 15.1 + } + }, + { + "revision": "756637200cb10943871f6c51171f33a340addb6e", + "label": "base-756637200cb1", + "round": 2, + "kinstr_per_frame": 133.9, + "kcycles_per_frame": 256.6, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 3.41, + "samples": 1330, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 6.4, + "cursor_probe": 0.0, + "syscall_leaf": 18.4, + "mutex_leaf": 16.9 + } + }, + { + "revision": "756637200cb10943871f6c51171f33a340addb6e", + "label": "base-756637200cb1", + "round": 3, + "kinstr_per_frame": 133.7, + "kcycles_per_frame": 266.5, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 15.73, + "samples": 1331, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 7.2, + "cursor_probe": 0.0, + "syscall_leaf": 21.5, + "mutex_leaf": 15.2 + } + }, + { + "revision": "7648ce03908df33c168f7471787f3572e87eeddb", + "label": "base-7648ce03908d", + "round": 1, + "kinstr_per_frame": 121.2, + "kcycles_per_frame": 231.2, + "recv_fps": 19998, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 4.22, + "samples": 1204, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 7.1, + "cursor_probe": 0.0, + "syscall_leaf": 22.8, + "mutex_leaf": 17.4 + } + }, + { + "revision": "7648ce03908df33c168f7471787f3572e87eeddb", + "label": "base-7648ce03908d", + "round": 2, + "kinstr_per_frame": 126.1, + "kcycles_per_frame": 230.6, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 2.31, + "samples": 1255, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 8.0, + "cursor_probe": 0.0, + "syscall_leaf": 25.7, + "mutex_leaf": 15.9 + } + }, + { + "revision": "7648ce03908df33c168f7471787f3572e87eeddb", + "label": "base-7648ce03908d", + "round": 3, + "kinstr_per_frame": 156.9, + "kcycles_per_frame": 384.1, + "recv_fps": 13977, + "groups_lost": 35553, + "host_load_1m": null, + "host_load_1m_logged": 55.74, + "samples": 1087, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 4.5, + "cursor_probe": 0.0, + "syscall_leaf": 20.8, + "mutex_leaf": 17.7 + } + }, + { + "revision": "a9a85580a4488a3ad3739d8d130ebb0f66e80510", + "label": "base-a9a85580a448", + "round": 1, + "kinstr_per_frame": 132.3, + "kcycles_per_frame": 268.1, + "recv_fps": 20001, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 4.9, + "samples": 1316, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 7.6, + "cursor_probe": 0.0, + "syscall_leaf": 22.1, + "mutex_leaf": 17.3 + } + }, + { + "revision": "a9a85580a4488a3ad3739d8d130ebb0f66e80510", + "label": "base-a9a85580a448", + "round": 2, + "kinstr_per_frame": 133.6, + "kcycles_per_frame": 262.5, + "recv_fps": 19999, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 3.31, + "samples": 1330, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 8.3, + "cursor_probe": 0.0, + "syscall_leaf": 22.3, + "mutex_leaf": 16.7 + } + }, + { + "revision": "a9a85580a4488a3ad3739d8d130ebb0f66e80510", + "label": "base-a9a85580a448", + "round": 3, + "kinstr_per_frame": 124.0, + "kcycles_per_frame": 283.8, + "recv_fps": 19998, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 18.89, + "samples": 1237, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 7.7, + "cursor_probe": 0.0, + "syscall_leaf": 17.3, + "mutex_leaf": 17.8 + } + }, + { + "revision": "b4dfafac8879ffc4cb661c0d47200aeccea7a2ba", + "label": "base-b4dfafac8879", + "round": 1, + "kinstr_per_frame": 130.1, + "kcycles_per_frame": 243.7, + "recv_fps": 20004, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 4.07, + "samples": 1290, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 9.5, + "cursor_probe": 0.0, + "syscall_leaf": 22.7, + "mutex_leaf": 15.4 + } + }, + { + "revision": "b4dfafac8879ffc4cb661c0d47200aeccea7a2ba", + "label": "base-b4dfafac8879", + "round": 2, + "kinstr_per_frame": 128.5, + "kcycles_per_frame": 229.6, + "recv_fps": 20001, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 3.77, + "samples": 1278, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 7.5, + "cursor_probe": 0.0, + "syscall_leaf": 24.2, + "mutex_leaf": 14.9 + } + }, + { + "revision": "b4dfafac8879ffc4cb661c0d47200aeccea7a2ba", + "label": "base-b4dfafac8879", + "round": 3, + "kinstr_per_frame": 117.7, + "kcycles_per_frame": 282.4, + "recv_fps": 20001, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 11.64, + "samples": 1170, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 5.9, + "cursor_probe": 0.0, + "syscall_leaf": 18.4, + "mutex_leaf": 18.3 + } + }, + { + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "label": "base-e84ffb3de313", + "round": 1, + "kinstr_per_frame": 136.9, + "kcycles_per_frame": 260.8, + "recv_fps": 20003, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 3.43, + "samples": 1363, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 9.6, + "cursor_probe": 0.0, + "syscall_leaf": 22.5, + "mutex_leaf": 16.7 + } + }, + { + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "label": "base-e84ffb3de313", + "round": 2, + "kinstr_per_frame": 134.9, + "kcycles_per_frame": 262.6, + "recv_fps": 20001, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 3.26, + "samples": 1341, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 7.8, + "cursor_probe": 0.0, + "syscall_leaf": 23.3, + "mutex_leaf": 15.5 + } + }, + { + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "label": "base-e84ffb3de313", + "round": 3, + "kinstr_per_frame": 134.9, + "kcycles_per_frame": 267.8, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 11.14, + "samples": 1342, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 8.5, + "cursor_probe": 0.0, + "syscall_leaf": 20.1, + "mutex_leaf": 17.2 + } + }, + { + "revision": "82194441a92bb89fbc69f3759ef336d676fad643", + "label": "current", + "round": 1, + "kinstr_per_frame": 147.4, + "kcycles_per_frame": 264.3, + "recv_fps": 20001, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 3.09, + "samples": 1466, + "markers": { + "learn_end": 0.4, + "poll_forward_drop": 15.8, + "cursor_probe": 8.0, + "syscall_leaf": 17.9, + "mutex_leaf": 17.9 + } + }, + { + "revision": "82194441a92bb89fbc69f3759ef336d676fad643", + "label": "current", + "round": 2, + "kinstr_per_frame": 148.0, + "kcycles_per_frame": 261.4, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 3.13, + "samples": 1474, + "markers": { + "learn_end": 0.1, + "poll_forward_drop": 14.1, + "cursor_probe": 7.7, + "syscall_leaf": 18.7, + "mutex_leaf": 19.3 + } + }, + { + "revision": "82194441a92bb89fbc69f3759ef336d676fad643", + "label": "current", + "round": 3, + "kinstr_per_frame": 148.0, + "kcycles_per_frame": 266.1, + "recv_fps": 19997, + "groups_lost": 0, + "host_load_1m": null, + "host_load_1m_logged": 7.96, + "samples": 1472, + "markers": { + "learn_end": 0.6, + "poll_forward_drop": 15.4, + "cursor_probe": 8.4, + "syscall_leaf": 16.6, + "mutex_leaf": 18.9 + } + } + ] + }, + { + "artifact": "run.BIL6LP", + "harness_revision": "d935d13e8dba7d36e06f836a04984d940dbf0a57", + "workload": "chat", + "runtime": "io-uring-workers", + "workers": 4, + "generator_revision": "d935d13e8dba7d36e06f836a04984d940dbf0a57", + "instruction_period": 2000003, + "sampling": "concurrent with the counter window", + "relay_cpus": "4-7", + "generator_cpus": "0-3", + "cells": [ + { + "revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", + "label": "base-0c2a798de5c2", + "round": 1, + "kinstr_per_frame": 160.0, + "kcycles_per_frame": 331.8, + "recv_fps": 19997, + "groups_lost": 0, + "host_load_1m": 7.69, + "host_load_1m_logged": 8.5, + "samples": 7997, + "markers": { + "learn_end": 14.7, + "poll_forward_drop": 9.7, + "cursor_probe": 0.0, + "syscall_leaf": 24.1, + "mutex_leaf": 16.7 + } + }, + { + "revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", + "label": "base-0c2a798de5c2", + "round": 2, + "kinstr_per_frame": 166.2, + "kcycles_per_frame": 323.7, + "recv_fps": 20003, + "groups_lost": 0, + "host_load_1m": 2.31, + "host_load_1m_logged": 2.29, + "samples": 8320, + "markers": { + "learn_end": 15.5, + "poll_forward_drop": 11.3, + "cursor_probe": 0.0, + "syscall_leaf": 27.1, + "mutex_leaf": 16.2 + } + }, + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 1, + "kinstr_per_frame": 173.9, + "kcycles_per_frame": 310.8, + "recv_fps": 19997, + "groups_lost": 0, + "host_load_1m": 3.01, + "host_load_1m_logged": 3.01, + "samples": 8696, + "markers": { + "learn_end": 0.9, + "poll_forward_drop": 20.9, + "cursor_probe": 6.9, + "syscall_leaf": 21.9, + "mutex_leaf": 21.4 + } + }, + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 2, + "kinstr_per_frame": 178.0, + "kcycles_per_frame": 316.8, + "recv_fps": 19997, + "groups_lost": 0, + "host_load_1m": 2.89, + "host_load_1m_logged": 2.74, + "samples": 8898, + "markers": { + "learn_end": 1.0, + "poll_forward_drop": 20.9, + "cursor_probe": 7.4, + "syscall_leaf": 23.1, + "mutex_leaf": 21.8 + } + }, + { + "revision": "2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3", + "label": "base-2fff7ea7ae4e", + "round": 1, + "kinstr_per_frame": 209.5, + "kcycles_per_frame": 415.5, + "recv_fps": 19997, + "groups_lost": 0, + "host_load_1m": 2.45, + "host_load_1m_logged": 2.43, + "samples": 10473, + "markers": { + "learn_end": 16.6, + "poll_forward_drop": 23.4, + "cursor_probe": 7.3, + "syscall_leaf": 31.6, + "mutex_leaf": 17.2 + } + }, + { + "revision": "2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3", + "label": "base-2fff7ea7ae4e", + "round": 2, + "kinstr_per_frame": 213.5, + "kcycles_per_frame": 430.9, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 2.55, + "host_load_1m_logged": 2.51, + "samples": 10671, + "markers": { + "learn_end": 16.6, + "poll_forward_drop": 23.4, + "cursor_probe": 7.7, + "syscall_leaf": 32.4, + "mutex_leaf": 17.4 + } + }, + { + "revision": "49b5ec9b5630c54f2c5e99657b114535087b7da2", + "label": "base-49b5ec9b5630", + "round": 1, + "kinstr_per_frame": 211.8, + "kcycles_per_frame": 415.6, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 4.47, + "host_load_1m_logged": 4.47, + "samples": 10583, + "markers": { + "learn_end": 13.1, + "poll_forward_drop": 19.1, + "cursor_probe": 8.2, + "syscall_leaf": 31.8, + "mutex_leaf": 17.5 + } + }, + { + "revision": "49b5ec9b5630c54f2c5e99657b114535087b7da2", + "label": "base-49b5ec9b5630", + "round": 2, + "kinstr_per_frame": 211.3, + "kcycles_per_frame": 395.3, + "recv_fps": 20003, + "groups_lost": 0, + "host_load_1m": 1.87, + "host_load_1m_logged": 2.03, + "samples": 10560, + "markers": { + "learn_end": 13.2, + "poll_forward_drop": 19.3, + "cursor_probe": 8.0, + "syscall_leaf": 31.2, + "mutex_leaf": 18.0 + } + }, + { + "revision": "55f6ed229250d17fc99389237659906d011d8a94", + "label": "base-55f6ed229250", + "round": 1, + "kinstr_per_frame": 161.1, + "kcycles_per_frame": 313.9, + "recv_fps": 19998, + "groups_lost": 0, + "host_load_1m": 5.46, + "host_load_1m_logged": 5.32, + "samples": 8050, + "markers": { + "learn_end": 15.6, + "poll_forward_drop": 10.9, + "cursor_probe": 0.0, + "syscall_leaf": 26.5, + "mutex_leaf": 18.6 + } + }, + { + "revision": "55f6ed229250d17fc99389237659906d011d8a94", + "label": "base-55f6ed229250", + "round": 2, + "kinstr_per_frame": 165.5, + "kcycles_per_frame": 320.9, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 3.07, + "host_load_1m_logged": 2.99, + "samples": 8282, + "markers": { + "learn_end": 15.9, + "poll_forward_drop": 11.0, + "cursor_probe": 0.0, + "syscall_leaf": 28.6, + "mutex_leaf": 16.7 + } + }, + { + "revision": "991572d636d264c6189a3477849d2969e689f55c", + "label": "base-991572d636d2", + "round": 1, + "kinstr_per_frame": 206.0, + "kcycles_per_frame": 402.2, + "recv_fps": 20003, + "groups_lost": 0, + "host_load_1m": 3.6, + "host_load_1m_logged": 3.6, + "samples": 10295, + "markers": { + "learn_end": 15.4, + "poll_forward_drop": 19.7, + "cursor_probe": 8.0, + "syscall_leaf": 36.2, + "mutex_leaf": 16.3 + } + }, + { + "revision": "991572d636d264c6189a3477849d2969e689f55c", + "label": "base-991572d636d2", + "round": 2, + "kinstr_per_frame": 207.0, + "kcycles_per_frame": 402.5, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 2.61, + "host_load_1m_logged": 2.51, + "samples": 10344, + "markers": { + "learn_end": 16.0, + "poll_forward_drop": 19.6, + "cursor_probe": 7.0, + "syscall_leaf": 35.0, + "mutex_leaf": 16.5 + } + }, + { + "revision": "e45064e3ccb43a535adb949b7db9a5a0aeb32413", + "label": "base-e45064e3ccb4", + "round": 1, + "kinstr_per_frame": 158.5, + "kcycles_per_frame": 316.3, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 5.21, + "host_load_1m_logged": 5.97, + "samples": 7921, + "markers": { + "learn_end": 14.5, + "poll_forward_drop": 10.0, + "cursor_probe": 0.0, + "syscall_leaf": 24.8, + "mutex_leaf": 17.0 + } + }, + { + "revision": "e45064e3ccb43a535adb949b7db9a5a0aeb32413", + "label": "base-e45064e3ccb4", + "round": 2, + "kinstr_per_frame": 168.4, + "kcycles_per_frame": 329.1, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 2.27, + "host_load_1m_logged": 2.27, + "samples": 8424, + "markers": { + "learn_end": 16.3, + "poll_forward_drop": 11.4, + "cursor_probe": 0.0, + "syscall_leaf": 27.9, + "mutex_leaf": 16.2 + } + }, + { + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "label": "base-e84ffb3de313", + "round": 1, + "kinstr_per_frame": 136.2, + "kcycles_per_frame": 277.1, + "recv_fps": 19998, + "groups_lost": 0, + "host_load_1m": 6.92, + "host_load_1m_logged": 7.17, + "samples": 6802, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 7.5, + "cursor_probe": 0.0, + "syscall_leaf": 21.4, + "mutex_leaf": 16.4 + } + }, + { + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "label": "base-e84ffb3de313", + "round": 2, + "kinstr_per_frame": 133.8, + "kcycles_per_frame": 258.5, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 3.14, + "host_load_1m_logged": 3.21, + "samples": 6691, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 7.5, + "cursor_probe": 0.0, + "syscall_leaf": 21.1, + "mutex_leaf": 16.6 + } + }, + { + "revision": "f206893047dc2f1db7b3422622dbb505470d0c70", + "label": "base-f206893047dc", + "round": 1, + "kinstr_per_frame": 168.5, + "kcycles_per_frame": 338.2, + "recv_fps": 20004, + "groups_lost": 0, + "host_load_1m": 9.21, + "host_load_1m_logged": 9.23, + "samples": 8424, + "markers": { + "learn_end": 16.6, + "poll_forward_drop": 10.8, + "cursor_probe": 0.0, + "syscall_leaf": 29.4, + "mutex_leaf": 16.6 + } + }, + { + "revision": "f206893047dc2f1db7b3422622dbb505470d0c70", + "label": "base-f206893047dc", + "round": 2, + "kinstr_per_frame": 165.9, + "kcycles_per_frame": 325.0, + "recv_fps": 19997, + "groups_lost": 0, + "host_load_1m": 2.56, + "host_load_1m_logged": 2.4, + "samples": 8293, + "markers": { + "learn_end": 16.4, + "poll_forward_drop": 11.2, + "cursor_probe": 0.0, + "syscall_leaf": 27.2, + "mutex_leaf": 16.5 + } + }, + { + "revision": "d935d13e8dba7d36e06f836a04984d940dbf0a57", + "label": "current", + "round": 1, + "kinstr_per_frame": 147.6, + "kcycles_per_frame": 260.9, + "recv_fps": 19997, + "groups_lost": 0, + "host_load_1m": 2.72, + "host_load_1m_logged": 2.83, + "samples": 7372, + "markers": { + "learn_end": 0.4, + "poll_forward_drop": 14.5, + "cursor_probe": 7.3, + "syscall_leaf": 17.8, + "mutex_leaf": 19.0 + } + }, + { + "revision": "d935d13e8dba7d36e06f836a04984d940dbf0a57", + "label": "current", + "round": 2, + "kinstr_per_frame": 147.0, + "kcycles_per_frame": 251.2, + "recv_fps": 19999, + "groups_lost": 0, + "host_load_1m": 2.99, + "host_load_1m_logged": 3.16, + "samples": 7347, + "markers": { + "learn_end": 0.5, + "poll_forward_drop": 14.5, + "cursor_probe": 6.9, + "syscall_leaf": 17.5, + "mutex_leaf": 18.9 + } + } + ] + }, + { + "artifact": "run.1KRu4S", + "harness_revision": "a6cd6f91b6a0c79a81f2a012a2683bc77bd6c7d7", + "workload": "chat", + "runtime": "io-uring-workers", + "workers": 4, + "generator_revision": "21d6da9fc1", + "instruction_period": 2000003, + "sampling": "concurrent with the counter window", + "relay_cpus": "4-7", + "generator_cpus": "0-3", + "cells": [ + { + "revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", + "label": "base-0c2a798de5c2", + "round": 1, + "kinstr_per_frame": 165.1, + "kcycles_per_frame": 318.0, + "recv_fps": 19999, + "groups_lost": 0, + "host_load_1m": 2.58, + "host_load_1m_logged": 2.58, + "samples": 8253, + "markers": { + "learn_end": 15.7, + "poll_forward_drop": 10.8, + "cursor_probe": 0.0, + "syscall_leaf": 26.7, + "mutex_leaf": 16.7 + } + }, + { + "revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", + "label": "base-0c2a798de5c2", + "round": 2, + "kinstr_per_frame": 166.1, + "kcycles_per_frame": 322.2, + "recv_fps": 20001, + "groups_lost": 0, + "host_load_1m": 4.23, + "host_load_1m_logged": 3.99, + "samples": 8304, + "markers": { + "learn_end": 15.0, + "poll_forward_drop": 10.7, + "cursor_probe": 0.0, + "syscall_leaf": 27.0, + "mutex_leaf": 17.0 + } + }, + { + "revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", + "label": "base-0c2a798de5c2", + "round": 3, + "kinstr_per_frame": 162.1, + "kcycles_per_frame": 327.9, + "recv_fps": 19997, + "groups_lost": 0, + "host_load_1m": 7.69, + "host_load_1m_logged": 7.69, + "samples": 8100, + "markers": { + "learn_end": 14.9, + "poll_forward_drop": 10.9, + "cursor_probe": 0.0, + "syscall_leaf": 26.4, + "mutex_leaf": 17.0 + } + }, + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 1, + "kinstr_per_frame": 175.2, + "kcycles_per_frame": 327.1, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 3.17, + "host_load_1m_logged": 2.99, + "samples": 8769, + "markers": { + "learn_end": 1.3, + "poll_forward_drop": 22.0, + "cursor_probe": 7.4, + "syscall_leaf": 22.8, + "mutex_leaf": 20.7 + } + }, + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 2, + "kinstr_per_frame": 173.5, + "kcycles_per_frame": 322.5, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 3.03, + "host_load_1m_logged": 3.04, + "samples": 8677, + "markers": { + "learn_end": 1.2, + "poll_forward_drop": 21.4, + "cursor_probe": 7.7, + "syscall_leaf": 21.9, + "mutex_leaf": 21.8 + } + }, + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 3, + "kinstr_per_frame": 125.6, + "kcycles_per_frame": 279.7, + "recv_fps": 20569, + "groups_lost": 32897, + "host_load_1m": 24.77, + "host_load_1m_logged": 24.77, + "samples": 6509, + "markers": { + "learn_end": 0.4, + "poll_forward_drop": 14.9, + "cursor_probe": 6.5, + "syscall_leaf": 12.4, + "mutex_leaf": 23.5 + } + }, + { + "revision": "2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3", + "label": "base-2fff7ea7ae4e", + "round": 1, + "kinstr_per_frame": 207.1, + "kcycles_per_frame": 412.6, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 2.9, + "host_load_1m_logged": 2.98, + "samples": 10354, + "markers": { + "learn_end": 16.7, + "poll_forward_drop": 22.5, + "cursor_probe": 7.0, + "syscall_leaf": 29.5, + "mutex_leaf": 17.7 + } + }, + { + "revision": "2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3", + "label": "base-2fff7ea7ae4e", + "round": 2, + "kinstr_per_frame": 202.8, + "kcycles_per_frame": 412.8, + "recv_fps": 20001, + "groups_lost": 0, + "host_load_1m": 3.67, + "host_load_1m_logged": 4.13, + "samples": 10136, + "markers": { + "learn_end": 16.3, + "poll_forward_drop": 22.0, + "cursor_probe": 6.7, + "syscall_leaf": 29.1, + "mutex_leaf": 18.1 + } + }, + { + "revision": "2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3", + "label": "base-2fff7ea7ae4e", + "round": 3, + "kinstr_per_frame": 189.9, + "kcycles_per_frame": 401.7, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 11.36, + "host_load_1m_logged": 11.56, + "samples": 9511, + "markers": { + "learn_end": 15.0, + "poll_forward_drop": 21.0, + "cursor_probe": 7.0, + "syscall_leaf": 24.8, + "mutex_leaf": 19.0 + } + }, + { + "revision": "55f6ed229250d17fc99389237659906d011d8a94", + "label": "base-55f6ed229250", + "round": 1, + "kinstr_per_frame": 171.1, + "kcycles_per_frame": 332.6, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 2.19, + "host_load_1m_logged": 2.06, + "samples": 8555, + "markers": { + "learn_end": 17.3, + "poll_forward_drop": 11.6, + "cursor_probe": 0.0, + "syscall_leaf": 30.6, + "mutex_leaf": 16.7 + } + }, + { + "revision": "55f6ed229250d17fc99389237659906d011d8a94", + "label": "base-55f6ed229250", + "round": 2, + "kinstr_per_frame": 167.3, + "kcycles_per_frame": 337.6, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 3.78, + "host_load_1m_logged": 3.72, + "samples": 8366, + "markers": { + "learn_end": 16.0, + "poll_forward_drop": 10.1, + "cursor_probe": 0.0, + "syscall_leaf": 28.8, + "mutex_leaf": 16.7 + } + }, + { + "revision": "55f6ed229250d17fc99389237659906d011d8a94", + "label": "base-55f6ed229250", + "round": 3, + "kinstr_per_frame": 169.2, + "kcycles_per_frame": 337.9, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 5.82, + "host_load_1m_logged": 6.05, + "samples": 8458, + "markers": { + "learn_end": 16.8, + "poll_forward_drop": 11.5, + "cursor_probe": 0.0, + "syscall_leaf": 30.4, + "mutex_leaf": 16.4 + } + }, + { + "revision": "991572d636d264c6189a3477849d2969e689f55c", + "label": "base-991572d636d2", + "round": 1, + "kinstr_per_frame": 202.7, + "kcycles_per_frame": 392.8, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 2.06, + "host_load_1m_logged": 2.15, + "samples": 10130, + "markers": { + "learn_end": 15.3, + "poll_forward_drop": 19.7, + "cursor_probe": 7.3, + "syscall_leaf": 33.7, + "mutex_leaf": 16.7 + } + }, + { + "revision": "991572d636d264c6189a3477849d2969e689f55c", + "label": "base-991572d636d2", + "round": 2, + "kinstr_per_frame": 195.1, + "kcycles_per_frame": 379.0, + "recv_fps": 20002, + "groups_lost": 0, + "host_load_1m": 4.73, + "host_load_1m_logged": 4.91, + "samples": 9752, + "markers": { + "learn_end": 15.2, + "poll_forward_drop": 18.3, + "cursor_probe": 6.9, + "syscall_leaf": 32.3, + "mutex_leaf": 16.6 + } + }, + { + "revision": "991572d636d264c6189a3477849d2969e689f55c", + "label": "base-991572d636d2", + "round": 3, + "kinstr_per_frame": 184.4, + "kcycles_per_frame": 365.8, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 6.87, + "host_load_1m_logged": 7.05, + "samples": 9215, + "markers": { + "learn_end": 14.6, + "poll_forward_drop": 18.3, + "cursor_probe": 7.1, + "syscall_leaf": 27.8, + "mutex_leaf": 18.1 + } + }, + { + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "label": "base-cdff3c8f952b", + "round": 1, + "kinstr_per_frame": 118.7, + "kcycles_per_frame": 212.2, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 2.22, + "host_load_1m_logged": 2.44, + "samples": 5942, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 6.8, + "cursor_probe": 0.0, + "syscall_leaf": 25.2, + "mutex_leaf": 16.1 + } + }, + { + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "label": "base-cdff3c8f952b", + "round": 2, + "kinstr_per_frame": 111.5, + "kcycles_per_frame": 222.3, + "recv_fps": 19999, + "groups_lost": 0, + "host_load_1m": 4.8, + "host_load_1m_logged": 5.2, + "samples": 5579, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 7.4, + "cursor_probe": 0.0, + "syscall_leaf": 22.4, + "mutex_leaf": 18.1 + } + }, + { + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "label": "base-cdff3c8f952b", + "round": 3, + "kinstr_per_frame": 109.8, + "kcycles_per_frame": 235.5, + "recv_fps": 20003, + "groups_lost": 0, + "host_load_1m": 21.62, + "host_load_1m_logged": 24.0, + "samples": 5484, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 6.9, + "cursor_probe": 0.0, + "syscall_leaf": 21.3, + "mutex_leaf": 17.3 + } + }, + { + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "label": "base-e84ffb3de313", + "round": 1, + "kinstr_per_frame": 135.0, + "kcycles_per_frame": 266.2, + "recv_fps": 19997, + "groups_lost": 0, + "host_load_1m": 2.84, + "host_load_1m_logged": 3.41, + "samples": 6746, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 7.8, + "cursor_probe": 0.0, + "syscall_leaf": 22.2, + "mutex_leaf": 16.0 + } + }, + { + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "label": "base-e84ffb3de313", + "round": 2, + "kinstr_per_frame": 136.9, + "kcycles_per_frame": 267.9, + "recv_fps": 19997, + "groups_lost": 0, + "host_load_1m": 4.93, + "host_load_1m_logged": 5.19, + "samples": 6846, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 8.0, + "cursor_probe": 0.0, + "syscall_leaf": 22.6, + "mutex_leaf": 16.1 + } + }, + { + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "label": "base-e84ffb3de313", + "round": 3, + "kinstr_per_frame": 116.1, + "kcycles_per_frame": 310.9, + "recv_fps": 22018, + "groups_lost": 31280, + "host_load_1m": 27.7, + "host_load_1m_logged": 27.7, + "samples": 6404, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 4.8, + "cursor_probe": 0.0, + "syscall_leaf": 16.0, + "mutex_leaf": 14.5 + } + }, + { + "revision": "a6cd6f91b6a0c79a81f2a012a2683bc77bd6c7d7", + "label": "current", + "round": 1, + "kinstr_per_frame": 145.6, + "kcycles_per_frame": 272.3, + "recv_fps": 19997, + "groups_lost": 0, + "host_load_1m": 9.76, + "host_load_1m_logged": 11.0, + "samples": 7274, + "markers": { + "learn_end": 0.5, + "poll_forward_drop": 14.8, + "cursor_probe": 6.8, + "syscall_leaf": 17.3, + "mutex_leaf": 19.5 + } + }, + { + "revision": "a6cd6f91b6a0c79a81f2a012a2683bc77bd6c7d7", + "label": "current", + "round": 2, + "kinstr_per_frame": 146.5, + "kcycles_per_frame": 281.0, + "recv_fps": 19998, + "groups_lost": 0, + "host_load_1m": 3.64, + "host_load_1m_logged": 3.63, + "samples": 7315, + "markers": { + "learn_end": 0.5, + "poll_forward_drop": 14.3, + "cursor_probe": 6.9, + "syscall_leaf": 16.8, + "mutex_leaf": 19.6 + } + }, + { + "revision": "a6cd6f91b6a0c79a81f2a012a2683bc77bd6c7d7", + "label": "current", + "round": 3, + "kinstr_per_frame": 143.1, + "kcycles_per_frame": 313.0, + "recv_fps": 19304, + "groups_lost": 0, + "host_load_1m": 32.43, + "host_load_1m_logged": 32.92, + "samples": 6910, + "markers": { + "learn_end": 0.4, + "poll_forward_drop": 14.1, + "cursor_probe": 7.4, + "syscall_leaf": 14.0, + "mutex_leaf": 19.5 + } + } + ] + }, + { + "artifact": "run.qYPSii", + "harness_revision": "74f46e79b75bb2cbb4c5d4c4d75e390249ba22d5", + "workload": "chat", + "runtime": "io-uring-workers", + "workers": 4, + "generator_revision": "21d6da9fc1", + "instruction_period": 2000003, + "sampling": "after the counter window", + "relay_cpus": "4-7", + "generator_cpus": "0-3", + "cells": [ + { + "revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", + "label": "base-0c2a798de5c2", + "round": 1, + "kinstr_per_frame": 246.0, + "kcycles_per_frame": 599.4, + "recv_fps": 7525, + "groups_lost": 16750, + "host_load_1m": 45.62, + "host_load_1m_logged": 54.66, + "samples": 5208, + "markers": { + "learn_end": 8.2, + "poll_forward_drop": 11.1, + "cursor_probe": 0.0, + "syscall_leaf": 14.2, + "mutex_leaf": 15.9 + } + }, + { + "revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", + "label": "base-0c2a798de5c2", + "round": 2, + "kinstr_per_frame": 149.9, + "kcycles_per_frame": 304.2, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 19.15, + "host_load_1m_logged": 19.77, + "samples": 7163, + "markers": { + "learn_end": 12.1, + "poll_forward_drop": 8.9, + "cursor_probe": 0.0, + "syscall_leaf": 18.0, + "mutex_leaf": 20.2 + } + }, + { + "revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", + "label": "base-0c2a798de5c2", + "round": 3, + "kinstr_per_frame": 169.8, + "kcycles_per_frame": 325.1, + "recv_fps": 19999, + "groups_lost": 0, + "host_load_1m": 4.38, + "host_load_1m_logged": 4.75, + "samples": 8337, + "markers": { + "learn_end": 15.4, + "poll_forward_drop": 11.7, + "cursor_probe": 0.0, + "syscall_leaf": 27.5, + "mutex_leaf": 16.8 + } + }, + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 1, + "kinstr_per_frame": 160.3, + "kcycles_per_frame": 305.2, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 17.94, + "host_load_1m_logged": 18.93, + "samples": 7895, + "markers": { + "learn_end": 0.9, + "poll_forward_drop": 19.7, + "cursor_probe": 7.6, + "syscall_leaf": 15.5, + "mutex_leaf": 23.8 + } + }, + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 2, + "kinstr_per_frame": 167.9, + "kcycles_per_frame": 337.0, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 9.63, + "host_load_1m_logged": 9.43, + "samples": 8307, + "markers": { + "learn_end": 1.1, + "poll_forward_drop": 21.0, + "cursor_probe": 7.2, + "syscall_leaf": 19.2, + "mutex_leaf": 22.7 + } + }, + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 3, + "kinstr_per_frame": 176.2, + "kcycles_per_frame": 328.6, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 3.1, + "host_load_1m_logged": 3.64, + "samples": 8758, + "markers": { + "learn_end": 1.3, + "poll_forward_drop": 21.2, + "cursor_probe": 7.3, + "syscall_leaf": 22.3, + "mutex_leaf": 20.7 + } + }, + { + "revision": "2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3", + "label": "base-2fff7ea7ae4e", + "round": 1, + "kinstr_per_frame": 169.1, + "kcycles_per_frame": 409.6, + "recv_fps": 17369, + "groups_lost": 5362, + "host_load_1m": 24.92, + "host_load_1m_logged": 23.96, + "samples": 9126, + "markers": { + "learn_end": 13.8, + "poll_forward_drop": 18.9, + "cursor_probe": 6.2, + "syscall_leaf": 17.4, + "mutex_leaf": 21.8 + } + }, + { + "revision": "2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3", + "label": "base-2fff7ea7ae4e", + "round": 2, + "kinstr_per_frame": 161.7, + "kcycles_per_frame": 349.3, + "recv_fps": 19820, + "groups_lost": 0, + "host_load_1m": 11.62, + "host_load_1m_logged": 10.84, + "samples": 8749, + "markers": { + "learn_end": 14.1, + "poll_forward_drop": 18.8, + "cursor_probe": 6.7, + "syscall_leaf": 19.0, + "mutex_leaf": 21.1 + } + }, + { + "revision": "2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3", + "label": "base-2fff7ea7ae4e", + "round": 3, + "kinstr_per_frame": 214.0, + "kcycles_per_frame": 415.4, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 2.35, + "host_load_1m_logged": 2.56, + "samples": 10709, + "markers": { + "learn_end": 17.0, + "poll_forward_drop": 24.0, + "cursor_probe": 7.6, + "syscall_leaf": 32.7, + "mutex_leaf": 16.9 + } + }, + { + "revision": "55f6ed229250d17fc99389237659906d011d8a94", + "label": "base-55f6ed229250", + "round": 1, + "kinstr_per_frame": 245.2, + "kcycles_per_frame": 655.0, + "recv_fps": 7433, + "groups_lost": 28655, + "host_load_1m": 53.37, + "host_load_1m_logged": 53.25, + "samples": 5224, + "markers": { + "learn_end": 7.2, + "poll_forward_drop": 8.3, + "cursor_probe": 0.0, + "syscall_leaf": 15.9, + "mutex_leaf": 16.9 + } + }, + { + "revision": "55f6ed229250d17fc99389237659906d011d8a94", + "label": "base-55f6ed229250", + "round": 2, + "kinstr_per_frame": 166.6, + "kcycles_per_frame": 340.8, + "recv_fps": 19997, + "groups_lost": 0, + "host_load_1m": 11.77, + "host_load_1m_logged": 12.27, + "samples": 8277, + "markers": { + "learn_end": 16.1, + "poll_forward_drop": 10.8, + "cursor_probe": 0.0, + "syscall_leaf": 27.6, + "mutex_leaf": 17.9 + } + }, + { + "revision": "55f6ed229250d17fc99389237659906d011d8a94", + "label": "base-55f6ed229250", + "round": 3, + "kinstr_per_frame": 174.6, + "kcycles_per_frame": 343.7, + "recv_fps": 19999, + "groups_lost": 0, + "host_load_1m": 3.23, + "host_load_1m_logged": 3.42, + "samples": 8758, + "markers": { + "learn_end": 17.5, + "poll_forward_drop": 11.7, + "cursor_probe": 0.0, + "syscall_leaf": 31.6, + "mutex_leaf": 16.3 + } + }, + { + "revision": "991572d636d264c6189a3477849d2969e689f55c", + "label": "base-991572d636d2", + "round": 1, + "kinstr_per_frame": 164.2, + "kcycles_per_frame": 331.5, + "recv_fps": 19825, + "groups_lost": 0, + "host_load_1m": 30.42, + "host_load_1m_logged": 31.94, + "samples": 8019, + "markers": { + "learn_end": 11.0, + "poll_forward_drop": 15.1, + "cursor_probe": 6.5, + "syscall_leaf": 17.6, + "mutex_leaf": 20.5 + } + }, + { + "revision": "991572d636d264c6189a3477849d2969e689f55c", + "label": "base-991572d636d2", + "round": 2, + "kinstr_per_frame": 181.6, + "kcycles_per_frame": 363.0, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 9.45, + "host_load_1m_logged": 9.45, + "samples": 9524, + "markers": { + "learn_end": 14.6, + "poll_forward_drop": 19.1, + "cursor_probe": 7.1, + "syscall_leaf": 30.3, + "mutex_leaf": 17.2 + } + }, + { + "revision": "991572d636d264c6189a3477849d2969e689f55c", + "label": "base-991572d636d2", + "round": 3, + "kinstr_per_frame": 204.7, + "kcycles_per_frame": 382.8, + "recv_fps": 20001, + "groups_lost": 0, + "host_load_1m": 2.13, + "host_load_1m_logged": 2.32, + "samples": 10294, + "markers": { + "learn_end": 15.5, + "poll_forward_drop": 20.1, + "cursor_probe": 7.7, + "syscall_leaf": 35.0, + "mutex_leaf": 16.6 + } + }, + { + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "label": "base-cdff3c8f952b", + "round": 1, + "kinstr_per_frame": 124.4, + "kcycles_per_frame": 287.4, + "recv_fps": 16559, + "groups_lost": 8554, + "host_load_1m": 21.13, + "host_load_1m_logged": 22.93, + "samples": 5763, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 5.3, + "cursor_probe": 0.0, + "syscall_leaf": 19.8, + "mutex_leaf": 16.7 + } + }, + { + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "label": "base-cdff3c8f952b", + "round": 2, + "kinstr_per_frame": 115.3, + "kcycles_per_frame": 219.5, + "recv_fps": 20001, + "groups_lost": 0, + "host_load_1m": 4.79, + "host_load_1m_logged": 5.05, + "samples": 5711, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 7.5, + "cursor_probe": 0.0, + "syscall_leaf": 23.9, + "mutex_leaf": 16.1 + } + }, + { + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "label": "base-cdff3c8f952b", + "round": 3, + "kinstr_per_frame": 117.7, + "kcycles_per_frame": 214.6, + "recv_fps": 19998, + "groups_lost": 0, + "host_load_1m": 2.38, + "host_load_1m_logged": 2.7, + "samples": 5823, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 7.1, + "cursor_probe": 0.0, + "syscall_leaf": 24.5, + "mutex_leaf": 16.0 + } + }, + { + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "label": "base-e84ffb3de313", + "round": 1, + "kinstr_per_frame": 125.5, + "kcycles_per_frame": 294.0, + "recv_fps": 19998, + "groups_lost": 0, + "host_load_1m": 21.29, + "host_load_1m_logged": 22.36, + "samples": 6352, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 6.3, + "cursor_probe": 0.0, + "syscall_leaf": 16.8, + "mutex_leaf": 17.2 + } + }, + { + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "label": "base-e84ffb3de313", + "round": 2, + "kinstr_per_frame": 128.2, + "kcycles_per_frame": 355.6, + "recv_fps": 19590, + "groups_lost": 0, + "host_load_1m": 24.24, + "host_load_1m_logged": 27.66, + "samples": 6356, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 5.0, + "cursor_probe": 0.0, + "syscall_leaf": 17.9, + "mutex_leaf": 15.0 + } + }, + { + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "label": "base-e84ffb3de313", + "round": 3, + "kinstr_per_frame": 135.7, + "kcycles_per_frame": 264.2, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 2.24, + "host_load_1m_logged": 2.24, + "samples": 6774, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 7.6, + "cursor_probe": 0.0, + "syscall_leaf": 21.6, + "mutex_leaf": 16.2 + } + }, + { + "revision": "74f46e79b75bb2cbb4c5d4c4d75e390249ba22d5", + "label": "current", + "round": 1, + "kinstr_per_frame": 145.2, + "kcycles_per_frame": 278.0, + "recv_fps": 19999, + "groups_lost": 0, + "host_load_1m": 11.7, + "host_load_1m_logged": 11.76, + "samples": 7185, + "markers": { + "learn_end": 0.4, + "poll_forward_drop": 14.3, + "cursor_probe": 6.6, + "syscall_leaf": 15.5, + "mutex_leaf": 19.8 + } + }, + { + "revision": "74f46e79b75bb2cbb4c5d4c4d75e390249ba22d5", + "label": "current", + "round": 2, + "kinstr_per_frame": 147.0, + "kcycles_per_frame": 278.5, + "recv_fps": 20003, + "groups_lost": 0, + "host_load_1m": 6.81, + "host_load_1m_logged": 6.97, + "samples": 7385, + "markers": { + "learn_end": 0.6, + "poll_forward_drop": 15.5, + "cursor_probe": 7.1, + "syscall_leaf": 17.7, + "mutex_leaf": 18.0 + } + }, + { + "revision": "74f46e79b75bb2cbb4c5d4c4d75e390249ba22d5", + "label": "current", + "round": 3, + "kinstr_per_frame": 148.8, + "kcycles_per_frame": 269.2, + "recv_fps": 20001, + "groups_lost": 0, + "host_load_1m": 3.07, + "host_load_1m_logged": 3.07, + "samples": 7392, + "markers": { + "learn_end": 0.5, + "poll_forward_drop": 14.4, + "cursor_probe": 6.6, + "syscall_leaf": 17.7, + "mutex_leaf": 19.3 + } + } + ] + }, + { + "artifact": "run.16AJUb", + "harness_revision": "bd812a762c149a4345430b53530a597d261144b8", + "workload": "chat", + "runtime": "io-uring-workers", + "workers": 4, + "generator_revision": "21d6da9fc1", + "instruction_period": 2000003, + "sampling": "after the counter window", + "relay_cpus": "4-7", + "generator_cpus": "0-3", + "cells": [ + { + "revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", + "label": "base-0c2a798de5c2", + "round": 1, + "kinstr_per_frame": 158.5, + "kcycles_per_frame": 330.8, + "recv_fps": 20001, + "groups_lost": 0, + "host_load_1m": 13.91, + "host_load_1m_logged": 14.83, + "samples": 7839, + "markers": { + "learn_end": 14.9, + "poll_forward_drop": 10.7, + "cursor_probe": 0.0, + "syscall_leaf": 23.5, + "mutex_leaf": 17.1 + } + }, + { + "revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", + "label": "base-0c2a798de5c2", + "round": 2, + "kinstr_per_frame": 139.5, + "kcycles_per_frame": 330.1, + "recv_fps": 19753, + "groups_lost": 0, + "host_load_1m": 25.02, + "host_load_1m_logged": 25.76, + "samples": 6782, + "markers": { + "learn_end": 11.1, + "poll_forward_drop": 8.5, + "cursor_probe": 0.0, + "syscall_leaf": 15.1, + "mutex_leaf": 18.7 + } + }, + { + "revision": "0c2a798de5c2e9abb29815b86f945b0a980e5345", + "label": "base-0c2a798de5c2", + "round": 3, + "kinstr_per_frame": 140.8, + "kcycles_per_frame": 335.0, + "recv_fps": 19906, + "groups_lost": 0, + "host_load_1m": 21.62, + "host_load_1m_logged": 21.62, + "samples": 7065, + "markers": { + "learn_end": 12.0, + "poll_forward_drop": 9.0, + "cursor_probe": 0.0, + "syscall_leaf": 16.1, + "mutex_leaf": 19.6 + } + }, + { + "revision": "1e05129753c720d0b2318cedbab345d745227d2b", + "label": "base-1e05129753c7", + "round": 1, + "kinstr_per_frame": 216.4, + "kcycles_per_frame": 513.7, + "recv_fps": 9349, + "groups_lost": 22438, + "host_load_1m": 32.14, + "host_load_1m_logged": 32.14, + "samples": 5175, + "markers": { + "learn_end": 8.5, + "poll_forward_drop": 6.8, + "cursor_probe": 0.0, + "syscall_leaf": 13.7, + "mutex_leaf": 17.7 + } + }, + { + "revision": "1e05129753c720d0b2318cedbab345d745227d2b", + "label": "base-1e05129753c7", + "round": 2, + "kinstr_per_frame": 139.6, + "kcycles_per_frame": 328.5, + "recv_fps": 19696, + "groups_lost": 6849, + "host_load_1m": 29.39, + "host_load_1m_logged": 29.39, + "samples": 7166, + "markers": { + "learn_end": 12.1, + "poll_forward_drop": 8.4, + "cursor_probe": 0.0, + "syscall_leaf": 17.0, + "mutex_leaf": 19.3 + } + }, + { + "revision": "1e05129753c720d0b2318cedbab345d745227d2b", + "label": "base-1e05129753c7", + "round": 3, + "kinstr_per_frame": 150.4, + "kcycles_per_frame": 299.4, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 7.22, + "host_load_1m_logged": 7.12, + "samples": 7901, + "markers": { + "learn_end": 14.5, + "poll_forward_drop": 11.2, + "cursor_probe": 0.0, + "syscall_leaf": 24.7, + "mutex_leaf": 17.7 + } + }, + { + "revision": "45007f6fb14fb44adc02823a8035f4f39de8ab94", + "label": "base-45007f6fb14f", + "round": 1, + "kinstr_per_frame": 144.6, + "kcycles_per_frame": 353.6, + "recv_fps": 16712, + "groups_lost": 13701, + "host_load_1m": 23.05, + "host_load_1m_logged": 28.3, + "samples": 5098, + "markers": { + "learn_end": 9.1, + "poll_forward_drop": 6.8, + "cursor_probe": 0.0, + "syscall_leaf": 14.3, + "mutex_leaf": 18.0 + } + }, + { + "revision": "45007f6fb14fb44adc02823a8035f4f39de8ab94", + "label": "base-45007f6fb14f", + "round": 2, + "kinstr_per_frame": 156.9, + "kcycles_per_frame": 330.2, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 20.44, + "host_load_1m_logged": 22.17, + "samples": 7645, + "markers": { + "learn_end": 13.6, + "poll_forward_drop": 9.5, + "cursor_probe": 0.0, + "syscall_leaf": 21.8, + "mutex_leaf": 19.1 + } + }, + { + "revision": "45007f6fb14fb44adc02823a8035f4f39de8ab94", + "label": "base-45007f6fb14f", + "round": 3, + "kinstr_per_frame": 160.8, + "kcycles_per_frame": 329.1, + "recv_fps": 19998, + "groups_lost": 0, + "host_load_1m": 252.79, + "host_load_1m_logged": 254.35, + "samples": 7942, + "markers": { + "learn_end": 15.2, + "poll_forward_drop": 11.2, + "cursor_probe": 0.0, + "syscall_leaf": 25.0, + "mutex_leaf": 17.2 + } + }, + { + "revision": "87515b48c693457b7541db12c66836fd6fc6b1e8", + "label": "base-87515b48c693", + "round": 1, + "kinstr_per_frame": 141.7, + "kcycles_per_frame": 332.4, + "recv_fps": 19983, + "groups_lost": 0, + "host_load_1m": 31.75, + "host_load_1m_logged": 32.37, + "samples": 6666, + "markers": { + "learn_end": 11.0, + "poll_forward_drop": 8.6, + "cursor_probe": 0.0, + "syscall_leaf": 15.3, + "mutex_leaf": 20.4 + } + }, + { + "revision": "87515b48c693457b7541db12c66836fd6fc6b1e8", + "label": "base-87515b48c693", + "round": 2, + "kinstr_per_frame": 141.6, + "kcycles_per_frame": 303.1, + "recv_fps": 19765, + "groups_lost": 0, + "host_load_1m": 15.34, + "host_load_1m_logged": 16.9, + "samples": 6887, + "markers": { + "learn_end": 11.0, + "poll_forward_drop": 8.4, + "cursor_probe": 0.0, + "syscall_leaf": 14.7, + "mutex_leaf": 20.3 + } + }, + { + "revision": "87515b48c693457b7541db12c66836fd6fc6b1e8", + "label": "base-87515b48c693", + "round": 3, + "kinstr_per_frame": 169.8, + "kcycles_per_frame": 340.9, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 96.24, + "host_load_1m_logged": 112.61, + "samples": 8412, + "markers": { + "learn_end": 14.8, + "poll_forward_drop": 11.1, + "cursor_probe": 0.0, + "syscall_leaf": 28.2, + "mutex_leaf": 16.3 + } + }, + { + "revision": "8ee3c80894186483b3c0d8f7e84b37ab0f525390", + "label": "base-8ee3c8089418", + "round": 1, + "kinstr_per_frame": 141.9, + "kcycles_per_frame": 356.1, + "recv_fps": 15002, + "groups_lost": 38842, + "host_load_1m": 30.58, + "host_load_1m_logged": 31.49, + "samples": 6099, + "markers": { + "learn_end": 10.8, + "poll_forward_drop": 8.8, + "cursor_probe": 0.0, + "syscall_leaf": 12.7, + "mutex_leaf": 18.4 + } + }, + { + "revision": "8ee3c80894186483b3c0d8f7e84b37ab0f525390", + "label": "base-8ee3c8089418", + "round": 2, + "kinstr_per_frame": 152.6, + "kcycles_per_frame": 349.8, + "recv_fps": 20012, + "groups_lost": 0, + "host_load_1m": 29.14, + "host_load_1m_logged": 29.13, + "samples": 8262, + "markers": { + "learn_end": 15.4, + "poll_forward_drop": 11.5, + "cursor_probe": 0.0, + "syscall_leaf": 27.4, + "mutex_leaf": 16.7 + } + }, + { + "revision": "8ee3c80894186483b3c0d8f7e84b37ab0f525390", + "label": "base-8ee3c8089418", + "round": 3, + "kinstr_per_frame": 150.4, + "kcycles_per_frame": 303.5, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 10.23, + "host_load_1m_logged": 10.69, + "samples": 7586, + "markers": { + "learn_end": 13.8, + "poll_forward_drop": 9.4, + "cursor_probe": 0.0, + "syscall_leaf": 20.6, + "mutex_leaf": 17.5 + } + }, + { + "revision": "c3c88aec81350cece608599cef5c33322547d146", + "label": "base-c3c88aec8135", + "round": 1, + "kinstr_per_frame": 150.7, + "kcycles_per_frame": 322.3, + "recv_fps": 19997, + "groups_lost": 0, + "host_load_1m": 20.63, + "host_load_1m_logged": 22.23, + "samples": 7644, + "markers": { + "learn_end": 14.9, + "poll_forward_drop": 10.1, + "cursor_probe": 0.0, + "syscall_leaf": 22.2, + "mutex_leaf": 17.7 + } + }, + { + "revision": "c3c88aec81350cece608599cef5c33322547d146", + "label": "base-c3c88aec8135", + "round": 2, + "kinstr_per_frame": 123.9, + "kcycles_per_frame": 272.4, + "recv_fps": 22352, + "groups_lost": 10324, + "host_load_1m": 23.92, + "host_load_1m_logged": 24.14, + "samples": 6730, + "markers": { + "learn_end": 11.5, + "poll_forward_drop": 8.4, + "cursor_probe": 0.0, + "syscall_leaf": 15.8, + "mutex_leaf": 19.8 + } + }, + { + "revision": "c3c88aec81350cece608599cef5c33322547d146", + "label": "base-c3c88aec8135", + "round": 3, + "kinstr_per_frame": 160.4, + "kcycles_per_frame": 329.0, + "recv_fps": 20003, + "groups_lost": 0, + "host_load_1m": 36.05, + "host_load_1m_logged": 44.87, + "samples": 8046, + "markers": { + "learn_end": 15.6, + "poll_forward_drop": 10.2, + "cursor_probe": 0.0, + "syscall_leaf": 25.8, + "mutex_leaf": 16.7 + } + }, + { + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "label": "base-e84ffb3de313", + "round": 1, + "kinstr_per_frame": 129.7, + "kcycles_per_frame": 275.4, + "recv_fps": 20002, + "groups_lost": 0, + "host_load_1m": 32.54, + "host_load_1m_logged": 32.54, + "samples": 6495, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 6.6, + "cursor_probe": 0.0, + "syscall_leaf": 19.1, + "mutex_leaf": 16.6 + } + }, + { + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "label": "base-e84ffb3de313", + "round": 2, + "kinstr_per_frame": 131.0, + "kcycles_per_frame": 259.4, + "recv_fps": 19883, + "groups_lost": 0, + "host_load_1m": 15.61, + "host_load_1m_logged": 15.61, + "samples": 6344, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 6.3, + "cursor_probe": 0.0, + "syscall_leaf": 16.6, + "mutex_leaf": 17.1 + } + }, + { + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "label": "base-e84ffb3de313", + "round": 3, + "kinstr_per_frame": 130.0, + "kcycles_per_frame": 267.5, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 9.84, + "host_load_1m_logged": 9.84, + "samples": 6231, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 5.7, + "cursor_probe": 0.0, + "syscall_leaf": 16.4, + "mutex_leaf": 17.0 + } + }, + { + "revision": "fd59eb18a50f6e0ac819e73d941a3832cbdac91f", + "label": "base-fd59eb18a50f", + "round": 1, + "kinstr_per_frame": 136.4, + "kcycles_per_frame": 357.7, + "recv_fps": 21611, + "groups_lost": 3490, + "host_load_1m": 20.4, + "host_load_1m_logged": 20.99, + "samples": 6836, + "markers": { + "learn_end": 10.5, + "poll_forward_drop": 8.2, + "cursor_probe": 0.0, + "syscall_leaf": 15.4, + "mutex_leaf": 18.7 + } + }, + { + "revision": "fd59eb18a50f6e0ac819e73d941a3832cbdac91f", + "label": "base-fd59eb18a50f", + "round": 2, + "kinstr_per_frame": 146.0, + "kcycles_per_frame": 364.8, + "recv_fps": 20004, + "groups_lost": 0, + "host_load_1m": 37.85, + "host_load_1m_logged": 37.14, + "samples": 7347, + "markers": { + "learn_end": 12.6, + "poll_forward_drop": 8.9, + "cursor_probe": 0.0, + "syscall_leaf": 17.4, + "mutex_leaf": 17.7 + } + }, + { + "revision": "fd59eb18a50f6e0ac819e73d941a3832cbdac91f", + "label": "base-fd59eb18a50f", + "round": 3, + "kinstr_per_frame": 159.4, + "kcycles_per_frame": 311.3, + "recv_fps": 20003, + "groups_lost": 0, + "host_load_1m": 7.45, + "host_load_1m_logged": 7.49, + "samples": 8229, + "markers": { + "learn_end": 14.9, + "poll_forward_drop": 10.2, + "cursor_probe": 0.0, + "syscall_leaf": 26.3, + "mutex_leaf": 16.3 + } + }, + { + "revision": "bd812a762c149a4345430b53530a597d261144b8", + "label": "current", + "round": 1, + "kinstr_per_frame": 144.6, + "kcycles_per_frame": 354.3, + "recv_fps": 19955, + "groups_lost": 0, + "host_load_1m": 15.03, + "host_load_1m_logged": 15.03, + "samples": 6179, + "markers": { + "learn_end": 0.3, + "poll_forward_drop": 12.2, + "cursor_probe": 6.4, + "syscall_leaf": 14.2, + "mutex_leaf": 19.0 + } + }, + { + "revision": "bd812a762c149a4345430b53530a597d261144b8", + "label": "current", + "round": 2, + "kinstr_per_frame": 141.5, + "kcycles_per_frame": 298.8, + "recv_fps": 20036, + "groups_lost": 0, + "host_load_1m": 29.44, + "host_load_1m_logged": 28.52, + "samples": 7167, + "markers": { + "learn_end": 0.3, + "poll_forward_drop": 13.6, + "cursor_probe": 7.4, + "syscall_leaf": 14.2, + "mutex_leaf": 20.2 + } + }, + { + "revision": "bd812a762c149a4345430b53530a597d261144b8", + "label": "current", + "round": 3, + "kinstr_per_frame": 142.7, + "kcycles_per_frame": 281.2, + "recv_fps": 19996, + "groups_lost": 0, + "host_load_1m": 16.58, + "host_load_1m_logged": 17.57, + "samples": 7166, + "markers": { + "learn_end": 0.3, + "poll_forward_drop": 14.8, + "cursor_probe": 6.7, + "syscall_leaf": 16.0, + "mutex_leaf": 18.4 + } + } + ] + }, + { + "artifact": "run.SkAD0k", + "harness_revision": "bd812a762c149a4345430b53530a597d261144b8", + "workload": "video", + "runtime": "io-uring-workers", + "workers": 4, + "generator_revision": "21d6da9fc1", + "instruction_period": 2000003, + "sampling": "after the counter window", + "relay_cpus": "4-7", + "generator_cpus": "0-3", + "cells": [ + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 1, + "kinstr_per_frame": 141.1, + "kcycles_per_frame": 330.6, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 5.42, + "host_load_1m_logged": 5.31, + "samples": 331, + "markers": { + "learn_end": 0.6, + "poll_forward_drop": 6.0, + "cursor_probe": 1.8, + "syscall_leaf": 35.6, + "mutex_leaf": 9.7 + } + }, + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 2, + "kinstr_per_frame": 139.5, + "kcycles_per_frame": 336.0, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 7.27, + "host_load_1m_logged": 7.17, + "samples": 324, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 5.6, + "cursor_probe": 1.2, + "syscall_leaf": 33.6, + "mutex_leaf": 9.6 + } + }, + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 3, + "kinstr_per_frame": 138.3, + "kcycles_per_frame": 339.8, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 7.29, + "host_load_1m_logged": 7.18, + "samples": 320, + "markers": { + "learn_end": 0.3, + "poll_forward_drop": 5.9, + "cursor_probe": 1.6, + "syscall_leaf": 32.8, + "mutex_leaf": 8.4 + } + }, + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 4, + "kinstr_per_frame": 146.8, + "kcycles_per_frame": 455.4, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 26.0, + "host_load_1m_logged": 27.34, + "samples": 343, + "markers": { + "learn_end": 0.3, + "poll_forward_drop": 4.7, + "cursor_probe": 1.7, + "syscall_leaf": 37.6, + "mutex_leaf": 9.3 + } + }, + { + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "label": "base-cdff3c8f952b", + "round": 1, + "kinstr_per_frame": 109.6, + "kcycles_per_frame": 291.5, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 4.97, + "host_load_1m_logged": 5.13, + "samples": 260, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 1.5, + "cursor_probe": 0.0, + "syscall_leaf": 43.8, + "mutex_leaf": 9.6 + } + }, + { + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "label": "base-cdff3c8f952b", + "round": 2, + "kinstr_per_frame": 105.6, + "kcycles_per_frame": 273.9, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 6.76, + "host_load_1m_logged": 6.91, + "samples": 253, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 2.8, + "cursor_probe": 0.0, + "syscall_leaf": 42.7, + "mutex_leaf": 7.9 + } + }, + { + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "label": "base-cdff3c8f952b", + "round": 3, + "kinstr_per_frame": 109.9, + "kcycles_per_frame": 286.4, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 6.84, + "host_load_1m_logged": 7.46, + "samples": 256, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 1.2, + "cursor_probe": 0.0, + "syscall_leaf": 41.0, + "mutex_leaf": 7.4 + } + }, + { + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "label": "base-cdff3c8f952b", + "round": 4, + "kinstr_per_frame": 119.9, + "kcycles_per_frame": 437.1, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 12.82, + "host_load_1m_logged": 18.92, + "samples": 282, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 1.4, + "cursor_probe": 0.0, + "syscall_leaf": 42.6, + "mutex_leaf": 8.2 + } + }, + { + "revision": "bd812a762c149a4345430b53530a597d261144b8", + "label": "current", + "round": 1, + "kinstr_per_frame": 120.5, + "kcycles_per_frame": 278.0, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 7.16, + "host_load_1m_logged": 7.16, + "samples": 279, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 0.4, + "cursor_probe": 0.0, + "syscall_leaf": 35.1, + "mutex_leaf": 4.3 + } + }, + { + "revision": "bd812a762c149a4345430b53530a597d261144b8", + "label": "current", + "round": 2, + "kinstr_per_frame": 124.2, + "kcycles_per_frame": 302.8, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 7.5, + "host_load_1m_logged": 7.63, + "samples": 284, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 0.4, + "cursor_probe": 0.4, + "syscall_leaf": 39.8, + "mutex_leaf": 4.2 + } + }, + { + "revision": "bd812a762c149a4345430b53530a597d261144b8", + "label": "current", + "round": 3, + "kinstr_per_frame": 125.6, + "kcycles_per_frame": 285.4, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 7.8, + "host_load_1m_logged": 7.8, + "samples": 288, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 0.3, + "cursor_probe": 0.0, + "syscall_leaf": 38.9, + "mutex_leaf": 3.1 + } + }, + { + "revision": "bd812a762c149a4345430b53530a597d261144b8", + "label": "current", + "round": 4, + "kinstr_per_frame": 138.3, + "kcycles_per_frame": 364.1, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 39.8, + "host_load_1m_logged": 40.13, + "samples": 326, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 0.3, + "cursor_probe": 0.0, + "syscall_leaf": 46.3, + "mutex_leaf": 5.2 + } + } + ] + }, + { + "artifact": "run.IV47ua", + "harness_revision": "bd812a762c149a4345430b53530a597d261144b8", + "workload": "chat", + "runtime": "tokio-workers", + "workers": 4, + "generator_revision": "21d6da9fc1", + "instruction_period": 2000003, + "sampling": "after the counter window", + "relay_cpus": "4-7", + "generator_cpus": "0-3", + "cells": [ + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 1, + "kinstr_per_frame": 185.1, + "kcycles_per_frame": 366.0, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 13.99, + "host_load_1m_logged": 13.99, + "samples": 9276, + "markers": { + "learn_end": 0.9, + "poll_forward_drop": 17.1, + "cursor_probe": 6.4, + "syscall_leaf": 4.3, + "mutex_leaf": 24.8 + } + }, + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 2, + "kinstr_per_frame": 189.6, + "kcycles_per_frame": 360.9, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 6.01, + "host_load_1m_logged": 6.21, + "samples": 9513, + "markers": { + "learn_end": 1.1, + "poll_forward_drop": 17.0, + "cursor_probe": 6.1, + "syscall_leaf": 6.5, + "mutex_leaf": 22.7 + } + }, + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 3, + "kinstr_per_frame": 184.3, + "kcycles_per_frame": 361.2, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 7.76, + "host_load_1m_logged": 7.98, + "samples": 9265, + "markers": { + "learn_end": 1.0, + "poll_forward_drop": 17.1, + "cursor_probe": 6.2, + "syscall_leaf": 4.0, + "mutex_leaf": 24.1 + } + }, + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 4, + "kinstr_per_frame": 187.1, + "kcycles_per_frame": 354.2, + "recv_fps": 19997, + "groups_lost": 0, + "host_load_1m": 6.29, + "host_load_1m_logged": 5.95, + "samples": 9465, + "markers": { + "learn_end": 0.8, + "poll_forward_drop": 18.0, + "cursor_probe": 6.8, + "syscall_leaf": 6.2, + "mutex_leaf": 23.3 + } + }, + { + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "label": "base-cdff3c8f952b", + "round": 1, + "kinstr_per_frame": 123.6, + "kcycles_per_frame": 273.2, + "recv_fps": 19998, + "groups_lost": 0, + "host_load_1m": 22.76, + "host_load_1m_logged": 22.7, + "samples": 6351, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 5.1, + "cursor_probe": 0.0, + "syscall_leaf": 0.3, + "mutex_leaf": 19.9 + } + }, + { + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "label": "base-cdff3c8f952b", + "round": 2, + "kinstr_per_frame": 131.1, + "kcycles_per_frame": 242.1, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 6.37, + "host_load_1m_logged": 7.05, + "samples": 6537, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 5.8, + "cursor_probe": 0.0, + "syscall_leaf": 2.8, + "mutex_leaf": 19.0 + } + }, + { + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "label": "base-cdff3c8f952b", + "round": 3, + "kinstr_per_frame": 131.1, + "kcycles_per_frame": 240.6, + "recv_fps": 20004, + "groups_lost": 0, + "host_load_1m": 5.3, + "host_load_1m_logged": 5.36, + "samples": 6642, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 6.9, + "cursor_probe": 0.0, + "syscall_leaf": 3.2, + "mutex_leaf": 20.7 + } + }, + { + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "label": "base-cdff3c8f952b", + "round": 4, + "kinstr_per_frame": 131.5, + "kcycles_per_frame": 241.5, + "recv_fps": 20003, + "groups_lost": 0, + "host_load_1m": 6.86, + "host_load_1m_logged": 7.27, + "samples": 6565, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 6.0, + "cursor_probe": 0.0, + "syscall_leaf": 2.6, + "mutex_leaf": 20.0 + } + }, + { + "revision": "bd812a762c149a4345430b53530a597d261144b8", + "label": "current", + "round": 1, + "kinstr_per_frame": 172.8, + "kcycles_per_frame": 324.4, + "recv_fps": 19996, + "groups_lost": 0, + "host_load_1m": 7.61, + "host_load_1m_logged": 8.28, + "samples": 8639, + "markers": { + "learn_end": 0.3, + "poll_forward_drop": 12.2, + "cursor_probe": 6.2, + "syscall_leaf": 1.6, + "mutex_leaf": 20.5 + } + }, + { + "revision": "bd812a762c149a4345430b53530a597d261144b8", + "label": "current", + "round": 2, + "kinstr_per_frame": 171.0, + "kcycles_per_frame": 362.7, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 11.35, + "host_load_1m_logged": 11.47, + "samples": 8545, + "markers": { + "learn_end": 0.3, + "poll_forward_drop": 12.4, + "cursor_probe": 6.3, + "syscall_leaf": 1.0, + "mutex_leaf": 20.1 + } + }, + { + "revision": "bd812a762c149a4345430b53530a597d261144b8", + "label": "current", + "round": 3, + "kinstr_per_frame": 172.2, + "kcycles_per_frame": 318.2, + "recv_fps": 20000, + "groups_lost": 0, + "host_load_1m": 5.71, + "host_load_1m_logged": 5.76, + "samples": 8638, + "markers": { + "learn_end": 0.4, + "poll_forward_drop": 11.6, + "cursor_probe": 6.0, + "syscall_leaf": 1.9, + "mutex_leaf": 19.8 + } + }, + { + "revision": "bd812a762c149a4345430b53530a597d261144b8", + "label": "current", + "round": 4, + "kinstr_per_frame": 167.8, + "kcycles_per_frame": 370.0, + "recv_fps": 19845, + "groups_lost": 0, + "host_load_1m": 10.76, + "host_load_1m_logged": 10.88, + "samples": 8671, + "markers": { + "learn_end": 0.2, + "poll_forward_drop": 12.1, + "cursor_probe": 6.1, + "syscall_leaf": 1.3, + "mutex_leaf": 20.4 + } + } + ] + }, + { + "artifact": "run.Z04ENl", + "harness_revision": "bd812a762c149a4345430b53530a597d261144b8", + "workload": "video", + "runtime": "tokio-workers", + "workers": 4, + "generator_revision": "21d6da9fc1", + "instruction_period": 2000003, + "sampling": "after the counter window", + "relay_cpus": "4-7", + "generator_cpus": "0-3", + "cells": [ + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 1, + "kinstr_per_frame": 170.2, + "kcycles_per_frame": 535.3, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 25.25, + "host_load_1m_logged": 27.59, + "samples": 370, + "markers": { + "learn_end": 0.5, + "poll_forward_drop": 9.5, + "cursor_probe": 2.4, + "syscall_leaf": 0.0, + "mutex_leaf": 15.4 + } + }, + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 2, + "kinstr_per_frame": 175.3, + "kcycles_per_frame": 483.3, + "recv_fps": 959, + "groups_lost": 0, + "host_load_1m": 30.01, + "host_load_1m_logged": 28.96, + "samples": 426, + "markers": { + "learn_end": 0.5, + "poll_forward_drop": 6.6, + "cursor_probe": 1.6, + "syscall_leaf": 0.0, + "mutex_leaf": 14.3 + } + }, + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 3, + "kinstr_per_frame": 163.2, + "kcycles_per_frame": 519.2, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 33.02, + "host_load_1m_logged": 33.58, + "samples": 384, + "markers": { + "learn_end": 0.3, + "poll_forward_drop": 9.1, + "cursor_probe": 2.3, + "syscall_leaf": 0.0, + "mutex_leaf": 17.7 + } + }, + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "label": "base-21d6da9fc108", + "round": 4, + "kinstr_per_frame": 164.5, + "kcycles_per_frame": 386.2, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 39.42, + "host_load_1m_logged": 39.65, + "samples": 385, + "markers": { + "learn_end": 0.3, + "poll_forward_drop": 9.1, + "cursor_probe": 2.3, + "syscall_leaf": 0.3, + "mutex_leaf": 14.3 + } + }, + { + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "label": "base-cdff3c8f952b", + "round": 1, + "kinstr_per_frame": 148.2, + "kcycles_per_frame": 378.2, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 13.51, + "host_load_1m_logged": 17.53, + "samples": 357, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 2.5, + "cursor_probe": 0.0, + "syscall_leaf": 0.0, + "mutex_leaf": 12.0 + } + }, + { + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "label": "base-cdff3c8f952b", + "round": 2, + "kinstr_per_frame": 149.5, + "kcycles_per_frame": 302.2, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 18.81, + "host_load_1m_logged": 19.83, + "samples": 354, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 2.8, + "cursor_probe": 0.0, + "syscall_leaf": 0.0, + "mutex_leaf": 13.0 + } + }, + { + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "label": "base-cdff3c8f952b", + "round": 3, + "kinstr_per_frame": 163.9, + "kcycles_per_frame": 506.6, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 23.81, + "host_load_1m_logged": 26.39, + "samples": 383, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 3.1, + "cursor_probe": 0.0, + "syscall_leaf": 0.0, + "mutex_leaf": 12.5 + } + }, + { + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "label": "base-cdff3c8f952b", + "round": 4, + "kinstr_per_frame": 156.0, + "kcycles_per_frame": 556.9, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 39.7, + "host_load_1m_logged": 39.65, + "samples": 368, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 3.5, + "cursor_probe": 0.0, + "syscall_leaf": 0.0, + "mutex_leaf": 14.4 + } + }, + { + "revision": "bd812a762c149a4345430b53530a597d261144b8", + "label": "current", + "round": 1, + "kinstr_per_frame": 147.3, + "kcycles_per_frame": 372.5, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 28.85, + "host_load_1m_logged": 28.75, + "samples": 348, + "markers": { + "learn_end": 0.3, + "poll_forward_drop": 0.3, + "cursor_probe": 0.3, + "syscall_leaf": 0.0, + "mutex_leaf": 9.8 + } + }, + { + "revision": "bd812a762c149a4345430b53530a597d261144b8", + "label": "current", + "round": 2, + "kinstr_per_frame": 142.9, + "kcycles_per_frame": 322.9, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 23.15, + "host_load_1m_logged": 24.73, + "samples": 335, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 0.6, + "cursor_probe": 0.0, + "syscall_leaf": 0.0, + "mutex_leaf": 8.7 + } + }, + { + "revision": "bd812a762c149a4345430b53530a597d261144b8", + "label": "current", + "round": 3, + "kinstr_per_frame": 146.4, + "kcycles_per_frame": 428.6, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 17.83, + "host_load_1m_logged": 18.09, + "samples": 347, + "markers": { + "learn_end": 0.0, + "poll_forward_drop": 0.3, + "cursor_probe": 0.3, + "syscall_leaf": 0.0, + "mutex_leaf": 11.0 + } + }, + { + "revision": "bd812a762c149a4345430b53530a597d261144b8", + "label": "current", + "round": 4, + "kinstr_per_frame": 147.2, + "kcycles_per_frame": 330.8, + "recv_fps": 960, + "groups_lost": 0, + "host_load_1m": 24.93, + "host_load_1m_logged": 24.93, + "samples": 345, + "markers": { + "learn_end": 0.3, + "poll_forward_drop": 0.3, + "cursor_probe": 0.0, + "syscall_leaf": 0.0, + "mutex_leaf": 9.6 + } + } + ] + } + ], + "fanout_method": "just bench-fanout: user instructions per delivered group on one pinned thread, a run of many cycles less a run of zero, at the listed subscriber counts; model = the track model alone, session = publisher, relay origin, and viewers over the in-memory transport", + "fanout_runs": [ + { + "artifact": "hist", + "note": "model counts per revision with frame pointers forced (RUSTFLAGS=-C force-frame-pointers=yes); compare only within this list", + "frame_pointers": true, + "counts": [ + { + "revision": "dcd7d2cbb891dc1cf72c68295ba70fbd17a4ec15", + "subscribers_512": 4411.1, + "subscribers_64": 4540.9, + "subscribers_1": 12552.6 + }, + { + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "subscribers_512": 4386.3, + "subscribers_64": 4512.5, + "subscribers_1": 12234.1 + }, + { + "revision": "fe722cec9f8432e0e46cb633cc5e79e59e7724cb", + "subscribers_512": 5313.0, + "subscribers_64": 5442.9, + "subscribers_1": 13455.1 + }, + { + "revision": "41cff87192983aae80db976d0058661525daa8ed", + "subscribers_512": 5612.2, + "subscribers_64": 5761.9, + "subscribers_1": 14591.6 + }, + { + "revision": "9477a589ccc5d0c3647df96788ebccd236523e19", + "subscribers_512": 5578.9, + "subscribers_64": 5723.1, + "subscribers_1": 14466.7 + }, + { + "revision": "2036225ed77a9cbea81d3e8c63f2912086a536b5", + "subscribers_512": 5604.1, + "subscribers_64": 5766.0, + "subscribers_1": 14607.6 + }, + { + "revision": "7648ce03908df33c168f7471787f3572e87eeddb", + "subscribers_512": 5787.2, + "subscribers_64": 5924.9, + "subscribers_1": 14778.8 + }, + { + "revision": "d44813cb703bf0d5366266ae8206f794856c69f8", + "subscribers_512": 5888.3, + "subscribers_64": 6041.8, + "subscribers_1": 15156.4 + }, + { + "revision": "b4dfafac8879ffc4cb661c0d47200aeccea7a2ba", + "subscribers_512": 5815.7, + "subscribers_64": 6041.8, + "subscribers_1": 15230.6 + }, + { + "revision": "28a8c9a7a665f5aef042a9a715e012bf17dac47a", + "subscribers_512": 5815.7, + "subscribers_64": 5969.6, + "subscribers_1": 15230.4 + }, + { + "revision": "9f9149812ce374bd66a14350d9a471f0150e691b", + "subscribers_512": 6016.2, + "subscribers_64": 6180.8, + "subscribers_1": 16033.6 + }, + { + "revision": "ce70844359e04f7d803111cd3763c21e7c85fdd5", + "subscribers_512": 6016.8, + "subscribers_64": 6180.5, + "subscribers_1": 16105.7 + }, + { + "revision": "a9a85580a4488a3ad3739d8d130ebb0f66e80510", + "subscribers_512": 6016.8, + "subscribers_64": 6108.5, + "subscribers_1": 16033.9 + }, + { + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "subscribers_512": 5943.6, + "subscribers_64": 6108.5, + "subscribers_1": 16033.6 + }, + { + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "subscribers_512": 12463.3, + "subscribers_64": 12670.9, + "subscribers_1": 26020.2 + }, + { + "revision": "a6f3d28216931c216d76b0ffd8e9ea2455741341", + "subscribers_512": 13681.1, + "subscribers_64": 13888.1, + "subscribers_1": 27299.5 + }, + { + "revision": "49e8eb0c6eb811521b5637a8d44a0947da294bc3", + "subscribers_512": 5170.5, + "subscribers_64": 5337.5, + "subscribers_1": 15445.4 + }, + { + "revision": "1bfbd89e1634ecd788c80f53aa85af1875a6fd97", + "subscribers_512": 5158.8, + "subscribers_64": 5320.8, + "subscribers_1": 15071.4 + }, + { + "revision": "1582fdc07d439b5a20a3a6caf8e3e865df23db72", + "subscribers_512": 5170.5, + "subscribers_64": 5337.5, + "subscribers_1": 15445.5 + }, + { + "revision": "318aa2f2e78e46d219dfca9b7d218e653137ecce", + "subscribers_512": 5166.8, + "subscribers_64": 5324.7, + "subscribers_1": 15111.8 + }, + { + "revision": "3d2530d74e7cbb23e9970d1ca0db6d898cece732", + "subscribers_512": 5166.8, + "subscribers_64": 5324.7, + "subscribers_1": 15111.8 + }, + { + "revision": "37b49d8042a6d8fc773045fd83e03621a4d52558", + "subscribers_512": 9342.4, + "subscribers_64": 9495.7, + "subscribers_1": 18970.1 + }, + { + "revision": "5b5fcf0116053b8c8d1fd957b30b5145bca03caa", + "subscribers_512": 9342.3, + "subscribers_64": 9495.2, + "subscribers_1": 18937.2 + }, + { + "revision": "fe08ce0f9ee4dd35566509b47a13308824f55890", + "subscribers_512": 9454.8, + "subscribers_64": 9637.0, + "subscribers_1": 21115.3 + }, + { + "revision": "c219c8db7755c5357eb9d47d873b6e9579a2be36", + "subscribers_512": 9693.7, + "subscribers_64": 9875.2, + "subscribers_1": 21298.0 + }, + { + "revision": "c764acebb90b5c102a8ea90207dd125ae4fd326b", + "subscribers_512": 9691.9, + "subscribers_64": 9874.3, + "subscribers_1": 21328.8 + }, + { + "revision": "3bbf2464a559ae453ac74013e969b803ea199a91", + "subscribers_512": 9777.2, + "subscribers_64": 9975.0, + "subscribers_1": 22322.1 + }, + { + "revision": "bdfa5b1d06394e05e30f167e297b45cc913af1f0", + "subscribers_512": 10151.3, + "subscribers_64": 10349.5, + "subscribers_1": 22700.9 + }, + { + "revision": "dffb2a43ba0f8da5a14bec35676f1d30ed978afd", + "subscribers_512": 10224.5, + "subscribers_64": 10417.9, + "subscribers_1": 22956.5 + }, + { + "revision": "267934b66dc097247fd878352bd8aae664ed873b", + "subscribers_512": 9927.4, + "subscribers_64": 10121.4, + "subscribers_1": 22684.1 + }, + { + "revision": "e01b0c0bdd4e0264b5e89ba0fe76c06d3b8caae7", + "subscribers_512": 9928.8, + "subscribers_64": 10133.1, + "subscribers_1": 22883.2 + }, + { + "revision": "145c61c62e10d764be6bf2e996591994495c2ba6", + "subscribers_512": 10081.0, + "subscribers_64": 10285.5, + "subscribers_1": 23084.6 + }, + { + "revision": "d44ee77b18b039fd8b72d410b82404ab8353db0e", + "subscribers_512": 10068.8, + "subscribers_64": 10273.5, + "subscribers_1": 23070.5 + }, + { + "revision": "14e9973dc8cd4c7fb932317f5dab7c3eb6bd58e5", + "subscribers_512": 11318.9, + "subscribers_64": 11517.4, + "subscribers_1": 24258.1 + }, + { + "revision": "3292489a46083d9ef75ac1ec83258fb040f59fba", + "subscribers_512": 12337.9, + "subscribers_64": 12542.6, + "subscribers_1": 25653.2 + }, + { + "revision": "7ad8d426e072a4b1f03483204ec2c9ae8d3d3ffb", + "subscribers_512": 12275.9, + "subscribers_64": 12480.5, + "subscribers_1": 25874.5 + }, + { + "revision": "64cba1337136c67ce77aaf6ae337f355c97a895e", + "subscribers_512": 12689.3, + "subscribers_64": 12872.9, + "subscribers_1": 26244.8 + }, + { + "revision": "ef2e423b93bfc189ed0305a7c6dee8453b590e5a", + "subscribers_512": 12302.0, + "subscribers_64": 12507.4, + "subscribers_1": 26015.0 + }, + { + "revision": "cc3c1961d849dd536dc203724bf6b6cca46f0d88", + "subscribers_512": 13213.4, + "subscribers_64": 13417.7, + "subscribers_1": 26900.8 + }, + { + "revision": "c9a43ad7b235932716fcb01879468f051a352e2b", + "subscribers_512": 13681.1, + "subscribers_64": 13888.0, + "subscribers_1": 27286.8 + } + ] + }, + { + "artifact": "run.vi0R1a", + "note": "model sweep of moq-net and kio commits in the large steps after e84ffb3de3", + "harness_revision": "44c0dc494ff3262de6f8bad38230f8ff09e6ebf1", + "frame_pointers": false, + "counts": [ + { + "label": "base-aff2e253a8ae", + "revision": "aff2e253a8aeb2de6f8de929ff11c5153ac58426", + "scenario": "model", + "subscribers": 512, + "per_group": 5010.427900300665 + }, + { + "label": "base-caff81eb7f64", + "revision": "caff81eb7f64fff28c554cf218ab393da182f55a", + "scenario": "model", + "subscribers": 512, + "per_group": 9274.04038890577 + }, + { + "label": "base-47bbe49ccd43", + "revision": "47bbe49ccd4324167274d9628d9d6f1d4d5a11aa", + "scenario": "model", + "subscribers": 512, + "per_group": 9274.040661783522 + }, + { + "label": "base-109235732c1f", + "revision": "109235732c1fec1a6e0284b3bd77d51fcc1074c2", + "scenario": "model", + "subscribers": 512, + "per_group": 9274.040714759787 + }, + { + "label": "base-5ad3652871f7", + "revision": "5ad3652871f7b62736e5cce0ca0338d2b162f3ee", + "scenario": "model", + "subscribers": 512, + "per_group": 9062.04057482248 + }, + { + "label": "base-2832aeb7b6c0", + "revision": "2832aeb7b6c06db26a26ac7437776a7a683d782e", + "scenario": "model", + "subscribers": 512, + "per_group": 9062.034914358368 + }, + { + "label": "base-fa59050f7064", + "revision": "fa59050f70642386bebcd7f0a043a2635dcc40d9", + "scenario": "model", + "subscribers": 512, + "per_group": 9062.533816850051 + }, + { + "label": "base-3a402c9b949c", + "revision": "3a402c9b949cf24822db1216473cbdf172d2e7bb", + "scenario": "model", + "subscribers": 512, + "per_group": 9062.533618938716 + }, + { + "label": "base-d4cf2f0ce097", + "revision": "d4cf2f0ce097a051acac3c62848e2b345ff8c7ab", + "scenario": "model", + "subscribers": 512, + "per_group": 9061.956926297018 + }, + { + "label": "base-e3a1cdf2328f", + "revision": "e3a1cdf2328f853254c467b2b3c9c7327753d31a", + "scenario": "model", + "subscribers": 512, + "per_group": 9813.631270190954 + }, + { + "label": "base-0c8b0924b579", + "revision": "0c8b0924b579da4c7fc34f9af40f866d0e4cd338", + "scenario": "model", + "subscribers": 512, + "per_group": 9813.631249200358 + }, + { + "label": "base-30f23c3f8af0", + "revision": "30f23c3f8af0b46eeebff687bfa45397ed70cf0a", + "scenario": "model", + "subscribers": 512, + "per_group": 9813.631152243795 + }, + { + "label": "base-deee150f8f64", + "revision": "deee150f8f6400c14632627601960a89fd8e2b26", + "scenario": "model", + "subscribers": 512, + "per_group": 10967.270665741748 + }, + { + "label": "base-69b44ddd2218", + "revision": "69b44ddd221819f93cb2bb0b8c8e724f5b7ae280", + "scenario": "model", + "subscribers": 512, + "per_group": 10967.270647749809 + }, + { + "label": "base-6cbb58f7903b", + "revision": "6cbb58f7903baa9eb98f9556ab811add85f760a5", + "scenario": "model", + "subscribers": 512, + "per_group": 10967.270635755182 + }, + { + "label": "base-17e00c8148d7", + "revision": "17e00c8148d76af6c1ee17a8efdb87c7e400bd2b", + "scenario": "model", + "subscribers": 512, + "per_group": 10967.270562787871 + }, + { + "label": "base-c1a141282003", + "revision": "c1a141282003e913c38f4b1953ba769c2030c884", + "scenario": "model", + "subscribers": 512, + "per_group": 10967.270676736822 + }, + { + "label": "base-d42abb2a8608", + "revision": "d42abb2a8608a43729c742feb550cc925c538d51", + "scenario": "model", + "subscribers": 512, + "per_group": 10967.27058977578 + }, + { + "label": "base-947c91d05b4b", + "revision": "947c91d05b4b12594e64b5f0ab84a28c1831a491", + "scenario": "model", + "subscribers": 512, + "per_group": 10985.500266880437 + }, + { + "label": "base-68fb455bac47", + "revision": "68fb455bac47023b19020266ac1e6d5618dd08bc", + "scenario": "model", + "subscribers": 512, + "per_group": 11189.308664718206 + }, + { + "label": "base-08e71cd3df56", + "revision": "08e71cd3df56d80d83f134844cf2147498b279fe", + "scenario": "model", + "subscribers": 512, + "per_group": 11189.303892856 + }, + { + "label": "base-32391c452792", + "revision": "32391c4527927c5e854b8139aebf2e2ae11f0714", + "scenario": "model", + "subscribers": 512, + "per_group": 11189.303752918693 + }, + { + "label": "base-664b1aad2e6f", + "revision": "664b1aad2e6ff7fe8ade4a57a272bcd5c0cc128b", + "scenario": "model", + "subscribers": 512, + "per_group": 11189.303716934814 + }, + { + "label": "base-81fe8a6631cf", + "revision": "81fe8a6631cf4c3402ab7e04a1dbf3ccb61ba264", + "scenario": "model", + "subscribers": 512, + "per_group": 11746.942647693833 + }, + { + "label": "base-bfccf91db866", + "revision": "bfccf91db8662fa8d36abd90da5cd54b07c36041", + "scenario": "model", + "subscribers": 512, + "per_group": 11746.942757644576 + }, + { + "label": "base-41118d073d47", + "revision": "41118d073d47e2aa3030dce40b8e93f48fa9ce27", + "scenario": "model", + "subscribers": 512, + "per_group": 11746.942336833099 + }, + { + "label": "base-33686a29725f", + "revision": "33686a29725f9785d85ab5926cc4a4f34eef5b9b", + "scenario": "model", + "subscribers": 512, + "per_group": 11746.947575486181 + }, + { + "label": "base-3c0dd8a2af6b", + "revision": "3c0dd8a2af6ba98c13965fa13c31c4e3185830ac", + "scenario": "model", + "subscribers": 512, + "per_group": 11959.76987609551 + }, + { + "label": "base-a039ca1be51f", + "revision": "a039ca1be51f28530b9acf519604039cbfd82504", + "scenario": "model", + "subscribers": 512, + "per_group": 11929.205388985734 + }, + { + "label": "base-e3e1cc884155", + "revision": "e3e1cc8841558a52c7811eb0fd8ebb0dcefb29cf", + "scenario": "model", + "subscribers": 512, + "per_group": 11929.205050137538 + }, + { + "label": "base-f1034b7cc973", + "revision": "f1034b7cc973009bb33abfd06404be1a1dea0266", + "scenario": "model", + "subscribers": 512, + "per_group": 12830.06617935165 + }, + { + "label": "base-d43ff4394baf", + "revision": "d43ff4394baff600c414ba2c6a2a02c230cb2c51", + "scenario": "model", + "subscribers": 512, + "per_group": 12834.068875143936 + }, + { + "label": "base-097dfc42257f", + "revision": "097dfc42257fc893dee5116006a265660fd71b01", + "scenario": "model", + "subscribers": 512, + "per_group": 12834.065782529427 + }, + { + "label": "base-5ab878666c31", + "revision": "5ab878666c312fc2a497a31656630efeb02ca55f", + "scenario": "model", + "subscribers": 512, + "per_group": 12784.511050049578 + }, + { + "label": "base-c68c93102534", + "revision": "c68c93102534b0f12382dcad20704fb57385bef1", + "scenario": "model", + "subscribers": 512, + "per_group": 13071.60937100179 + }, + { + "label": "base-be3918800076", + "revision": "be3918800076ea8685fc8e0468a097234516e3a9", + "scenario": "model", + "subscribers": 512, + "per_group": 13071.609004166134 + }, + { + "label": "base-1777a2d6f98d", + "revision": "1777a2d6f98d7b5e12787dbc0687a6a8cdc2ae38", + "scenario": "model", + "subscribers": 512, + "per_group": 13100.6231598244 + }, + { + "label": "base-ad3ca7784aa0", + "revision": "ad3ca7784aa00f7b8b93b8a8076304673c50e733", + "scenario": "model", + "subscribers": 512, + "per_group": 13100.492399405066 + }, + { + "label": "base-25ff6e97546f", + "revision": "25ff6e97546f59dcebde66f43b67ba55f590867c", + "scenario": "model", + "subscribers": 512, + "per_group": 13236.30172082907 + }, + { + "label": "base-651c30fee9d8", + "revision": "651c30fee9d88fd56f4079433dd3e1f98d77f21c", + "scenario": "model", + "subscribers": 512, + "per_group": 9534.986503046635 + }, + { + "label": "base-82f7159bb25c", + "revision": "82f7159bb25ca64a63d893296e97b83e1774fbea", + "scenario": "model", + "subscribers": 512, + "per_group": 9534.986494050665 + }, + { + "label": "base-767685e1ba0d", + "revision": "767685e1ba0dabfd921fe8439d7f55ccf65eeedb", + "scenario": "model", + "subscribers": 512, + "per_group": 9534.986662974987 + }, + { + "label": "base-e7ca859b5527", + "revision": "e7ca859b552748fbc97e588117fd1c2915258b6d", + "scenario": "model", + "subscribers": 512, + "per_group": 9535.916662335274 + }, + { + "label": "base-197bdae8eeeb", + "revision": "197bdae8eeeb32b325b6f5ad64aadf1477b0241e", + "scenario": "model", + "subscribers": 512, + "per_group": 9535.917204092566 + }, + { + "label": "base-6eff65506e81", + "revision": "6eff65506e817ca82c6ee3d408c55066cf811510", + "scenario": "model", + "subscribers": 512, + "per_group": 9535.917027171827 + }, + { + "label": "base-d6e175f8e15f", + "revision": "d6e175f8e15f0f9323e472b9746ca4346bfa7b35", + "scenario": "model", + "subscribers": 512, + "per_group": 9535.92292952757 + }, + { + "label": "base-7c7e1d1c0f31", + "revision": "7c7e1d1c0f31dccdb4adcc76acb9acf6515180e8", + "scenario": "model", + "subscribers": 512, + "per_group": 9535.922853561604 + }, + { + "label": "base-84d9b018b83d", + "revision": "84d9b018b83d880d901cfd84a9064b4923d284b5", + "scenario": "model", + "subscribers": 512, + "per_group": 9535.922934525333 + }, + { + "label": "current", + "revision": "44c0dc494ff3262de6f8bad38230f8ff09e6ebf1", + "scenario": "model", + "subscribers": 512, + "per_group": 10060.128887258508 + } + ] + }, + { + "artifact": "run.TmGdFy", + "note": "each culprit against its first parent", + "harness_revision": "b243c4046f472d0a547e401deb71036f1a5c94c3", + "frame_pointers": false, + "counts": [ + { + "label": "base-55f6ed229250", + "revision": "55f6ed229250d17fc99389237659906d011d8a94", + "scenario": "model", + "subscribers": 64, + "per_group": 5173.549239 + }, + { + "label": "base-55f6ed229250", + "revision": "55f6ed229250d17fc99389237659906d011d8a94", + "scenario": "model", + "subscribers": 512, + "per_group": 5015.741348875704 + }, + { + "label": "base-caff81eb7f64", + "revision": "caff81eb7f64fff28c554cf218ab393da182f55a", + "scenario": "model", + "subscribers": 64, + "per_group": 9437.337956 + }, + { + "label": "base-caff81eb7f64", + "revision": "caff81eb7f64fff28c554cf218ab393da182f55a", + "scenario": "model", + "subscribers": 512, + "per_group": 9279.352597036528 + }, + { + "label": "base-30f23c3f8af0", + "revision": "30f23c3f8af0b46eeebff687bfa45397ed70cf0a", + "scenario": "model", + "subscribers": 64, + "per_group": 10020.597976 + }, + { + "label": "base-30f23c3f8af0", + "revision": "30f23c3f8af0b46eeebff687bfa45397ed70cf0a", + "scenario": "model", + "subscribers": 512, + "per_group": 9818.072495522007 + }, + { + "label": "base-deee150f8f64", + "revision": "deee150f8f6400c14632627601960a89fd8e2b26", + "scenario": "model", + "subscribers": 64, + "per_group": 11171.813651 + }, + { + "label": "base-deee150f8f64", + "revision": "deee150f8f6400c14632627601960a89fd8e2b26", + "scenario": "model", + "subscribers": 512, + "per_group": 10972.215946256078 + }, + { + "label": "base-947c91d05b4b", + "revision": "947c91d05b4b12594e64b5f0ab84a28c1831a491", + "scenario": "model", + "subscribers": 64, + "per_group": 11190.094659 + }, + { + "label": "base-947c91d05b4b", + "revision": "947c91d05b4b12594e64b5f0ab84a28c1831a491", + "scenario": "model", + "subscribers": 512, + "per_group": 10990.25496077757 + }, + { + "label": "base-68fb455bac47", + "revision": "68fb455bac47023b19020266ac1e6d5618dd08bc", + "scenario": "model", + "subscribers": 64, + "per_group": 11394.092394 + }, + { + "label": "base-68fb455bac47", + "revision": "68fb455bac47023b19020266ac1e6d5618dd08bc", + "scenario": "model", + "subscribers": 512, + "per_group": 11194.250525764457 + }, + { + "label": "base-3f5939eb2ee3", + "revision": "3f5939eb2ee39f4ff9d52f6c061ff366110ced6a", + "scenario": "model", + "subscribers": 64, + "per_group": 11394.092954 + }, + { + "label": "base-3f5939eb2ee3", + "revision": "3f5939eb2ee39f4ff9d52f6c061ff366110ced6a", + "scenario": "model", + "subscribers": 512, + "per_group": 11194.25055874968 + }, + { + "label": "base-81fe8a6631cf", + "revision": "81fe8a6631cf4c3402ab7e04a1dbf3ccb61ba264", + "scenario": "model", + "subscribers": 64, + "per_group": 11957.435432 + }, + { + "label": "base-81fe8a6631cf", + "revision": "81fe8a6631cf4c3402ab7e04a1dbf3ccb61ba264", + "scenario": "model", + "subscribers": 512, + "per_group": 11751.995400060772 + }, + { + "label": "base-3d095a319cb6", + "revision": "3d095a319cb695e6812350cb09aef4ee712c247a", + "scenario": "model", + "subscribers": 64, + "per_group": 11957.435119 + }, + { + "label": "base-3d095a319cb6", + "revision": "3d095a319cb695e6812350cb09aef4ee712c247a", + "scenario": "model", + "subscribers": 512, + "per_group": 11751.995168164662 + }, + { + "label": "base-3c0dd8a2af6b", + "revision": "3c0dd8a2af6ba98c13965fa13c31c4e3185830ac", + "scenario": "model", + "subscribers": 64, + "per_group": 12171.447257 + }, + { + "label": "base-3c0dd8a2af6b", + "revision": "3c0dd8a2af6ba98c13965fa13c31c4e3185830ac", + "scenario": "model", + "subscribers": 512, + "per_group": 11965.12150656506 + }, + { + "label": "base-4eb9c23db5de", + "revision": "4eb9c23db5de2baa48ef838c357392fa6424fa2a", + "scenario": "model", + "subscribers": 64, + "per_group": 12117.255659 + }, + { + "label": "base-4eb9c23db5de", + "revision": "4eb9c23db5de2baa48ef838c357392fa6424fa2a", + "scenario": "model", + "subscribers": 512, + "per_group": 11911.038718654043 + }, + { + "label": "base-f1034b7cc973", + "revision": "f1034b7cc973009bb33abfd06404be1a1dea0266", + "scenario": "model", + "subscribers": 64, + "per_group": 12993.101633 + }, + { + "label": "base-f1034b7cc973", + "revision": "f1034b7cc973009bb33abfd06404be1a1dea0266", + "scenario": "model", + "subscribers": 512, + "per_group": 12786.68907129606 + }, + { + "label": "base-b7187a4559ed", + "revision": "b7187a4559ed2093b59c6a69b6058e10466cd37c", + "scenario": "model", + "subscribers": 64, + "per_group": 13044.710638 + }, + { + "label": "base-b7187a4559ed", + "revision": "b7187a4559ed2093b59c6a69b6058e10466cd37c", + "scenario": "model", + "subscribers": 512, + "per_group": 12835.771269471277 + }, + { + "label": "base-c68c93102534", + "revision": "c68c93102534b0f12382dcad20704fb57385bef1", + "scenario": "model", + "subscribers": 64, + "per_group": 13319.716747 + }, + { + "label": "base-c68c93102534", + "revision": "c68c93102534b0f12382dcad20704fb57385bef1", + "scenario": "model", + "subscribers": 512, + "per_group": 13110.767371217695 + }, + { + "label": "base-bd71f72b40d9", + "revision": "bd71f72b40d9a85001acc1b36ab40b6a5e1a6760", + "scenario": "model", + "subscribers": 64, + "per_group": 13348.304234 + }, + { + "label": "base-bd71f72b40d9", + "revision": "bd71f72b40d9a85001acc1b36ab40b6a5e1a6760", + "scenario": "model", + "subscribers": 512, + "per_group": 13139.74781497889 + }, + { + "label": "base-25ff6e97546f", + "revision": "25ff6e97546f59dcebde66f43b67ba55f590867c", + "scenario": "model", + "subscribers": 64, + "per_group": 13508.303632 + }, + { + "label": "base-25ff6e97546f", + "revision": "25ff6e97546f59dcebde66f43b67ba55f590867c", + "scenario": "model", + "subscribers": 512, + "per_group": 13299.747363181295 + }, + { + "label": "base-84d9b018b83d", + "revision": "84d9b018b83d880d901cfd84a9064b4923d284b5", + "scenario": "model", + "subscribers": 64, + "per_group": 9736.783127 + }, + { + "label": "base-84d9b018b83d", + "revision": "84d9b018b83d880d901cfd84a9064b4923d284b5", + "scenario": "model", + "subscribers": 512, + "per_group": 9540.758499192361 + }, + { + "label": "base-bdfa5b1d0639", + "revision": "bdfa5b1d06394e05e30f167e297b45cc913af1f0", + "scenario": "model", + "subscribers": 64, + "per_group": 10101.811168 + }, + { + "label": "base-bdfa5b1d0639", + "revision": "bdfa5b1d06394e05e30f167e297b45cc913af1f0", + "scenario": "model", + "subscribers": 512, + "per_group": 9905.499159376599 + }, + { + "label": "base-db92d18a24c2", + "revision": "db92d18a24c2dc4e72274bc04b4a49ed296dcad1", + "scenario": "model", + "subscribers": 64, + "per_group": 12171.450912 + }, + { + "label": "base-db92d18a24c2", + "revision": "db92d18a24c2dc4e72274bc04b4a49ed296dcad1", + "scenario": "model", + "subscribers": 512, + "per_group": 11965.120981800153 + }, + { + "label": "base-3292489a4608", + "revision": "3292489a46083d9ef75ac1ec83258fb040f59fba", + "scenario": "model", + "subscribers": 64, + "per_group": 12171.445025 + }, + { + "label": "base-3292489a4608", + "revision": "3292489a46083d9ef75ac1ec83258fb040f59fba", + "scenario": "model", + "subscribers": 512, + "per_group": 11965.122121289662 + }, + { + "label": "base-097dfc42257f", + "revision": "097dfc42257fc893dee5116006a265660fd71b01", + "scenario": "model", + "subscribers": 64, + "per_group": 12997.10948 + }, + { + "label": "base-097dfc42257f", + "revision": "097dfc42257fc893dee5116006a265660fd71b01", + "scenario": "model", + "subscribers": 512, + "per_group": 12790.695173562244 + }, + { + "label": "base-cc3c1961d849", + "revision": "cc3c1961d849dd536dc203724bf6b6cca46f0d88", + "scenario": "model", + "subscribers": 64, + "per_group": 12997.109092 + }, + { + "label": "base-cc3c1961d849", + "revision": "cc3c1961d849dd536dc203724bf6b6cca46f0d88", + "scenario": "model", + "subscribers": 512, + "per_group": 12790.690385707203 + }, + { + "label": "base-d44ee77b18b0", + "revision": "d44ee77b18b039fd8b72d410b82404ab8353db0e", + "scenario": "model", + "subscribers": 64, + "per_group": 10020.59734 + }, + { + "label": "base-d44ee77b18b0", + "revision": "d44ee77b18b039fd8b72d410b82404ab8353db0e", + "scenario": "model", + "subscribers": 512, + "per_group": 9818.073104249297 + }, + { + "label": "base-e3a1cdf2328f", + "revision": "e3a1cdf2328f853254c467b2b3c9c7327753d31a", + "scenario": "model", + "subscribers": 64, + "per_group": 10020.599767 + }, + { + "label": "base-e3a1cdf2328f", + "revision": "e3a1cdf2328f853254c467b2b3c9c7327753d31a", + "scenario": "model", + "subscribers": 512, + "per_group": 9818.07263745842 + }, + { + "label": "base-cdff3c8f952b", + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "scenario": "model", + "subscribers": 64, + "per_group": 4409.825208 + }, + { + "label": "base-cdff3c8f952b", + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "scenario": "model", + "subscribers": 512, + "per_group": 4285.009976530514 + }, + { + "label": "base-e84ffb3de313", + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "scenario": "model", + "subscribers": 64, + "per_group": 6004.348586 + }, + { + "label": "base-e84ffb3de313", + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "scenario": "model", + "subscribers": 512, + "per_group": 5840.058401835978 + }, + { + "label": "base-21d6da9fc108", + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "scenario": "model", + "subscribers": 64, + "per_group": 12288.032389 + }, + { + "label": "base-21d6da9fc108", + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "scenario": "model", + "subscribers": 512, + "per_group": 12079.328686748337 + }, + { + "label": "base-a6f3d2821693", + "revision": "a6f3d28216931c216d76b0ffd8e9ea2455741341", + "scenario": "model", + "subscribers": 64, + "per_group": 13508.303174 + }, + { + "label": "base-a6f3d2821693", + "revision": "a6f3d28216931c216d76b0ffd8e9ea2455741341", + "scenario": "model", + "subscribers": 512, + "per_group": 13299.745055215264 + }, + { + "label": "current", + "revision": "b243c4046f472d0a547e401deb71036f1a5c94c3", + "scenario": "model", + "subscribers": 64, + "per_group": 10399.519503 + }, + { + "label": "current", + "revision": "b243c4046f472d0a547e401deb71036f1a5c94c3", + "scenario": "model", + "subscribers": 512, + "per_group": 10189.137410440122 + }, + { + "label": "base-55f6ed229250", + "revision": "55f6ed229250d17fc99389237659906d011d8a94", + "scenario": "session", + "subscribers": 64, + "per_group": null + }, + { + "label": "base-55f6ed229250", + "revision": "55f6ed229250d17fc99389237659906d011d8a94", + "scenario": "session", + "subscribers": 512, + "per_group": null + }, + { + "label": "base-caff81eb7f64", + "revision": "caff81eb7f64fff28c554cf218ab393da182f55a", + "scenario": "session", + "subscribers": 64, + "per_group": null + }, + { + "label": "base-caff81eb7f64", + "revision": "caff81eb7f64fff28c554cf218ab393da182f55a", + "scenario": "session", + "subscribers": 512, + "per_group": null + }, + { + "label": "base-30f23c3f8af0", + "revision": "30f23c3f8af0b46eeebff687bfa45397ed70cf0a", + "scenario": "session", + "subscribers": 64, + "per_group": null + }, + { + "label": "base-30f23c3f8af0", + "revision": "30f23c3f8af0b46eeebff687bfa45397ed70cf0a", + "scenario": "session", + "subscribers": 512, + "per_group": null + }, + { + "label": "base-deee150f8f64", + "revision": "deee150f8f6400c14632627601960a89fd8e2b26", + "scenario": "session", + "subscribers": 64, + "per_group": null + }, + { + "label": "base-deee150f8f64", + "revision": "deee150f8f6400c14632627601960a89fd8e2b26", + "scenario": "session", + "subscribers": 512, + "per_group": null + }, + { + "label": "base-947c91d05b4b", + "revision": "947c91d05b4b12594e64b5f0ab84a28c1831a491", + "scenario": "session", + "subscribers": 64, + "per_group": 143788.77370241523 + }, + { + "label": "base-947c91d05b4b", + "revision": "947c91d05b4b12594e64b5f0ab84a28c1831a491", + "scenario": "session", + "subscribers": 512, + "per_group": 141133.06857860333 + }, + { + "label": "base-68fb455bac47", + "revision": "68fb455bac47023b19020266ac1e6d5618dd08bc", + "scenario": "session", + "subscribers": 64, + "per_group": 143324.79522552784 + }, + { + "label": "base-68fb455bac47", + "revision": "68fb455bac47023b19020266ac1e6d5618dd08bc", + "scenario": "session", + "subscribers": 512, + "per_group": 140678.44754464287 + }, + { + "label": "base-3f5939eb2ee3", + "revision": "3f5939eb2ee39f4ff9d52f6c061ff366110ced6a", + "scenario": "session", + "subscribers": 64, + "per_group": 142781.21071257198 + }, + { + "label": "base-3f5939eb2ee3", + "revision": "3f5939eb2ee39f4ff9d52f6c061ff366110ced6a", + "scenario": "session", + "subscribers": 512, + "per_group": 140313.2066127232 + }, + { + "label": "base-81fe8a6631cf", + "revision": "81fe8a6631cf4c3402ab7e04a1dbf3ccb61ba264", + "scenario": "session", + "subscribers": 64, + "per_group": 146287.1747840691 + }, + { + "label": "base-81fe8a6631cf", + "revision": "81fe8a6631cf4c3402ab7e04a1dbf3ccb61ba264", + "scenario": "session", + "subscribers": 512, + "per_group": 143603.11443718112 + }, + { + "label": "base-3d095a319cb6", + "revision": "3d095a319cb695e6812350cb09aef4ee712c247a", + "scenario": "session", + "subscribers": 64, + "per_group": 146079.13113803582 + }, + { + "label": "base-3d095a319cb6", + "revision": "3d095a319cb695e6812350cb09aef4ee712c247a", + "scenario": "session", + "subscribers": 512, + "per_group": 143397.84674944196 + }, + { + "label": "base-3c0dd8a2af6b", + "revision": "3c0dd8a2af6ba98c13965fa13c31c4e3185830ac", + "scenario": "session", + "subscribers": 64, + "per_group": 142925.80393274152 + }, + { + "label": "base-3c0dd8a2af6b", + "revision": "3c0dd8a2af6ba98c13965fa13c31c4e3185830ac", + "scenario": "session", + "subscribers": 512, + "per_group": 140256.0416135204 + }, + { + "label": "base-4eb9c23db5de", + "revision": "4eb9c23db5de2baa48ef838c357392fa6424fa2a", + "scenario": "session", + "subscribers": 64, + "per_group": 118998.90310100767 + }, + { + "label": "base-4eb9c23db5de", + "revision": "4eb9c23db5de2baa48ef838c357392fa6424fa2a", + "scenario": "session", + "subscribers": 512, + "per_group": 116906.18026546556 + }, + { + "label": "base-f1034b7cc973", + "revision": "f1034b7cc973009bb33abfd06404be1a1dea0266", + "scenario": "session", + "subscribers": 64, + "per_group": 122072.00230926104 + }, + { + "label": "base-f1034b7cc973", + "revision": "f1034b7cc973009bb33abfd06404be1a1dea0266", + "scenario": "session", + "subscribers": 512, + "per_group": 119959.71555125956 + }, + { + "label": "base-b7187a4559ed", + "revision": "b7187a4559ed2093b59c6a69b6058e10466cd37c", + "scenario": "session", + "subscribers": 64, + "per_group": 122503.40778950736 + }, + { + "label": "base-b7187a4559ed", + "revision": "b7187a4559ed2093b59c6a69b6058e10466cd37c", + "scenario": "session", + "subscribers": 512, + "per_group": 120387.56217115752 + }, + { + "label": "base-c68c93102534", + "revision": "c68c93102534b0f12382dcad20704fb57385bef1", + "scenario": "session", + "subscribers": 64, + "per_group": 122962.28769793666 + }, + { + "label": "base-c68c93102534", + "revision": "c68c93102534b0f12382dcad20704fb57385bef1", + "scenario": "session", + "subscribers": 512, + "per_group": 120837.39202008929 + }, + { + "label": "base-bd71f72b40d9", + "revision": "bd71f72b40d9a85001acc1b36ab40b6a5e1a6760", + "scenario": "session", + "subscribers": 64, + "per_group": 122839.86978166987 + }, + { + "label": "base-bd71f72b40d9", + "revision": "bd71f72b40d9a85001acc1b36ab40b6a5e1a6760", + "scenario": "session", + "subscribers": 512, + "per_group": 120719.14727160396 + }, + { + "label": "base-25ff6e97546f", + "revision": "25ff6e97546f59dcebde66f43b67ba55f590867c", + "scenario": "session", + "subscribers": 64, + "per_group": 122894.58472288867 + }, + { + "label": "base-25ff6e97546f", + "revision": "25ff6e97546f59dcebde66f43b67ba55f590867c", + "scenario": "session", + "subscribers": 512, + "per_group": 120761.91606544962 + }, + { + "label": "base-84d9b018b83d", + "revision": "84d9b018b83d880d901cfd84a9064b4923d284b5", + "scenario": "session", + "subscribers": 64, + "per_group": null + }, + { + "label": "base-84d9b018b83d", + "revision": "84d9b018b83d880d901cfd84a9064b4923d284b5", + "scenario": "session", + "subscribers": 512, + "per_group": null + }, + { + "label": "base-bdfa5b1d0639", + "revision": "bdfa5b1d06394e05e30f167e297b45cc913af1f0", + "scenario": "session", + "subscribers": 64, + "per_group": null + }, + { + "label": "base-bdfa5b1d0639", + "revision": "bdfa5b1d06394e05e30f167e297b45cc913af1f0", + "scenario": "session", + "subscribers": 512, + "per_group": null + }, + { + "label": "base-db92d18a24c2", + "revision": "db92d18a24c2dc4e72274bc04b4a49ed296dcad1", + "scenario": "session", + "subscribers": 64, + "per_group": 142925.57175703775 + }, + { + "label": "base-db92d18a24c2", + "revision": "db92d18a24c2dc4e72274bc04b4a49ed296dcad1", + "scenario": "session", + "subscribers": 512, + "per_group": 140256.04686503508 + }, + { + "label": "base-3292489a4608", + "revision": "3292489a46083d9ef75ac1ec83258fb040f59fba", + "scenario": "session", + "subscribers": 64, + "per_group": 142986.527331254 + }, + { + "label": "base-3292489a4608", + "revision": "3292489a46083d9ef75ac1ec83258fb040f59fba", + "scenario": "session", + "subscribers": 512, + "per_group": 140316.09682916134 + }, + { + "label": "base-097dfc42257f", + "revision": "097dfc42257fc893dee5116006a265660fd71b01", + "scenario": "session", + "subscribers": 64, + "per_group": 122237.25825735765 + }, + { + "label": "base-097dfc42257f", + "revision": "097dfc42257fc893dee5116006a265660fd71b01", + "scenario": "session", + "subscribers": 512, + "per_group": 120122.85361527423 + }, + { + "label": "base-cc3c1961d849", + "revision": "cc3c1961d849dd536dc203724bf6b6cca46f0d88", + "scenario": "session", + "subscribers": 64, + "per_group": 122259.8310240723 + }, + { + "label": "base-cc3c1961d849", + "revision": "cc3c1961d849dd536dc203724bf6b6cca46f0d88", + "scenario": "session", + "subscribers": 512, + "per_group": 120144.74912308673 + }, + { + "label": "base-d44ee77b18b0", + "revision": "d44ee77b18b039fd8b72d410b82404ab8353db0e", + "scenario": "session", + "subscribers": 64, + "per_group": null + }, + { + "label": "base-d44ee77b18b0", + "revision": "d44ee77b18b039fd8b72d410b82404ab8353db0e", + "scenario": "session", + "subscribers": 512, + "per_group": null + }, + { + "label": "base-e3a1cdf2328f", + "revision": "e3a1cdf2328f853254c467b2b3c9c7327753d31a", + "scenario": "session", + "subscribers": 64, + "per_group": null + }, + { + "label": "base-e3a1cdf2328f", + "revision": "e3a1cdf2328f853254c467b2b3c9c7327753d31a", + "scenario": "session", + "subscribers": 512, + "per_group": null + }, + { + "label": "base-cdff3c8f952b", + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "scenario": "session", + "subscribers": 64, + "per_group": null + }, + { + "label": "base-cdff3c8f952b", + "revision": "cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95", + "scenario": "session", + "subscribers": 512, + "per_group": null + }, + { + "label": "base-e84ffb3de313", + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "scenario": "session", + "subscribers": 64, + "per_group": null + }, + { + "label": "base-e84ffb3de313", + "revision": "e84ffb3de3139f7245bed5697754fd6add9a022e", + "scenario": "session", + "subscribers": 512, + "per_group": null + }, + { + "label": "base-21d6da9fc108", + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "scenario": "session", + "subscribers": 64, + "per_group": 135423.53441898592 + }, + { + "label": "base-21d6da9fc108", + "revision": "21d6da9fc108b48373a961f34e43c3fcd38b0146", + "scenario": "session", + "subscribers": 512, + "per_group": 132932.98566047513 + }, + { + "label": "base-a6f3d2821693", + "revision": "a6f3d28216931c216d76b0ffd8e9ea2455741341", + "scenario": "session", + "subscribers": 64, + "per_group": 123075.42752319257 + }, + { + "label": "base-a6f3d2821693", + "revision": "a6f3d28216931c216d76b0ffd8e9ea2455741341", + "scenario": "session", + "subscribers": 512, + "per_group": 120948.21283083546 + }, + { + "label": "current", + "revision": "b243c4046f472d0a547e401deb71036f1a5c94c3", + "scenario": "session", + "subscribers": 64, + "per_group": 110473.08423304543 + }, + { + "label": "current", + "revision": "b243c4046f472d0a547e401deb71036f1a5c94c3", + "scenario": "session", + "subscribers": 512, + "per_group": 108705.21113679848 + } + ] + }, + { + "artifact": "run.s6IjbN", + "note": "this branch's fixes, each on top of the last", + "harness_revision": "df74d8dbe07f760b2ff5eb6da9dda57d9ee8006d", + "frame_pointers": false, + "counts": [ + { + "label": "base-a6f3d2821693", + "revision": "a6f3d28216931c216d76b0ffd8e9ea2455741341", + "scenario": "model", + "subscribers": 64, + "per_group": 13508.284341 + }, + { + "label": "base-a6f3d2821693", + "revision": "a6f3d28216931c216d76b0ffd8e9ea2455741341", + "scenario": "model", + "subscribers": 512, + "per_group": 13299.745813875385 + }, + { + "label": "base-0db3f6c91b54", + "revision": "0db3f6c91b5482429b3bf3b6d40313c3c923ae98", + "scenario": "model", + "subscribers": 64, + "per_group": 10687.580642 + }, + { + "label": "base-0db3f6c91b54", + "revision": "0db3f6c91b5482429b3bf3b6d40313c3c923ae98", + "scenario": "model", + "subscribers": 512, + "per_group": 10476.74617571328 + }, + { + "label": "base-5132fc4a455b", + "revision": "5132fc4a455bf6a063112d1d5185c6e3b8a2a4f5", + "scenario": "model", + "subscribers": 64, + "per_group": 10336.580476 + }, + { + "label": "base-5132fc4a455b", + "revision": "5132fc4a455bf6a063112d1d5185c6e3b8a2a4f5", + "scenario": "model", + "subscribers": 512, + "per_group": 10125.742104537167 + }, + { + "label": "base-101ed5a8066d", + "revision": "101ed5a8066d4c17ec85ac3cc2d5ec40686c4962", + "scenario": "model", + "subscribers": 64, + "per_group": 10399.485894 + }, + { + "label": "base-101ed5a8066d", + "revision": "101ed5a8066d4c17ec85ac3cc2d5ec40686c4962", + "scenario": "model", + "subscribers": 512, + "per_group": 10188.911252758764 + }, + { + "label": "current", + "revision": "df74d8dbe07f760b2ff5eb6da9dda57d9ee8006d", + "scenario": "model", + "subscribers": 64, + "per_group": 10399.518113 + }, + { + "label": "current", + "revision": "df74d8dbe07f760b2ff5eb6da9dda57d9ee8006d", + "scenario": "model", + "subscribers": 512, + "per_group": 10189.138902771558 + }, + { + "label": "base-a6f3d2821693", + "revision": "a6f3d28216931c216d76b0ffd8e9ea2455741341", + "scenario": "session", + "subscribers": 64, + "per_group": 123077.5850227927 + }, + { + "label": "base-a6f3d2821693", + "revision": "a6f3d28216931c216d76b0ffd8e9ea2455741341", + "scenario": "session", + "subscribers": 512, + "per_group": 120947.86723732462 + }, + { + "label": "base-0db3f6c91b54", + "revision": "0db3f6c91b5482429b3bf3b6d40313c3c923ae98", + "scenario": "session", + "subscribers": 64, + "per_group": 116020.33326335573 + }, + { + "label": "base-0db3f6c91b54", + "revision": "0db3f6c91b5482429b3bf3b6d40313c3c923ae98", + "scenario": "session", + "subscribers": 512, + "per_group": 114139.33255939094 + }, + { + "label": "base-5132fc4a455b", + "revision": "5132fc4a455bf6a063112d1d5185c6e3b8a2a4f5", + "scenario": "session", + "subscribers": 64, + "per_group": 114999.65691978567 + }, + { + "label": "base-5132fc4a455b", + "revision": "5132fc4a455bf6a063112d1d5185c6e3b8a2a4f5", + "scenario": "session", + "subscribers": 512, + "per_group": 113132.77601841518 + }, + { + "label": "base-101ed5a8066d", + "revision": "101ed5a8066d4c17ec85ac3cc2d5ec40686c4962", + "scenario": "session", + "subscribers": 64, + "per_group": 110473.41742642355 + }, + { + "label": "base-101ed5a8066d", + "revision": "101ed5a8066d4c17ec85ac3cc2d5ec40686c4962", + "scenario": "session", + "subscribers": 512, + "per_group": 108709.31261957908 + }, + { + "label": "current", + "revision": "df74d8dbe07f760b2ff5eb6da9dda57d9ee8006d", + "scenario": "session", + "subscribers": 64, + "per_group": 110473.45486444338 + }, + { + "label": "current", + "revision": "df74d8dbe07f760b2ff5eb6da9dda57d9ee8006d", + "scenario": "session", + "subscribers": 512, + "per_group": 108709.42349928252 + } + ] + }, + { + "artifact": "run.evL1wx", + "note": "this branch against main at 619df6e1b3", + "harness_revision": "af83ff87fcb3cb0e2ff3969ed1f96fcebee62043", + "frame_pointers": false, + "counts": [ + { + "label": "base", + "revision": "619df6e1b3a27457b31be76d14d58ae7b5dc7afc", + "scenario": "model", + "subscribers": 64, + "per_group": 13576.320641 + }, + { + "label": "base", + "revision": "619df6e1b3a27457b31be76d14d58ae7b5dc7afc", + "scenario": "model", + "subscribers": 512, + "per_group": 13367.74955919748 + }, + { + "label": "current", + "revision": "af83ff87fcb3cb0e2ff3969ed1f96fcebee62043", + "scenario": "model", + "subscribers": 64, + "per_group": 10329.611195 + }, + { + "label": "current", + "revision": "af83ff87fcb3cb0e2ff3969ed1f96fcebee62043", + "scenario": "model", + "subscribers": 512, + "per_group": 10118.748023885299 + }, + { + "label": "base", + "revision": "619df6e1b3a27457b31be76d14d58ae7b5dc7afc", + "scenario": "session", + "subscribers": 64, + "per_group": 123241.99125279911 + }, + { + "label": "base", + "revision": "619df6e1b3a27457b31be76d14d58ae7b5dc7afc", + "scenario": "session", + "subscribers": 512, + "per_group": 121114.091796875 + }, + { + "label": "current", + "revision": "af83ff87fcb3cb0e2ff3969ed1f96fcebee62043", + "scenario": "session", + "subscribers": 64, + "per_group": 115007.12947856686 + }, + { + "label": "current", + "revision": "af83ff87fcb3cb0e2ff3969ed1f96fcebee62043", + "scenario": "session", + "subscribers": 512, + "per_group": 113142.93781887754 + } + ] + } + ], + "quiet_host_runs": [ + { + "artifact": "nixos2:run.we4umM", + "host": "nixos2 (i7-8700, idle; quest gate not installed there, so its load and other agents were checked first)", + "relay_cpus": "0-3", + "generator_cpus": "4,5,10,11", + "generator_revision": "21d6da9fc1", + "workload": "chat", + "runtime": "io-uring-workers", + "revisions": { "base": "a6f3d28216 (main merge)", "current": "44c0dc494f (the probe and window changes)" }, + "summary": { + "base": { + "4": { + "cells": [ + { + "cell": "relay/base/round-1/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 154075.30345533811, + "cycles_per_frame": 301337.768161681, + "recv_fps": 20002.865948772975, + "groups_lost": 0, + "host_load_1m": 4.68 + }, + { + "cell": "relay/base/round-2/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 155046.662319841, + "cycles_per_frame": 314087.22154518875, + "recv_fps": 19998.415463227855, + "groups_lost": 0, + "host_load_1m": 1.67 + }, + { + "cell": "relay/base/round-3/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 154380.03417891636, + "cycles_per_frame": 351534.3433222535, + "recv_fps": 20001.617599275352, + "groups_lost": 0, + "host_load_1m": 1.45 + }, + { + "cell": "relay/base/round-4/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 154590.6275877289, + "cycles_per_frame": 304639.855884305, + "recv_fps": 20001.623163712316, + "groups_lost": 0, + "host_load_1m": 2.64 + } + ], + "instructions_per_frame": { + "median": 154485.33088332263, + "min": 154075.30345533811, + "max": 155046.662319841 + }, + "cycles_per_frame": { + "median": 309363.5387147469, + "min": 301337.768161681, + "max": 351534.3433222535 + }, + "recv_fps": { + "median": 20001.620381493834, + "min": 19998.415463227855, + "max": 20002.865948772975 + }, + "groups_lost": { "median": 0.0, "min": 0, "max": 0 }, + "host_load_1m": { "median": 2.1550000000000002, "min": 1.45, "max": 4.68 } + } + }, + "current": { + "4": { + "cells": [ + { + "cell": "relay/current/round-1/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 148274.66412811857, + "cycles_per_frame": 337324.0407741121, + "recv_fps": 20000.0, + "groups_lost": 0, + "host_load_1m": 2.41 + }, + { + "cell": "relay/current/round-2/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 148357.681494429, + "cycles_per_frame": 274414.4307833131, + "recv_fps": 20003.999606288755, + "groups_lost": 0, + "host_load_1m": 1.54 + }, + { + "cell": "relay/current/round-3/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 150798.68041245252, + "cycles_per_frame": 322361.04395030695, + "recv_fps": 19997.632603729515, + "groups_lost": 0, + "host_load_1m": 2.75 + }, + { + "cell": "relay/current/round-4/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 149414.94788195015, + "cycles_per_frame": 315090.17196768953, + "recv_fps": 19999.995299019218, + "groups_lost": 0, + "host_load_1m": 1.32 + } + ], + "instructions_per_frame": { + "median": 148886.3146881896, + "min": 148274.66412811857, + "max": 150798.68041245252 + }, + "cycles_per_frame": { + "median": 318725.60795899824, + "min": 274414.4307833131, + "max": 337324.0407741121 + }, + "recv_fps": { + "median": 19999.99764950961, + "min": 19997.632603729515, + "max": 20003.999606288755 + }, + "groups_lost": { "median": 0.0, "min": 0, "max": 0 }, + "host_load_1m": { "median": 1.975, "min": 1.32, "max": 2.75 } + } + } + } + } + ], + "notes": [ + "nixos2:run.cAVCAV (in-tree qmux and receive leases against their parents) was stopped after one round when other agents started fuzzing and compiling there; its round-1 cells (qmux 116k to 122k, leases 130k to 132k) are not conclusive and are not used." + ] } diff --git a/.github/relay-cpu-bisect.md b/.github/relay-cpu-bisect.md index a12a8594f03..2d7d7bebe3e 100644 --- a/.github/relay-cpu-bisect.md +++ b/.github/relay-cpu-bisect.md @@ -79,7 +79,7 @@ At 512 subscribers, in instructions per delivered group, the model path cost 4,2 | Public ingest, [moq-dev/moq#6111](https://github.com/moq-dev/moq/pull/6111) | 5,816 | 6,016 | +200 | | The m0 simplification audit, [moq-dev/moq#6108](https://github.com/moq-dev/moq/pull/6108) | 6,017 | 6,017 | 0 | -In-tree qmux ([moq-dev/moq#6231](https://github.com/moq-dev/moq/pull/6231)) and receive leases ([moq-dev/moq#5878](https://github.com/moq-dev/moq/pull/5878)) are QUIC-level, which neither scenario reaches; their relay counts on the quiet test host are below. The model path barely moved before [e84ffb3de3](https://github.com/moq-dev/moq/commit/e84ffb3de3139f7245bed5697754fd6add9a022e) (5,840) and then grew in a few large steps, each against its parent (run.TmGdFy, one build profile throughout): +In-tree qmux ([moq-dev/moq#6231](https://github.com/moq-dev/moq/pull/6231)) and receive leases ([moq-dev/moq#5878](https://github.com/moq-dev/moq/pull/5878)) are QUIC-level, which neither scenario reaches. A relay run pairing each with its parent on the quiet test host got one round (qmux 116k to 122k, leases 130k to 132k per frame) before other agents started fuzzing and compiling there, so neither is attributed yet. The model path barely moved before [e84ffb3de3](https://github.com/moq-dev/moq/commit/e84ffb3de3139f7245bed5697754fd6add9a022e) (5,840) and then grew in a few large steps, each against its parent (run.TmGdFy, one build profile throughout): | Commit | Model | Session | | --- | ---: | ---: | @@ -105,6 +105,8 @@ Each is its own commit, counted in one run (run.s6IjbN in the evidence) at 512 s | A window with one set of runs reads without growing across two | 10,126 (-351) | 113,133 (-1,007) | | This branch | 10,126 (-23.9%) | 113,133 (-6.5%) | +Against `main` at [619df6e1b3](https://github.com/moq-dev/moq/commit/619df6e1b3) (run.evL1wx), after merging it: model 13,368 to 10,119 (-24.3%), session 121,114 to 113,143 (-6.6%). + `Cursor::probe` is the relay's check for groups a reader has missed. Each time a subscriber asks for its next group, the probe looks at what arrived on the track since its last look and decides whether any group the reader still wants has left the cache without the live feed delivering it (evicted, skipped, or from before the feed started), and asks upstream to fill those. A live reader that takes each group as it arrives never has such a gap, yet it paid for the whole check on every group: locking the subscription and the track's shared state, cloning the track's fills, walking the arrival ledger, checking each new group for holes, and running the ask-and-answer loop. Each time the cache aged out an old group, the eviction also landed with every reader, all of which had already taken it, and forced each one to re-run the full check. The fast path judges a group the arrival walk serves without the hole, backfill, and ledger passes, skips the subscription, fills, and answers when nothing is asked or owed, and starts each reader's eviction inbox above what it handed out. It recovers most of [moq-dev/moq#6171](https://github.com/moq-dev/moq/pull/6171)'s and [moq-dev/moq#7339](https://github.com/moq-dev/moq/pull/7339)'s cost. kio's `WaiterList::register` is the session path's second-largest self cost: 7.2k per group. A waiter records 8 lists, and a relay session's control and serving tasks park on more, so every list past the eighth takes a duplicate entry on every poll. Remembering 16 cut the session by 4,424 (-3.9%) and was reverted: with it, two moq-net integration tests (`errored_end::an_errored_end_keeps_its_history_across_a_hop`, `quiet_catalog::a_quiet_catalog_reaches_an_unfloored_relay_reader_when_a_peer_resumes_past_it`) fail, and they pass with 8. A read whose waiter parks on more lists than it records is retired and walks again; with more records those reads stay parked, and something they depend on doesn't wake them. That needs its own investigation before the waiter can remember more. From 77da888fe462b2445e277577e117a4616e78f518 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 13:51:16 -0800 Subject: [PATCH 35/36] quest: the QUIC-level feature steps still need quiet relay runs Co-Authored-By: Claude Opus 5.5 --- quest/m1/release-perf/relay-cpu-bisect.md | 1 + 1 file changed, 1 insertion(+) diff --git a/quest/m1/release-perf/relay-cpu-bisect.md b/quest/m1/release-perf/relay-cpu-bisect.md index 08c40414ab4..7c29224589d 100644 --- a/quest/m1/release-perf/relay-cpu-bisect.md +++ b/quest/m1/release-perf/relay-cpu-bisect.md @@ -33,6 +33,7 @@ Remaining: - Settle the attributed costs nothing here cut without changing behavior (the report's tables: the window, drift, and live-group walks under [moq-dev/moq#5929](https://github.com/moq-dev/moq/pull/5929), [moq-dev/moq#6995](https://github.com/moq-dev/moq/pull/6995), and [moq-dev/moq#7542](https://github.com/moq-dev/moq/pull/7542); the small flat steps of compressed tracks, ranged reads, and public ingest; the lite publisher's per-group stream open and close), and record each accepted one and its feature in the epic's Plan. - The session path's remaining largest self costs: moves of large values (about 9.4k per group: `group::Consumer` is 280 bytes, moved by value through `poll_peek_group`, `take`, and the viewer's serve loop), and waiter registration (about 7.2k). A waiter that remembered 16 lists instead of 8 cut the session by 3.9% but made two integration tests miss a wake, since reads it no longer retires stay parked; see the report. +- Attribute in-tree qmux ([moq-dev/moq#6231](https://github.com/moq-dev/moq/pull/6231)) and receive leases ([moq-dev/moq#5878](https://github.com/moq-dev/moq/pull/5878)), each against its parent, with relay runs on a quiet host: they are QUIC-level, which `just bench-fanout` doesn't reach, and the one attempt on `nixos2` lost the host to other agents after a round. - Re-pair `main` against [cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95) on chat and video over both runtimes on a quiet host, then delete this quest. Measure as the report does. For a model or session change, `just bench-fanout` against the base, which host load doesn't move. For the relay, `BENCH_GENERATOR=21d6da9fc1`, `BENCH_INSTRUCTION_PERIOD=2000003`, `kernel.perf_event_max_sample_rate` raised to 100,000 and restored after, on a quiet host such as `nixos2` (6 cores: relay on 0-3, generator on 4,5,10,11, `BENCH_BUILD_JOBS=2` for its 15 GiB), comparing only cells from one run; instructions per frame fall by up to 20% when other host work makes the relay batch more frames per wakeup. From 21b6ab173af6732e88ee97576c024630fadf3c74 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 14:19:21 -0800 Subject: [PATCH 36/36] quest: plan the waiter and group consumer follow-ups, and accept the attributed costs Records the maintainer's 2026-11-03 decisions on moq-dev/moq#7605: the flat per-group feature costs are accepted in the release-perf epic with their counts, the waiter record limit becomes an m0 correctness quest, and shrinking group::Consumer joins the epic. Co-Authored-By: Claude Opus 5.5 --- .github/relay-cpu-bisect.md | 2 +- quest/m0/README.md | 1 + quest/m0/kio-waiter-records.md | 35 ++++++++++++++++++++ quest/m1/release-perf/README.md | 28 ++++++++++++++++ quest/m1/release-perf/group-consumer-size.md | 21 ++++++++++++ quest/m1/release-perf/relay-cpu-bisect.md | 19 +++++++++-- 6 files changed, 103 insertions(+), 3 deletions(-) create mode 100644 quest/m0/kio-waiter-records.md create mode 100644 quest/m1/release-perf/group-consumer-size.md diff --git a/.github/relay-cpu-bisect.md b/.github/relay-cpu-bisect.md index 2d7d7bebe3e..12a1a385969 100644 --- a/.github/relay-cpu-bisect.md +++ b/.github/relay-cpu-bisect.md @@ -109,7 +109,7 @@ Against `main` at [619df6e1b3](https://github.com/moq-dev/moq/commit/619df6e1b3) `Cursor::probe` is the relay's check for groups a reader has missed. Each time a subscriber asks for its next group, the probe looks at what arrived on the track since its last look and decides whether any group the reader still wants has left the cache without the live feed delivering it (evicted, skipped, or from before the feed started), and asks upstream to fill those. A live reader that takes each group as it arrives never has such a gap, yet it paid for the whole check on every group: locking the subscription and the track's shared state, cloning the track's fills, walking the arrival ledger, checking each new group for holes, and running the ask-and-answer loop. Each time the cache aged out an old group, the eviction also landed with every reader, all of which had already taken it, and forced each one to re-run the full check. The fast path judges a group the arrival walk serves without the hole, backfill, and ledger passes, skips the subscription, fills, and answers when nothing is asked or owed, and starts each reader's eviction inbox above what it handed out. It recovers most of [moq-dev/moq#6171](https://github.com/moq-dev/moq/pull/6171)'s and [moq-dev/moq#7339](https://github.com/moq-dev/moq/pull/7339)'s cost. -kio's `WaiterList::register` is the session path's second-largest self cost: 7.2k per group. A waiter records 8 lists, and a relay session's control and serving tasks park on more, so every list past the eighth takes a duplicate entry on every poll. Remembering 16 cut the session by 4,424 (-3.9%) and was reverted: with it, two moq-net integration tests (`errored_end::an_errored_end_keeps_its_history_across_a_hop`, `quiet_catalog::a_quiet_catalog_reaches_an_unfloored_relay_reader_when_a_peer_resumes_past_it`) fail, and they pass with 8. A read whose waiter parks on more lists than it records is retired and walks again; with more records those reads stay parked, and something they depend on doesn't wake them. That needs its own investigation before the waiter can remember more. +kio's `WaiterList::register` is the session path's second-largest self cost: 7.2k per group. A waiter records 8 lists, and a relay session's control and serving tasks park on more, so every list past the eighth takes a duplicate entry on every poll. Remembering 16 cut the session by 4,424 (-3.9%) and was reverted: with it, two moq-net integration tests (`errored_end::an_errored_end_keeps_its_history_across_a_hop`, `quiet_catalog::a_quiet_catalog_reaches_an_unfloored_relay_reader_when_a_peer_resumes_past_it`) fail, and they pass with 8. A read whose waiter parks on more lists than it records is retired and walks again; with more records those reads stay parked, and something they depend on doesn't wake them. That is now an m0 correctness quest, [a waiter's record limit changes no behavior](../quest/m0/kio-waiter-records.md), and the per-group costs this report attributes and nothing here cut are accepted in the [release-perf epic](../quest/m1/release-perf/README.md). On the quiet test host (run.we4umM, one-minute load at most 4.7), chat over io\_uring went from 154.5k \[154.1-155.0] instructions per frame on `main` to 148.9k \[148.3-150.8] with the probe and window changes (-3.6%), four rounds each, every cell at 20,000 frames/s with no loss. The relay's QUIC, crypto, and syscalls are most of its cost, so the model's -23% shows as a few percent there. diff --git a/quest/m0/README.md b/quest/m0/README.md index 25a135c7f0d..b0090933bb2 100644 --- a/quest/m0/README.md +++ b/quest/m0/README.md @@ -35,6 +35,7 @@ since cutting segments from first-frame pts waits on nothing in m1. - [Backport the joined-head FETCH race fix](/quest/m0/backport-fill-head-race.md) - a `release` relay stops FETCHing a group head its own SUBSCRIBE is bringing, which left live readers hanging - [CI cost and latency](/quest/m0/ci/README.md) - cloud CI runs only in a batched merge gate as one `just ci` pass per batch, expensive checks run nightly and gate `release`, caches move to R2, and runners are chosen last - [More tests under load](/quest/m0/test-flakes-2/README.md) - load-only and queue failures, one quest per flake, fixed at the cause, since each one fails a whole batch +- [A waiter's record limit changes no behavior](/quest/m0/kio-waiter-records.md) - two moq-net tests stop failing when kio's waiters remember 16 lists, and the limit becomes a tuning knob worth 3.9% per delivered group - [Backport the relay and auth hardening to release](/quest/m0/backport-release-hardening.md) - `release`'s relay refuses dot-segment paths, serves its HTTP routes with main's response hardening, and withdraws advertisements a peer starves of send credit; its auth client refuses redirects - [Release's MKV importer bounds what it buffers](/quest/m0/backport-mkv-buffer.md) - on `release`, an MKV input that declares a huge element and trickles its bytes no longer grows the importer's memory without limit - [HLS export on a declared grid](/quest/m0/hls-media-grid.md) - the catalog's `hls` field and record tracks go away, and the export numbers segments on `media.join.interval` from media tracks diff --git a/quest/m0/kio-waiter-records.md b/quest/m0/kio-waiter-records.md new file mode 100644 index 00000000000..f4225dae312 --- /dev/null +++ b/quest/m0/kio-waiter-records.md @@ -0,0 +1,35 @@ +# [M] A waiter's record limit changes no behavior + +## Goal + +Raising kio's `PARKED`, the number of lists a `kio::Waiter` remembers parking on, from 8 to 16 changes only what it costs: no moq-net test or relay behavior depends on the limit. Today two moq-net integration tests fail at 16 and pass at 8, so a missed wake, or a test leaning on one by accident, hides behind it. Find and fix that, then set the limit to 16 and take the 3.9% per delivered group the chat fan-out measured. + +## Plan + +Decided 2026-11-03 (maintainer, on [moq-dev/moq#7605](https://github.com/moq-dev/moq/pull/7605)): m0, as a correctness bug rather than a performance tweak, since "the fact that tests fail if we change a constant is a HUGE red flag". + +Reproduce: set `const PARKED: usize = 16;` in `rs/kio/src/waiter.rs`, then + +```bash +nix develop --command cargo nextest run -p moq-net --all-features \ + -E 'test(an_errored_end_keeps_its_history_across_a_hop) | test(a_quiet_catalog_reaches_an_unfloored_relay_reader_when_a_peer_resumes_past_it)' +``` + +- `errored_end::an_errored_end_keeps_its_history_across_a_hop` fails with `subscribe: Stream(App(7))` at `rs/moq-net/tests/integration/errored_end.rs:35`. +- `quiet_catalog::a_quiet_catalog_reaches_an_unfloored_relay_reader_when_a_peer_resumes_past_it` fails with `resume subscribe: Unroutable` at `rs/moq-net/tests/integration/quiet_catalog.rs:178`. +- Both pass at 8, on [0e9b7ae6bb](https://github.com/moq-dev/moq/commit/0e9b7ae6bb4d22b50da244a3c1146acd9bb91f5a) and on `main` at the time. + +The lead: `rs/moq-net/src/util/idle.rs` keeps a read's waiter, and reports the read parked without walking it again, only while every list it parked on is recorded. Past the limit the waiter is marked lost and retired, and the read walks again. At 8 the reads in these tests overflowed and walked on every poll; at 16 they stay parked, so a change they depend on doesn't wake them, through a list not registered with this waiter or a wake that only a walk noticed. Find the read and the change, and fix the wake or the test's hidden dependency, not the limit. + +Then set the limit to 16: + +- `util::idle::test::a_waiter_parked_past_its_records_is_retired` parks on 9 lists, the old limit plus one; make it the limit plus one. +- The larger waiter makes clippy's `large_enum_variant` fire on `Sub` in `rs/moq-net/src/model/resume.rs` and `Reader` in `rs/moq-stats/src/aggregate.rs`; box their pending variants, as [moq-dev/moq#7605](https://github.com/moq-dev/moq/pull/7605)'s reverted commits [101ed5a806](https://github.com/moq-dev/moq/commit/101ed5a806) and [1a5ceeca84](https://github.com/moq-dev/moq/commit/1a5ceeca84) did. +- Measure with `just bench-fanout "" session "64 512"`. On [moq-dev/moq#7605](https://github.com/moq-dev/moq/pull/7605), 16 took the session scenario from 115,000 to 110,524 instructions per group at 64 viewers and from 113,200 to 108,747 at 512 (-3.9%); 12 and 32 each cost about 1-2% more than 16, since every registration searches the records. + +Public API: none expected. Wire: none. + +## Related + +- [Relay CPU per frame since #5926](/quest/m1/release-perf/relay-cpu-bisect.md) - found this, measuring the waiter's registration cost +- [Task sets release memory after a burst](/quest/m2/performance-survey/task-capacity.md) - a task slot's `Park` holds a whole `Waiter`, which 16 records grow diff --git a/quest/m1/release-perf/README.md b/quest/m1/release-perf/README.md index 1f974bc788d..9221840ffae 100644 --- a/quest/m1/release-perf/README.md +++ b/quest/m1/release-perf/README.md @@ -284,9 +284,37 @@ Decided 2026-11-01 (maintainer), in bench-only backport to `release`, which carries it once that PR lands. Bench-only backports are allowed from now on. +Decided 2026-11-03 (maintainer), in +[moq-dev/moq#7605](https://github.com/moq-dev/moq/pull/7605): these flat +per-group costs of the relay's chat fan-out are accepted as the features they +pay for. Each is instructions per delivered group at 512 subscribers through +the track model, counted by `just bench-fanout` against the merge's parent: + +- Per-reader ranges and time anchors + ([moq-dev/moq#5929](https://github.com/moq-dev/moq/pull/5929)): +902 when + it landed. moq-dev/moq#7605 took back 351 of the window reads it and + [moq-dev/moq#7542](https://github.com/moq-dev/moq/pull/7542) left on every + peek; what remains reads each reader's window and drift edge. +- A subscriber reads only its ranges' frames + ([moq-dev/moq#6129](https://github.com/moq-dev/moq/pull/6129)): +183. +- Public ingest ([moq-dev/moq#6111](https://github.com/moq-dev/moq/pull/6111)): + +200. +- Compressed tracks ([moq-dev/moq#5928](https://github.com/moq-dev/moq/pull/5928)): + +33. +- Closing the relay demand gaps + ([moq-dev/moq#7052](https://github.com/moq-dev/moq/pull/7052)): +365. +- Bounded reads end through routes + ([moq-dev/moq#7625](https://github.com/moq-dev/moq/pull/7625)): +275, and + +450 per group through the in-process session. + +The first four were counted with frame pointers forced and the last two +without, so compare each only with its own parent. None grows with +subscribers. + ## Required - [Relay CPU per frame since #5926](/quest/m1/release-perf/relay-cpu-bisect.md) - the 18-56% rise in relay instructions per delivered frame is bisected, attributed, and its avoidable cost fixed +- [A read moves a small group consumer](/quest/m1/release-perf/group-consumer-size.md) - `group::Consumer` shrinks from 280 bytes, so a read stops copying it through every layer ## Related diff --git a/quest/m1/release-perf/group-consumer-size.md b/quest/m1/release-perf/group-consumer-size.md new file mode 100644 index 00000000000..057debc66c2 --- /dev/null +++ b/quest/m1/release-perf/group-consumer-size.md @@ -0,0 +1,21 @@ +# [M] A read moves a small group consumer + +## Goal + +`moq_net::group::Consumer` shrinks from 280 bytes by boxing what a read rarely touches, so handing a group to each reader stops copying 280 bytes through every layer it passes. `just bench-fanout` shows the drop, and the common read path gains no allocation per group. + +## Plan + +Decided 2026-11-03 (maintainer, on [moq-dev/moq#7605](https://github.com/moq-dev/moq/pull/7605)): a follow-up in this epic. + +Found by [moq-dev/moq#7605](https://github.com/moq-dev/moq/pull/7605)'s session scenario (a publisher, a relay origin, and viewers over the in-memory transport): `memcpy` is its largest self cost, about 9.4k of 113k instructions per delivered group at 64 viewers, mostly moves of `group::Consumer` and the `Peeked` that carries one, by value through `Poll>>` layers. The largest callers per group: `track::Subscriber::poll_peek_group` (982), the viewer's `lite::subscriber::UniServe::poll_serve` (715), `track::Subscriber::take` (566, 442, and 428 across its inlined copies), `track::Cursor::poll_peek` (507), and `kio::Tasks::insert` (454). + +- Find which fields a read touches per group and box the rest, or the rarely-set state, behind one pointer. Check `size_of` before and after. +- Measure with `just bench-fanout "" "model session" "64 512"`, and `BENCH_FANOUT_CALLGRIND=1` with `bench/fanout.py compare` to see the moves shrink. Keep the session scenario's allocations per group flat: the [relay allocations](/quest/m2/perf/relay-allocs.md) quest tracks them. + +Public API: `group::Consumer`'s layout only; no signature change expected. Wire: none. + +## Related + +- [Relay CPU per frame since #5926](/quest/m1/release-perf/relay-cpu-bisect.md) - found this +- [The relay allocates less per delivered frame](/quest/m2/perf/relay-allocs.md) - boxing must not add an allocation per delivered group diff --git a/quest/m1/release-perf/relay-cpu-bisect.md b/quest/m1/release-perf/relay-cpu-bisect.md index 7c29224589d..c64a984f82d 100644 --- a/quest/m1/release-perf/relay-cpu-bisect.md +++ b/quest/m1/release-perf/relay-cpu-bisect.md @@ -29,10 +29,22 @@ Decided 2026-11-03 (maintainer, on [moq-dev/moq#7605](https://github.com/moq-dev Follow-up planning on 2026-11-03 (maintainer) added a load-independent mode to the driver: `just bench-fanout` counts the chat fan-out per delivered group on one pinned thread, through the model alone and through a relay over the in-memory transport, for any list of revisions. [The report](/.github/relay-cpu-bisect.md) attributes each step with it, and the fixes so far take `main`'s model path from 13,300 to 10,126 instructions per group (-23.9%) and the in-process session from 120,948 to 113,133 (-6.5%), at 512 subscribers. +Decided 2026-11-03 (maintainer, on [moq-dev/moq#7605](https://github.com/moq-dev/moq/pull/7605)), after the attribution: + +1. Costs measured and not cut: + - ✅ Accept the flat ones (per-reader ranges' remainder, ranged reads, public ingest, compressed tracks, [moq-dev/moq#7052](https://github.com/moq-dev/moq/pull/7052), and bounded reads), recorded with their numbers in the [epic](/quest/m1/release-perf/README.md). Keep [moq-dev/moq#6995](https://github.com/moq-dev/moq/pull/6995)'s and [moq-dev/moq#7542](https://github.com/moq-dev/moq/pull/7542)'s session cost here for a re-measure against this branch. + - Accept all of them now. + - Keep investigating all of them. +2. The waiter's list records: + - ✅ A follow-up at m0, as a correctness bug: [a waiter's record limit changes no behavior](/quest/m0/kio-waiter-records.md). + - Leave the limit at 8. +3. `group::Consumer` moves: + - ✅ A follow-up in this epic: [a read moves a small group consumer](/quest/m1/release-perf/group-consumer-size.md). + - Leave it. + Remaining: -- Settle the attributed costs nothing here cut without changing behavior (the report's tables: the window, drift, and live-group walks under [moq-dev/moq#5929](https://github.com/moq-dev/moq/pull/5929), [moq-dev/moq#6995](https://github.com/moq-dev/moq/pull/6995), and [moq-dev/moq#7542](https://github.com/moq-dev/moq/pull/7542); the small flat steps of compressed tracks, ranged reads, and public ingest; the lite publisher's per-group stream open and close), and record each accepted one and its feature in the epic's Plan. -- The session path's remaining largest self costs: moves of large values (about 9.4k per group: `group::Consumer` is 280 bytes, moved by value through `poll_peek_group`, `take`, and the viewer's serve loop), and waiter registration (about 7.2k). A waiter that remembered 16 lists instead of 8 cut the session by 3.9% but made two integration tests miss a wake, since reads it no longer retires stay parked; see the report. +- Re-measure [moq-dev/moq#6995](https://github.com/moq-dev/moq/pull/6995)'s and [moq-dev/moq#7542](https://github.com/moq-dev/moq/pull/7542)'s session cost against `main` with the probe fast path (about +3.3k and +3.1k per group when each landed; much of it was the probe's walk of the live groups, which the fast path now skips), then fix what is accidental or record what remains in the epic. - Attribute in-tree qmux ([moq-dev/moq#6231](https://github.com/moq-dev/moq/pull/6231)) and receive leases ([moq-dev/moq#5878](https://github.com/moq-dev/moq/pull/5878)), each against its parent, with relay runs on a quiet host: they are QUIC-level, which `just bench-fanout` doesn't reach, and the one attempt on `nixos2` lost the host to other agents after a round. - Re-pair `main` against [cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95) on chat and video over both runtimes on a quiet host, then delete this quest. @@ -42,5 +54,8 @@ Public API: none expected. Wire: none. ## Related +- [A waiter's record limit changes no behavior](/quest/m0/kio-waiter-records.md) - found here; the session's waiter registration cost waits on it +- [A read moves a small group consumer](/quest/m1/release-perf/group-consumer-size.md) - found here; the session's largest remaining self cost + - [Calibrate the paired nightly benchmarks](/quest/m1/bench-ci.md) - the nightly series that should have caught this growth - [Performance audit](/quest/m2/audit-perf.md) - counts instructions when bisecting, for the same reason