From 634197cb9ac8a85764ff2f408d4a2513f6e87ab7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 10 Oct 2026 14:17:56 -0700 Subject: [PATCH 1/8] chore(quest): claim relay profiling harness Co-Authored-By: GPT-6 From 7e8a20a8c1db4dc54edbd5f6b9a5598411de1d4b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 10 Oct 2026 14:45:25 -0700 Subject: [PATCH 2/8] feat(bench): probe profiling support and classify worker lock waits Co-Authored-By: GPT-6 --- .github/justfile | 1 + bench/README.md | 22 +++++++ bench/profile.py | 116 ++++++++++++++++++++++++++++++++++ bench/profile.test.py | 38 +++++++++++ flake.nix | 6 +- justfile | 4 ++ quest/m1/perf/lock-profile.md | 7 ++ 7 files changed, 193 insertions(+), 1 deletion(-) create mode 100644 bench/profile.py create mode 100644 bench/profile.test.py diff --git a/.github/justfile b/.github/justfile index 1bccfc53380..93957c9c6dd 100644 --- a/.github/justfile +++ b/.github/justfile @@ -8,6 +8,7 @@ check: # Tests of the release tooling itself; nightly, not on every pull request. test: + python3 bench/profile.test.py python3 sh/gh/install-formula.test.py sh/gh/release.test.sh python3 sh/rs/package-rename.test.py diff --git a/bench/README.md b/bench/README.md index 2ed1eb44aa3..a94332650fc 100644 --- a/bench/README.md +++ b/bench/README.md @@ -68,3 +68,25 @@ work. Benchmark output is informational and machine-specific. Crashes, zero delivery, and invalid samples still fail the command. + +## Profiling capability draft + +`nix develop --command just bench-profile-probe lock OUTPUT` probes bpftrace +permissions before a costly build; use `cpu` to probe perf's CPU clock event. +A refused probe retains stdout, stderr and its command in OUTPUT and exits +unsuccessfully. Linux lock capture needs sudo or CAP_BPF plus tracefs access; +CPU capture needs perf_event permission. The probe captures no profile. + +`profile.py thread-cpu PID OUTPUT` records relay worker CPU nanoseconds from +Linux schedstat. The reducer accepts normalized records with `tid`, `comm`, +`wait_ns` and symbolized `stack` fields, plus before/after CPU counters keyed +by TID. Only std mutex `lock_contended` frames contribute to the lock share; +allocator, condvar and runtime parking waits are counted separately. The +denominator is scheduled CPU time, so wait/CPU can exceed one. Missing CPU or +user stacks are refused. Spinning does not reach the futex and is excluded. + +This is a partial foundation, not the profiling launcher. Still required: +integrate captures with run.sh's owned-child lifecycle, preserve metadata and +control runs, normalize bpftrace output, validate resolved MoQ stacks, capture +CPU profiles and jemalloc snapshots/churn, and exercise macOS samply. Heap +capture must verify the existing SIGUSR1 listener before sending any signal. diff --git a/bench/profile.py b/bench/profile.py new file mode 100644 index 00000000000..107aa120629 --- /dev/null +++ b/bench/profile.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Probe capture privileges and reduce explicitly classified worker futex waits.""" +import argparse +import json +import platform +import re +import shutil +import subprocess +from pathlib import Path + +# Condvar, allocator and runtime parking futexes are not std mutex acquisitions. +MUTEX = re.compile(r"std::sys::(?:sync|pal::unix)::mutex::futex::Mutex::lock_contended") +WORKER = re.compile(r"moq-(?:uring|quic)-[0-9]+$") + + +def reduce(samples, before, after): + """Normalize mutex waits by thread CPU counters, never elapsed time.""" + result = {} + for sample in samples: + tid, name = str(sample['tid']), sample['comm'] + if not WORKER.fullmatch(name): + continue + wait = sample['wait_ns'] + if not isinstance(wait, int) or isinstance(wait, bool) or wait < 0: + raise ValueError('wait_ns must be a nonnegative integer') + stack = sample['stack'] + if not isinstance(stack, list) or not stack or not all(isinstance(frame, str) for frame in stack): + raise ValueError('a nonempty symbolized user stack is required') + row = result.setdefault(tid, {'comm': name, 'mutex_wait_ns': 0, 'other_futex_wait_ns': 0, 'stacks': []}) + if row['comm'] != name: + raise ValueError('thread identity changed inside the capture') + if any(MUTEX.search(frame) for frame in stack): + row['mutex_wait_ns'] += wait + row['stacks'].append({'wait_ns': wait, 'stack': stack}) + else: + row['other_futex_wait_ns'] += wait + if not result: + raise ValueError('capture has no symbolized relay-worker futex samples') + for tid, row in result.items(): + if tid not in before or tid not in after: + raise ValueError(f'missing CPU counter for worker {tid}') + if any(not isinstance(value, int) or isinstance(value, bool) or value < 0 for value in (before[tid], after[tid])): + raise ValueError(f'invalid CPU counter for worker {tid}') + cpu = after[tid] - before[tid] + if cpu <= 0: + raise ValueError(f'worker {tid} has no advancing CPU counter') + row['cpu_ns'] = cpu + row['mutex_wait_per_cpu'] = row['mutex_wait_ns'] / cpu + return {'version': 1, 'workers': result, + 'scope': 'completed futex waits with std mutex lock_contended frames; spinning is excluded'} + + +def thread_cpu(pid): + """Read Linux per-thread scheduled CPU nanoseconds for the matching workers.""" + if platform.system() != 'Linux': + raise ValueError('thread CPU capture requires Linux /proc schedstat') + result = {} + for task in (Path('/proc') / str(pid) / 'task').iterdir(): + if WORKER.fullmatch((task / 'comm').read_text().strip()): + result[task.name] = int((task / 'schedstat').read_text().split()[0]) + if not result: + raise ValueError('PID has no moq-uring-N or moq-quic-N workers') + return result + + +def probe(mode, directory): + """Retain both successful and refused capability probes before any build.""" + directory.mkdir(parents=True, exist_ok=True) + if platform.system() != 'Linux': + raise ValueError('this draft probes Linux only; macOS samply capture remains unimplemented') + tool = {'lock': 'bpftrace', 'cpu': 'perf'}[mode] + executable = shutil.which(tool) + if not executable: + raise ValueError(f'{tool} is unavailable; enter the Nix dev shell') + command = ([executable, '-e', 'BEGIN { exit(); }'] if mode == 'lock' else + [executable, 'stat', '-e', 'cpu-clock', '--', 'true']) + completed = subprocess.run(command, capture_output=True, text=True, timeout=30) + (directory / 'probe.stdout').write_text(completed.stdout) + (directory / 'probe.stderr').write_text(completed.stderr) + evidence = {'version': 1, 'mode': mode, 'command': command, + 'kernel': platform.platform(), 'exit_code': completed.returncode, + 'capture': 'not attempted; this command only probes capability'} + (directory / 'probe.json').write_text(json.dumps(evidence, indent=2) + '\n') + if completed.returncode: + requirement = 'sudo/CAP_BPF, tracefs and BPF permissions' if mode == 'lock' else 'perf_event permissions' + raise ValueError(f'{tool} capture refused: requires {requirement}; see {directory / "probe.stderr"}') + print('Capability probe passed. No profile was captured.') + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest='command', required=True) + capability = commands.add_parser('probe') + capability.add_argument('mode', choices=('lock', 'cpu')) + capability.add_argument('directory', type=Path) + cpu = commands.add_parser('thread-cpu') + cpu.add_argument('pid', type=int) + cpu.add_argument('output', type=Path) + merge = commands.add_parser('reduce') + for name in ('samples', 'before', 'after', 'output'): + merge.add_argument(name, type=Path) + args = parser.parse_args() + try: + if args.command == 'probe': + probe(args.mode, args.directory) + elif args.command == 'thread-cpu': + args.output.write_text(json.dumps(thread_cpu(args.pid), indent=2) + '\n') + else: + result = reduce(*(json.loads(getattr(args, name).read_text()) for name in ('samples', 'before', 'after'))) + args.output.write_text(json.dumps(result, indent=2) + '\n') + except (ValueError, OSError, subprocess.TimeoutExpired) as error: + parser.exit(1, f'profile: {error}\n') + + +if __name__ == '__main__': + main() diff --git a/bench/profile.test.py b/bench/profile.test.py new file mode 100644 index 00000000000..b013cc0d6ca --- /dev/null +++ b/bench/profile.test.py @@ -0,0 +1,38 @@ +import importlib.util +import unittest +from pathlib import Path + +spec = importlib.util.spec_from_file_location('profile', Path(__file__).with_name('profile.py')) +profile = importlib.util.module_from_spec(spec) +spec.loader.exec_module(profile) + + +class Waits(unittest.TestCase): + def sample(self, stack, wait=300): + return {'tid': 1, 'comm': 'moq-quic-0', 'wait_ns': wait, 'stack': stack} + + def test_cpu_denominator_can_exceed_one(self): + row = profile.reduce([self.sample(['std::sys::sync::mutex::futex::Mutex::lock_contended', 'moq_net::serve'])], + {'1': 100}, {'1': 200})['workers']['1'] + self.assertEqual(row['mutex_wait_per_cpu'], 3) + self.assertEqual(row['cpu_ns'], 100) + + def test_condvar_and_allocator_waits_are_separate(self): + row = profile.reduce([self.sample(['std::sys::sync::condvar::futex::Condvar::wait']), + self.sample(['jemalloc::background_thread'])], {'1': 10}, {'1': 20})['workers']['1'] + self.assertEqual(row['mutex_wait_ns'], 0) + self.assertEqual(row['other_futex_wait_ns'], 600) + + def test_missing_or_reset_cpu_refused(self): + for before, after in [({}, {'1': 2}), ({'1': 2}, {'1': 1}), ({'1': 2}, {'1': 2})]: + with self.assertRaises(ValueError): + profile.reduce([self.sample(['moq_net::serve'])], before, after) + + def test_empty_and_unsymbolized_refused(self): + for samples in [[], [self.sample([])]]: + with self.assertRaises(ValueError): + profile.reduce(samples, {'1': 1}, {'1': 2}) + + +if __name__ == '__main__': + unittest.main() diff --git a/flake.nix b/flake.nix index 703ec58e1c9..c3de5ff8d21 100644 --- a/flake.nix +++ b/flake.nix @@ -562,7 +562,11 @@ ++ cppDeps ++ dartDeps ++ devTools - ++ [ quest-cli ]; + ++ [ quest-cli ] + ++ pkgs.lib.optionals pkgs.stdenv.hostPlatform.isLinux [ + pkgs.bpftrace + pkgs.perf + ]; # jemalloc's configure uses -O0 test builds, which conflict with # Nix's _FORTIFY_SOURCE hardening (requires -O). diff --git a/justfile b/justfile index 9a48ff498d6..a24e76a75c7 100644 --- a/justfile +++ b/justfile @@ -56,6 +56,10 @@ bench-audio: bench-runtime $ROUNDS="3" $WORKERS="": MOQ_BENCH_RUNTIME_ROUNDS="$ROUNDS" MOQ_BENCH_RUNTIME_WORKERS="$WORKERS" bench/run.sh --runtime +# Probe capture permissions without starting a relay or reporting a profile. +bench-profile-probe MODE="lock" OUTPUT="target/profile-probe": + python3 bench/profile.py probe {{quote(MODE)}} {{quote(OUTPUT)}} + # Install repo-wide tooling. Per-language deps install on first check. install: bun install diff --git a/quest/m1/perf/lock-profile.md b/quest/m1/perf/lock-profile.md index 03da0a80415..9b0908cf352 100644 --- a/quest/m1/perf/lock-profile.md +++ b/quest/m1/perf/lock-profile.md @@ -17,6 +17,13 @@ Profiling is opt-in and costs nothing when disabled. ## Plan +Draft foundation adds fail-closed Linux tool/permission probes and a tested +worker mutex-wait/CPU reducer. This host has perf_event_paranoid=4 and +unprivileged_bpf_disabled=2. Actual capture/lifecycle integration, normalized +bpftrace input, control overhead, symbolized MoQ stacks, heap snapshots/churn +and macOS validation remain. Keep this quest; probes are not smoke captures. + + Decided 2026-10-08: the reproducible CPU and allocation profile quest merged here, so the relay has one profiling recipe with capture modes rather than two launchers. The lock-wait mode lands first, since From 5aade775fbfbc4f0daf6c34a0155b4c4d05f72c8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 10 Oct 2026 21:47:14 -0700 Subject: [PATCH 3/8] Validate profiling capabilities and retain refusal evidence Co-Authored-By: GPT-6 Astra --- bench/README.md | 6 ++-- bench/profile.py | 68 +++++++++++++++++++++++++---------- bench/profile.test.py | 39 +++++++++++++++++++- justfile | 2 +- quest/m1/perf/lock-profile.md | 5 ++- 5 files changed, 93 insertions(+), 27 deletions(-) diff --git a/bench/README.md b/bench/README.md index 87d9f19725a..594490039ae 100644 --- a/bench/README.md +++ b/bench/README.md @@ -75,8 +75,8 @@ and invalid samples still fail the command. `nix develop --command just bench-profile-probe lock OUTPUT` probes bpftrace permissions before a costly build; use `cpu` to probe perf's CPU clock event. A refused probe retains stdout, stderr and its command in OUTPUT and exits -unsuccessfully. Linux lock capture needs sudo or CAP_BPF plus tracefs access; -CPU capture needs perf_event permission. The probe captures no profile. +unsuccessfully. Linux lock capture needs sudo or CAP\_BPF plus tracefs access; +CPU capture needs perf\_event permission. The lock probe loads both futex tracepoints and user-stack collection before exiting; a BEGIN-only program would not establish that those facilities are usable. Missing tools and process-start failures retain refusal diagnostics too. The probe captures no profile. `profile.py thread-cpu PID OUTPUT` records relay worker CPU nanoseconds from Linux schedstat. The reducer accepts normalized records with `tid`, `comm`, @@ -84,7 +84,7 @@ Linux schedstat. The reducer accepts normalized records with `tid`, `comm`, by TID. Only std mutex `lock_contended` frames contribute to the lock share; allocator, condvar and runtime parking waits are counted separately. The denominator is scheduled CPU time, so wait/CPU can exceed one. Missing CPU or -user stacks are refused. Spinning does not reach the futex and is excluded. +user stacks and address-only unresolved stacks are refused. Spinning does not reach the futex and is excluded. This is a partial foundation, not the profiling launcher. Still required: integrate captures with run.sh's owned-child lifecycle, preserve metadata and diff --git a/bench/profile.py b/bench/profile.py index 107aa120629..ef0f5575d0b 100644 --- a/bench/profile.py +++ b/bench/profile.py @@ -10,6 +10,18 @@ # Condvar, allocator and runtime parking futexes are not std mutex acquisitions. MUTEX = re.compile(r"std::sys::(?:sync|pal::unix)::mutex::futex::Mutex::lock_contended") +SYMBOL = re.compile(r"[A-Za-z_][A-Za-z_0-9]*::|(?:^|\s)[A-Za-z_][A-Za-z_0-9]*\+") +LOCK_PROBE = """ +tracepoint:syscalls:sys_enter_futex /pid == cpid/ { + @started[tid] = nsecs; + @stack[tid] = ustack; +} +tracepoint:syscalls:sys_exit_futex /pid == cpid && @started[tid]/ { + @waits[@stack[tid]] = sum(nsecs - @started[tid]); + delete(@started, tid); + delete(@stack, tid); +} +""" WORKER = re.compile(r"moq-(?:uring|quic)-[0-9]+$") @@ -26,6 +38,8 @@ def reduce(samples, before, after): stack = sample['stack'] if not isinstance(stack, list) or not stack or not all(isinstance(frame, str) for frame in stack): raise ValueError('a nonempty symbolized user stack is required') + if not any(SYMBOL.search(frame) for frame in stack): + raise ValueError('user stack contains no resolved symbols') row = result.setdefault(tid, {'comm': name, 'mutex_wait_ns': 0, 'other_futex_wait_ns': 0, 'stacks': []}) if row['comm'] != name: raise ValueError('thread identity changed inside the capture') @@ -66,24 +80,40 @@ def thread_cpu(pid): def probe(mode, directory): """Retain both successful and refused capability probes before any build.""" directory.mkdir(parents=True, exist_ok=True) - if platform.system() != 'Linux': - raise ValueError('this draft probes Linux only; macOS samply capture remains unimplemented') - tool = {'lock': 'bpftrace', 'cpu': 'perf'}[mode] - executable = shutil.which(tool) - if not executable: - raise ValueError(f'{tool} is unavailable; enter the Nix dev shell') - command = ([executable, '-e', 'BEGIN { exit(); }'] if mode == 'lock' else - [executable, 'stat', '-e', 'cpu-clock', '--', 'true']) - completed = subprocess.run(command, capture_output=True, text=True, timeout=30) - (directory / 'probe.stdout').write_text(completed.stdout) - (directory / 'probe.stderr').write_text(completed.stderr) - evidence = {'version': 1, 'mode': mode, 'command': command, - 'kernel': platform.platform(), 'exit_code': completed.returncode, - 'capture': 'not attempted; this command only probes capability'} - (directory / 'probe.json').write_text(json.dumps(evidence, indent=2) + '\n') - if completed.returncode: - requirement = 'sudo/CAP_BPF, tracefs and BPF permissions' if mode == 'lock' else 'perf_event permissions' - raise ValueError(f'{tool} capture refused: requires {requirement}; see {directory / "probe.stderr"}') + evidence = {'version': 1, 'mode': mode, 'command': None, + 'kernel': platform.platform(), 'exit_code': None, + 'status': 'unavailable', 'capture': 'not attempted; capability probe only'} + stdout, stderr = '', '' + try: + if platform.system() != 'Linux': + raise ValueError('this draft probes Linux only; macOS samply capture remains unimplemented') + tool = {'lock': 'bpftrace', 'cpu': 'perf'}[mode] + executable = shutil.which(tool) + if not executable: + raise ValueError(f'{tool} is unavailable; enter the Nix dev shell') + command = ([executable, '-e', LOCK_PROBE, '-c', 'true'] if mode == 'lock' else + [executable, 'stat', '-e', 'cpu-clock', '--', 'true']) + evidence['command'] = command + evidence['status'] = 'spawn failed' + # The traced child starts after attachment and its exit ends the probe. + # BEGIN exits before attachment and cannot prove these capabilities. + # Both probes exit by construction. No deadline hides slow setup or + # substitutes for the child process's real result. + completed = subprocess.run(command, capture_output=True, text=True) + stdout, stderr = completed.stdout, completed.stderr + evidence.update(exit_code=completed.returncode, + status='passed' if completed.returncode == 0 else 'refused') + if completed.returncode: + requirement = 'tracefs, BPF and user-stack permissions' if mode == 'lock' else 'perf_event permissions' + raise ValueError(f'{tool} capture refused: requires {requirement}; see {directory / "probe.stderr"}') + except (ValueError, OSError) as error: + if not stderr: + stderr = str(error) + '\n' + raise + finally: + (directory / 'probe.stdout').write_text(stdout) + (directory / 'probe.stderr').write_text(stderr) + (directory / 'probe.json').write_text(json.dumps(evidence, indent=2) + '\n') print('Capability probe passed. No profile was captured.') @@ -108,7 +138,7 @@ def main(): else: result = reduce(*(json.loads(getattr(args, name).read_text()) for name in ('samples', 'before', 'after'))) args.output.write_text(json.dumps(result, indent=2) + '\n') - except (ValueError, OSError, subprocess.TimeoutExpired) as error: + except (ValueError, OSError) as error: parser.exit(1, f'profile: {error}\n') diff --git a/bench/profile.test.py b/bench/profile.test.py index b013cc0d6ca..d971bc57317 100644 --- a/bench/profile.test.py +++ b/bench/profile.test.py @@ -1,5 +1,9 @@ import importlib.util import unittest +import json +import subprocess +import tempfile +from unittest.mock import patch from pathlib import Path spec = importlib.util.spec_from_file_location('profile', Path(__file__).with_name('profile.py')) @@ -29,10 +33,43 @@ def test_missing_or_reset_cpu_refused(self): profile.reduce([self.sample(['moq_net::serve'])], before, after) def test_empty_and_unsymbolized_refused(self): - for samples in [[], [self.sample([])]]: + for samples in [[], [self.sample([])], [self.sample(['0x7f123abc'])], [self.sample(['[unknown]'])]]: with self.assertRaises(ValueError): profile.reduce(samples, {'1': 1}, {'1': 2}) +class Probes(unittest.TestCase): + def setUp(self): + kernel = patch.object(profile.platform, "platform", return_value="test-kernel") + kernel.start() + self.addCleanup(kernel.stop) + + def test_lock_probe_loads_futex_and_stack_programs(self): + with tempfile.TemporaryDirectory() as directory, patch.object(profile.platform, 'system', return_value='Linux'), patch.object(profile.shutil, 'which', return_value='/bpftrace'), patch.object(profile.subprocess, 'run', return_value=subprocess.CompletedProcess([], 0, '', '')) as run: + profile.probe('lock', Path(directory)) + program = run.call_args.args[0][2] + self.assertIn('sys_enter_futex', program) + self.assertIn('sys_exit_futex', program) + self.assertIn('ustack', program) + self.assertNotIn('BEGIN', program) + self.assertEqual(run.call_args.args[0][-2:], ['-c', 'true']) + + def test_unavailable_tool_retains_refusal(self): + with tempfile.TemporaryDirectory() as directory, patch.object(profile.platform, 'system', return_value='Linux'), patch.object(profile.shutil, 'which', return_value=None): + with self.assertRaises(ValueError): + profile.probe('lock', Path(directory)) + evidence = json.loads((Path(directory) / 'probe.json').read_text()) + self.assertEqual(evidence['status'], 'unavailable') + self.assertTrue((Path(directory) / 'probe.stderr').read_text()) + + def test_spawn_failure_retains_refusal(self): + with tempfile.TemporaryDirectory() as directory, patch.object(profile.platform, 'system', return_value='Linux'), patch.object(profile.shutil, 'which', return_value='/bpftrace'), patch.object(profile.subprocess, 'run', side_effect=PermissionError('denied')): + with self.assertRaises(OSError): + profile.probe('lock', Path(directory)) + evidence = json.loads((Path(directory) / 'probe.json').read_text()) + self.assertEqual(evidence['status'], 'spawn failed') + self.assertIn('denied', (Path(directory) / 'probe.stderr').read_text()) + + if __name__ == '__main__': unittest.main() diff --git a/justfile b/justfile index eb2c5175388..3294bd82d3c 100644 --- a/justfile +++ b/justfile @@ -58,7 +58,7 @@ bench-runtime $ROUNDS="3" $WORKERS="": # Probe capture permissions without starting a relay or reporting a profile. bench-profile-probe MODE="lock" OUTPUT="target/profile-probe": - python3 bench/profile.py probe {{quote(MODE)}} {{quote(OUTPUT)}} + python3 bench/profile.py probe {{ quote(MODE) }} {{ quote(OUTPUT) }} # Install repo-wide tooling. Per-language deps install on first check. install: diff --git a/quest/m1/perf/lock-profile.md b/quest/m1/perf/lock-profile.md index 9b0908cf352..405784367b6 100644 --- a/quest/m1/perf/lock-profile.md +++ b/quest/m1/perf/lock-profile.md @@ -18,11 +18,10 @@ Profiling is opt-in and costs nothing when disabled. ## Plan Draft foundation adds fail-closed Linux tool/permission probes and a tested -worker mutex-wait/CPU reducer. This host has perf_event_paranoid=4 and +worker mutex-wait/CPU reducer. The lock probe now exercises the futex tracepoints and user-stack program, retains unavailable-tool and spawn failures, and rejects address-only stacks. This host has perf_event_paranoid=4 and unprivileged_bpf_disabled=2. Actual capture/lifecycle integration, normalized bpftrace input, control overhead, symbolized MoQ stacks, heap snapshots/churn -and macOS validation remain. Keep this quest; probes are not smoke captures. - +and macOS validation remain. Keep this quest; probes are not smoke captures. The benchmark lifecycle and retained-artifact implementation in the comparison PR must land before this integrates with it; do not copy that branch into a second launcher. Decided 2026-10-08: the reproducible CPU and allocation profile quest merged here, so the relay has one profiling recipe with capture modes rather than From 695fd7cec372abed765699eb635892f4ef42e9c8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 12 Oct 2026 08:43:40 -0700 Subject: [PATCH 4/8] feat(bench): capture relay lock waits without root via eu-stack sampling Replace the bpftrace/perf capability probe with a capture path that works on hosts with perf_event_paranoid=4 and unprivileged BPF disabled, and in CI: the relay starts with PR_SET_PTRACER naming the harness, eu-stack samples worker stacks at 20 Hz, and addr2line symbolizes once with inlined frames. `just bench-lock` pairs each profiled run with an unprofiled control and nightly smoke-tests it. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly.yml | 5 + bench/README.md | 25 ++++ bench/metadata.py | 2 +- bench/profile.py | 251 ++++++++++++++++++---------------- bench/profile.test.py | 115 ++++++++-------- bench/relay.sh | 22 ++- bench/run.sh | 73 +++++++++- flake.nix | 6 +- justfile | 6 +- quest/m1/perf/lock-profile.md | 42 +++--- 10 files changed, 339 insertions(+), 208 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 340c6e0dbfc..aa25904e7cb 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -91,6 +91,11 @@ jobs: if: ${{ !cancelled() }} run: nix develop --command just bench-audio + # Smoke-tests the unprivileged capture path: resolved worker stacks and CPU. + - name: Relay lock-wait profile + if: ${{ !cancelled() }} + run: nix develop --command just bench-lock fanout-heavy 4 + - name: Latency observation benchmark if: ${{ !cancelled() }} run: nix develop --command just rs bench-latency diff --git a/bench/README.md b/bench/README.md index 17db78c1b7b..7ae04dfc8dc 100644 --- a/bench/README.md +++ b/bench/README.md @@ -48,6 +48,31 @@ Runtime comparison requires Linux because io\_uring and relay process metrics come from Linux interfaces. The default worker count is the number of online logical CPUs. +Attribute relay worker lock waits to the stacks that blocked, as a share of +each worker's CPU: + +```bash +nix develop --command just bench-lock +nix develop --command just bench-lock fanout-heavy "4 16" +``` + +This builds the relay with `[profile.profiling]` and frame pointers, then runs +the workload on Tokio workers at each count twice: an unprofiled control, then +a profiled run, so the throughput and CPU difference shows the profiler's +overhead. During the profiled run's steady window, `eu-stack` samples every +thread's stack back to back. A `moq-quic-N` sample counts as a mutex wait only +when it is blocked in the futex under std's `Mutex::lock_contended` +(`kio::Lock`); condvar, allocator and parking waits are counted separately, and +spinning is excluded. Wait time is the wall window scaled by the share of +samples, and `/proc` schedstat supplies CPU, so wait/CPU can exceed one. +`lock/mutex.folded` holds the blocking stacks in flamegraph folded format. + +It needs no root or perf/BPF permissions: the relay starts with +`PR_SET_PTRACER` naming the harness, which lets Yama's default +`ptrace_scope=1` admit the sampler. Each sample stops the relay while it +unwinds, so treat the stacks as attribution and the absolute wait as an +estimate. Linux only. + ## Workloads The `workloads/` TOML files contain only traffic shape. The harness supplies the diff --git a/bench/metadata.py b/bench/metadata.py index 5a996ce59e9..5497cacbf0c 100644 --- a/bench/metadata.py +++ b/bench/metadata.py @@ -28,7 +28,7 @@ def command(*args): "os": platform.platform(), "machine": platform.machine(), "processor": platform.processor(), "affinity": sorted(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else None, "environment": {key: value for key, value in os.environ.items() if key.startswith(("MOQ_BENCH_", "CARGO_PROFILE_")) or key in ("RUSTFLAGS", "CARGO_BUILD_TARGET", "CARGO_BUILD_RUSTFLAGS", "TOKIO_WORKER_THREADS", "LD_PRELOAD", "MALLOC_CONF", "GLIBC_TUNABLES", "RUSTC_WRAPPER", "RUSTC_WORKSPACE_WRAPPER")}, - "profile": "release", "features": "default + moq-relay/io-uring,moq-bench/uring" if mode == "--runtime" else "default", + "profile": "profiling" if mode == "--lock" else "release", "features": "default + moq-relay/io-uring,moq-bench/uring" if mode == "--runtime" else "default", "mode": mode, "runtime_workers": sys.argv[4] if len(sys.argv) > 4 else None, "rounds": int(sys.argv[5]) if len(sys.argv) > 5 else None, "allocator": "system by build; preload and allocator tunables are recorded in environment", diff --git a/bench/profile.py b/bench/profile.py index ef0f5575d0b..37193aa0bce 100644 --- a/bench/profile.py +++ b/bench/profile.py @@ -1,144 +1,159 @@ #!/usr/bin/env python3 -"""Probe capture privileges and reduce explicitly classified worker futex waits.""" +"""Sample relay worker stacks with eu-stack and attribute std mutex waits.""" import argparse +import ctypes import json -import platform +import os import re -import shutil import subprocess +import time +from collections import Counter, defaultdict from pathlib import Path -# Condvar, allocator and runtime parking futexes are not std mutex acquisitions. -MUTEX = re.compile(r"std::sys::(?:sync|pal::unix)::mutex::futex::Mutex::lock_contended") -SYMBOL = re.compile(r"[A-Za-z_][A-Za-z_0-9]*::|(?:^|\s)[A-Za-z_][A-Za-z_0-9]*\+") -LOCK_PROBE = """ -tracepoint:syscalls:sys_enter_futex /pid == cpid/ { - @started[tid] = nsecs; - @stack[tid] = ustack; -} -tracepoint:syscalls:sys_exit_futex /pid == cpid && @started[tid]/ { - @waits[@stack[tid]] = sum(nsecs - @started[tid]); - delete(@started, tid); - delete(@stack, tid); -} -""" -WORKER = re.compile(r"moq-(?:uring|quic)-[0-9]+$") - - -def reduce(samples, before, after): - """Normalize mutex waits by thread CPU counters, never elapsed time.""" +PR_SET_PTRACER = 0x59616D61 +# Each sample stops the relay while it unwinds; 20 Hz bounds that, and the control run measures it. +INTERVAL_NS = 50_000_000 +WORKER = re.compile(r"moq-(?:uring|quic)-[0-9]+") +# The blocking call itself: std's futex wrapper or glibc's (allocator, pthread). +BLOCKED = re.compile(r"futex_wait|futex_abstimed_wait|lll_lock_wait") +# kio::Lock is a std Mutex; condvar, allocator and parking waits are separate. +# An inlined frame carries only its short name. +MUTEX = re.compile(r"(?:\bMutex>?::|^)lock_contended$") +FRAME = re.compile(r"#\d+\s+0x([0-9a-f]+)\s+-\s+(\S+)") + + +def parse(text): + """Split `eu-stack -q -m` output into innermost-first (module, address) lists keyed by TID.""" + stacks, frames = {}, None + for line in text.splitlines(): + if line.startswith('TID '): + frames = stacks.setdefault(line[4:].rstrip(':'), []) + elif frames is not None and (match := FRAME.match(line.strip())): + frames.append((match.group(2), int(match.group(1), 16))) + return stacks + + +def bases(pid): + """Load address of each mapped file, so addresses become file-relative for addr2line.""" result = {} - for sample in samples: - tid, name = str(sample['tid']), sample['comm'] - if not WORKER.fullmatch(name): + for line in (Path('/proc') / str(pid) / 'maps').read_text().splitlines(): + fields = line.split(maxsplit=5) + if len(fields) == 6 and fields[2] == '00000000' and fields[5].startswith('/'): + result.setdefault(fields[5], int(fields[0].split('-')[0], 16)) + return result + + +def addr2line(module, offsets): + """Resolve file offsets to innermost-first function names, inlined frames included.""" + done = subprocess.run(['addr2line', '-a', '-f', '-i', '-C', '-e', module], + input=''.join(f'{offset:#x}\n' for offset in offsets), + capture_output=True, text=True, check=True) + # Each address line is followed by (function, file:line) pairs, innermost first. + result, names, position = {}, [], 0 + for line in done.stdout.splitlines(): + if line.startswith('0x'): + names, position = result.setdefault(int(line, 16), []), 0 continue - wait = sample['wait_ns'] - if not isinstance(wait, int) or isinstance(wait, bool) or wait < 0: - raise ValueError('wait_ns must be a nonnegative integer') - stack = sample['stack'] - if not isinstance(stack, list) or not stack or not all(isinstance(frame, str) for frame in stack): - raise ValueError('a nonempty symbolized user stack is required') - if not any(SYMBOL.search(frame) for frame in stack): - raise ValueError('user stack contains no resolved symbols') - row = result.setdefault(tid, {'comm': name, 'mutex_wait_ns': 0, 'other_futex_wait_ns': 0, 'stacks': []}) - if row['comm'] != name: - raise ValueError('thread identity changed inside the capture') - if any(MUTEX.search(frame) for frame in stack): - row['mutex_wait_ns'] += wait - row['stacks'].append({'wait_ns': wait, 'stack': stack}) - else: - row['other_futex_wait_ns'] += wait - if not result: - raise ValueError('capture has no symbolized relay-worker futex samples') - for tid, row in result.items(): - if tid not in before or tid not in after: - raise ValueError(f'missing CPU counter for worker {tid}') - if any(not isinstance(value, int) or isinstance(value, bool) or value < 0 for value in (before[tid], after[tid])): - raise ValueError(f'invalid CPU counter for worker {tid}') + if position % 2 == 0: + names.append(line) + position += 1 + return result + + +def symbolize(samples, loaded): + """Replace (module, address) frames with names; caller frames use their call instruction.""" + wanted = defaultdict(set) + for sample in samples: + for stack in sample.values(): + for depth, (module, address) in enumerate(stack): + if module in loaded: + wanted[module].add(address - loaded[module] - (depth > 0)) + names = {module: addr2line(module, sorted(offsets)) for module, offsets in wanted.items()} + return [{tid: [name for depth, (module, address) in enumerate(stack) + for name in names.get(module, {}).get(address - loaded.get(module, 0) - (depth > 0), ['??'])] + for tid, stack in sample.items()} for sample in samples] + + +def classify(stack): + """A sample is a mutex wait only while blocked in the futex under lock_contended.""" + if not any(BLOCKED.search(frame) for frame in stack): + return 'other' + return 'mutex' if any(MUTEX.search(frame) for frame in stack) else 'futex' + + +def reduce(names, samples, before, after, window_ns): + """Turn per-sample stacks into each worker's wait share of wall and CPU time.""" + if not any(re.search(r'\bmoq_|\bkio::', frame) for sample in samples for tid in names for frame in sample.get(tid, [])): + raise ValueError('worker stacks contain no resolved MoQ frames; build with the profiling profile') + workers, folded = {}, Counter() + for tid, name in names.items(): + stacks = [sample[tid] for sample in samples if tid in sample] + if not stacks: + raise ValueError(f'worker {name} ({tid}) was never sampled') cpu = after[tid] - before[tid] if cpu <= 0: - raise ValueError(f'worker {tid} has no advancing CPU counter') - row['cpu_ns'] = cpu - row['mutex_wait_per_cpu'] = row['mutex_wait_ns'] / cpu - return {'version': 1, 'workers': result, - 'scope': 'completed futex waits with std mutex lock_contended frames; spinning is excluded'} + raise ValueError(f'worker {name} ({tid}) has no advancing CPU counter') + counts = Counter(classify(stack) for stack in stacks) + wait = window_ns * counts['mutex'] // len(stacks) + workers[tid] = {'comm': name, 'samples': len(stacks), 'mutex_samples': counts['mutex'], + 'other_futex_samples': counts['futex'], 'cpu_ns': cpu, + 'mutex_wait_ns': wait, 'mutex_wait_per_cpu': wait / cpu} + folded.update(';'.join(reversed(stack)) for stack in stacks if classify(stack) == 'mutex') + return {'version': 1, 'window_ns': window_ns, 'samples': len(samples), 'workers': workers, + 'scope': 'wall-clock stack samples blocked in a futex under std Mutex::lock_contended; spinning is excluded'}, folded -def thread_cpu(pid): - """Read Linux per-thread scheduled CPU nanoseconds for the matching workers.""" - if platform.system() != 'Linux': - raise ValueError('thread CPU capture requires Linux /proc schedstat') - result = {} - for task in (Path('/proc') / str(pid) / 'task').iterdir(): - if WORKER.fullmatch((task / 'comm').read_text().strip()): - result[task.name] = int((task / 'schedstat').read_text().split()[0]) - if not result: - raise ValueError('PID has no moq-uring-N or moq-quic-N workers') - return result +def schedstat(pid, tids): + return {tid: int((Path('/proc') / str(pid) / 'task' / tid / 'schedstat').read_text().split()[0]) for tid in tids} -def probe(mode, directory): - """Retain both successful and refused capability probes before any build.""" +def sample(pid, delay, duration, directory): + """Capture one steady window, then symbolize once after the relay is released.""" directory.mkdir(parents=True, exist_ok=True) - evidence = {'version': 1, 'mode': mode, 'command': None, - 'kernel': platform.platform(), 'exit_code': None, - 'status': 'unavailable', 'capture': 'not attempted; capability probe only'} - stdout, stderr = '', '' - try: - if platform.system() != 'Linux': - raise ValueError('this draft probes Linux only; macOS samply capture remains unimplemented') - tool = {'lock': 'bpftrace', 'cpu': 'perf'}[mode] - executable = shutil.which(tool) - if not executable: - raise ValueError(f'{tool} is unavailable; enter the Nix dev shell') - command = ([executable, '-e', LOCK_PROBE, '-c', 'true'] if mode == 'lock' else - [executable, 'stat', '-e', 'cpu-clock', '--', 'true']) - evidence['command'] = command - evidence['status'] = 'spawn failed' - # The traced child starts after attachment and its exit ends the probe. - # BEGIN exits before attachment and cannot prove these capabilities. - # Both probes exit by construction. No deadline hides slow setup or - # substitutes for the child process's real result. - completed = subprocess.run(command, capture_output=True, text=True) - stdout, stderr = completed.stdout, completed.stderr - evidence.update(exit_code=completed.returncode, - status='passed' if completed.returncode == 0 else 'refused') - if completed.returncode: - requirement = 'tracefs, BPF and user-stack permissions' if mode == 'lock' else 'perf_event permissions' - raise ValueError(f'{tool} capture refused: requires {requirement}; see {directory / "probe.stderr"}') - except (ValueError, OSError) as error: - if not stderr: - stderr = str(error) + '\n' - raise - finally: - (directory / 'probe.stdout').write_text(stdout) - (directory / 'probe.stderr').write_text(stderr) - (directory / 'probe.json').write_text(json.dumps(evidence, indent=2) + '\n') - print('Capability probe passed. No profile was captured.') + tasks = Path('/proc') / str(pid) / 'task' + names = {task.name: name for task in tasks.iterdir() if WORKER.fullmatch(name := (task / 'comm').read_text().strip())} + if not names: + raise ValueError(f'PID {pid} has no moq-uring-N or moq-quic-N workers') + loaded = bases(pid) + time.sleep(delay) + before, start = schedstat(pid, names), time.monotonic_ns() + raw = [] + with open(directory / 'eu-stack.log', 'w') as log: + while (now := time.monotonic_ns()) - start < duration * 1_000_000_000: + # Exit 1 marks a partial backtrace, such as a frame without CFI; 2 is no frames at all. + done = subprocess.run(['eu-stack', '-q', '-m', '-p', str(pid)], capture_output=True, text=True) + log.write(done.stderr) + if done.returncode > 1: + raise ValueError(f'eu-stack failed ({done.returncode}): {done.stderr.strip()}') + raw.append(parse(done.stdout)) + time.sleep(max(0, now + INTERVAL_NS - time.monotonic_ns()) / 1e9) + after, window = schedstat(pid, names), time.monotonic_ns() - start + result, folded = reduce(names, symbolize(raw, loaded), before, after, window) + (directory / 'lock.json').write_text(json.dumps(result, indent=2) + '\n') + (directory / 'mutex.folded').write_text(''.join(f'{stack} {count}\n' for stack, count in folded.most_common())) def main(): parser = argparse.ArgumentParser(description=__doc__) commands = parser.add_subparsers(dest='command', required=True) - capability = commands.add_parser('probe') - capability.add_argument('mode', choices=('lock', 'cpu')) - capability.add_argument('directory', type=Path) - cpu = commands.add_parser('thread-cpu') - cpu.add_argument('pid', type=int) - cpu.add_argument('output', type=Path) - merge = commands.add_parser('reduce') - for name in ('samples', 'before', 'after', 'output'): - merge.add_argument(name, type=Path) + traceable = commands.add_parser('traceable', help='let TRACER and its descendants ptrace COMMAND, then exec it') + traceable.add_argument('tracer', type=int) + traceable.add_argument('argv', nargs=argparse.REMAINDER) + lock = commands.add_parser('lock', help='sample worker stacks of PID into DIRECTORY') + lock.add_argument('pid', type=int) + lock.add_argument('directory', type=Path) + lock.add_argument('--delay', type=float, default=0) + lock.add_argument('--duration', type=float, required=True) args = parser.parse_args() try: - if args.command == 'probe': - probe(args.mode, args.directory) - elif args.command == 'thread-cpu': - args.output.write_text(json.dumps(thread_cpu(args.pid), indent=2) + '\n') - else: - result = reduce(*(json.loads(getattr(args, name).read_text()) for name in ('samples', 'before', 'after'))) - args.output.write_text(json.dumps(result, indent=2) + '\n') - except (ValueError, OSError) as error: + if args.command == 'traceable': + # Yama's default ptrace_scope=1 only lets ancestors attach, and eu-stack is not one. + if ctypes.CDLL(None, use_errno=True).prctl(PR_SET_PTRACER, args.tracer, 0, 0, 0): + raise OSError(ctypes.get_errno(), 'PR_SET_PTRACER failed') + argv = args.argv[1:] if args.argv[:1] == ['--'] else args.argv + os.execvp(argv[0], argv) + sample(args.pid, args.delay, args.duration, args.directory) + except (ValueError, OSError, subprocess.CalledProcessError) as error: parser.exit(1, f'profile: {error}\n') diff --git a/bench/profile.test.py b/bench/profile.test.py index d971bc57317..bde1064af70 100644 --- a/bench/profile.test.py +++ b/bench/profile.test.py @@ -1,74 +1,81 @@ import importlib.util -import unittest -import json import subprocess -import tempfile -from unittest.mock import patch +import unittest from pathlib import Path +from unittest.mock import patch spec = importlib.util.spec_from_file_location('profile', Path(__file__).with_name('profile.py')) profile = importlib.util.module_from_spec(spec) spec.loader.exec_module(profile) +BLOCKED = ['syscall', 'std::sys::pal::unix::futex::futex_wait', '::lock_contended', + 'lock>', '::is_expired'] +INLINED = ['syscall', 'std::sys::pal::unix::futex::futex_wait', 'lock_contended', 'moq_net::serve'] +SPINNING = ['::lock_contended', 'moq_net::serve'] +CONDVAR = ['syscall', 'std::sys::pal::unix::futex::futex_wait', + '::wait', 'moq_net::serve'] +IDLE = ['epoll_wait', '::poll', 'moq_net::serve'] -class Waits(unittest.TestCase): - def sample(self, stack, wait=300): - return {'tid': 1, 'comm': 'moq-quic-0', 'wait_ns': wait, 'stack': stack} - def test_cpu_denominator_can_exceed_one(self): - row = profile.reduce([self.sample(['std::sys::sync::mutex::futex::Mutex::lock_contended', 'moq_net::serve'])], - {'1': 100}, {'1': 200})['workers']['1'] - self.assertEqual(row['mutex_wait_per_cpu'], 3) - self.assertEqual(row['cpu_ns'], 100) - - def test_condvar_and_allocator_waits_are_separate(self): - row = profile.reduce([self.sample(['std::sys::sync::condvar::futex::Condvar::wait']), - self.sample(['jemalloc::background_thread'])], {'1': 10}, {'1': 20})['workers']['1'] - self.assertEqual(row['mutex_wait_ns'], 0) - self.assertEqual(row['other_futex_wait_ns'], 600) +class Parse(unittest.TestCase): + def test_threads_modules_and_addresses(self): + text = '''PID 7 - process +TID 7: +#0 0x00007ffff7d2966d - /lib/libc.so.6 +#1 0x00005555567d3259 - /bin/moq-relay +TID 8: +#0 0x00007ffff7d2966d - /lib/libc.so.6 +''' + self.assertEqual(profile.parse(text), { + '7': [('/lib/libc.so.6', 0x7ffff7d2966d), ('/bin/moq-relay', 0x5555567d3259)], + '8': [('/lib/libc.so.6', 0x7ffff7d2966d)]}) - def test_missing_or_reset_cpu_refused(self): - for before, after in [({}, {'1': 2}), ({'1': 2}, {'1': 1}), ({'1': 2}, {'1': 2})]: - with self.assertRaises(ValueError): - profile.reduce([self.sample(['moq_net::serve'])], before, after) + def test_inlined_frames_and_caller_offsets(self): + output = '''0x0000000000001000 +inner +/src/a.rs:1 +outer +/src/b.rs:2 +0x0000000000002000 +?? +??:0 +''' + done = subprocess.CompletedProcess([], 0, output, '') + with patch.object(profile.subprocess, 'run', return_value=done) as run: + stacks = profile.symbolize([{'1': [('/bin/relay', 0x2100), ('/bin/relay', 0x1101)]}], {'/bin/relay': 0x100}) + # A caller frame holds a return address, so it resolves one byte earlier. + self.assertEqual(run.call_args.kwargs['input'], '0x1000\n0x2000\n') + self.assertEqual(stacks, [{'1': ['??', 'inner', 'outer']}]) - def test_empty_and_unsymbolized_refused(self): - for samples in [[], [self.sample([])], [self.sample(['0x7f123abc'])], [self.sample(['[unknown]'])]]: - with self.assertRaises(ValueError): - profile.reduce(samples, {'1': 1}, {'1': 2}) +class Classify(unittest.TestCase): + def test_only_blocked_lock_contended_is_a_mutex_wait(self): + self.assertEqual(profile.classify(BLOCKED), 'mutex') + self.assertEqual(profile.classify(INLINED), 'mutex') + self.assertEqual(profile.classify(SPINNING), 'other') + self.assertEqual(profile.classify(CONDVAR), 'futex') + self.assertEqual(profile.classify(IDLE), 'other') -class Probes(unittest.TestCase): - def setUp(self): - kernel = patch.object(profile.platform, "platform", return_value="test-kernel") - kernel.start() - self.addCleanup(kernel.stop) - def test_lock_probe_loads_futex_and_stack_programs(self): - with tempfile.TemporaryDirectory() as directory, patch.object(profile.platform, 'system', return_value='Linux'), patch.object(profile.shutil, 'which', return_value='/bpftrace'), patch.object(profile.subprocess, 'run', return_value=subprocess.CompletedProcess([], 0, '', '')) as run: - profile.probe('lock', Path(directory)) - program = run.call_args.args[0][2] - self.assertIn('sys_enter_futex', program) - self.assertIn('sys_exit_futex', program) - self.assertIn('ustack', program) - self.assertNotIn('BEGIN', program) - self.assertEqual(run.call_args.args[0][-2:], ['-c', 'true']) +class Reduce(unittest.TestCase): + def test_wait_is_window_share_and_can_exceed_cpu(self): + samples = [{'1': BLOCKED}, {'1': BLOCKED}, {'1': BLOCKED}, {'1': CONDVAR}] + result, folded = profile.reduce({'1': 'moq-quic-0'}, samples, {'1': 0}, {'1': 100}, 400) + row = result['workers']['1'] + self.assertEqual((row['samples'], row['mutex_samples'], row['other_futex_samples']), (4, 3, 1)) + self.assertEqual(row['mutex_wait_ns'], 300) + self.assertEqual(row['mutex_wait_per_cpu'], 3) + self.assertEqual(folded, {';'.join(reversed(BLOCKED)): 3}) - def test_unavailable_tool_retains_refusal(self): - with tempfile.TemporaryDirectory() as directory, patch.object(profile.platform, 'system', return_value='Linux'), patch.object(profile.shutil, 'which', return_value=None): - with self.assertRaises(ValueError): - profile.probe('lock', Path(directory)) - evidence = json.loads((Path(directory) / 'probe.json').read_text()) - self.assertEqual(evidence['status'], 'unavailable') - self.assertTrue((Path(directory) / 'probe.stderr').read_text()) + def test_unsampled_or_idle_cpu_refused(self): + with self.assertRaises(ValueError): + profile.reduce({'1': 'moq-quic-0', '2': 'moq-quic-1'}, [{'1': IDLE}], {'1': 0, '2': 0}, {'1': 1, '2': 1}, 1) + with self.assertRaises(ValueError): + profile.reduce({'1': 'moq-quic-0'}, [{'1': IDLE}], {'1': 5}, {'1': 5}, 1) - def test_spawn_failure_retains_refusal(self): - with tempfile.TemporaryDirectory() as directory, patch.object(profile.platform, 'system', return_value='Linux'), patch.object(profile.shutil, 'which', return_value='/bpftrace'), patch.object(profile.subprocess, 'run', side_effect=PermissionError('denied')): - with self.assertRaises(OSError): - profile.probe('lock', Path(directory)) - evidence = json.loads((Path(directory) / 'probe.json').read_text()) - self.assertEqual(evidence['status'], 'spawn failed') - self.assertIn('denied', (Path(directory) / 'probe.stderr').read_text()) + def test_unresolved_stacks_refused(self): + with self.assertRaises(ValueError): + profile.reduce({'1': 'moq-quic-0'}, [{'1': ['??', 'syscall']}], {'1': 0}, {'1': 1}, 1) if __name__ == '__main__': diff --git a/bench/relay.sh b/bench/relay.sh index ba70781c193..cb0fbb9e3c2 100755 --- a/bench/relay.sh +++ b/bench/relay.sh @@ -3,6 +3,11 @@ # Sourced by run.sh, which owns these process and output paths. # shellcheck disable=SC2153,SC2154 +# Prepended to the relay command line; the lock profile makes the relay traceable. +RELAY_PREFIX=() +# Set to sample the relay's worker stacks during each workload's steady window. +PROFILE_LOCK= + relay_config() { local path=$1 local port=$2 @@ -89,9 +94,9 @@ start_relay() { if [[ -n $tokio_threads ]]; then env TOKIO_WORKER_THREADS="$tokio_threads" RUST_LOG=warn \ - "$relay" "$config" >"$log" 2>&1 & + "${RELAY_PREFIX[@]}" "$relay" "$config" >"$log" 2>&1 & else - RUST_LOG=warn "$relay" "$config" >"$log" 2>&1 & + RUST_LOG=warn "${RELAY_PREFIX[@]}" "$relay" "$config" >"$log" 2>&1 & fi RELAY_PID=$! @@ -229,6 +234,12 @@ run_workload() { --interval 500ms --duration 10s --output "$host" \ >"$directory/host.log" 2>&1 & HOST_PID=$! + if [[ -n $PROFILE_LOCK ]]; then + # The second half of the host window, matching the summary's throughput window. + python3 "$ROOT/bench/profile.py" lock "$relay_pid" "$directory/lock" --delay 5 --duration 5 \ + >"$directory/lock.log" 2>&1 & + PROFILE_PID=$! + fi if ! wait -n -p finished "$HOST_PID" "$LOAD_PID"; then cat "$directory/host.log" "$directory/load.log" >&2 return 1 @@ -238,6 +249,13 @@ run_workload() { return 1 fi HOST_PID= + if [[ -n $PROFILE_LOCK ]]; then + if ! wait "$PROFILE_PID"; then + cat "$directory/lock.log" >&2 + return 1 + fi + PROFILE_PID= + fi kill -INT "$LOAD_PID" if ! wait "$LOAD_PID"; then cat "$directory/load.log" >&2 diff --git a/bench/run.sh b/bench/run.sh index 31ff01e8c73..35a544be405 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -7,6 +7,7 @@ set -euo pipefail # runtime with the same number of independent Tokio/epoll and io_uring workers # on the current tree. The current load generator drives every relay so the # workloads stay identical. `--audio` isolates the audio grouping/load-shape matrix. +# `--lock` samples relay worker stacks to attribute std mutex waits. MODE=${1:-} ROUNDS=${MOQ_BENCH_RUNTIME_ROUNDS:-3} @@ -19,6 +20,13 @@ case $MODE in --audio) BASE= ;; + --lock) + BASE= + LOCK_WORKLOAD=${MOQ_BENCH_LOCK_WORKLOAD:-fanout} + LOCK_WORKERS=${MOQ_BENCH_LOCK_WORKERS:-1 4 16} + # Without frame pointers or symbols the stacks stop at libc. + export RUSTFLAGS="${RUSTFLAGS:+$RUSTFLAGS }-C force-frame-pointers=yes" + ;; --runtime) BASE= RUNTIME_WORKERS=${MOQ_BENCH_RUNTIME_WORKERS:-} @@ -62,10 +70,15 @@ WORKTREE= RELAY_PID= HOST_PID= LOAD_PID= +PROFILE_PID= STDOUT_PID= STDERR_PID= cleanup() { + if [[ -n $PROFILE_PID ]]; then + kill "$PROFILE_PID" 2>/dev/null || true + wait "$PROFILE_PID" 2>/dev/null || true + fi if [[ -n $LOAD_PID ]]; then kill "$LOAD_PID" 2>/dev/null || true wait "$LOAD_PID" 2>/dev/null || true @@ -419,13 +432,10 @@ print_audio_sample() { ] | @tsv' "$RUN/relay/$label/$workload/summary.json" } -run_runtime_comparison() { - if [[ $(uname -s) != Linux ]]; then - printf 'runtime comparison needs Linux for io_uring and /proc metrics\n' >&2 - return 1 - fi +# Worker runtimes read their certificate from disk instead of generating one. +generate_certificate() { if ! command -v openssl >/dev/null; then - printf 'runtime comparison needs openssl to generate its temporary certificate\n' >&2 + printf 'worker runtimes need openssl to generate a temporary certificate\n' >&2 return 1 fi @@ -439,6 +449,14 @@ run_runtime_comparison() { cat "$RUN/openssl.log" >&2 return 1 fi +} + +run_runtime_comparison() { + if [[ $(uname -s) != Linux ]]; then + printf 'runtime comparison needs Linux for io_uring and /proc metrics\n' >&2 + return 1 + fi + generate_certificate printf 'Building one relay binary with the Tokio and io_uring paths...\n' CARGO_TARGET_DIR=$CURRENT_TARGET cargo build --locked --release \ @@ -473,6 +491,46 @@ run_runtime_comparison() { print_runtime_comparison } +run_lock_profile() { + if [[ $(uname -s) != Linux ]]; then + printf 'lock profiling needs Linux for eu-stack and /proc schedstat\n' >&2 + return 1 + fi + generate_certificate + + printf 'Building the relay with the profiling profile and frame pointers...\n' + CARGO_TARGET_DIR=$TARGET/profile cargo build --locked --profile profiling -p moq-relay -p moq-bench + LOAD_BIN=$TARGET/profile/profiling/moq-bench + HOST_BIN=$TARGET/profile/profiling/moq-bench-host + local relay=$TARGET/profile/profiling/moq-relay + # Yama's default ptrace_scope lets only ancestors attach; eu-stack is this shell's grandchild. + RELAY_PREFIX=(python3 "$ROOT/bench/profile.py" traceable "$$" --) + + printf '\nLock wait: %s, tokio workers; each profiled run follows an unprofiled control\n' "$LOCK_WORKLOAD" + printf '%-8s %8s %13s %13s %10s %10s %13s %9s\n' \ + workers samples control-fps profiled-fps control-CPU profiled-CPU mutex-wait-ms wait/CPU + local workers + for workers in $LOCK_WORKERS; do + run_workload "control-$workers" "$relay" "$LOCK_WORKLOAD" tokio-workers "$workers" + PROFILE_LOCK=true + run_workload "lock-$workers" "$relay" "$LOCK_WORKLOAD" tokio-workers "$workers" + PROFILE_LOCK= + jq -r -s '[ + .[0].recv_fps, .[1].recv_fps, .[0].cpu_cores, .[1].cpu_cores, + ([.[2].workers[].samples] | min), + ([.[2].workers[].mutex_wait_ns] | add / 1000000), + ([.[2].workers[].mutex_wait_ns] | add) / ([.[2].workers[].cpu_ns] | add) + ] | @tsv' "$RUN/relay/control-$workers/$LOCK_WORKLOAD/summary.json" \ + "$RUN/relay/lock-$workers/$LOCK_WORKLOAD/summary.json" \ + "$RUN/relay/lock-$workers/$LOCK_WORKLOAD/lock/lock.json" | + while IFS=$'\t' read -r control profiled control_cpu profiled_cpu samples wait share; do + printf '%-8s %8s %13.1f %13.1f %10.3f %10.3f %13.1f %9.4f\n' \ + "$workers" "$samples" "$control" "$profiled" "$control_cpu" "$profiled_cpu" "$wait" "$share" + done + done + printf '\nMutex stacks (flamegraph folded format): %s/relay/lock-*/%s/lock/mutex.folded\n' "$RUN" "$LOCK_WORKLOAD" +} + cd "$ROOT" if [[ $MODE == --audio ]]; then @@ -481,6 +539,9 @@ if [[ $MODE == --audio ]]; then elif [[ $MODE == --runtime ]]; then run_runtime_comparison exit 0 +elif [[ $MODE == --lock ]]; then + run_lock_profile + exit 0 elif [[ -n $BASE ]]; then BASE_COMMIT=$(git rev-parse --verify "$BASE^{commit}") WORKTREE=$RUN/base diff --git a/flake.nix b/flake.nix index b5fbef0d0a5..485b6122374 100644 --- a/flake.nix +++ b/flake.nix @@ -564,10 +564,8 @@ ++ dartDeps ++ devTools ++ [ quest-cli ] - ++ pkgs.lib.optionals pkgs.stdenv.hostPlatform.isLinux [ - pkgs.bpftrace - pkgs.perf - ]; + # eu-stack samples relay worker stacks for `just bench-lock`. + ++ pkgs.lib.optionals pkgs.stdenv.hostPlatform.isLinux [ pkgs.elfutils ]; # jemalloc's configure uses -O0 test builds, which conflict with # Nix's _FORTIFY_SOURCE hardening (requires -O). diff --git a/justfile b/justfile index 3294bd82d3c..05488be8ba3 100644 --- a/justfile +++ b/justfile @@ -56,9 +56,9 @@ bench-audio: bench-runtime $ROUNDS="3" $WORKERS="": MOQ_BENCH_RUNTIME_ROUNDS="$ROUNDS" MOQ_BENCH_RUNTIME_WORKERS="$WORKERS" bench/run.sh --runtime -# Probe capture permissions without starting a relay or reporting a profile. -bench-profile-probe MODE="lock" OUTPUT="target/profile-probe": - python3 bench/profile.py probe {{ quote(MODE) }} {{ quote(OUTPUT) }} +# Attribute relay worker std mutex waits by stack, for each worker count. +bench-lock $WORKLOAD="fanout" $WORKERS="1 4 16": + MOQ_BENCH_LOCK_WORKLOAD="$WORKLOAD" MOQ_BENCH_LOCK_WORKERS="$WORKERS" bench/run.sh --lock # Install repo-wide tooling. Per-language deps install on first check. install: diff --git a/quest/m1/perf/lock-profile.md b/quest/m1/perf/lock-profile.md index ff9f9665a18..ce60f3132ba 100644 --- a/quest/m1/perf/lock-profile.md +++ b/quest/m1/perf/lock-profile.md @@ -6,7 +6,7 @@ One local `just` recipe runs a bench shape and captures a relay profile in one of three modes, with enough metadata to reproduce it and to tell relay cost from load-generator cost: -- **Lock wait** (first): bpftrace reports, per relay worker thread +- **Lock wait** (first): sampled stacks report, per relay worker thread (`moq-uring-N`, `moq-quic-N`), the time spent blocked in futex waits and the user stacks that blocked, as a share of that thread's CPU. Lock wait becomes a number anyone can reproduce without code in `kio`. @@ -17,11 +17,8 @@ Profiling is opt-in and costs nothing when disabled. ## Plan -Draft foundation adds fail-closed Linux tool/permission probes and a tested -worker mutex-wait/CPU reducer. The lock probe now exercises the futex tracepoints and user-stack program, retains unavailable-tool and spawn failures, and rejects address-only stacks. This host has perf_event_paranoid=4 and -unprivileged_bpf_disabled=2. Actual capture/lifecycle integration, normalized -bpftrace input, control overhead, symbolized MoQ stacks, heap snapshots/churn -and macOS validation remain. Keep this quest; probes are not smoke captures. The benchmark lifecycle and retained-artifact implementation in the comparison PR must land before this integrates with it; do not copy that branch into a second launcher. +`just bench-lock` (`bench/run.sh --lock`) delivers the lock-wait mode on +Linux; the CPU and heap modes remain. Decided 2026-10-08: the reproducible CPU and allocation profile quest merged here, so the relay has one profiling recipe with capture modes rather than @@ -45,30 +42,35 @@ Shared by every mode: OS/kernel, CPU, runtime, affinity, allocator, and resolved workload with the result, in a caller-selected artifact directory. On cancellation or a failed capture, stop every owned child and keep the diagnostics. -- Local only, documented next to `just bench` and `just bench-runtime`. Not - in CI, since the modes need privileges. Probe tool and permission support - and refuse an unsupported capture clearly; never substitute wall-clock - timing for CPU samples. +- Documented next to `just bench` and `just bench-runtime`. A mode that + needs privileges stays local and refuses an unsupported capture clearly; + one that does not runs nightly. Never substitute wall-clock timing for CPU + samples. Lock wait (decided by the maintainer 2026-10-07, after closing #5031, which timed every contended `kio::Lock` acquire in code and added public API in three crates plus five Prometheus families): -- Measure from outside the process: bpftrace on the futex syscalls of the - relay's worker threads, aggregated by thread name and user stack. No - instrumentation in `kio`, no new public API, no metrics. -- `Consumer::poll` is generic and inlined, and bpftrace doesn't expand inline - frames, so the stacks name its callers. +- Measure from outside the process, aggregated by thread name and user + stack. No instrumentation in `kio`, no new public API, no metrics. +- Capture without root, so it runs on a developer host with + `perf_event_paranoid=4` and `unprivileged_bpf_disabled=2` and nightly in CI + (decided 2026-10-12, replacing bpftrace, which needs `CAP_BPF`). The relay + starts with `PR_SET_PTRACER` naming the harness; `eu-stack -q -m` samples + every thread at 20 Hz in the steady window, and `addr2line -i -C` + symbolizes once afterwards, expanding inlined frames such as + `Consumer::poll`. Wait is the window scaled by the share of samples blocked + in the futex; `/proc` schedstat supplies CPU. Each sample briefly stops the + relay, so an unprofiled control run precedes every profiled run. - Count only stacks under the std mutex's contended path (`kio::Lock` is a `std::sync::Mutex`): futex waits also include allocator, condvar, and tokio waits, and a contended acquire that resolves while spinning never reaches the futex. -- Add bpftrace to the Nix dev shell for Linux only - (`lib.optionals stdenv.isLinux`), since the shell also builds for - `aarch64-darwin`. The mode needs sudo (or `CAP_BPF`) and says so. - Shapes and worker counts follow #5031: fanout (`bench/workloads/fanout.toml`) - and chat (`rs/moq-bench/config/chat.toml`) at 1, 4, and 16 workers. Wait is - off-CPU, so a thread's share can exceed 1 when it blocks longer than it runs. + at 1, 4, and 16 workers. Chat (`rs/moq-bench/config/chat.toml`) needs its + 60 s startup and 2,001 connections, which the harness's fixed ramp does not + allow yet. Wait is off-CPU, so a thread's share can exceed 1 when it blocks + longer than it runs. - Check that it roughly reproduces #5031's wait time (see kio channel contention), not its contended-acquire counts, and that its stacks lead to the same call sites. From 1b9ad556b1426ed167e8fbe93d26d138c8ac2c30 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 12 Oct 2026 09:32:44 -0700 Subject: [PATCH 5/8] fix(bench): require resolved MoQ frames for every profiled worker One worker's resolved stack no longer vouches for another whose unwinding failed, which would have silently reported zero contention for it. Co-Authored-By: Claude Opus 5.5 --- bench/profile.py | 4 ++-- bench/profile.test.py | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/bench/profile.py b/bench/profile.py index 37193aa0bce..21368ebb403 100644 --- a/bench/profile.py +++ b/bench/profile.py @@ -83,13 +83,13 @@ def classify(stack): def reduce(names, samples, before, after, window_ns): """Turn per-sample stacks into each worker's wait share of wall and CPU time.""" - if not any(re.search(r'\bmoq_|\bkio::', frame) for sample in samples for tid in names for frame in sample.get(tid, [])): - raise ValueError('worker stacks contain no resolved MoQ frames; build with the profiling profile') workers, folded = {}, Counter() for tid, name in names.items(): stacks = [sample[tid] for sample in samples if tid in sample] if not stacks: raise ValueError(f'worker {name} ({tid}) was never sampled') + if not any(re.search(r'\bmoq_|\bkio::', frame) for stack in stacks for frame in stack): + raise ValueError(f'worker {name} ({tid}) has no resolved MoQ frames; build with the profiling profile') cpu = after[tid] - before[tid] if cpu <= 0: raise ValueError(f'worker {name} ({tid}) has no advancing CPU counter') diff --git a/bench/profile.test.py b/bench/profile.test.py index bde1064af70..4e86c532211 100644 --- a/bench/profile.test.py +++ b/bench/profile.test.py @@ -73,9 +73,11 @@ def test_unsampled_or_idle_cpu_refused(self): with self.assertRaises(ValueError): profile.reduce({'1': 'moq-quic-0'}, [{'1': IDLE}], {'1': 5}, {'1': 5}, 1) - def test_unresolved_stacks_refused(self): + def test_unresolved_stacks_refused_per_worker(self): + # One resolved worker must not vouch for another whose unwinding failed. with self.assertRaises(ValueError): - profile.reduce({'1': 'moq-quic-0'}, [{'1': ['??', 'syscall']}], {'1': 0}, {'1': 1}, 1) + profile.reduce({'1': 'moq-quic-0', '2': 'moq-quic-1'}, [{'1': IDLE, '2': ['??', 'syscall']}], + {'1': 0, '2': 0}, {'1': 1, '2': 1}, 1) if __name__ == '__main__': From f70254f218c76ecb123c0670f8ed4a8ce7f36b93 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 12 Oct 2026 13:07:43 -0700 Subject: [PATCH 6/8] feat(bench): record relay lock waits with perf sched_switch Replace the eu-stack sampler, whose ptrace stops likely inflated the waits it measured, with perf: sched:sched_switch stacks plus switch-in events give each worker an exact off-CPU interval, classified by std's Mutex::lock_contended futex path. User stacks unwind by DWARF because glibc's frameless syscall wrapper hides lock_contended from frame-pointer unwinding. The recipe refuses with the exact sysctl and tracefs commands until the kernel allows the capture; nightly sets them with sudo. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly.yml | 9 +- bench/README.md | 32 +++--- bench/profile.py | 184 ++++++++++++++++------------------ bench/profile.test.py | 118 +++++++++++----------- bench/profile.test.txt | 104 +++++++++++++++++++ bench/relay.sh | 10 +- bench/run.sh | 18 ++-- flake.nix | 4 +- quest/m1/perf/lock-profile.md | 31 +++--- 9 files changed, 311 insertions(+), 199 deletions(-) create mode 100644 bench/profile.test.txt diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index aa25904e7cb..e1ad40d8892 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -91,7 +91,14 @@ jobs: if: ${{ !cancelled() }} run: nix develop --command just bench-audio - # Smoke-tests the unprivileged capture path: resolved worker stacks and CPU. + # perf needs the sched_switch tracepoint and kernel stacks; bench-lock refuses without them. + - name: Allow perf tracing + if: ${{ !cancelled() }} + run: | + sudo sysctl -w kernel.perf_event_paranoid=-1 kernel.kptr_restrict=0 + sudo mount -o remount,mode=755 /sys/kernel/tracing + + # Smoke-tests the capture path: exact off-CPU waits, resolved worker stacks and CPU. - name: Relay lock-wait profile if: ${{ !cancelled() }} run: nix develop --command just bench-lock fanout-heavy 4 diff --git a/bench/README.md b/bench/README.md index 7ae04dfc8dc..5d6e790c79e 100644 --- a/bench/README.md +++ b/bench/README.md @@ -59,19 +59,25 @@ nix develop --command just bench-lock fanout-heavy "4 16" This builds the relay with `[profile.profiling]` and frame pointers, then runs the workload on Tokio workers at each count twice: an unprofiled control, then a profiled run, so the throughput and CPU difference shows the profiler's -overhead. During the profiled run's steady window, `eu-stack` samples every -thread's stack back to back. A `moq-quic-N` sample counts as a mutex wait only -when it is blocked in the futex under std's `Mutex::lock_contended` -(`kio::Lock`); condvar, allocator and parking waits are counted separately, and -spinning is excluded. Wait time is the wall window scaled by the share of -samples, and `/proc` schedstat supplies CPU, so wait/CPU can exceed one. -`lock/mutex.folded` holds the blocking stacks in flamegraph folded format. - -It needs no root or perf/BPF permissions: the relay starts with -`PR_SET_PTRACER` naming the harness, which lets Yama's default -`ptrace_scope=1` admit the sampler. Each sample stops the relay while it -unwinds, so treat the stacks as attribution and the absolute wait as an -estimate. Linux only. +overhead. During the profiled run's steady window, `perf record` captures every +worker switch-out on the `sched:sched_switch` tracepoint with a DWARF-unwound +stack, plus each switch-in, so every off-CPU interval has an exact duration. +glibc's `syscall` wrapper keeps no frame pointer, so frame-pointer unwinding +would drop `Mutex::lock_contended`, which is why the stacks are unwound by +DWARF. An interval counts as a mutex wait only when the worker is blocked in +the futex under std's `Mutex::lock_contended` (`kio::Lock`). Condvar, +allocator and parking waits are counted separately, and spinning is excluded. +`/proc` schedstat supplies CPU, so wait/CPU can exceed one. +`lock/mutex.folded` holds the blocking stacks, weighted by microseconds +waited, in flamegraph folded format. + +Linux only. The kernel must expose the tracepoint and kernel stacks, and the +recipe refuses before building until it does: + +```bash +sudo sysctl -w kernel.perf_event_paranoid=-1 kernel.kptr_restrict=0 +sudo mount -o remount,mode=755 /sys/kernel/tracing +``` ## Workloads diff --git a/bench/profile.py b/bench/profile.py index 21368ebb403..0f4bffd6900 100644 --- a/bench/profile.py +++ b/bench/profile.py @@ -1,134 +1,127 @@ #!/usr/bin/env python3 -"""Sample relay worker stacks with eu-stack and attribute std mutex waits.""" +"""Record relay worker off-CPU stacks with perf and attribute std mutex waits.""" import argparse -import ctypes import json import os import re import subprocess import time -from collections import Counter, defaultdict +from collections import Counter from pathlib import Path -PR_SET_PTRACER = 0x59616D61 -# Each sample stops the relay while it unwinds; 20 Hz bounds that, and the control run measures it. -INTERVAL_NS = 50_000_000 +EVENT = 'sched:sched_switch' WORKER = re.compile(r"moq-(?:uring|quic)-[0-9]+") # The blocking call itself: std's futex wrapper or glibc's (allocator, pthread). BLOCKED = re.compile(r"futex_wait|futex_abstimed_wait|lll_lock_wait") # kio::Lock is a std Mutex; condvar, allocator and parking waits are separate. # An inlined frame carries only its short name. MUTEX = re.compile(r"(?:\bMutex>?::|^)lock_contended$") -FRAME = re.compile(r"#\d+\s+0x([0-9a-f]+)\s+-\s+(\S+)") +HEADER = re.compile(r"\s*(\d+)\s+(\d+)\.(\d+):\s+(.*)") +FRAME = re.compile(r"\s+[0-9a-f]+\s+(.*?)\s+\((.*)\)$") +# Without these, perf refuses the tracepoint or records unresolvable stacks. +SETUP = """sudo sysctl -w kernel.perf_event_paranoid=-1 kernel.kptr_restrict=0 +sudo mount -o remount,mode=755 /sys/kernel/tracing""" -def parse(text): - """Split `eu-stack -q -m` output into innermost-first (module, address) lists keyed by TID.""" - stacks, frames = {}, None - for line in text.splitlines(): - if line.startswith('TID '): - frames = stacks.setdefault(line[4:].rstrip(':'), []) - elif frames is not None and (match := FRAME.match(line.strip())): - frames.append((match.group(2), int(match.group(1), 16))) - return stacks - - -def bases(pid): - """Load address of each mapped file, so addresses become file-relative for addr2line.""" - result = {} - for line in (Path('/proc') / str(pid) / 'maps').read_text().splitlines(): - fields = line.split(maxsplit=5) - if len(fields) == 6 and fields[2] == '00000000' and fields[5].startswith('/'): - result.setdefault(fields[5], int(fields[0].split('-')[0], 16)) - return result +def require(): + """Refuse to record unless the kernel exposes the tracepoint and stacks to this user.""" + settings = {name: Path('/proc/sys/kernel', name).read_text().strip() for name in ('perf_event_paranoid', 'kptr_restrict')} + tracing = os.access(f'/sys/kernel/tracing/events/{EVENT.replace(":", "/")}/format', os.R_OK) + if settings != {'perf_event_paranoid': '-1', 'kptr_restrict': '0'} or not tracing: + state = ', '.join(f'{name}={value}' for name, value in settings.items()) + raise ValueError(f'perf needs {EVENT} and kernel stacks ({state}, tracefs readable={tracing}); run:\n{SETUP}') -def addr2line(module, offsets): - """Resolve file offsets to innermost-first function names, inlined frames included.""" - done = subprocess.run(['addr2line', '-a', '-f', '-i', '-C', '-e', module], - input=''.join(f'{offset:#x}\n' for offset in offsets), - capture_output=True, text=True, check=True) - # Each address line is followed by (function, file:line) pairs, innermost first. - result, names, position = {}, [], 0 - for line in done.stdout.splitlines(): - if line.startswith('0x'): - names, position = result.setdefault(int(line, 16), []), 0 - continue - if position % 2 == 0: - names.append(line) - position += 1 +def parse(text): + """Split `perf script` output into (tid, nanoseconds, event, innermost-first frames) records.""" + records, frames = [], None + for line in text.splitlines(): + if match := HEADER.match(line): + tid, seconds, fraction, event = match.groups() + frames = [] + nanoseconds = int(seconds) * 1_000_000_000 + int(fraction.ljust(9, '0')) + records.append((tid, nanoseconds, event.strip().rstrip(':'), frames)) + elif frames is not None and (match := FRAME.match(line)): + symbol, dso = match.groups() + # Kernel frames all show the scheduler; the user stack names the wait. + if not dso.startswith('['): + frames.append(re.sub(r'\+0x[0-9a-f]+$', '', symbol)) + return records + + +def waits(records): + """Pair each switch-out stack with the thread's next switch-in for an exact off-CPU duration.""" + end = max((record[1] for record in records), default=0) + pending, result = {}, [] + for tid, at, event, frames in records: + if event == EVENT: + pending[tid] = (at, frames) + elif event.startswith('PERF_RECORD_SWITCH IN') and tid in pending: + start, stack = pending.pop(tid) + result.append((tid, at - start, stack)) + # A wait still open when recording stops counts up to the last event. + result.extend((tid, end - start, stack) for tid, (start, stack) in pending.items()) return result -def symbolize(samples, loaded): - """Replace (module, address) frames with names; caller frames use their call instruction.""" - wanted = defaultdict(set) - for sample in samples: - for stack in sample.values(): - for depth, (module, address) in enumerate(stack): - if module in loaded: - wanted[module].add(address - loaded[module] - (depth > 0)) - names = {module: addr2line(module, sorted(offsets)) for module, offsets in wanted.items()} - return [{tid: [name for depth, (module, address) in enumerate(stack) - for name in names.get(module, {}).get(address - loaded.get(module, 0) - (depth > 0), ['??'])] - for tid, stack in sample.items()} for sample in samples] - - def classify(stack): - """A sample is a mutex wait only while blocked in the futex under lock_contended.""" + """An off-CPU interval is a mutex wait only when blocked in the futex under lock_contended.""" if not any(BLOCKED.search(frame) for frame in stack): return 'other' return 'mutex' if any(MUTEX.search(frame) for frame in stack) else 'futex' -def reduce(names, samples, before, after, window_ns): - """Turn per-sample stacks into each worker's wait share of wall and CPU time.""" +def reduce(names, intervals, before, after, window_ns): + """Sum each worker's off-CPU time by class, as a share of its CPU time.""" workers, folded = {}, Counter() for tid, name in names.items(): - stacks = [sample[tid] for sample in samples if tid in sample] - if not stacks: - raise ValueError(f'worker {name} ({tid}) was never sampled') - if not any(re.search(r'\bmoq_|\bkio::', frame) for stack in stacks for frame in stack): + mine = [(wait, stack) for owner, wait, stack in intervals if owner == tid] + if not mine: + raise ValueError(f'worker {name} ({tid}) never switched out') + if not any(re.search(r'\bmoq_|\bkio::', frame) for _, stack in mine for frame in stack): raise ValueError(f'worker {name} ({tid}) has no resolved MoQ frames; build with the profiling profile') cpu = after[tid] - before[tid] if cpu <= 0: raise ValueError(f'worker {name} ({tid}) has no advancing CPU counter') - counts = Counter(classify(stack) for stack in stacks) - wait = window_ns * counts['mutex'] // len(stacks) - workers[tid] = {'comm': name, 'samples': len(stacks), 'mutex_samples': counts['mutex'], - 'other_futex_samples': counts['futex'], 'cpu_ns': cpu, - 'mutex_wait_ns': wait, 'mutex_wait_per_cpu': wait / cpu} - folded.update(';'.join(reversed(stack)) for stack in stacks if classify(stack) == 'mutex') - return {'version': 1, 'window_ns': window_ns, 'samples': len(samples), 'workers': workers, - 'scope': 'wall-clock stack samples blocked in a futex under std Mutex::lock_contended; spinning is excluded'}, folded + total = Counter() + for wait, stack in mine: + kind = classify(stack) + total[kind] += wait + if kind == 'mutex': + folded[';'.join(reversed(stack))] += wait // 1000 + workers[tid] = {'comm': name, 'switches': len(mine), 'cpu_ns': cpu, 'mutex_wait_ns': total['mutex'], + 'other_futex_wait_ns': total['futex'], 'other_off_cpu_ns': total['other'], + 'mutex_wait_per_cpu': total['mutex'] / cpu} + return {'version': 2, 'window_ns': window_ns, 'workers': workers, + 'scope': 'exact off-CPU time blocked in a futex under std Mutex::lock_contended; spinning is excluded'}, folded def schedstat(pid, tids): return {tid: int((Path('/proc') / str(pid) / 'task' / tid / 'schedstat').read_text().split()[0]) for tid in tids} -def sample(pid, delay, duration, directory): - """Capture one steady window, then symbolize once after the relay is released.""" +def record(pid, delay, duration, directory): + """Record one steady window of worker switch-outs, then symbolize after the relay is released.""" + require() directory.mkdir(parents=True, exist_ok=True) tasks = Path('/proc') / str(pid) / 'task' names = {task.name: name for task in tasks.iterdir() if WORKER.fullmatch(name := (task / 'comm').read_text().strip())} if not names: raise ValueError(f'PID {pid} has no moq-uring-N or moq-quic-N workers') - loaded = bases(pid) time.sleep(delay) + data = directory / 'perf.data' before, start = schedstat(pid, names), time.monotonic_ns() - raw = [] - with open(directory / 'eu-stack.log', 'w') as log: - while (now := time.monotonic_ns()) - start < duration * 1_000_000_000: - # Exit 1 marks a partial backtrace, such as a frame without CFI; 2 is no frames at all. - done = subprocess.run(['eu-stack', '-q', '-m', '-p', str(pid)], capture_output=True, text=True) - log.write(done.stderr) - if done.returncode > 1: - raise ValueError(f'eu-stack failed ({done.returncode}): {done.stderr.strip()}') - raw.append(parse(done.stdout)) - time.sleep(max(0, now + INTERVAL_NS - time.monotonic_ns()) / 1e9) + # The tracepoint fires in the outgoing thread, so -p never sees a worker's switch-in; + # --switch-events records it. glibc's syscall wrapper keeps no frame pointer, so + # frame-pointer unwinding drops its caller, lock_contended; DWARF unwinding keeps it. + subprocess.run(['perf', 'record', '--quiet', '--no-buildid-cache', '-e', EVENT, '--call-graph', 'dwarf', '--switch-events', + '-p', str(pid), '-o', str(data), '--', 'sleep', str(duration)], check=True) after, window = schedstat(pid, names), time.monotonic_ns() - start - result, folded = reduce(names, symbolize(raw, loaded), before, after, window) + # Symbols come from the local binaries; never wait on a debuginfod server. + script = subprocess.run(['perf', 'script', '-i', str(data), '--show-switch-events', '--inline', '--ns', + '-F', 'tid,time,event,ip,sym,dso'], capture_output=True, text=True, check=True, + env=dict(os.environ, DEBUGINFOD_URLS='')) + result, folded = reduce(names, waits(parse(script.stdout)), before, after, window) (directory / 'lock.json').write_text(json.dumps(result, indent=2) + '\n') (directory / 'mutex.folded').write_text(''.join(f'{stack} {count}\n' for stack, count in folded.most_common())) @@ -136,23 +129,18 @@ def sample(pid, delay, duration, directory): def main(): parser = argparse.ArgumentParser(description=__doc__) commands = parser.add_subparsers(dest='command', required=True) - traceable = commands.add_parser('traceable', help='let TRACER and its descendants ptrace COMMAND, then exec it') - traceable.add_argument('tracer', type=int) - traceable.add_argument('argv', nargs=argparse.REMAINDER) - lock = commands.add_parser('lock', help='sample worker stacks of PID into DIRECTORY') - lock.add_argument('pid', type=int) - lock.add_argument('directory', type=Path) - lock.add_argument('--delay', type=float, default=0) - lock.add_argument('--duration', type=float, required=True) + commands.add_parser('check', help='refuse unless perf can record the tracepoint with kernel stacks') + capture = commands.add_parser('record', help='record worker off-CPU stacks of PID into DIRECTORY') + capture.add_argument('pid', type=int) + capture.add_argument('directory', type=Path) + capture.add_argument('--delay', type=float, default=0) + capture.add_argument('--duration', type=float, required=True) args = parser.parse_args() try: - if args.command == 'traceable': - # Yama's default ptrace_scope=1 only lets ancestors attach, and eu-stack is not one. - if ctypes.CDLL(None, use_errno=True).prctl(PR_SET_PTRACER, args.tracer, 0, 0, 0): - raise OSError(ctypes.get_errno(), 'PR_SET_PTRACER failed') - argv = args.argv[1:] if args.argv[:1] == ['--'] else args.argv - os.execvp(argv[0], argv) - sample(args.pid, args.delay, args.duration, args.directory) + if args.command == 'check': + require() + else: + record(args.pid, args.delay, args.duration, args.directory) except (ValueError, OSError, subprocess.CalledProcessError) as error: parser.exit(1, f'profile: {error}\n') diff --git a/bench/profile.test.py b/bench/profile.test.py index 4e86c532211..946b167f18e 100644 --- a/bench/profile.test.py +++ b/bench/profile.test.py @@ -1,5 +1,4 @@ import importlib.util -import subprocess import unittest from pathlib import Path from unittest.mock import patch @@ -8,76 +7,83 @@ profile = importlib.util.module_from_spec(spec) spec.loader.exec_module(profile) -BLOCKED = ['syscall', 'std::sys::pal::unix::futex::futex_wait', '::lock_contended', - 'lock>', '::is_expired'] -INLINED = ['syscall', 'std::sys::pal::unix::futex::futex_wait', 'lock_contended', 'moq_net::serve'] -SPINNING = ['::lock_contended', 'moq_net::serve'] -CONDVAR = ['syscall', 'std::sys::pal::unix::futex::futex_wait', - '::wait', 'moq_net::serve'] -IDLE = ['epoll_wait', '::poll', 'moq_net::serve'] +FIXTURE = Path(__file__).with_name('profile.test.txt').read_text() +IDLE, MUTEX, ALLOCATOR = '2152705', '2152709', '2152707' class Parse(unittest.TestCase): - def test_threads_modules_and_addresses(self): - text = '''PID 7 - process -TID 7: -#0 0x00007ffff7d2966d - /lib/libc.so.6 -#1 0x00005555567d3259 - /bin/moq-relay -TID 8: -#0 0x00007ffff7d2966d - /lib/libc.so.6 -''' - self.assertEqual(profile.parse(text), { - '7': [('/lib/libc.so.6', 0x7ffff7d2966d), ('/bin/moq-relay', 0x5555567d3259)], - '8': [('/lib/libc.so.6', 0x7ffff7d2966d)]}) - - def test_inlined_frames_and_caller_offsets(self): - output = '''0x0000000000001000 -inner -/src/a.rs:1 -outer -/src/b.rs:2 -0x0000000000002000 -?? -??:0 -''' - done = subprocess.CompletedProcess([], 0, output, '') - with patch.object(profile.subprocess, 'run', return_value=done) as run: - stacks = profile.symbolize([{'1': [('/bin/relay', 0x2100), ('/bin/relay', 0x1101)]}], {'/bin/relay': 0x100}) - # A caller frame holds a return address, so it resolves one byte earlier. - self.assertEqual(run.call_args.kwargs['input'], '0x1000\n0x2000\n') - self.assertEqual(stacks, [{'1': ['??', 'inner', 'outer']}]) + def test_recorded_switches_pair_into_exact_waits(self): + records = profile.parse(FIXTURE) + self.assertEqual([event for _, _, event, _ in records], ['sched:sched_switch', 'PERF_RECORD_SWITCH IN'] * 3) + # Kernel frames are dropped; inlined user frames are kept innermost first. + self.assertFalse(any('__schedule' in frame for _, _, _, frames in records for frame in frames)) + stacks = {tid: (wait, stack) for tid, wait, stack in profile.waits(records)} + self.assertEqual(stacks[IDLE][0], 21661) + self.assertEqual(stacks[MUTEX][0], 6315) + self.assertEqual(stacks[ALLOCATOR][0], 13709) + # DWARF unwinding keeps the caller of glibc's frameless syscall wrapper. + self.assertEqual(stacks[MUTEX][1][:3], ['syscall', 'std::sys::pal::unix::futex::futex_wait', + '::lock_contended']) + + def test_open_wait_runs_to_the_last_event(self): + records = [('1', 10, 'sched:sched_switch', ['a']), ('2', 50, 'PERF_RECORD_SWITCH IN', [])] + self.assertEqual(profile.waits(records), [('1', 40, ['a'])]) class Classify(unittest.TestCase): - def test_only_blocked_lock_contended_is_a_mutex_wait(self): - self.assertEqual(profile.classify(BLOCKED), 'mutex') - self.assertEqual(profile.classify(INLINED), 'mutex') - self.assertEqual(profile.classify(SPINNING), 'other') - self.assertEqual(profile.classify(CONDVAR), 'futex') - self.assertEqual(profile.classify(IDLE), 'other') + def test_recorded_stacks(self): + stacks = {tid: stack for tid, _, stack in profile.waits(profile.parse(FIXTURE))} + self.assertEqual(profile.classify(stacks[MUTEX]), 'mutex') + self.assertEqual(profile.classify(stacks[ALLOCATOR]), 'futex') + self.assertEqual(profile.classify(stacks[IDLE]), 'other') + + def test_spinning_is_not_a_wait(self): + self.assertEqual(profile.classify(['::lock_contended', 'moq_net::serve']), 'other') class Reduce(unittest.TestCase): - def test_wait_is_window_share_and_can_exceed_cpu(self): - samples = [{'1': BLOCKED}, {'1': BLOCKED}, {'1': BLOCKED}, {'1': CONDVAR}] - result, folded = profile.reduce({'1': 'moq-quic-0'}, samples, {'1': 0}, {'1': 100}, 400) - row = result['workers']['1'] - self.assertEqual((row['samples'], row['mutex_samples'], row['other_futex_samples']), (4, 3, 1)) - self.assertEqual(row['mutex_wait_ns'], 300) - self.assertEqual(row['mutex_wait_per_cpu'], 3) - self.assertEqual(folded, {';'.join(reversed(BLOCKED)): 3}) - - def test_unsampled_or_idle_cpu_refused(self): + def test_recorded_waits_by_class_over_cpu(self): + intervals = profile.waits(profile.parse(FIXTURE)) + names = {IDLE: 'moq-quic-0', MUTEX: 'moq-quic-1', ALLOCATOR: 'moq-quic-2'} + before = dict.fromkeys(names, 0) + result, folded = profile.reduce(names, intervals, before, dict.fromkeys(names, 3000), 1) + rows = result['workers'] + self.assertEqual(rows[MUTEX]['mutex_wait_ns'], 6315) + self.assertAlmostEqual(rows[MUTEX]['mutex_wait_per_cpu'], 6315 / 3000) + self.assertEqual(rows[ALLOCATOR]['other_futex_wait_ns'], 13709) + self.assertEqual(rows[IDLE]['other_off_cpu_ns'], 21661) + [(stack, weight)] = folded.items() + self.assertTrue(stack.endswith('lock_contended;std::sys::pal::unix::futex::futex_wait;syscall')) + self.assertEqual(weight, 6) + + def test_silent_or_idle_worker_refused(self): + intervals = [('1', 5, ['moq_net::serve'])] with self.assertRaises(ValueError): - profile.reduce({'1': 'moq-quic-0', '2': 'moq-quic-1'}, [{'1': IDLE}], {'1': 0, '2': 0}, {'1': 1, '2': 1}, 1) + profile.reduce({'1': 'moq-quic-0', '2': 'moq-quic-1'}, intervals, {'1': 0, '2': 0}, {'1': 1, '2': 1}, 1) with self.assertRaises(ValueError): - profile.reduce({'1': 'moq-quic-0'}, [{'1': IDLE}], {'1': 5}, {'1': 5}, 1) + profile.reduce({'1': 'moq-quic-0'}, intervals, {'1': 5}, {'1': 5}, 1) def test_unresolved_stacks_refused_per_worker(self): # One resolved worker must not vouch for another whose unwinding failed. + intervals = [('1', 5, ['moq_net::serve']), ('2', 5, ['[unknown]', 'syscall'])] with self.assertRaises(ValueError): - profile.reduce({'1': 'moq-quic-0', '2': 'moq-quic-1'}, [{'1': IDLE, '2': ['??', 'syscall']}], - {'1': 0, '2': 0}, {'1': 1, '2': 1}, 1) + profile.reduce({'1': 'moq-quic-0', '2': 'moq-quic-1'}, intervals, {'1': 0, '2': 0}, {'1': 1, '2': 1}, 1) + + +class Require(unittest.TestCase): + def check(self, paranoid, kptr, tracing): + values = {'perf_event_paranoid': paranoid, 'kptr_restrict': kptr} + with patch.object(profile.Path, 'read_text', lambda path: values[path.name] + '\n'), \ + patch.object(profile.os, 'access', return_value=tracing): + profile.require() + + def test_missing_setup_names_the_commands(self): + for case in [('2', '0', True), ('-1', '1', True), ('-1', '0', False)]: + with self.assertRaises(ValueError) as refused: + self.check(*case) + self.assertIn('sudo sysctl -w kernel.perf_event_paranoid=-1 kernel.kptr_restrict=0', str(refused.exception)) + self.assertIn('sudo mount -o remount,mode=755 /sys/kernel/tracing', str(refused.exception)) + self.check('-1', '0', True) if __name__ == '__main__': diff --git a/bench/profile.test.txt b/bench/profile.test.txt new file mode 100644 index 00000000000..85da970cac9 --- /dev/null +++ b/bench/profile.test.txt @@ -0,0 +1,104 @@ +# `perf script --show-switch-events --inline --ns -F tid,time,event,ip,sym,dso` cut to three +# switch-outs of a fanout-heavy relay: epoll idle, a std mutex wait and a malloc arena wait. +# Recorded with the context-switches software event, since tracefs was not readable on the +# recording host, and relabeled sched:sched_switch; the stacks are what perf printed. +2152705 271706.866241331: sched:sched_switch: + 7ffff7ca84f2 __syscall_cancel_arch_end (/nix/store/h4wfwic161kxrr74jlzla5lsm28hgary-glibc-2.44-25/lib/libc.so.6) + 7ffff7d2c07a epoll_wait (/nix/store/h4wfwic161kxrr74jlzla5lsm28hgary-glibc-2.44-25/lib/libc.so.6) + 5555569d8ae4 select (inlined) + 5555569d8ae4 poll (inlined) + 5555569d8ae4 ::turn (/moq/moq-relay) + 5555569d8608 ::park_internal (/moq/moq-relay) + 5555569dc84c park (inlined) + 5555569dc84c park (inlined) + 5555569dc84c park (inlined) + 5555569dc84c {closure#0} (inlined) + 5555569dc84c enter<(), tokio::runtime::scheduler::current_thread::{impl#3}::park_internal::{closure_env#0}> (inlined) + 5555569dc84c ::park_internal (/moq/moq-relay) + 5555569dcae9 ::park (/moq/moq-relay) + 55555637f51c {closure#0}>> (inlined) + 55555637f51c {closure#0}>>, core::option::Option<()>> (inlined) + 55555637f51c set>>, core::option::Option<()>>, (alloc::boxed::Box, core::option::Option<()>)> (inlined) + 55555637f51c {closure#0}<(alloc::boxed::Box, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}>>, core::option::Option<()>>> (inlined) + 55555637f51c try_with, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}>>, core::option::Option<()>>>, (alloc::boxed::Box, core::option::Option<()>)> (inlined) + 55555637f51c with, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}>>, core::option::Option<()>>>, (alloc::boxed::Box, core::option::Option<()>)> (inlined) + 55555637f51c set_scheduler<(alloc::boxed::Box, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}>>, core::option::Option<()>>> (inlined) + 55555637f51c enter>>, core::option::Option<()>> (inlined) + 55555637f51c block_on>> (inlined) + 55555637f51c {closure#0}> (inlined) + 55555637f51c enter_runtime>, ()> (inlined) + 55555637f51c block_on> (inlined) + 55555637f51c block_on_inner> (inlined) + 55555637f51c block_on> (inlined) + 55555637f51c block_on (inlined) + 55555637f51c run (inlined) + 55555637f51c {closure#0} (inlined) + 55555637f51c std::sys::backtrace::__rust_begin_short_backtrace::<::spawn::{closure#0}, ()> (/moq/moq-relay) + 5555567aa964 {closure#0} (inlined) + 5555567aa964 call_once<(), std::thread::lifecycle::spawn_unchecked::{closure#1}::{closure_env#0}> (inlined) + 5555567aa964 do_call>, ()> (inlined) + 5555567aa964 catch_unwind<(), core::panic::unwind_safe::AssertUnwindSafe>> (inlined) + 5555567aa964 catch_unwind>, ()> (inlined) + 5555567aa964 {closure#1} (inlined) + 5555567aa964 ::spawn::{closure#0}, ()>::{closure#1} as core::ops::function::FnOnce<()>>::call_once::{shim:vtable#0} (/moq/moq-relay) + ffffffff91d308fd __schedule ([kernel.kallsyms]) + ffffffff91d30d97 schedule ([kernel.kallsyms]) + +2152705 271706.866262992: PERF_RECORD_SWITCH IN +2152709 271706.880076160: sched:sched_switch: + 7ffff7d2966d syscall (/nix/store/h4wfwic161kxrr74jlzla5lsm28hgary-glibc-2.44-25/lib/libc.so.6) + 55555698c3f5 std::sys::pal::unix::futex::futex_wait (inlined) + 55555698c3f5 ::lock_contended (/moq/moq-relay) + 555555ea6fd5 lock (inlined) + 555555ea6fd5 lock> (inlined) + 555555ea6fd5 lock> (inlined) + 555555ea6fd5 poll> (inlined) + 555555ea6fd5 poll (inlined) + 555555ea6fd5 poll_drift (inlined) + 555555ea6fd5 poll_recv_group (inlined) + 555555ea6fd5 ::poll_recv_group (/moq/moq-relay) + 555555ea2e46 poll_group_once (inlined) + 555555ea2e46 poll_group (inlined) + 555555ea2e46 poll_group (inlined) + 555555ea2e46 ::poll_recv_group (/moq/moq-relay) + ffffffff91d308fd __schedule ([kernel.kallsyms]) + ffffffff91d30d97 schedule ([kernel.kallsyms]) + +2152709 271706.880082475: PERF_RECORD_SWITCH IN +2152707 271706.914931010: sched:sched_switch: + 7ffff7c9c218 __lll_lock_wait_private (/nix/store/h4wfwic161kxrr74jlzla5lsm28hgary-glibc-2.44-25/lib/libc.so.6) + 7ffff7cae500 _int_free_chunk (/nix/store/h4wfwic161kxrr74jlzla5lsm28hgary-glibc-2.44-25/lib/libc.so.6) + 5555558b0e87 ::dealloc (inlined) + 5555558b0e87 __rustc::__rdl_dealloc (inlined) + 5555558b0e87 dealloc_nonnull (inlined) + 5555558b0e87 deallocate_impl_runtime (inlined) + 5555558b0e87 deallocate_impl (inlined) + 5555558b0e87 deallocate (inlined) + 5555558b0e87 deallocate (inlined) + 5555558b0e87 drop (inlined) + 5555558b0e87 drop_glue> (inlined) + 5555558b0e87 drop_glue> (inlined) + 5555558b0e87 drop_glue (inlined) + 5555558b0e87 drop_glue<[alloc::string::String]> (inlined) + 5555558b0e87 drop_in_place<[alloc::string::String]> (inlined) + 5555558b0e87 drop_in_place<[alloc::string::String]> (inlined) + 5555558b0e87 drop (inlined) + 5555558b0e87 drop_glue> (inlined) + 5555558b0e87 drop_glue (inlined) + 5555558b0e87 drop_glue> (inlined) + 5555558b0e87 core::ptr::drop_glue:: (/moq/moq-relay) + 5555563adba1 core::ptr::drop_glue::> (/moq/moq-relay) + 555555b8e788 drop_glue>> (inlined) + 555555b8e788 drop_glue>> (inlined) + 555555b8e788 drop_glue>>> (inlined) + 555555b8e788 drop_glue>>>> (inlined) + 555555b8e788 >>>::poll (/moq/moq-relay) + 555555bbd392 poll_step> (inlined) + 555555bbd392 poll> (inlined) + 555555bbd392 , moq_net::lite::publisher::SubscribeServe>>>::poll_serve (/moq/moq-relay) + 555555b37bed poll, moq_net::lite::publisher::SubscribeServe>> (inlined) + 555555b37bed >>::poll_serve (/moq/moq-relay) + ffffffff91d308fd __schedule ([kernel.kallsyms]) + ffffffff91d30d97 schedule ([kernel.kallsyms]) + +2152707 271706.914944719: PERF_RECORD_SWITCH IN diff --git a/bench/relay.sh b/bench/relay.sh index cb0fbb9e3c2..2c643fb9156 100755 --- a/bench/relay.sh +++ b/bench/relay.sh @@ -3,9 +3,7 @@ # Sourced by run.sh, which owns these process and output paths. # shellcheck disable=SC2153,SC2154 -# Prepended to the relay command line; the lock profile makes the relay traceable. -RELAY_PREFIX=() -# Set to sample the relay's worker stacks during each workload's steady window. +# Set to record the relay's worker off-CPU stacks during each workload's steady window. PROFILE_LOCK= relay_config() { @@ -94,9 +92,9 @@ start_relay() { if [[ -n $tokio_threads ]]; then env TOKIO_WORKER_THREADS="$tokio_threads" RUST_LOG=warn \ - "${RELAY_PREFIX[@]}" "$relay" "$config" >"$log" 2>&1 & + "$relay" "$config" >"$log" 2>&1 & else - RUST_LOG=warn "${RELAY_PREFIX[@]}" "$relay" "$config" >"$log" 2>&1 & + RUST_LOG=warn "$relay" "$config" >"$log" 2>&1 & fi RELAY_PID=$! @@ -236,7 +234,7 @@ run_workload() { HOST_PID=$! if [[ -n $PROFILE_LOCK ]]; then # The second half of the host window, matching the summary's throughput window. - python3 "$ROOT/bench/profile.py" lock "$relay_pid" "$directory/lock" --delay 5 --duration 5 \ + python3 "$ROOT/bench/profile.py" record "$relay_pid" "$directory/lock" --delay 5 --duration 5 \ >"$directory/lock.log" 2>&1 & PROFILE_PID=$! fi diff --git a/bench/run.sh b/bench/run.sh index 35a544be405..e1d413e36c8 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -7,7 +7,7 @@ set -euo pipefail # runtime with the same number of independent Tokio/epoll and io_uring workers # on the current tree. The current load generator drives every relay so the # workloads stay identical. `--audio` isolates the audio grouping/load-shape matrix. -# `--lock` samples relay worker stacks to attribute std mutex waits. +# `--lock` records relay worker off-CPU stacks to attribute std mutex waits. MODE=${1:-} ROUNDS=${MOQ_BENCH_RUNTIME_ROUNDS:-3} @@ -24,7 +24,7 @@ case $MODE in BASE= LOCK_WORKLOAD=${MOQ_BENCH_LOCK_WORKLOAD:-fanout} LOCK_WORKERS=${MOQ_BENCH_LOCK_WORKERS:-1 4 16} - # Without frame pointers or symbols the stacks stop at libc. + # For frame-pointer profilers; the lock capture itself unwinds by DWARF (see profile.py). export RUSTFLAGS="${RUSTFLAGS:+$RUSTFLAGS }-C force-frame-pointers=yes" ;; --runtime) @@ -493,9 +493,11 @@ run_runtime_comparison() { run_lock_profile() { if [[ $(uname -s) != Linux ]]; then - printf 'lock profiling needs Linux for eu-stack and /proc schedstat\n' >&2 + printf 'lock profiling needs Linux for perf and /proc schedstat\n' >&2 return 1 fi + # Refuse before the build when the kernel would refuse the capture. + python3 "$ROOT/bench/profile.py" check generate_certificate printf 'Building the relay with the profiling profile and frame pointers...\n' @@ -503,12 +505,10 @@ run_lock_profile() { LOAD_BIN=$TARGET/profile/profiling/moq-bench HOST_BIN=$TARGET/profile/profiling/moq-bench-host local relay=$TARGET/profile/profiling/moq-relay - # Yama's default ptrace_scope lets only ancestors attach; eu-stack is this shell's grandchild. - RELAY_PREFIX=(python3 "$ROOT/bench/profile.py" traceable "$$" --) printf '\nLock wait: %s, tokio workers; each profiled run follows an unprofiled control\n' "$LOCK_WORKLOAD" printf '%-8s %8s %13s %13s %10s %10s %13s %9s\n' \ - workers samples control-fps profiled-fps control-CPU profiled-CPU mutex-wait-ms wait/CPU + workers switches control-fps profiled-fps control-CPU profiled-CPU mutex-wait-ms wait/CPU local workers for workers in $LOCK_WORKERS; do run_workload "control-$workers" "$relay" "$LOCK_WORKLOAD" tokio-workers "$workers" @@ -517,15 +517,15 @@ run_lock_profile() { PROFILE_LOCK= jq -r -s '[ .[0].recv_fps, .[1].recv_fps, .[0].cpu_cores, .[1].cpu_cores, - ([.[2].workers[].samples] | min), + ([.[2].workers[].switches] | min), ([.[2].workers[].mutex_wait_ns] | add / 1000000), ([.[2].workers[].mutex_wait_ns] | add) / ([.[2].workers[].cpu_ns] | add) ] | @tsv' "$RUN/relay/control-$workers/$LOCK_WORKLOAD/summary.json" \ "$RUN/relay/lock-$workers/$LOCK_WORKLOAD/summary.json" \ "$RUN/relay/lock-$workers/$LOCK_WORKLOAD/lock/lock.json" | - while IFS=$'\t' read -r control profiled control_cpu profiled_cpu samples wait share; do + while IFS=$'\t' read -r control profiled control_cpu profiled_cpu switches wait share; do printf '%-8s %8s %13.1f %13.1f %10.3f %10.3f %13.1f %9.4f\n' \ - "$workers" "$samples" "$control" "$profiled" "$control_cpu" "$profiled_cpu" "$wait" "$share" + "$workers" "$switches" "$control" "$profiled" "$control_cpu" "$profiled_cpu" "$wait" "$share" done done printf '\nMutex stacks (flamegraph folded format): %s/relay/lock-*/%s/lock/mutex.folded\n' "$RUN" "$LOCK_WORKLOAD" diff --git a/flake.nix b/flake.nix index 485b6122374..cbfab4c0933 100644 --- a/flake.nix +++ b/flake.nix @@ -564,8 +564,8 @@ ++ dartDeps ++ devTools ++ [ quest-cli ] - # eu-stack samples relay worker stacks for `just bench-lock`. - ++ pkgs.lib.optionals pkgs.stdenv.hostPlatform.isLinux [ pkgs.elfutils ]; + # perf records relay worker off-CPU stacks for `just bench-lock`. + ++ pkgs.lib.optionals pkgs.stdenv.hostPlatform.isLinux [ pkgs.perf ]; # jemalloc's configure uses -O0 test builds, which conflict with # Nix's _FORTIFY_SOURCE hardening (requires -O). diff --git a/quest/m1/perf/lock-profile.md b/quest/m1/perf/lock-profile.md index ce60f3132ba..bf7c96b22c8 100644 --- a/quest/m1/perf/lock-profile.md +++ b/quest/m1/perf/lock-profile.md @@ -6,7 +6,7 @@ One local `just` recipe runs a bench shape and captures a relay profile in one of three modes, with enough metadata to reproduce it and to tell relay cost from load-generator cost: -- **Lock wait** (first): sampled stacks report, per relay worker thread +- **Lock wait** (first): perf reports, per relay worker thread (`moq-uring-N`, `moq-quic-N`), the time spent blocked in futex waits and the user stacks that blocked, as a share of that thread's CPU. Lock wait becomes a number anyone can reproduce without code in `kio`. @@ -42,10 +42,10 @@ Shared by every mode: OS/kernel, CPU, runtime, affinity, allocator, and resolved workload with the result, in a caller-selected artifact directory. On cancellation or a failed capture, stop every owned child and keep the diagnostics. -- Documented next to `just bench` and `just bench-runtime`. A mode that - needs privileges stays local and refuses an unsupported capture clearly; - one that does not runs nightly. Never substitute wall-clock timing for CPU - samples. +- Documented next to `just bench` and `just bench-runtime`, and smoke-tested + nightly, where the runner can grant privileges with sudo. Refuse an + unsupported capture with the exact setup commands; never fall back silently + or substitute wall-clock timing for CPU samples. Lock wait (decided by the maintainer 2026-10-07, after closing #5031, which timed every contended `kio::Lock` acquire in code and added public API in @@ -53,15 +53,18 @@ three crates plus five Prometheus families): - Measure from outside the process, aggregated by thread name and user stack. No instrumentation in `kio`, no new public API, no metrics. -- Capture without root, so it runs on a developer host with - `perf_event_paranoid=4` and `unprivileged_bpf_disabled=2` and nightly in CI - (decided 2026-10-12, replacing bpftrace, which needs `CAP_BPF`). The relay - starts with `PR_SET_PTRACER` naming the harness; `eu-stack -q -m` samples - every thread at 20 Hz in the steady window, and `addr2line -i -C` - symbolizes once afterwards, expanding inlined frames such as - `Consumer::poll`. Wait is the window scaled by the share of samples blocked - in the futex; `/proc` schedstat supplies CPU. Each sample briefly stops the - relay, so an unprofiled control run precedes every profiled run. +- perf records each worker switch-out on `sched:sched_switch` with its + stack, and `--switch-events` adds the switch-in, so each off-CPU interval + has an exact duration; `/proc` schedstat supplies CPU (decided by the + maintainer 2026-10-12, replacing bpftrace and an eu-stack sampler; its + ptrace stops likely inflated waits, reading 0.15 wait/CPU on fanout-heavy + at 4 workers where exact intervals read 0.0013). It needs + `kernel.perf_event_paranoid=-1`, `kernel.kptr_restrict=0` and a readable + `/sys/kernel/tracing`, and refuses with the commands until they hold; the + nightly sets them with sudo. User stacks unwind by DWARF, because glibc's + frameless `syscall` wrapper hides `Mutex::lock_contended` from + frame-pointer unwinding. `perf script --inline` expands inlined frames such + as `Consumer::poll`. - Count only stacks under the std mutex's contended path (`kio::Lock` is a `std::sync::Mutex`): futex waits also include allocator, condvar, and tokio waits, and a contended acquire that resolves while spinning never reaches From c9b7089683403640841cf836c78d2ce2d7355173 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 12 Oct 2026 18:23:13 -0700 Subject: [PATCH 7/8] feat(bench): sample switch-outs without tracefs, drop frame pointers Sample the context-switches software event with a period of 1 instead of the sched:sched_switch tracepoint. It records the same switch-out stacks but needs only perf_event_paranoid <= 1, not a tracefs remount or kptr_restrict=0. DWARF unwinding makes the frame-pointer build flag unnecessary. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly.yml | 8 +++----- bench/README.md | 20 +++++++++---------- bench/profile.py | 28 +++++++++++++-------------- bench/profile.test.py | 23 +++++++++++----------- bench/profile.test.txt | 8 +++----- bench/run.sh | 4 +--- quest/m1/perf/lock-profile.md | 36 +++++++++++++++++++---------------- 7 files changed, 61 insertions(+), 66 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index e1ad40d8892..9bb3e59dedd 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -91,12 +91,10 @@ jobs: if: ${{ !cancelled() }} run: nix develop --command just bench-audio - # perf needs the sched_switch tracepoint and kernel stacks; bench-lock refuses without them. - - name: Allow perf tracing + # perf samples worker switch-outs in kernel context; bench-lock refuses without it. + - name: Allow perf kernel sampling if: ${{ !cancelled() }} - run: | - sudo sysctl -w kernel.perf_event_paranoid=-1 kernel.kptr_restrict=0 - sudo mount -o remount,mode=755 /sys/kernel/tracing + run: sudo sysctl -w kernel.perf_event_paranoid=1 # Smoke-tests the capture path: exact off-CPU waits, resolved worker stacks and CPU. - name: Relay lock-wait profile diff --git a/bench/README.md b/bench/README.md index 5d6e790c79e..288494c01ae 100644 --- a/bench/README.md +++ b/bench/README.md @@ -56,12 +56,12 @@ nix develop --command just bench-lock nix develop --command just bench-lock fanout-heavy "4 16" ``` -This builds the relay with `[profile.profiling]` and frame pointers, then runs -the workload on Tokio workers at each count twice: an unprofiled control, then -a profiled run, so the throughput and CPU difference shows the profiler's -overhead. During the profiled run's steady window, `perf record` captures every -worker switch-out on the `sched:sched_switch` tracepoint with a DWARF-unwound -stack, plus each switch-in, so every off-CPU interval has an exact duration. +This builds the relay with `[profile.profiling]`, then runs the workload on +Tokio workers at each count twice: an unprofiled control, then a profiled run, +so the throughput and CPU difference shows the profiler's overhead. During the +profiled run's steady window, `perf record` samples every worker switch-out +(the `context-switches` event with a period of 1) with a DWARF-unwound stack, +plus each switch-in, so every off-CPU interval has an exact duration. glibc's `syscall` wrapper keeps no frame pointer, so frame-pointer unwinding would drop `Mutex::lock_contended`, which is why the stacks are unwound by DWARF. An interval counts as a mutex wait only when the worker is blocked in @@ -71,12 +71,12 @@ allocator and parking waits are counted separately, and spinning is excluded. `lock/mutex.folded` holds the blocking stacks, weighted by microseconds waited, in flamegraph folded format. -Linux only. The kernel must expose the tracepoint and kernel stacks, and the -recipe refuses before building until it does: +Linux only. A switch-out is sampled in kernel context, which perf allows only +at `perf_event_paranoid` 1 or lower, so the recipe refuses before building +until then: ```bash -sudo sysctl -w kernel.perf_event_paranoid=-1 kernel.kptr_restrict=0 -sudo mount -o remount,mode=755 /sys/kernel/tracing +sudo sysctl -w kernel.perf_event_paranoid=1 ``` ## Workloads diff --git a/bench/profile.py b/bench/profile.py index 0f4bffd6900..d5bbbf684ad 100644 --- a/bench/profile.py +++ b/bench/profile.py @@ -9,7 +9,7 @@ from collections import Counter from pathlib import Path -EVENT = 'sched:sched_switch' +EVENT = 'context-switches' WORKER = re.compile(r"moq-(?:uring|quic)-[0-9]+") # The blocking call itself: std's futex wrapper or glibc's (allocator, pthread). BLOCKED = re.compile(r"futex_wait|futex_abstimed_wait|lll_lock_wait") @@ -18,18 +18,15 @@ MUTEX = re.compile(r"(?:\bMutex>?::|^)lock_contended$") HEADER = re.compile(r"\s*(\d+)\s+(\d+)\.(\d+):\s+(.*)") FRAME = re.compile(r"\s+[0-9a-f]+\s+(.*?)\s+\((.*)\)$") -# Without these, perf refuses the tracepoint or records unresolvable stacks. -SETUP = """sudo sysctl -w kernel.perf_event_paranoid=-1 kernel.kptr_restrict=0 -sudo mount -o remount,mode=755 /sys/kernel/tracing""" +# A switch-out is sampled in kernel context, which perf allows only at paranoid <= 1. +SETUP = 'sudo sysctl -w kernel.perf_event_paranoid=1' def require(): - """Refuse to record unless the kernel exposes the tracepoint and stacks to this user.""" - settings = {name: Path('/proc/sys/kernel', name).read_text().strip() for name in ('perf_event_paranoid', 'kptr_restrict')} - tracing = os.access(f'/sys/kernel/tracing/events/{EVENT.replace(":", "/")}/format', os.R_OK) - if settings != {'perf_event_paranoid': '-1', 'kptr_restrict': '0'} or not tracing: - state = ', '.join(f'{name}={value}' for name, value in settings.items()) - raise ValueError(f'perf needs {EVENT} and kernel stacks ({state}, tracefs readable={tracing}); run:\n{SETUP}') + """Refuse to record unless perf may sample this user's processes in kernel context.""" + paranoid = int(Path('/proc/sys/kernel/perf_event_paranoid').read_text()) + if paranoid > 1: + raise ValueError(f'perf needs kernel samples of the relay (perf_event_paranoid={paranoid}); run:\n{SETUP}') def parse(text): @@ -111,10 +108,11 @@ def record(pid, delay, duration, directory): time.sleep(delay) data = directory / 'perf.data' before, start = schedstat(pid, names), time.monotonic_ns() - # The tracepoint fires in the outgoing thread, so -p never sees a worker's switch-in; - # --switch-events records it. glibc's syscall wrapper keeps no frame pointer, so - # frame-pointer unwinding drops its caller, lock_contended; DWARF unwinding keeps it. - subprocess.run(['perf', 'record', '--quiet', '--no-buildid-cache', '-e', EVENT, '--call-graph', 'dwarf', '--switch-events', + # A period of 1 samples every switch-out. The sample fires in the outgoing thread, so -p + # never sees a worker's switch-in; --switch-events records it. glibc's syscall wrapper + # keeps no frame pointer, so frame-pointer unwinding drops its caller, lock_contended; + # DWARF unwinding keeps it. + subprocess.run(['perf', 'record', '--quiet', '--no-buildid-cache', '-e', EVENT, '-c', '1', '--call-graph', 'dwarf', '--switch-events', '-p', str(pid), '-o', str(data), '--', 'sleep', str(duration)], check=True) after, window = schedstat(pid, names), time.monotonic_ns() - start # Symbols come from the local binaries; never wait on a debuginfod server. @@ -129,7 +127,7 @@ def record(pid, delay, duration, directory): def main(): parser = argparse.ArgumentParser(description=__doc__) commands = parser.add_subparsers(dest='command', required=True) - commands.add_parser('check', help='refuse unless perf can record the tracepoint with kernel stacks') + commands.add_parser('check', help='refuse unless perf can sample the relay in kernel context') capture = commands.add_parser('record', help='record worker off-CPU stacks of PID into DIRECTORY') capture.add_argument('pid', type=int) capture.add_argument('directory', type=Path) diff --git a/bench/profile.test.py b/bench/profile.test.py index 946b167f18e..848b0e46bcd 100644 --- a/bench/profile.test.py +++ b/bench/profile.test.py @@ -14,7 +14,7 @@ class Parse(unittest.TestCase): def test_recorded_switches_pair_into_exact_waits(self): records = profile.parse(FIXTURE) - self.assertEqual([event for _, _, event, _ in records], ['sched:sched_switch', 'PERF_RECORD_SWITCH IN'] * 3) + self.assertEqual([event for _, _, event, _ in records], ['context-switches', 'PERF_RECORD_SWITCH IN'] * 3) # Kernel frames are dropped; inlined user frames are kept innermost first. self.assertFalse(any('__schedule' in frame for _, _, _, frames in records for frame in frames)) stacks = {tid: (wait, stack) for tid, wait, stack in profile.waits(records)} @@ -26,7 +26,7 @@ def test_recorded_switches_pair_into_exact_waits(self): '::lock_contended']) def test_open_wait_runs_to_the_last_event(self): - records = [('1', 10, 'sched:sched_switch', ['a']), ('2', 50, 'PERF_RECORD_SWITCH IN', [])] + records = [('1', 10, 'context-switches', ['a']), ('2', 50, 'PERF_RECORD_SWITCH IN', [])] self.assertEqual(profile.waits(records), [('1', 40, ['a'])]) @@ -71,19 +71,18 @@ def test_unresolved_stacks_refused_per_worker(self): class Require(unittest.TestCase): - def check(self, paranoid, kptr, tracing): - values = {'perf_event_paranoid': paranoid, 'kptr_restrict': kptr} - with patch.object(profile.Path, 'read_text', lambda path: values[path.name] + '\n'), \ - patch.object(profile.os, 'access', return_value=tracing): + def check(self, paranoid): + with patch.object(profile.Path, 'read_text', return_value=paranoid + '\n'): profile.require() - def test_missing_setup_names_the_commands(self): - for case in [('2', '0', True), ('-1', '1', True), ('-1', '0', False)]: + def test_kernel_sampling_refusal_names_the_command(self): + # Ubuntu's default is 4; 2 already forbids kernel samples. + for paranoid in ['4', '2']: with self.assertRaises(ValueError) as refused: - self.check(*case) - self.assertIn('sudo sysctl -w kernel.perf_event_paranoid=-1 kernel.kptr_restrict=0', str(refused.exception)) - self.assertIn('sudo mount -o remount,mode=755 /sys/kernel/tracing', str(refused.exception)) - self.check('-1', '0', True) + self.check(paranoid) + self.assertIn('sudo sysctl -w kernel.perf_event_paranoid=1', str(refused.exception)) + for paranoid in ['1', '0', '-1']: + self.check(paranoid) if __name__ == '__main__': diff --git a/bench/profile.test.txt b/bench/profile.test.txt index 85da970cac9..4b5815859c7 100644 --- a/bench/profile.test.txt +++ b/bench/profile.test.txt @@ -1,8 +1,6 @@ # `perf script --show-switch-events --inline --ns -F tid,time,event,ip,sym,dso` cut to three # switch-outs of a fanout-heavy relay: epoll idle, a std mutex wait and a malloc arena wait. -# Recorded with the context-switches software event, since tracefs was not readable on the -# recording host, and relabeled sched:sched_switch; the stacks are what perf printed. -2152705 271706.866241331: sched:sched_switch: +2152705 271706.866241331: context-switches: 7ffff7ca84f2 __syscall_cancel_arch_end (/nix/store/h4wfwic161kxrr74jlzla5lsm28hgary-glibc-2.44-25/lib/libc.so.6) 7ffff7d2c07a epoll_wait (/nix/store/h4wfwic161kxrr74jlzla5lsm28hgary-glibc-2.44-25/lib/libc.so.6) 5555569d8ae4 select (inlined) @@ -45,7 +43,7 @@ ffffffff91d30d97 schedule ([kernel.kallsyms]) 2152705 271706.866262992: PERF_RECORD_SWITCH IN -2152709 271706.880076160: sched:sched_switch: +2152709 271706.880076160: context-switches: 7ffff7d2966d syscall (/nix/store/h4wfwic161kxrr74jlzla5lsm28hgary-glibc-2.44-25/lib/libc.so.6) 55555698c3f5 std::sys::pal::unix::futex::futex_wait (inlined) 55555698c3f5 ::lock_contended (/moq/moq-relay) @@ -65,7 +63,7 @@ ffffffff91d30d97 schedule ([kernel.kallsyms]) 2152709 271706.880082475: PERF_RECORD_SWITCH IN -2152707 271706.914931010: sched:sched_switch: +2152707 271706.914931010: context-switches: 7ffff7c9c218 __lll_lock_wait_private (/nix/store/h4wfwic161kxrr74jlzla5lsm28hgary-glibc-2.44-25/lib/libc.so.6) 7ffff7cae500 _int_free_chunk (/nix/store/h4wfwic161kxrr74jlzla5lsm28hgary-glibc-2.44-25/lib/libc.so.6) 5555558b0e87 ::dealloc (inlined) diff --git a/bench/run.sh b/bench/run.sh index e1d413e36c8..8bc702b0951 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -24,8 +24,6 @@ case $MODE in BASE= LOCK_WORKLOAD=${MOQ_BENCH_LOCK_WORKLOAD:-fanout} LOCK_WORKERS=${MOQ_BENCH_LOCK_WORKERS:-1 4 16} - # For frame-pointer profilers; the lock capture itself unwinds by DWARF (see profile.py). - export RUSTFLAGS="${RUSTFLAGS:+$RUSTFLAGS }-C force-frame-pointers=yes" ;; --runtime) BASE= @@ -500,7 +498,7 @@ run_lock_profile() { python3 "$ROOT/bench/profile.py" check generate_certificate - printf 'Building the relay with the profiling profile and frame pointers...\n' + printf 'Building the relay with the profiling profile...\n' CARGO_TARGET_DIR=$TARGET/profile cargo build --locked --profile profiling -p moq-relay -p moq-bench LOAD_BIN=$TARGET/profile/profiling/moq-bench HOST_BIN=$TARGET/profile/profiling/moq-bench-host diff --git a/quest/m1/perf/lock-profile.md b/quest/m1/perf/lock-profile.md index bf7c96b22c8..7b1d6388477 100644 --- a/quest/m1/perf/lock-profile.md +++ b/quest/m1/perf/lock-profile.md @@ -31,10 +31,11 @@ Shared by every mode: - Reuse `bench/run.sh`'s lifecycle (builds, relay PID, workload, host samples) rather than adding a second launcher. Select workload, duration, and mode through one configuration. -- Build the relay with `[profile.profiling]` and - `RUSTFLAGS="-C force-frame-pointers=yes"`: the release build `bench/run.sh` - uses has neither debug info nor frame pointers, so stacks would stop at the - libc frame. +- Build the relay with `[profile.profiling]`: the release build + `bench/run.sh` uses has no debug info to symbolize or unwind with. Add + `RUSTFLAGS="-C force-frame-pointers=yes"` only for a mode that unwinds by + frame pointers; lock wait unwinds by DWARF, so it builds without them + (decided 2026-10-12). - Capture the relay process and all its workers, never the load generator. Start after readiness and mark a steady window. Keep an uninstrumented control run to quantify profiler overhead. @@ -53,18 +54,21 @@ three crates plus five Prometheus families): - Measure from outside the process, aggregated by thread name and user stack. No instrumentation in `kio`, no new public API, no metrics. -- perf records each worker switch-out on `sched:sched_switch` with its - stack, and `--switch-events` adds the switch-in, so each off-CPU interval - has an exact duration; `/proc` schedstat supplies CPU (decided by the - maintainer 2026-10-12, replacing bpftrace and an eu-stack sampler; its - ptrace stops likely inflated waits, reading 0.15 wait/CPU on fanout-heavy - at 4 workers where exact intervals read 0.0013). It needs - `kernel.perf_event_paranoid=-1`, `kernel.kptr_restrict=0` and a readable - `/sys/kernel/tracing`, and refuses with the commands until they hold; the - nightly sets them with sudo. User stacks unwind by DWARF, because glibc's - frameless `syscall` wrapper hides `Mutex::lock_contended` from - frame-pointer unwinding. `perf script --inline` expands inlined frames such - as `Consumer::poll`. +- perf samples each worker switch-out with its stack, and `--switch-events` + adds the switch-in, so each off-CPU interval has an exact duration; + `/proc` schedstat supplies CPU (decided by the maintainer 2026-10-12, + replacing bpftrace and an eu-stack sampler; its ptrace stops likely + inflated waits, reading 0.15 wait/CPU on fanout-heavy at 4 workers where + exact intervals read 0.0013). It samples the `context-switches` software + event with a period of 1, not the `sched:sched_switch` tracepoint, since + it records the same switch-outs without tracefs access (decided + 2026-10-12). It needs `kernel.perf_event_paranoid` at 1 or lower for + kernel-context samples, and refuses with the command until then; the + nightly sets it with sudo. Kernel frames are discarded, so `kptr_restrict` + does not matter. User stacks unwind by DWARF, because glibc's frameless + `syscall` wrapper hides `Mutex::lock_contended` from frame-pointer + unwinding. `perf script --inline` expands inlined frames such as + `Consumer::poll`. - Count only stacks under the std mutex's contended path (`kio::Lock` is a `std::sync::Mutex`): futex waits also include allocator, condvar, and tokio waits, and a contended acquire that resolves while spinning never reaches From 3f02d0a18067b37c6c8c9f2be962ce2b883cb3c2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 12 Oct 2026 19:14:24 -0700 Subject: [PATCH 8/8] fix(bench): refuse dropped perf events and keep 16-worker stacks resolved Address review: refuse a recording with PERF_RECORD_LOST, close open waits at the window end on a monotonic clock, and run the profiler in its own process group so cleanup stops perf. A 2 MiB ring buffer stops drops at 16 workers, which needs perf_event_paranoid=-1. Restore frame pointers: without them perf's DWARF unwinder stops at the first relay frame. Report workers asleep for the whole window as idle instead of refusing. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly.yml | 4 +-- bench/README.md | 28 ++++++++++---------- bench/profile.py | 49 +++++++++++++++++++++-------------- bench/profile.test.py | 43 ++++++++++++++++++++---------- bench/relay.sh | 3 ++- bench/run.sh | 19 ++++++++------ quest/m1/perf/lock-profile.md | 20 +++++++------- 7 files changed, 101 insertions(+), 65 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index c22bb52a457..a9eb85d699e 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -91,10 +91,10 @@ jobs: if: ${{ !cancelled() }} run: nix develop --command just bench-audio - # perf samples worker switch-outs in kernel context; bench-lock refuses without it. + # perf samples worker switch-outs in kernel context into a large buffer; bench-lock refuses without it. - name: Allow perf kernel sampling if: ${{ !cancelled() }} - run: sudo sysctl -w kernel.perf_event_paranoid=1 + run: sudo sysctl -w kernel.perf_event_paranoid=-1 # Smoke-tests the capture path: exact off-CPU waits, resolved worker stacks and CPU. - name: Relay lock-wait profile diff --git a/bench/README.md b/bench/README.md index 288494c01ae..39d2e17d5fa 100644 --- a/bench/README.md +++ b/bench/README.md @@ -56,27 +56,29 @@ nix develop --command just bench-lock nix develop --command just bench-lock fanout-heavy "4 16" ``` -This builds the relay with `[profile.profiling]`, then runs the workload on -Tokio workers at each count twice: an unprofiled control, then a profiled run, -so the throughput and CPU difference shows the profiler's overhead. During the -profiled run's steady window, `perf record` samples every worker switch-out -(the `context-switches` event with a period of 1) with a DWARF-unwound stack, -plus each switch-in, so every off-CPU interval has an exact duration. -glibc's `syscall` wrapper keeps no frame pointer, so frame-pointer unwinding -would drop `Mutex::lock_contended`, which is why the stacks are unwound by -DWARF. An interval counts as a mutex wait only when the worker is blocked in +This builds the relay with `[profile.profiling]` and frame pointers, then runs +the workload on Tokio workers at each count twice: an unprofiled control, then +a profiled run, so the throughput and CPU difference shows the profiler's +overhead. During the profiled run's steady window, `perf record` samples every +worker switch-out (the `context-switches` event with a period of 1) with a +DWARF-unwound stack, plus each switch-in, so every off-CPU interval has an +exact duration. glibc's `syscall` wrapper keeps no frame pointer, so +frame-pointer unwinding alone would drop `Mutex::lock_contended`; without +frame pointers in the relay, perf's DWARF unwinder stops at the first relay +frame. Both are needed. An interval counts as a mutex wait only when the worker is blocked in the futex under std's `Mutex::lock_contended` (`kio::Lock`). Condvar, allocator and parking waits are counted separately, and spinning is excluded. `/proc` schedstat supplies CPU, so wait/CPU can exceed one. `lock/mutex.folded` holds the blocking stacks, weighted by microseconds waited, in flamegraph folded format. -Linux only. A switch-out is sampled in kernel context, which perf allows only -at `perf_event_paranoid` 1 or lower, so the recipe refuses before building -until then: +Linux only. A switch-out is sampled in kernel context, and 16 workers need a +larger ring buffer than the unprivileged locked-memory limit allows, so the +recipe refuses before building until `perf_event_paranoid` is -1. A recording +that still drops events is refused rather than reported. ```bash -sudo sysctl -w kernel.perf_event_paranoid=1 +sudo sysctl -w kernel.perf_event_paranoid=-1 ``` ## Workloads diff --git a/bench/profile.py b/bench/profile.py index d5bbbf684ad..23d61e90999 100644 --- a/bench/profile.py +++ b/bench/profile.py @@ -18,15 +18,16 @@ MUTEX = re.compile(r"(?:\bMutex>?::|^)lock_contended$") HEADER = re.compile(r"\s*(\d+)\s+(\d+)\.(\d+):\s+(.*)") FRAME = re.compile(r"\s+[0-9a-f]+\s+(.*?)\s+\((.*)\)$") -# A switch-out is sampled in kernel context, which perf allows only at paranoid <= 1. -SETUP = 'sudo sysctl -w kernel.perf_event_paranoid=1' +# A switch-out is sampled in kernel context (paranoid <= 1), and the ring buffer that keeps +# 16 workers from dropping samples exceeds the locked-memory limit unless paranoid is -1. +SETUP = 'sudo sysctl -w kernel.perf_event_paranoid=-1' def require(): - """Refuse to record unless perf may sample this user's processes in kernel context.""" + """Refuse to record unless perf may sample the relay in kernel context with a large buffer.""" paranoid = int(Path('/proc/sys/kernel/perf_event_paranoid').read_text()) - if paranoid > 1: - raise ValueError(f'perf needs kernel samples of the relay (perf_event_paranoid={paranoid}); run:\n{SETUP}') + if paranoid != -1: + raise ValueError(f'perf needs kernel samples and unlimited buffers (perf_event_paranoid={paranoid}); run:\n{SETUP}') def parse(text): @@ -37,7 +38,11 @@ def parse(text): tid, seconds, fraction, event = match.groups() frames = [] nanoseconds = int(seconds) * 1_000_000_000 + int(fraction.ljust(9, '0')) - records.append((tid, nanoseconds, event.strip().rstrip(':'), frames)) + event = event.strip().rstrip(':') + # A dropped switch would silently shorten or lose a wait. + if event.startswith('PERF_RECORD_LOST'): + raise ValueError(f'perf dropped events ({event}); the recording is incomplete') + records.append((tid, nanoseconds, event, frames)) elif frames is not None and (match := FRAME.match(line)): symbol, dso = match.groups() # Kernel frames all show the scheduler; the user stack names the wait. @@ -46,9 +51,8 @@ def parse(text): return records -def waits(records): +def waits(records, end): """Pair each switch-out stack with the thread's next switch-in for an exact off-CPU duration.""" - end = max((record[1] for record in records), default=0) pending, result = {}, [] for tid, at, event, frames in records: if event == EVENT: @@ -56,7 +60,7 @@ def waits(records): elif event.startswith('PERF_RECORD_SWITCH IN') and tid in pending: start, stack = pending.pop(tid) result.append((tid, at - start, stack)) - # A wait still open when recording stops counts up to the last event. + # A wait still open when recording stops counts up to the end of the window. result.extend((tid, end - start, stack) for tid, (start, stack) in pending.items()) return result @@ -70,14 +74,18 @@ def classify(stack): def reduce(names, intervals, before, after, window_ns): """Sum each worker's off-CPU time by class, as a share of its CPU time.""" - workers, folded = {}, Counter() + workers, idle, folded = {}, [], Counter() for tid, name in names.items(): mine = [(wait, stack) for owner, wait, stack in intervals if owner == tid] + cpu = after[tid] - before[tid] + # Asleep for the whole window, such as a worker with no connections. + if not mine and cpu == 0: + idle.append(name) + continue if not mine: - raise ValueError(f'worker {name} ({tid}) never switched out') + raise ValueError(f'worker {name} ({tid}) ran but never switched out') if not any(re.search(r'\bmoq_|\bkio::', frame) for _, stack in mine for frame in stack): - raise ValueError(f'worker {name} ({tid}) has no resolved MoQ frames; build with the profiling profile') - cpu = after[tid] - before[tid] + raise ValueError(f'worker {name} ({tid}) has no resolved MoQ frames; build with the profiling profile and frame pointers') if cpu <= 0: raise ValueError(f'worker {name} ({tid}) has no advancing CPU counter') total = Counter() @@ -89,7 +97,9 @@ def reduce(names, intervals, before, after, window_ns): workers[tid] = {'comm': name, 'switches': len(mine), 'cpu_ns': cpu, 'mutex_wait_ns': total['mutex'], 'other_futex_wait_ns': total['futex'], 'other_off_cpu_ns': total['other'], 'mutex_wait_per_cpu': total['mutex'] / cpu} - return {'version': 2, 'window_ns': window_ns, 'workers': workers, + if not workers: + raise ValueError('no worker ran during the window') + return {'version': 2, 'window_ns': window_ns, 'workers': workers, 'idle': sorted(idle), 'scope': 'exact off-CPU time blocked in a futex under std Mutex::lock_contended; spinning is excluded'}, folded @@ -111,15 +121,16 @@ def record(pid, delay, duration, directory): # A period of 1 samples every switch-out. The sample fires in the outgoing thread, so -p # never sees a worker's switch-in; --switch-events records it. glibc's syscall wrapper # keeps no frame pointer, so frame-pointer unwinding drops its caller, lock_contended; - # DWARF unwinding keeps it. + # DWARF unwinding keeps it. A monotonic clock lets open waits close at the window's end. + # The default 512 KiB buffer drops samples at 16 workers; 2 MiB does not. subprocess.run(['perf', 'record', '--quiet', '--no-buildid-cache', '-e', EVENT, '-c', '1', '--call-graph', 'dwarf', '--switch-events', - '-p', str(pid), '-o', str(data), '--', 'sleep', str(duration)], check=True) - after, window = schedstat(pid, names), time.monotonic_ns() - start + '-k', 'monotonic', '-m', '2M', '-p', str(pid), '-o', str(data), '--', 'sleep', str(duration)], check=True) + after, end = schedstat(pid, names), time.monotonic_ns() # Symbols come from the local binaries; never wait on a debuginfod server. - script = subprocess.run(['perf', 'script', '-i', str(data), '--show-switch-events', '--inline', '--ns', + script = subprocess.run(['perf', 'script', '-i', str(data), '--show-switch-events', '--show-lost-events', '--inline', '--ns', '-F', 'tid,time,event,ip,sym,dso'], capture_output=True, text=True, check=True, env=dict(os.environ, DEBUGINFOD_URLS='')) - result, folded = reduce(names, waits(parse(script.stdout)), before, after, window) + result, folded = reduce(names, waits(parse(script.stdout), end), before, after, end - start) (directory / 'lock.json').write_text(json.dumps(result, indent=2) + '\n') (directory / 'mutex.folded').write_text(''.join(f'{stack} {count}\n' for stack, count in folded.most_common())) diff --git a/bench/profile.test.py b/bench/profile.test.py index 848b0e46bcd..1dfdae109c0 100644 --- a/bench/profile.test.py +++ b/bench/profile.test.py @@ -9,6 +9,8 @@ FIXTURE = Path(__file__).with_name('profile.test.txt').read_text() IDLE, MUTEX, ALLOCATOR = '2152705', '2152709', '2152707' +# Every fixture wait closes, so the window end only has to follow the last event. +END = 271707 * 1_000_000_000 class Parse(unittest.TestCase): @@ -17,7 +19,7 @@ def test_recorded_switches_pair_into_exact_waits(self): self.assertEqual([event for _, _, event, _ in records], ['context-switches', 'PERF_RECORD_SWITCH IN'] * 3) # Kernel frames are dropped; inlined user frames are kept innermost first. self.assertFalse(any('__schedule' in frame for _, _, _, frames in records for frame in frames)) - stacks = {tid: (wait, stack) for tid, wait, stack in profile.waits(records)} + stacks = {tid: (wait, stack) for tid, wait, stack in profile.waits(records, END)} self.assertEqual(stacks[IDLE][0], 21661) self.assertEqual(stacks[MUTEX][0], 6315) self.assertEqual(stacks[ALLOCATOR][0], 13709) @@ -25,14 +27,19 @@ def test_recorded_switches_pair_into_exact_waits(self): self.assertEqual(stacks[MUTEX][1][:3], ['syscall', 'std::sys::pal::unix::futex::futex_wait', '::lock_contended']) - def test_open_wait_runs_to_the_last_event(self): - records = [('1', 10, 'context-switches', ['a']), ('2', 50, 'PERF_RECORD_SWITCH IN', [])] - self.assertEqual(profile.waits(records), [('1', 40, ['a'])]) + def test_open_wait_runs_to_the_window_end(self): + # Even when the switch-out is the last recorded event. + records = [('2', 5, 'PERF_RECORD_SWITCH IN', []), ('1', 10, 'context-switches', ['a'])] + self.assertEqual(profile.waits(records, 50), [('1', 40, ['a'])]) + + def test_lost_events_refused(self): + with self.assertRaises(ValueError): + profile.parse(FIXTURE + '2152705 271706.914950000: PERF_RECORD_LOST lost 3\n') class Classify(unittest.TestCase): def test_recorded_stacks(self): - stacks = {tid: stack for tid, _, stack in profile.waits(profile.parse(FIXTURE))} + stacks = {tid: stack for tid, _, stack in profile.waits(profile.parse(FIXTURE), END)} self.assertEqual(profile.classify(stacks[MUTEX]), 'mutex') self.assertEqual(profile.classify(stacks[ALLOCATOR]), 'futex') self.assertEqual(profile.classify(stacks[IDLE]), 'other') @@ -43,7 +50,7 @@ def test_spinning_is_not_a_wait(self): class Reduce(unittest.TestCase): def test_recorded_waits_by_class_over_cpu(self): - intervals = profile.waits(profile.parse(FIXTURE)) + intervals = profile.waits(profile.parse(FIXTURE), END) names = {IDLE: 'moq-quic-0', MUTEX: 'moq-quic-1', ALLOCATOR: 'moq-quic-2'} before = dict.fromkeys(names, 0) result, folded = profile.reduce(names, intervals, before, dict.fromkeys(names, 3000), 1) @@ -56,13 +63,24 @@ def test_recorded_waits_by_class_over_cpu(self): self.assertTrue(stack.endswith('lock_contended;std::sys::pal::unix::futex::futex_wait;syscall')) self.assertEqual(weight, 6) - def test_silent_or_idle_worker_refused(self): + def test_silent_worker_refused(self): intervals = [('1', 5, ['moq_net::serve'])] + # Worker 2 ran but recorded no switch-out, so the capture missed it. with self.assertRaises(ValueError): profile.reduce({'1': 'moq-quic-0', '2': 'moq-quic-1'}, intervals, {'1': 0, '2': 0}, {'1': 1, '2': 1}, 1) + # Switched out without a CPU advance: the counter is broken. with self.assertRaises(ValueError): profile.reduce({'1': 'moq-quic-0'}, intervals, {'1': 5}, {'1': 5}, 1) + def test_sleeping_worker_reported_idle(self): + # Light load at 16 workers leaves some without connections, asleep for the whole window. + intervals = [('1', 5, ['moq_net::serve'])] + result, _ = profile.reduce({'1': 'moq-quic-0', '2': 'moq-quic-1'}, intervals, {'1': 0, '2': 7}, {'1': 1, '2': 7}, 1) + self.assertEqual(list(result['workers']), ['1']) + self.assertEqual(result['idle'], ['moq-quic-1']) + with self.assertRaises(ValueError): + profile.reduce({'2': 'moq-quic-1'}, [], {'2': 7}, {'2': 7}, 1) + def test_unresolved_stacks_refused_per_worker(self): # One resolved worker must not vouch for another whose unwinding failed. intervals = [('1', 5, ['moq_net::serve']), ('2', 5, ['[unknown]', 'syscall'])] @@ -75,14 +93,13 @@ def check(self, paranoid): with patch.object(profile.Path, 'read_text', return_value=paranoid + '\n'): profile.require() - def test_kernel_sampling_refusal_names_the_command(self): - # Ubuntu's default is 4; 2 already forbids kernel samples. - for paranoid in ['4', '2']: + def test_refusal_names_the_command(self): + # Ubuntu's default is 4; 2 forbids kernel samples; 1 and 0 cap the locked buffer. + for paranoid in ['4', '2', '1', '0']: with self.assertRaises(ValueError) as refused: self.check(paranoid) - self.assertIn('sudo sysctl -w kernel.perf_event_paranoid=1', str(refused.exception)) - for paranoid in ['1', '0', '-1']: - self.check(paranoid) + self.assertIn('sudo sysctl -w kernel.perf_event_paranoid=-1', str(refused.exception)) + self.check('-1') if __name__ == '__main__': diff --git a/bench/relay.sh b/bench/relay.sh index 2c643fb9156..d14df4e638c 100755 --- a/bench/relay.sh +++ b/bench/relay.sh @@ -234,7 +234,8 @@ run_workload() { HOST_PID=$! if [[ -n $PROFILE_LOCK ]]; then # The second half of the host window, matching the summary's throughput window. - python3 "$ROOT/bench/profile.py" record "$relay_pid" "$directory/lock" --delay 5 --duration 5 \ + # Its own process group, so cleanup also stops perf and its workload. + setsid python3 "$ROOT/bench/profile.py" record "$relay_pid" "$directory/lock" --delay 5 --duration 5 \ >"$directory/lock.log" 2>&1 & PROFILE_PID=$! fi diff --git a/bench/run.sh b/bench/run.sh index 8bc702b0951..d1fc701c7db 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -24,6 +24,8 @@ case $MODE in BASE= LOCK_WORKLOAD=${MOQ_BENCH_LOCK_WORKLOAD:-fanout} LOCK_WORKERS=${MOQ_BENCH_LOCK_WORKERS:-1 4 16} + # Without frame pointers, perf's DWARF unwinder stops at the first relay frame. + export RUSTFLAGS="${RUSTFLAGS:+$RUSTFLAGS }-C force-frame-pointers=yes" ;; --runtime) BASE= @@ -74,7 +76,8 @@ STDERR_PID= cleanup() { if [[ -n $PROFILE_PID ]]; then - kill "$PROFILE_PID" 2>/dev/null || true + # setsid made the profiler a group leader without forking, so its PID is the group. + kill -- -"$PROFILE_PID" 2>/dev/null || true wait "$PROFILE_PID" 2>/dev/null || true fi if [[ -n $LOAD_PID ]]; then @@ -498,15 +501,15 @@ run_lock_profile() { python3 "$ROOT/bench/profile.py" check generate_certificate - printf 'Building the relay with the profiling profile...\n' + printf 'Building the relay with the profiling profile and frame pointers...\n' CARGO_TARGET_DIR=$TARGET/profile cargo build --locked --profile profiling -p moq-relay -p moq-bench LOAD_BIN=$TARGET/profile/profiling/moq-bench HOST_BIN=$TARGET/profile/profiling/moq-bench-host local relay=$TARGET/profile/profiling/moq-relay printf '\nLock wait: %s, tokio workers; each profiled run follows an unprofiled control\n' "$LOCK_WORKLOAD" - printf '%-8s %8s %13s %13s %10s %10s %13s %9s\n' \ - workers switches control-fps profiled-fps control-CPU profiled-CPU mutex-wait-ms wait/CPU + printf '%-8s %5s %8s %13s %13s %10s %10s %13s %9s\n' \ + workers idle switches control-fps profiled-fps control-CPU profiled-CPU mutex-wait-ms wait/CPU local workers for workers in $LOCK_WORKERS; do run_workload "control-$workers" "$relay" "$LOCK_WORKLOAD" tokio-workers "$workers" @@ -514,16 +517,16 @@ run_lock_profile() { run_workload "lock-$workers" "$relay" "$LOCK_WORKLOAD" tokio-workers "$workers" PROFILE_LOCK= jq -r -s '[ - .[0].recv_fps, .[1].recv_fps, .[0].cpu_cores, .[1].cpu_cores, + .[0].recv_fps, .[1].recv_fps, .[0].cpu_cores, .[1].cpu_cores, (.[2].idle | length), ([.[2].workers[].switches] | min), ([.[2].workers[].mutex_wait_ns] | add / 1000000), ([.[2].workers[].mutex_wait_ns] | add) / ([.[2].workers[].cpu_ns] | add) ] | @tsv' "$RUN/relay/control-$workers/$LOCK_WORKLOAD/summary.json" \ "$RUN/relay/lock-$workers/$LOCK_WORKLOAD/summary.json" \ "$RUN/relay/lock-$workers/$LOCK_WORKLOAD/lock/lock.json" | - while IFS=$'\t' read -r control profiled control_cpu profiled_cpu switches wait share; do - printf '%-8s %8s %13.1f %13.1f %10.3f %10.3f %13.1f %9.4f\n' \ - "$workers" "$switches" "$control" "$profiled" "$control_cpu" "$profiled_cpu" "$wait" "$share" + while IFS=$'\t' read -r control profiled control_cpu profiled_cpu idle switches wait share; do + printf '%-8s %5s %8s %13.1f %13.1f %10.3f %10.3f %13.1f %9.4f\n' \ + "$workers" "$idle" "$switches" "$control" "$profiled" "$control_cpu" "$profiled_cpu" "$wait" "$share" done done printf '\nMutex stacks (flamegraph folded format): %s/relay/lock-*/%s/lock/mutex.folded\n' "$RUN" "$LOCK_WORKLOAD" diff --git a/quest/m1/perf/lock-profile.md b/quest/m1/perf/lock-profile.md index 7b1d6388477..f2e997b4090 100644 --- a/quest/m1/perf/lock-profile.md +++ b/quest/m1/perf/lock-profile.md @@ -31,11 +31,11 @@ Shared by every mode: - Reuse `bench/run.sh`'s lifecycle (builds, relay PID, workload, host samples) rather than adding a second launcher. Select workload, duration, and mode through one configuration. -- Build the relay with `[profile.profiling]`: the release build - `bench/run.sh` uses has no debug info to symbolize or unwind with. Add - `RUSTFLAGS="-C force-frame-pointers=yes"` only for a mode that unwinds by - frame pointers; lock wait unwinds by DWARF, so it builds without them - (decided 2026-10-12). +- Build the relay with `[profile.profiling]` and + `RUSTFLAGS="-C force-frame-pointers=yes"`: the release build `bench/run.sh` + uses has neither debug info nor frame pointers, so stacks would stop at the + libc frame. Lock wait unwinds by DWARF, but without frame pointers perf's + unwinder still stops at the first relay frame (checked 2026-10-12). - Capture the relay process and all its workers, never the load generator. Start after readiness and mark a steady window. Keep an uninstrumented control run to quantify profiler overhead. @@ -62,10 +62,12 @@ three crates plus five Prometheus families): exact intervals read 0.0013). It samples the `context-switches` software event with a period of 1, not the `sched:sched_switch` tracepoint, since it records the same switch-outs without tracefs access (decided - 2026-10-12). It needs `kernel.perf_event_paranoid` at 1 or lower for - kernel-context samples, and refuses with the command until then; the - nightly sets it with sudo. Kernel frames are discarded, so `kptr_restrict` - does not matter. User stacks unwind by DWARF, because glibc's frameless + 2026-10-12). It needs `kernel.perf_event_paranoid=-1`: kernel-context + samples need 1 or lower, and the 2 MiB ring buffer that keeps 16 workers + from dropping samples exceeds the locked-memory limit above -1. It refuses + with the command until then, and refuses any recording that dropped + events; the nightly sets it with sudo. Kernel frames are discarded, so + `kptr_restrict` does not matter. User stacks unwind by DWARF, because glibc's frameless `syscall` wrapper hides `Mutex::lock_contended` from frame-pointer unwinding. `perf script --inline` expands inlined frames such as `Consumer::poll`.