From ec4803af07661df0ce8e4da0694e12a836f07bb9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 11:00:32 -0700 Subject: [PATCH 1/7] quest: claim m1/ffi-bench Co-Authored-By: Claude Opus 5.5 From 378e8f3ea267f569851d6b9214326f845bf5cdcb Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 12:28:00 -0700 Subject: [PATCH 2/7] bench: track binding FFI overhead nightly against Rust baselines `just bench-ffi` runs the same in-process publish/subscribe rounds in Go, Python, and Kotlin and in two Rust baselines (moq-ffi's API called from Rust, and moq-net), then prints each binding's ratio. The nightly replaces the loopback-QUIC Go smoke with it and keeps the JSON results. Co-Authored-By: Claude Opus 5.5 --- .github/justfile | 1 + .github/workflows/nightly.yml | 14 +- Cargo.lock | 1 + bench/README.md | 28 +- bench/ffi.py | 97 ++++++ bench/ffi.sh | 15 + bench/ffi.test.py | 54 +++ go/wrapper/runtime_bench_test.go | 324 +++++++++++------- justfile | 4 + kt/justfile | 4 + kt/moq/src/jvmTest/kotlin/dev/moq/FfiBench.kt | 117 +++++++ py/justfile | 7 + py/moq-rs/benches/ffi.py | 123 +++++++ quest/m1/README.md | 1 - quest/m1/bench-site.md | 6 +- quest/m1/boltffi.md | 2 +- quest/m1/ffi-bench-wrappers.md | 10 +- quest/m1/ffi-bench.md | 58 ---- quest/m1/ffi-bytes.md | 10 +- quest/m1/ffi-poll.md | 6 +- quest/m2/ffi-bytes-return.md | 2 +- quest/m2/ffi-size-profile.md | 6 +- rs/moq-ffi/Cargo.toml | 4 + rs/moq-ffi/src/bench.rs | 276 +++++++++++++++ rs/moq-ffi/src/lib.rs | 4 + sh/go/bench.sh | 2 +- sh/kt/bench.sh | 14 + 27 files changed, 972 insertions(+), 218 deletions(-) create mode 100644 bench/ffi.py create mode 100755 bench/ffi.sh create mode 100644 bench/ffi.test.py create mode 100644 kt/moq/src/jvmTest/kotlin/dev/moq/FfiBench.kt create mode 100644 py/moq-rs/benches/ffi.py delete mode 100644 quest/m1/ffi-bench.md create mode 100644 rs/moq-ffi/src/bench.rs create mode 100755 sh/kt/bench.sh diff --git a/.github/justfile b/.github/justfile index 7df3cb98190..979945d743f 100644 --- a/.github/justfile +++ b/.github/justfile @@ -15,6 +15,7 @@ test: python3 bench/summary.test.py python3 bench/relay.test.py python3 bench/profile.test.py + python3 bench/ffi.test.py python3 sh/gh/gst-formula.test.py python3 sh/changed.test.py python3 sh/dispatch-query.test.py diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index c39b95e165b..3b6b5606dcd 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -62,8 +62,10 @@ jobs: - name: Features recipe: rs features profile: ci - - name: Go publisher runtime benchmark - recipe: go bench -benchtime=1x + - name: Binding benchmark + recipe: bench-ffi + artifact: bench-ffi + artifact-path: target/bench/ffi - name: Audio grouping relay benchmark recipe: bench-audio # Smoke-tests the capture path: exact off-CPU waits, resolved worker stacks and CPU. @@ -127,6 +129,14 @@ jobs: env: NEXTEST_PROFILE: ${{ matrix.profile || 'default' }} + - name: Preserve results + if: ${{ matrix.artifact && !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ${{ matrix.artifact }} + path: ${{ matrix.artifact-path }} + retention-days: 45 + tests: name: Tests (${{ matrix.recipe }}) runs-on: ubuntu-24.04-arm diff --git a/Cargo.lock b/Cargo.lock index 3bf788a746d..83fa3150ceb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4461,6 +4461,7 @@ dependencies = [ "getrandom 0.4.3", "hang", "kio 0.6.2", + "libc", "moq-audio", "moq-json", "moq-mux", diff --git a/bench/README.md b/bench/README.md index 65ce0278517..358a0c8c94b 100644 --- a/bench/README.md +++ b/bench/README.md @@ -131,16 +131,26 @@ hide a relay improvement behind generator saturation. CPU efficiency remains withheld until offered load and delivery/loss are comparable. Zero-baseline metrics retain absolute deltas when their percentage ratio is undefined. -Measure delivered payload through the Go bindings and a local QUIC relay: +Measure what each binding adds over Rust: ```bash -nix develop --command just go bench -benchtime=3s -count=3 +nix develop --command just bench-ffi ``` -This independently sweeps 1/4 publishers and 1/4 subscribers per publisher, -checks every received frame's payload and timestamp, and acknowledges each batch before publishing the -next. Reported MB/s counts delivery to every subscriber. The optimized native -library, relay and clients share the process, so pin the benchmark process with -`taskset` to compare one CPU with multiple CPUs. Compare identical CPU sets and -traffic shapes across revisions; the nightly smoke runs each shape once without -a timing threshold. +Every runner (Rust, Go, Python, Kotlin) runs the same in-process rounds with no +network: each publisher writes one group from the driving thread while every +subscriber reads it on its own task, and a round waits for every subscriber. It +sweeps frame sizes from 100 B to 1 MiB at one publisher and subscriber, then +1/4 publishers by 1/4 subscribers at 1 KiB, and times two sync calls +(`sequence`, `info`). Each case reports delivered MB/s and process CPU (user + +system, every thread) per MB. `MOQ_BENCH_SECONDS` sets each case's time budget +(default 1). + +Each runner writes `target/bench/ffi/.json`; `bench/ffi.py` combines +them into a table and `report.json`. Ratios are against `rust-ffi`, moq-ffi's +API called from Rust without crossing the FFI, so a wrapper's ratio is what its +boundary and wrapper cost. `rust-net` runs the same rounds on moq-net, so its +ratio is what moq-ffi's handles and runtime cost. Ratios hold only within one +machine. The nightly runs it, prints the table in the job summary, and keeps the +results as the `bench-ffi` artifact. Each language also has its own recipe +(`just go bench`, `just py bench`, `just kt bench`). diff --git a/bench/ffi.py b/bench/ffi.py new file mode 100644 index 00000000000..7f7703aa6c4 --- /dev/null +++ b/bench/ffi.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Combine the binding benchmark's per-runner results into one table and report. + +Each runner writes `/.json`, a list of {name, unit, value}. The table shows +every runner beside its ratio to the `rust-ffi` baseline, run on the same machine, which +cancels most of the runner's own speed. `report.json` holds every value and ratio, named +`/`, for the trend history. +""" +import argparse +import json +import os +from pathlib import Path + +BASELINE = 'rust-ffi' +# Baselines first, then the wrappers in a stable order. +ORDER = ('rust-net', 'rust-ffi') +SMALLER_IS_BETTER = {'ms/MB', 'ns'} + + +def load(directory): + runners = {} + for path in sorted(Path(directory).glob('*.json')): + if path.name == 'report.json': + continue + runners[path.stem] = {entry['name']: entry for entry in json.loads(path.read_text())} + if BASELINE not in runners: + raise ValueError(f'missing the {BASELINE} baseline in {directory}') + return runners + + +def order(runners): + return sorted(runners, key=lambda r: (ORDER.index(r) if r in ORDER else len(ORDER), r)) + + +def ratio(entry, baseline): + """How many times worse than the baseline: above 1 is slower or costlier.""" + if baseline is None or not baseline['value'] or not entry['value']: + return None + if entry['unit'] in SMALLER_IS_BETTER: + return entry['value'] / baseline['value'] + return baseline['value'] / entry['value'] + + +def report(runners): + entries = [] + baseline = runners[BASELINE] + for runner in order(runners): + for name, entry in runners[runner].items(): + entries.append({'name': f'{runner}/{name}', 'unit': entry['unit'], 'value': entry['value']}) + if runner != BASELINE and (r := ratio(entry, baseline.get(name))) is not None: + entries.append({'name': f'{runner}/{name} vs {BASELINE}', 'unit': 'x', 'value': r}) + return entries + + +def table(runners): + columns = order(runners) + names = list(dict.fromkeys(name for runner in columns for name in runners[runner])) + baseline = runners[BASELINE] + lines = [ + '| benchmark | unit | ' + ' | '.join(columns) + ' |', + '|---|---|' + '---|' * len(columns), + ] + for name in names: + unit = next(runners[r][name]['unit'] for r in columns if name in runners[r]) + cells = [] + for runner in columns: + entry = runners[runner].get(name) + if entry is None: + cells.append('') + continue + cell = f"{entry['value']:.4g}" + if runner != BASELINE and (r := ratio(entry, baseline.get(name))) is not None: + cell += f' ({r:.2f}x)' + cells.append(cell) + lines.append(f'| {name} | {unit} | ' + ' | '.join(cells) + ' |') + return '\n'.join(lines) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('directory') + args = parser.parse_args() + runners = load(args.directory) + (Path(args.directory) / 'report.json').write_text(json.dumps(report(runners), indent=2)) + markdown = ( + f'## Binding benchmark\n\nRatios are against `{BASELINE}`; above 1x is slower or costlier.\n\n' + + table(runners) + + '\n' + ) + print(markdown) + if summary := os.environ.get('GITHUB_STEP_SUMMARY'): + with open(summary, 'a') as f: + f.write(markdown) + + +if __name__ == '__main__': + main() diff --git a/bench/ffi.sh b/bench/ffi.sh new file mode 100755 index 00000000000..ccfd9badd77 --- /dev/null +++ b/bench/ffi.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Run the binding benchmark: the Rust baselines and every wrapper's runner on this +# machine, then the ratio table. Ratios only hold within one machine, so stale +# results from an earlier run are cleared first. +set -euo pipefail +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +export MOQ_BENCH_DIR="${MOQ_BENCH_DIR:-$root/target/bench/ffi}" +rm -rf "$MOQ_BENCH_DIR" + +cd "$root" +cargo test --locked --release -p moq-ffi --lib bench::benchmark_ffi -- --ignored --nocapture +just go bench -benchtime="${MOQ_BENCH_SECONDS:-1}s" +just py bench +just kt bench +python3 bench/ffi.py "$MOQ_BENCH_DIR" diff --git a/bench/ffi.test.py b/bench/ffi.test.py new file mode 100644 index 00000000000..029d5a23824 --- /dev/null +++ b/bench/ffi.test.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path + +spec = importlib.util.spec_from_file_location('ffi', Path(__file__).with_name('ffi.py')) +ffi = importlib.util.module_from_spec(spec) +spec.loader.exec_module(ffi) + + +def write(directory, runner, entries): + (Path(directory) / f'{runner}.json').write_text( + json.dumps([{'name': n, 'unit': u, 'value': v} for n, u, v in entries]) + ) + + +class FfiTest(unittest.TestCase): + def runners(self, directory): + write(directory, 'rust-ffi', [('throughput/a', 'MB/s', 100.0), ('cpu/a', 'ms/MB', 2.0)]) + write(directory, 'rust-net', [('throughput/a', 'MB/s', 200.0)]) + write(directory, 'python', [('throughput/a', 'MB/s', 25.0), ('cpu/a', 'ms/MB', 8.0)]) + return ffi.load(directory) + + def test_ratio_is_worse_above_one(self): + with tempfile.TemporaryDirectory() as directory: + report = {e['name']: e['value'] for e in ffi.report(self.runners(directory))} + self.assertEqual(report['python/throughput/a vs rust-ffi'], 4.0) + self.assertEqual(report['python/cpu/a vs rust-ffi'], 4.0) + self.assertEqual(report['rust-net/throughput/a vs rust-ffi'], 0.5) + self.assertNotIn('rust-ffi/throughput/a vs rust-ffi', report) + + def test_table_orders_baselines_first(self): + with tempfile.TemporaryDirectory() as directory: + table = ffi.table(self.runners(directory)) + self.assertTrue(table.splitlines()[0].startswith('| benchmark | unit | rust-net | rust-ffi | python |')) + self.assertIn('| throughput/a | MB/s | 200 (0.50x) | 100 | 25 (4.00x) |', table) + + def test_missing_baseline_fails(self): + with tempfile.TemporaryDirectory() as directory: + write(directory, 'python', [('throughput/a', 'MB/s', 25.0)]) + with self.assertRaises(ValueError): + ffi.load(directory) + + def test_skips_its_own_report(self): + with tempfile.TemporaryDirectory() as directory: + self.runners(directory) + (Path(directory) / 'report.json').write_text('[]') + self.assertNotIn('report', ffi.load(directory)) + + +if __name__ == '__main__': + unittest.main() diff --git a/go/wrapper/runtime_bench_test.go b/go/wrapper/runtime_bench_test.go index 26b14335940..7971a345a4e 100644 --- a/go/wrapper/runtime_bench_test.go +++ b/go/wrapper/runtime_bench_test.go @@ -3,137 +3,227 @@ package moq_test import ( "bytes" "context" + "encoding/json" "fmt" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "syscall" "testing" "time" "moq.dev/moq" ) -// BenchmarkPublisher measures delivered payload through real QUIC sessions and -// generated Go callbacks. Each round waits for every subscriber, so dropping -// groups or merely buffering writes cannot improve the reported throughput. -func BenchmarkPublisher(b *testing.B) { - for _, publishers := range []int{1, 4} { - for _, subscribers := range []int{1, 4} { - b.Run(fmt.Sprintf("publishers=%d/subscribers=%d", publishers, subscribers), func(b *testing.B) { - ctx, cancel := context.WithCancel(context.Background()) - server, err := moq.Listen(ctx, "127.0.0.1:0", moq.WithTLSGenerate("localhost")) - if err != nil { - cancel() - b.Fatal(err) +// benchCase is one throughput case of the binding benchmark (see bench/README.md): +// publishers tracks, each read by subscribers consumers. +type benchCase struct{ size, publishers, subscribers int } + +// Frame sizes at one publisher and subscriber, then fan-out at 1 KiB. +var benchCases = []benchCase{ + {100, 1, 1}, {1024, 1, 1}, {64 * 1024, 1, 1}, {1024 * 1024, 1, 1}, + {1024, 1, 4}, {1024, 4, 1}, {1024, 4, 4}, +} + +// frames per group, holding a group under moq-net's 32 MB group cache. +func (c benchCase) frames() int { return min(max((8<<20)/c.size, 1), 64) } + +func (c benchCase) name() string { + return fmt.Sprintf("size=%d/pub=%d/sub=%d", c.size, c.publishers, c.subscribers) +} + +// benchResult is one entry of the shared JSON report. +type benchResult struct { + Name string `json:"name"` + Unit string `json:"unit"` + Value float64 `json:"value"` +} + +// benchResults holds each benchmark's latest run: testing.B reruns a benchmark with a +// growing b.N, and the last run is the one it reports. +var benchResults sync.Map + +func benchRecord(name, unit string, value float64) { + benchResults.Store(name, benchResult{name, unit, value}) +} + +func cpuTime() time.Duration { + var usage syscall.Rusage + if err := syscall.Getrusage(syscall.RUSAGE_SELF, &usage); err != nil { + panic(err) + } + return time.Duration(usage.Utime.Nano() + usage.Stime.Nano()) +} + +// TestMain writes BenchmarkFFI's results to $MOQ_BENCH_DIR/go.json after the run. +func TestMain(m *testing.M) { + code := m.Run() + if dir := os.Getenv("MOQ_BENCH_DIR"); dir != "" && code == 0 { + var results []benchResult + benchResults.Range(func(_, value any) bool { + results = append(results, value.(benchResult)) + return true + }) + sort.Slice(results, func(i, j int) bool { return results[i].Name < results[j].Name }) + data, err := json.MarshalIndent(results, "", " ") + if err == nil { + err = os.MkdirAll(dir, 0o755) + } + if err == nil { + err = os.WriteFile(filepath.Join(dir, "go.json"), data, 0o644) + } + if err != nil { + fmt.Fprintln(os.Stderr, err) + code = 1 + } + } + os.Exit(code) +} + +// BenchmarkFFI is the Go runner of the binding benchmark: the same in-process rounds as +// every wrapper, so dividing by the Rust baselines isolates what Go adds. +func BenchmarkFFI(b *testing.B) { + for _, c := range benchCases { + b.Run("throughput/"+c.name(), func(b *testing.B) { benchThroughput(b, c) }) + } + b.Run("call/sequence", func(b *testing.B) { + track := benchTrack(b, "calls") + producer, err := track.AppendGroup() + if err != nil { + b.Fatal(err) + } + group, err := producer.Consume() + if err != nil { + b.Fatal(err) + } + b.Cleanup(group.Cancel) + benchCall(b, func() { group.Sequence() }) + }) + b.Run("call/info", func(b *testing.B) { + consumer, err := benchTrack(b, "calls").Consume(nil) + if err != nil { + b.Fatal(err) + } + b.Cleanup(consumer.Cancel) + benchCall(b, func() { + if _, err := consumer.Info(); err != nil { + b.Fatal(err) + } + }) + }) +} + +func benchTrack(b *testing.B, name string) *moq.TrackProducer { + broadcast, err := moq.NewBroadcastProducer() + if err != nil { + b.Fatal(err) + } + b.Cleanup(func() { _ = broadcast.Close() }) + track, err := broadcast.PublishTrack(name, nil) + if err != nil { + b.Fatal(err) + } + return track +} + +func benchCall(b *testing.B, call func()) { + b.ResetTimer() + for range b.N { + call() + } + benchRecord(strings.TrimPrefix(b.Name(), "BenchmarkFFI/"), "ns", float64(b.Elapsed().Nanoseconds())/float64(b.N)) +} + +// benchThroughput runs one round per iteration: each publisher writes a group from this +// goroutine in turn while every consumer reads it on its own goroutine. A round waits for +// every consumer, so buffering writes cannot improve the reported throughput. +func benchThroughput(b *testing.B, c benchCase) { + ctx, cancel := context.WithCancel(context.Background()) + b.Cleanup(cancel) + frames := c.frames() + tracks := make([]*moq.TrackProducer, c.publishers) + consumers := make([]*moq.TrackConsumer, 0, c.publishers*c.subscribers) + for p := range tracks { + tracks[p] = benchTrack(b, fmt.Sprint(p)) + for range c.subscribers { + consumer, err := tracks[p].Consume(nil) + if err != nil { + b.Fatal(err) + } + b.Cleanup(consumer.Cancel) + consumers = append(consumers, consumer) + } + } + payload := bytes.Repeat([]byte{0x5a}, c.size) + delivered := int64(len(consumers) * frames * c.size) + b.SetBytes(delivered) + + round := func(n int) error { + done := make(chan error, len(consumers)) + for _, consumer := range consumers { + go func() { + group, err := consumer.RecvGroup(ctx) + if err == nil && group == nil { + err = fmt.Errorf("track ended") } - served := make(chan error, 1) - go func() { served <- server.Serve(ctx) }() - b.Cleanup(func() { cancel(); _ = server.Close(); <-served }) - url := "https://" + server.LocalAddr() - dial := func() *moq.Client { - client, err := moq.Dial(ctx, url, moq.WithTLSVerify(false)) - if err != nil { - b.Fatal(err) - } - b.Cleanup(func() { _ = client.Close() }) - return client + if err != nil { + done <- err + return } - tracks := make([]*moq.TrackProducer, publishers) - consumers := make([]*moq.TrackConsumer, 0, publishers*subscribers) - for p := range tracks { - client := dial() - name := fmt.Sprintf("runtime/%d", p) - broadcast, err := client.CreateBroadcast(name) - if err != nil { - b.Fatal(err) - } - b.Cleanup(func() { _ = broadcast.Close() }) - tracks[p], err = broadcast.PublishTrack("payload", nil) + defer group.Cancel() + for f := range frames { + frame, err := group.ReadFrame(ctx) if err != nil { - b.Fatal(err) - } - if err := broadcast.Announce(moq.Route{}); err != nil { - b.Fatal(err) + done <- err + return } - for s := 0; s < subscribers; s++ { - subscriber := dial() - announced, err := subscriber.AnnouncedBroadcast(name) - if err != nil { - b.Fatal(err) - } - b.Cleanup(announced.Cancel) - remote, err := announced.Available(ctx) - if err != nil { - b.Fatal(err) - } - consumer, err := remote.SubscribeTrack(ctx, "payload", nil) - if err != nil { - b.Fatal(err) - } - b.Cleanup(consumer.Cancel) - consumers = append(consumers, consumer) + want := time.Duration(n*frames+f) * time.Microsecond + if frame == nil || len(frame.Payload) != c.size || frame.Timestamp == nil || *frame.Timestamp != want { + done <- fmt.Errorf("missing, reordered, or truncated frame %d of round %d", f, n) + return } } - const frames = 64 - payload := bytes.Repeat([]byte{0x5a}, 1024) - b.SetBytes(int64(publishers * subscribers * frames * len(payload))) - b.ResetTimer() - for n := 0; n < b.N; n++ { - done := make(chan error, len(tracks)+len(consumers)) - for _, consumer := range consumers { - go func(consumer *moq.TrackConsumer) { - group, err := consumer.RecvGroup(ctx) - if err != nil { - done <- err - return - } - if group == nil { - done <- fmt.Errorf("track ended before batch") - return - } - defer group.Cancel() - for f := 0; f < frames; f++ { - frame, err := group.ReadFrame(ctx) - if err != nil { - done <- err - return - } - want := time.Duration(n*frames+f) * time.Millisecond - if frame == nil || frame.Timestamp == nil || *frame.Timestamp != want || !bytes.Equal(frame.Payload, payload) { - done <- fmt.Errorf("missing, reordered, or corrupt frame %d", f) - return - } - } - done <- nil - }(consumer) - } - for _, track := range tracks { - go func(track *moq.TrackProducer) { - group, err := track.AppendGroup() - if err != nil { - done <- err - return - } - for f := 0; f < frames; f++ { - if err := group.WriteFrame(moq.Frame{Payload: payload, Timestamp: ts(time.Duration(n*frames+f) * time.Millisecond)}); err != nil { - done <- err - return - } - } - done <- group.Finish() - }(track) - } - var failure error - for range len(tracks) + len(consumers) { - if err := <-done; err != nil { - if failure == nil { - failure = err - } - cancel() - } - } - if failure != nil { - b.Fatal(failure) - } + done <- nil + }() + } + for _, track := range tracks { + group, err := track.AppendGroup() + if err != nil { + return err + } + for f := range frames { + timestamp := time.Duration(n*frames+f) * time.Microsecond + if err := group.WriteFrame(moq.Frame{Payload: payload, Timestamp: ×tamp}); err != nil { + return err } - }) + } + if err := group.Finish(); err != nil { + return err + } + } + var failure error + for range consumers { + if err := <-done; err != nil && failure == nil { + failure = err + cancel() + } + } + return failure + } + + b.ResetTimer() + cpu := cpuTime() + for n := range b.N { + if err := round(n); err != nil { + b.Fatal(err) } } + mb := float64(int64(b.N)*delivered) / 1e6 + cpuPerMB := (cpuTime() - cpu).Seconds() * 1e3 / mb + b.ReportMetric(cpuPerMB, "cpu-ms/MB") + name := c.name() + benchRecord("throughput/"+name, "MB/s", mb/b.Elapsed().Seconds()) + benchRecord("cpu/"+name, "ms/MB", cpuPerMB) } diff --git a/justfile b/justfile index 55111153e6f..1b926cd1fc8 100644 --- a/justfile +++ b/justfile @@ -46,6 +46,10 @@ dev: bench $BASE="": bench/run.sh "$BASE" +# Benchmark the Rust, Go, Python, and Kotlin bindings against two Rust baselines. +bench-ffi: + bench/ffi.sh + # Measure audio group sizes across relay publisher/subscriber and fanout shapes. bench-audio: bench/run.sh --audio diff --git a/kt/justfile b/kt/justfile index 2a3994710a6..5ef639e0768 100644 --- a/kt/justfile +++ b/kt/justfile @@ -21,6 +21,10 @@ check: generate: bash ../sh/kt/generate.sh +# Run the binding benchmark's Kotlin runner (see bench/README.md) on a release build. +bench: + bash ../sh/kt/bench.sh + # Runs in the caller's directory, so a relative `--lib-dir` or `--output` means # what it would to the script itself. See kt/README.md for the --lib-dir layout. diff --git a/kt/moq/src/jvmTest/kotlin/dev/moq/FfiBench.kt b/kt/moq/src/jvmTest/kotlin/dev/moq/FfiBench.kt new file mode 100644 index 00000000000..ab16fc0b89d --- /dev/null +++ b/kt/moq/src/jvmTest/kotlin/dev/moq/FfiBench.kt @@ -0,0 +1,117 @@ +package dev.moq + +import java.io.File +import java.lang.management.ManagementFactory +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.runBlocking +import kotlinx.serialization.json.buildJsonArray +import kotlinx.serialization.json.put +import kotlinx.serialization.json.addJsonObject +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNotNull + +/** One throughput case: [publishers] tracks, each read by [subscribers] consumers. */ +private data class Case(val size: Int, val publishers: Int, val subscribers: Int) { + /** Frames per group, holding a group under moq-net's 32 MB group cache. */ + val frames = ((8 shl 20) / size).coerceIn(1, 64) + val delivered = publishers.toLong() * subscribers * frames * size + val name = "size=$size/pub=$publishers/sub=$subscribers" +} + +// Frame sizes at one publisher and subscriber, then fan-out at 1 KiB. +private val cases = listOf(100, 1024, 64 * 1024, 1024 * 1024).map { Case(it, 1, 1) } + + listOf(1 to 4, 4 to 1, 4 to 4).map { (p, s) -> Case(1024, p, s) } + +private const val WARMUP = 3 +private val budgetNs = ((System.getenv("MOQ_BENCH_SECONDS")?.toDouble() ?: 1.0) * 1e9).toLong() + +/** Process CPU time, user plus system, across every thread. */ +private fun cpuNs(): Long = + (ManagementFactory.getOperatingSystemMXBean() as com.sun.management.OperatingSystemMXBean).processCpuTime + +private class Result(val name: String, val unit: String, val value: Double) + +/** + * The Kotlin runner of the binding benchmark (`just bench-ffi`, see bench/README.md). + * + * The same in-process rounds as every wrapper, so dividing by the Rust baselines isolates + * what Kotlin adds. Runs only when `MOQ_BENCH_DIR` is set, so `just kt check` skips it. + */ +class FfiBench { + @Test + fun benchmark() { + val dir = System.getenv("MOQ_BENCH_DIR") ?: return + val results = mutableListOf() + runBlocking { cases.forEach { throughput(it, results) } } + + val track = BroadcastProducer().publishTrack("calls", null) + val group = track.appendGroup().consume() + call("sequence", results) { group.sequence() } + val consumer = track.consume(null) + call("info", results) { consumer.info() } + + val json = buildJsonArray { + results.forEach { + addJsonObject { + put("name", it.name) + put("unit", it.unit) + put("value", it.value) + } + } + } + File(dir).mkdirs() + File(dir, "kotlin.json").writeText(json.toString()) + } + + private suspend fun throughput(case: Case, results: MutableList) = kotlinx.coroutines.coroutineScope { + val tracks = (0 until case.publishers).map { BroadcastProducer().publishTrack("$it", null) } + val consumers = tracks.flatMap { track -> (0 until case.subscribers).map { track.consume(null) } } + val payload = ByteArray(case.size) { 0x5a } + + // Each consumer reads on its own coroutine while this one writes each group in turn. + suspend fun step(n: Long) { + val readers = consumers.map { consumer -> + async { + val group = assertNotNull(consumer.recvGroup(), "track ended") + repeat(case.frames) { f -> + val frame = assertNotNull(group.readFrame(), "group ended") + assertEquals(case.size, frame.payload.size, "frame $f of round $n") + assertEquals((n * case.frames + f).toULong(), frame.timestampUs) + } + } + } + tracks.forEach { track -> + val group = track.appendGroup() + repeat(case.frames) { f -> + group.writeFrame(Frame(payload = payload, timestampUs = (n * case.frames + f).toULong())) + } + group.finish() + } + readers.awaitAll() + } + + repeat(WARMUP) { step(it.toLong()) } + val (start, cpuStart) = System.nanoTime() to cpuNs() + var n = WARMUP.toLong() + while (System.nanoTime() - start < budgetNs) { + step(n) + n++ + } + val (wallNs, cpuUsedNs) = (System.nanoTime() - start) to (cpuNs() - cpuStart) + val mb = (n - WARMUP) * case.delivered / 1e6 + results += Result("throughput/${case.name}", "MB/s", mb / (wallNs / 1e9)) + results += Result("cpu/${case.name}", "ms/MB", cpuUsedNs / 1e6 / mb) + } + + private inline fun call(name: String, results: MutableList, fn: () -> Unit) { + val start = System.nanoTime() + var calls = 0L + while (System.nanoTime() - start < budgetNs) { + repeat(1000) { fn() } + calls += 1000 + } + results += Result("call/$name", "ns", (System.nanoTime() - start).toDouble() / calls) + } +} diff --git a/py/justfile b/py/justfile index da3fc8cfbf7..fd5bde95d3c 100644 --- a/py/justfile +++ b/py/justfile @@ -62,6 +62,13 @@ build: uv sync --locked --no-install-workspace just _develop +# Run the binding benchmark's Python runner (see bench/README.md) on a release build. +bench: + uv sync --locked --no-install-workspace + cd moq-ffi && uv run --no-sync maturin develop --locked --uv --release + uv pip install --no-deps -e moq-rs + uv run --no-sync python moq-rs/benches/ffi.py + # Build the pure-python moq-rs wrapper sdist + wheel into py/dist (for release). # moq-ffi is built separately by maturin (see release-py.yml); the wrapper is diff --git a/py/moq-rs/benches/ffi.py b/py/moq-rs/benches/ffi.py new file mode 100644 index 00000000000..b1d69e5f5f2 --- /dev/null +++ b/py/moq-rs/benches/ffi.py @@ -0,0 +1,123 @@ +"""Python runner of the binding benchmark (`just bench-ffi`, see bench/README.md). + +The same in-process rounds as every wrapper, so dividing by the Rust baselines +isolates what Python adds: the FFI boundary, asyncio bridging, and the wrapper. +""" + +import asyncio +import json +import os +import resource +import time +from dataclasses import dataclass +from datetime import timedelta +from pathlib import Path +from typing import Callable + +import moq + +WARMUP = 3 +BUDGET = float(os.environ.get("MOQ_BENCH_SECONDS", "1")) + + +@dataclass(frozen=True) +class Case: + """One throughput case: `publishers` tracks, each read by `subscribers` consumers.""" + + size: int + publishers: int + subscribers: int + + @property + def frames(self) -> int: + """Frames per group, holding a group under moq-net's 32 MB group cache.""" + return min(max((8 << 20) // self.size, 1), 64) + + @property + def delivered(self) -> int: + return self.publishers * self.subscribers * self.frames * self.size + + @property + def name(self) -> str: + return f"size={self.size}/pub={self.publishers}/sub={self.subscribers}" + + +# Frame sizes at one publisher and subscriber, then fan-out at 1 KiB. +CASES = [Case(size, 1, 1) for size in (100, 1024, 64 * 1024, 1024 * 1024)] + [ + Case(1024, p, s) for p, s in ((1, 4), (4, 1), (4, 4)) +] + + +def cpu() -> float: + """Process CPU seconds, user plus system, across every thread.""" + usage = resource.getrusage(resource.RUSAGE_SELF) + return usage.ru_utime + usage.ru_stime + + +async def read(case: Case, consumer: moq.TrackConsumer, n: int) -> None: + group = await consumer.recv_group() + assert group is not None, "track ended" + for f in range(case.frames): + frame = await group.read_frame() + assert frame is not None, "group ended" + assert len(frame.payload) == case.size, f"frame {f} of round {n}" + assert frame.timestamp == timedelta(microseconds=n * case.frames + f) + + +async def throughput(case: Case, results: list[dict]) -> None: + tracks = [moq.BroadcastProducer().publish_track(str(p)) for p in range(case.publishers)] + consumers = [track.consume() for track in tracks for _ in range(case.subscribers)] + payload = b"\x5a" * case.size + + async def step(n: int) -> None: + # Each consumer reads on its own task while this one writes each group in turn. + readers = [asyncio.create_task(read(case, consumer, n)) for consumer in consumers] + for track in tracks: + group = track.append_group() + for f in range(case.frames): + group.write_frame(payload, timedelta(microseconds=n * case.frames + f)) + group.finish() + await asyncio.gather(*readers) + + for n in range(WARMUP): + await step(n) + start, cpu_start, n = time.perf_counter(), cpu(), WARMUP + while time.perf_counter() - start < BUDGET: + await step(n) + n += 1 + wall, cpu_used = time.perf_counter() - start, cpu() - cpu_start + mb = (n - WARMUP) * case.delivered / 1e6 + results.append({"name": f"throughput/{case.name}", "unit": "MB/s", "value": mb / wall}) + results.append({"name": f"cpu/{case.name}", "unit": "ms/MB", "value": cpu_used * 1e3 / mb}) + + +def call(name: str, fn: Callable[[], object], results: list[dict]) -> None: + start, calls = time.perf_counter(), 0 + while time.perf_counter() - start < BUDGET: + for _ in range(1000): + fn() + calls += 1000 + results.append({"name": f"call/{name}", "unit": "ns", "value": (time.perf_counter() - start) * 1e9 / calls}) + + +async def main() -> None: + results: list[dict] = [] + for case in CASES: + await throughput(case, results) + + track = moq.BroadcastProducer().publish_track("calls") + group = track.append_group().consume() + call("sequence", lambda: group.sequence, results) + consumer = track.consume() + call("info", consumer.info, results) + + report = json.dumps(results, indent=2) + if out := os.environ.get("MOQ_BENCH_DIR"): + Path(out).mkdir(parents=True, exist_ok=True) + (Path(out) / "python.json").write_text(report) + else: + print(report) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/quest/m1/README.md b/quest/m1/README.md index c0de027c689..f5899f1d507 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -115,7 +115,6 @@ above them. - [Generated C bindings](/quest/m1/c/README.md) - C generated from moq-ffi ships as `moq-c` 0.8.0 and replaces the hand-written libmoq - [BoltFFI or UniFFI](/quest/m1/boltffi.md) - BoltFFI vs UniFFI on a moq-ffi slice in Python, Kotlin, and Swift decides which one the moq-dev fork starts from - [Hard-fork the binding generator](/quest/m1/uniffi-fork.md) - one moq-dev repo holds the chosen generator's core and every language backend, released under one tag and one pin -- [Binding benchmark harness](/quest/m1/ffi-bench.md) - Python, Kotlin, and Go benchmark their wrappers nightly against two Rust baselines, so FFI overhead is tracked - [Benchmark trend site](/quest/m1/bench-site.md) - a GitHub Pages site charts the binding, Rust, and JS nightly benchmarks over time - [Link the benchmark trends](/quest/m1/bench-site-doc.md) - the docs point language choosers at the trend site - [Swift, Dart, and C++ benchmarks](/quest/m1/ffi-bench-wrappers.md) - the other three wrappers join the nightly binding benchmark diff --git a/quest/m1/bench-site.md b/quest/m1/bench-site.md index 8800cf00429..4d9e56f7228 100644 --- a/quest/m1/bench-site.md +++ b/quest/m1/bench-site.md @@ -4,7 +4,7 @@ A public GitHub Pages site charts nightly benchmarks over time, so a regression or an optimization shows as a step in the trend. It carries the -[binding benchmarks](/quest/m1/ffi-bench.md) and the existing nightly Rust +[binding benchmarks](/bench/README.md) and the existing nightly Rust Criterion and JS benchmarks, and other sources can join without a new site. @@ -37,7 +37,3 @@ Decided with the maintainer on 2026-10-15: `bench/README.md`. Public API: none. Wire: none. - -## Required - -- [Binding benchmark harness](/quest/m1/ffi-bench.md) - the first source the site charts diff --git a/quest/m1/boltffi.md b/quest/m1/boltffi.md index af7b681290a..3dc62eb9815 100644 --- a/quest/m1/boltffi.md +++ b/quest/m1/boltffi.md @@ -37,7 +37,7 @@ Decided with the maintainer on 2026-10-13: `read_frame`, and one async read loop. Bind them with both generators side by side via `cfg_attr` in a spike crate, not in moq-ffi. The spike crate carries its own benchmarks, so it does not wait on the - [binding benchmark](/quest/m1/ffi-bench.md). It also builds for wasm32: + [binding benchmark](/bench/README.md). It also builds for wasm32: the browser comparison in #5478 reuses this slice for per-call numbers, so tell that quest if the crate's location or shape changes. - **Measure**: no-op and record call overhead, payload throughput across diff --git a/quest/m1/ffi-bench-wrappers.md b/quest/m1/ffi-bench-wrappers.md index f5c4de5096a..98678e30c1c 100644 --- a/quest/m1/ffi-bench-wrappers.md +++ b/quest/m1/ffi-bench-wrappers.md @@ -2,7 +2,7 @@ ## Goal -Swift, Dart, and C++ run the [binding benchmark](/quest/m1/ffi-bench.md) +Swift, Dart, and C++ run the [binding benchmark](/bench/README.md) nightly with the same shape, metrics, and output as Python, Kotlin, and Go, so every wrapper's FFI cost is tracked and shows on the trend site. @@ -16,11 +16,7 @@ all six to prove its fix. - Swift has no Linux toolchain in the dev shell; run it in the macOS job in `platform.yml`. Its numbers come from a different runner, so report its ratio against a Rust baseline measured in that same job. -- Follow whatever the harness quest settled on for shape and output; extend - it rather than fork it. +- Follow the shape and output in `bench/README.md` (`bench/ffi.sh` runs + each runner, `bench/ffi.py` combines them); extend it rather than fork it. Public API: none. Wire: none. - -## Required - -- [Binding benchmark harness](/quest/m1/ffi-bench.md) - the shape, baselines, and output these join diff --git a/quest/m1/ffi-bench.md b/quest/m1/ffi-bench.md deleted file mode 100644 index 62eb81b3ad0..00000000000 --- a/quest/m1/ffi-bench.md +++ /dev/null @@ -1,58 +0,0 @@ -# [L] Binding benchmark harness - -## Goal - -Python, Kotlin (JVM), and Go each run a benchmark of their wrapper's -publish and subscribe path every night, beside two Rust baselines measured -on the same runner, so FFI overhead is tracked and optimizations are -measured instead of guessed. The nightly job summary shows each binding's -numbers and its ratio to the baselines. This is the shared harness later -binding performance quests measure on. - -Swift, Dart, and C++ join in [their own quest](/quest/m1/ffi-bench-wrappers.md). -The trend website is [the site quest](/quest/m1/bench-site.md). - -## Plan - -Decided with the maintainer on 2026-10-15: - -- **Split from [Payload bytes](/quest/m1/ffi-bytes.md)**, which planned this - benchmark but is blocked behind the UniFFI fork. The harness needs no - fork, so it starts now and the fork's patches are measured on it. -- **Shape**: an in-process publisher and subscriber, no network - (`MoqTrackProducer::consume` connects them without a session). - `write_frame` and `read_frame` throughput over frame sizes from 100 B to - 1 MB, a no-op call, and a record round trip. Sweep publisher and - subscriber counts independently, as the Go bench it replaces does, so a - cost that grows with either axis shows as a slope. -- **Metrics**: throughput and per-call time, plus process CPU (user + sys) - per MB. Async bridging (asyncio, coroutines, goroutines) burns CPU on - other threads that wall time hides. -- **Layer**: each language benches its idiomatic wrapper (`py/moq-rs`, - `kt/moq`, `go/wrapper`), which is what users feel. The no-op call goes - through the raw generated binding to isolate the generator. -- **Two Rust baselines**: the same loop against moq-ffi's Rust API called - directly (binding / moq-ffi is the boundary's cost) and against moq-net - (moq-ffi / moq-net is the `ffi::Task` and mutex machinery that - [the poll spike](/quest/m1/ffi-poll.md) targets). Criterion, like the - other Rust benches. -- **Tracking**: report ratios to the same-job Rust baselines, which cancel - most hosted-runner noise. No thresholds. Emit every result in one JSON - shape (github-action-benchmark's custom format is a good fit) as a nightly - artifact; [the site](/quest/m1/bench-site.md) publishes the history. -- **Go**: replace the loopback-QUIC `BenchmarkPublisher` in - `go/wrapper/runtime_bench_test.go` with the in-process harness; the - nightly "Go publisher runtime benchmark" slot runs the new one. Delete the - QUIC version. -- Use each language's idiomatic bench tooling where it gives sound - statistics (Go `testing.B`, Criterion), but avoid heavy new build - plumbing just for a framework. A plain timed loop is fine. -- Local recipes per language plus one that runs them all, documented in - `bench/README.md`. - -Public API: none. Wire: none. - -## Related - -- [Payload bytes](/quest/m1/ffi-bytes.md) - the first optimization measured on this harness -- [Benchmark regressions in CI](/quest/m2/bench-ci.md) - the Criterion side of benchmark tracking diff --git a/quest/m1/ffi-bytes.md b/quest/m1/ffi-bytes.md index 1d5d817d08f..9777380d7e0 100644 --- a/quest/m1/ffi-bytes.md +++ b/quest/m1/ffi-bytes.md @@ -4,17 +4,16 @@ A binding moves a frame payload across the FFI with no per-byte loop: every copy on the path is a bulk copy, and the report counts them per direction. -Proven on the nightly [binding benchmark](/quest/m1/ffi-bench.md) in every +Proven on the nightly [binding benchmark](/bench/README.md) in every wrapper (Python, Go, Kotlin, Swift, Dart, and C++). ## Plan Decided with the maintainer on 2026-10-13: -- **Benchmark first.** The per-wrapper benchmark moved to - [its own quest](/quest/m1/ffi-bench.md) on 2026-10-15 so it could start - before the fork. Add a Rust microbenchmark of UniFFI's lift and lower for - `Vec` here, beside the patch it proves. +- **Benchmark first.** `just bench-ffi` ([bench/README.md](/bench/README.md)) + measures every wrapper before and after. Add a Rust microbenchmark of + UniFFI's lift and lower for `Vec` here, beside the patch it proves. - **Bulk copy in the fork's core.** UniFFI lifts and lowers `Vec` through the generic `Vec` path, one byte at a time with a bounds check each (`uniffi_core` `ffi_converter_impls.rs`), in both directions. That @@ -43,6 +42,5 @@ type. Wire: none. ## Required -- [Binding benchmark harness](/quest/m1/ffi-bench.md) - Python, Kotlin, and Go numbers before and after - [Swift, Dart, and C++ benchmarks](/quest/m1/ffi-bench-wrappers.md) - the other three wrappers' numbers - [Hard fork](/quest/m1/uniffi-fork.md) - the repository the core patch lands in diff --git a/quest/m1/ffi-poll.md b/quest/m1/ffi-poll.md index e3fbb57c9b1..5f54bbc37ec 100644 --- a/quest/m1/ffi-poll.md +++ b/quest/m1/ffi-poll.md @@ -32,7 +32,7 @@ Decided with the maintainer on 2026-10-13: the numbers justify it. - **Hot path**: track `next` and group `read_frame`, prototyped in both shapes beside the existing `Task` version. -- **Measure** in Python and Go, on the [binding benchmark](/quest/m1/ffi-bench.md): +- **Measure** in Python and Go, on the [binding benchmark](/bench/README.md): per-call latency, wakeups per frame, throughput, and CPU. Sweep subscriber count so a cost that grows with the fan-out shows up. Test cancellation semantics too: a cancelled read must not swallow the next @@ -42,10 +42,6 @@ Decided with the maintainer on 2026-10-13: Public API: none. Wire: none. -## Required - -- [Binding benchmark harness](/quest/m1/ffi-bench.md) - the per-wrapper benchmark this measures on - ## Related - [Runtime trait](/quest/m2/runtime-trait.md) - replaces web-async; moq-ffi's tokio runtime may move behind it diff --git a/quest/m2/ffi-bytes-return.md b/quest/m2/ffi-bytes-return.md index b7ebf86d163..ee9f05ee6ac 100644 --- a/quest/m2/ffi-bytes-return.md +++ b/quest/m2/ffi-bytes-return.md @@ -5,7 +5,7 @@ A frame payload returned to a binding (`read_frame`, `recv_datagram`, decoded media) reaches the host without a copy, through a Rust-owned buffer the host releases, in every generator of [moq-dev/uniffi](/quest/m1/uniffi-fork.md). -Proven on the [binding benchmark](/quest/m1/ffi-bench.md). +Proven on the [binding benchmark](/bench/README.md). ## Plan diff --git a/quest/m2/ffi-size-profile.md b/quest/m2/ffi-size-profile.md index b192e1e36ea..665b0069e2b 100644 --- a/quest/m2/ffi-size-profile.md +++ b/quest/m2/ffi-size-profile.md @@ -19,13 +19,9 @@ Guidance: opt-level 3 with per-package overrides, and compare against building everything at "s". - Measure what the bindings actually do: publish and subscribe throughput - through moq-ffi on the [binding benchmark](/quest/m1/ffi-bench.md), plus + through moq-ffi on the [binding benchmark](/bench/README.md), plus audio and video encode and decode. Wire the benchmark into CI, at least nightly. - If it's adopted, use a named profile (for example `[profile.ffi]`, inheriting release) that every binding build path selects, so the relay and self-builders keep opt-level 3. - -## Required - -- [Binding benchmark harness](/quest/m1/ffi-bench.md) - the per-wrapper benchmark this measures with diff --git a/rs/moq-ffi/Cargo.toml b/rs/moq-ffi/Cargo.toml index ccf3dbcae80..1820ca2612c 100644 --- a/rs/moq-ffi/Cargo.toml +++ b/rs/moq-ffi/Cargo.toml @@ -86,3 +86,7 @@ uniffi = { workspace = true, features = ["build"] } pollster = { workspace = true } tokio = { workspace = true, features = ["test-util"] } web-transport-trait = { workspace = true } + +[target.'cfg(unix)'.dev-dependencies] +# Process CPU time for the benchmark's baselines. +libc = { workspace = true } diff --git a/rs/moq-ffi/src/bench.rs b/rs/moq-ffi/src/bench.rs new file mode 100644 index 00000000000..048e2143cae --- /dev/null +++ b/rs/moq-ffi/src/bench.rs @@ -0,0 +1,276 @@ +//! The Rust baselines for the binding benchmark (`just bench-ffi`, see `bench/README.md`). +//! +//! Every wrapper runs the same loop, so dividing a wrapper's numbers by these isolates +//! what it adds: `rust-ffi` calls the exported API without crossing the FFI, and `rust-net` +//! runs the same rounds on moq-net, below moq-ffi's handles and runtime. + +use crate::consumer::MoqTrackConsumer; +use crate::media::MoqFrame; +use crate::producer::{MoqBroadcastProducer, MoqTrackProducer}; + +use std::future::Future; +use std::hint::black_box; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +const WARMUP: u64 = 3; + +/// One throughput case: `publishers` tracks, each read by `subscribers` consumers. +struct Case { + size: usize, + publishers: usize, + subscribers: usize, +} + +impl Case { + /// Frame sizes at one publisher and subscriber, then fan-out at 1 KiB. + fn all() -> Vec { + let sizes = [100, 1024, 64 * 1024, 1024 * 1024].map(|size| Case { + size, + publishers: 1, + subscribers: 1, + }); + let fanout = [(1, 4), (4, 1), (4, 4)].map(|(publishers, subscribers)| Case { + size: 1024, + publishers, + subscribers, + }); + sizes.into_iter().chain(fanout).collect() + } + + /// Frames per group, holding a group under moq-net's 32 MB group cache. + fn frames(&self) -> usize { + ((8 << 20) / self.size).clamp(1, 64) + } + + fn delivered(&self) -> u64 { + (self.publishers * self.subscribers * self.frames() * self.size) as u64 + } + + fn name(&self) -> String { + format!("size={}/pub={}/sub={}", self.size, self.publishers, self.subscribers) + } +} + +fn budget() -> Duration { + let seconds = std::env::var("MOQ_BENCH_SECONDS").map_or(1.0, |s| s.parse().expect("MOQ_BENCH_SECONDS")); + Duration::from_secs_f64(seconds) +} + +/// Process CPU time, user plus system, across every thread. +fn cpu() -> Duration { + let mut usage = std::mem::MaybeUninit::::uninit(); + // SAFETY: getrusage fills the struct it is handed and returns non-zero only for a bad `who`. + let usage = unsafe { + assert_eq!(libc::getrusage(libc::RUSAGE_SELF, usage.as_mut_ptr()), 0); + usage.assume_init() + }; + let time = |t: libc::timeval| Duration::new(t.tv_sec as u64, t.tv_usec as u32 * 1000); + time(usage.ru_utime) + time(usage.ru_stime) +} + +#[derive(Default)] +struct Report(Vec); + +impl Report { + fn push(&mut self, name: String, unit: &str, value: f64) { + self.0 + .push(serde_json::json!({ "name": name, "unit": unit, "value": value })); + } + + /// Run `round` for the time budget after a warmup, recording throughput and CPU per MB. + async fn throughput(&mut self, case: &Case, mut round: F) + where + F: FnMut(u64) -> Fut, + Fut: Future, + { + for n in 0..WARMUP { + round(n).await; + } + let (start, cpu_start, budget) = (Instant::now(), cpu(), budget()); + let mut n = WARMUP; + while start.elapsed() < budget { + round(n).await; + n += 1; + } + let (wall, cpu) = (start.elapsed(), cpu() - cpu_start); + let mb = ((n - WARMUP) * case.delivered()) as f64 / 1e6; + self.push(format!("throughput/{}", case.name()), "MB/s", mb / wall.as_secs_f64()); + self.push(format!("cpu/{}", case.name()), "ms/MB", cpu.as_secs_f64() * 1e3 / mb); + } + + /// Time `call` over the budget, recording nanoseconds per call. + fn call(&mut self, name: &str, mut call: impl FnMut()) { + let (start, budget) = (Instant::now(), budget()); + let mut calls = 0u64; + while start.elapsed() < budget { + for _ in 0..1000 { + call(); + } + calls += 1000; + } + self.push( + format!("call/{name}"), + "ns", + start.elapsed().as_nanos() as f64 / calls as f64, + ); + } + + /// Write the report to `$MOQ_BENCH_DIR/.json`, or print it without one. + fn write(self, runner: &str) { + let json = serde_json::to_string_pretty(&self.0).unwrap(); + match std::env::var_os("MOQ_BENCH_DIR") { + Some(dir) => { + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(std::path::Path::new(&dir).join(format!("{runner}.json")), json).unwrap(); + } + None => println!("{runner}: {json}"), + } + } +} + +fn check(case: &Case, round: u64, frame: usize, payload: &[u8], timestamp_us: Option) { + assert_eq!(payload.len(), case.size, "frame {frame} of round {round}"); + assert_eq!(timestamp_us, Some(round * case.frames() as u64 + frame as u64)); +} + +async fn ffi_round(case: &Case, tracks: &[Arc], consumers: &[Arc], n: u64) { + let readers: Vec<_> = consumers + .iter() + .map(|consumer| { + let (consumer, frames, size) = (consumer.clone(), case.frames(), case.size); + tokio::spawn(async move { + let group = consumer.recv_group().await.unwrap().expect("track ended"); + let mut read = Vec::with_capacity(frames); + for _ in 0..frames { + let frame = group.read_frame().await.unwrap().expect("group ended"); + read.push((frame.payload.len(), frame.timestamp_us)); + } + assert!(read.iter().all(|(len, _)| *len == size)); + read + }) + }) + .collect(); + let payload = vec![0x5a; case.size]; + for track in tracks { + let group = track.append_group().unwrap(); + for f in 0..case.frames() { + let frame = MoqFrame { + payload: payload.clone(), + timestamp_us: Some(n * case.frames() as u64 + f as u64), + }; + group.write_frame(frame).unwrap(); + } + group.finish().unwrap(); + } + for reader in readers { + for (f, (len, timestamp_us)) in reader.await.unwrap().into_iter().enumerate() { + check(case, n, f, &payload[..len], timestamp_us); + } + } +} + +/// Subscribers are not shared handles, so each reader task borrows one for the round and returns it. +async fn net_round( + case: &Case, + mut tracks: Vec, + consumers: Vec, + n: u64, +) -> Vec { + let readers: Vec<_> = consumers + .into_iter() + .map(|mut consumer| { + let frames = case.frames(); + tokio::spawn(async move { + let mut group = consumer.recv_group().await.unwrap().expect("track ended"); + let mut read = Vec::with_capacity(frames); + for _ in 0..frames { + let frame = group.read_frame().await.unwrap().expect("group ended"); + read.push((frame.payload, frame.timestamp.map(|t| t.as_micros() as u64))); + } + (consumer, read) + }) + }) + .collect(); + let payload = bytes::Bytes::from(vec![0x5a; case.size]); + for track in &mut tracks { + let mut group = track.append_group().unwrap(); + for f in 0..case.frames() { + let timestamp = moq_net::Timestamp::from_micros(n * case.frames() as u64 + f as u64).unwrap(); + // Copy, as a binding's write does, so the baseline does not get zero-copy for free. + group.write_frame(timestamp, payload.to_vec()).unwrap(); + } + group.finish().unwrap(); + } + let mut consumers = Vec::with_capacity(readers.len()); + for reader in readers { + let (consumer, read) = reader.await.unwrap(); + for (f, (payload, timestamp_us)) in read.into_iter().enumerate() { + check(case, n, f, &payload, timestamp_us); + } + consumers.push(consumer); + } + consumers +} + +async fn rust_ffi() { + let mut report = Report::default(); + for case in Case::all() { + let broadcast = MoqBroadcastProducer::new().unwrap(); + let tracks: Vec<_> = (0..case.publishers) + .map(|p| broadcast.publish_track(p.to_string(), None).unwrap()) + .collect(); + let consumers: Vec<_> = tracks + .iter() + .flat_map(|track| (0..case.subscribers).map(|_| track.consume(None).unwrap())) + .collect(); + report + .throughput(&case, |n| ffi_round(&case, &tracks, &consumers, n)) + .await; + } + + let broadcast = MoqBroadcastProducer::new().unwrap(); + let track = broadcast.publish_track("calls".into(), None).unwrap(); + let group = track.append_group().unwrap().consume().unwrap(); + report.call("sequence", || { + black_box(black_box(&group).sequence()); + }); + let consumer = track.consume(None).unwrap(); + report.call("info", || { + black_box(black_box(&consumer).info().unwrap()); + }); + report.write("rust-ffi"); +} + +async fn rust_net() { + let mut report = Report::default(); + for case in Case::all() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let info = moq_net::track::Info::default().with_timescale(moq_net::Timescale::MICRO); + let tracks: Vec<_> = (0..case.publishers) + .map(|p| broadcast.create_track(p.to_string(), Some(info.clone())).unwrap()) + .collect(); + let consumers = std::cell::RefCell::new( + tracks + .iter() + .flat_map(|track| (0..case.subscribers).map(|_| track.subscribe(None))) + .collect::>(), + ); + // Producers are shared handles, so each round appends through its own clones. + let (case, tracks, consumers) = (&case, &tracks, &consumers); + report + .throughput(case, |n| async move { + let returned = net_round(case, tracks.clone(), consumers.take(), n).await; + consumers.replace(returned); + }) + .await; + } + report.write("rust-net"); +} + +#[tokio::test] +#[ignore = "benchmark: run by `just bench-ffi`"] +async fn benchmark_ffi() { + rust_ffi().await; + rust_net().await; +} diff --git a/rs/moq-ffi/src/lib.rs b/rs/moq-ffi/src/lib.rs index e0c2a5f12e7..090bd368e06 100644 --- a/rs/moq-ffi/src/lib.rs +++ b/rs/moq-ffi/src/lib.rs @@ -60,3 +60,7 @@ pub extern "C" fn moq_ffi_shutdown() { // The test suite drives a native relay over a tokio runtime. #[cfg(all(test, not(target_arch = "wasm32")))] mod test; + +// The binding benchmark's Rust baselines; ignored unless `just bench-ffi` asks for them. +#[cfg(all(test, unix))] +mod bench; diff --git a/sh/go/bench.sh b/sh/go/bench.sh index af274ae4292..c155bbf04c0 100755 --- a/sh/go/bench.sh +++ b/sh/go/bench.sh @@ -6,4 +6,4 @@ export MOQ_FFI_PROFILE=release staged=$(bash "$here/stage.sh" --output "$root/dist/runtime-bench") wrapper=$(printf '%s\n' "$staged" | sed -n 2p) cd "$wrapper" -CGO_ENABLED=1 GOFLAGS=-mod=mod go test -run '^$' -bench '^BenchmarkPublisher$' "$@" +CGO_ENABLED=1 GOFLAGS=-mod=mod go test -run '^$' -bench '^BenchmarkFFI$' "$@" diff --git a/sh/kt/bench.sh b/sh/kt/bench.sh new file mode 100755 index 00000000000..57e61a43cdd --- /dev/null +++ b/sh/kt/bench.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Run the binding benchmark's Kotlin runner (see bench/README.md) on a release +# moq-ffi. The runner is a JVM test that skips itself without MOQ_BENCH_DIR, so +# `just kt check` never pays for it. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +KT_DIR="$(cd "$SCRIPT_DIR/../../kt" && pwd)" +export MOQ_BENCH_DIR="${MOQ_BENCH_DIR:-$KT_DIR/../target/bench/ffi}" + +MOQ_FFI_PROFILE=release bash "$SCRIPT_DIR/generate.sh" +gradle -p "$KT_DIR" -Pmoqffi.version=0.0.0-dev :moq:jvmTest --tests dev.moq.FfiBench --rerun +echo "kt bench: wrote $MOQ_BENCH_DIR/kotlin.json" From fcb778d5f4837ae593f3a806d1a0aed11a2d7eb3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 12:37:01 -0700 Subject: [PATCH 3/7] quest: plan Kotlin binding call overhead from the first numbers Co-Authored-By: Claude Opus 5.5 --- quest/m1/README.md | 1 + quest/m1/boltffi.md | 4 ++++ quest/m1/kt-ffi-calls.md | 34 ++++++++++++++++++++++++++++++++++ quest/m1/uniffi-fork.md | 4 +++- 4 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 quest/m1/kt-ffi-calls.md diff --git a/quest/m1/README.md b/quest/m1/README.md index f5899f1d507..90761689ae0 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -119,6 +119,7 @@ above them. - [Link the benchmark trends](/quest/m1/bench-site-doc.md) - the docs point language choosers at the trend site - [Swift, Dart, and C++ benchmarks](/quest/m1/ffi-bench-wrappers.md) - the other three wrappers join the nightly binding benchmark - [Payload bytes](/quest/m1/ffi-bytes.md) - payloads cross the FFI in one bulk copy, proven on the binding benchmark in every wrapper +- [Kotlin calls](/quest/m1/kt-ffi-calls.md) - Kotlin calls into moq-ffi cost about what Python's and Go's do, sync and async, once the fork's Kotlin generator drops JNA's per-call overhead - [Poll-native async spike](/quest/m1/ffi-poll.md) - measure kio poll-native exports and foreign poll against `ffi::Task`, then pick the shape - [Rendition preference](/quest/m1/rendition-preference.md) - automatic selection by ``, `Video::ranked`, and WHEP keeps the highest `preference` that decodes, so a compatibility transcode is only picked when nothing preferred decodes - [Rust PROBE on demand](/quest/m1/probe-on-demand-rs.md) - a Rust client opens PROBE only while a `recv_bandwidth()` consumer is held, and `Stats` drops `estimated_recv_rate` diff --git a/quest/m1/boltffi.md b/quest/m1/boltffi.md index 3dc62eb9815..b6b3d852817 100644 --- a/quest/m1/boltffi.md +++ b/quest/m1/boltffi.md @@ -43,6 +43,10 @@ Decided with the maintainer on 2026-10-13: - **Measure**: no-op and record call overhead, payload throughput across frame sizes from 100 B to 1 MB, and the cost of each async await. Re-measure UniFFI rather than trusting BoltFFI's numbers for it. + `just bench-ffi` gives UniFFI's numbers through the real moq-ffi. A known + UniFFI cost to weigh: Kotlin sync calls measured 11-22 us against 1.3 us + from Python on 2026-10-15, from JNA per-call structures and per-wakeup + thread attach ([Kotlin calls](/quest/m1/kt-ffi-calls.md)). - **Size table**: native library, generated bindings, and wasm plus JS glue (gzip), for both generators. Measure the native library with moq-ffi's default features on and off; the toggle changes nothing on wasm32, where diff --git a/quest/m1/kt-ffi-calls.md b/quest/m1/kt-ffi-calls.md new file mode 100644 index 00000000000..2f0db3cfad7 --- /dev/null +++ b/quest/m1/kt-ffi-calls.md @@ -0,0 +1,34 @@ +# [M] Kotlin binding calls without JNA overhead + +## Goal + +A Kotlin call into moq-ffi costs about what the same call costs from Python or +Go, sync and async alike, proven on the +[binding benchmark](/bench/README.md). Measured 2026-10-15: a sync +`sequence()` took 11-22 us from Kotlin against 1.3 us from Python and 0.12 us +from Go, and the 100 B throughput case kept about 5.8 cores busy for 2 MB/s. + +## Plan + +Decided with the maintainer on 2026-10-15: + +- **Profile first** (JFR or async-profiler) and confirm each cause before + changing anything. The two suspects in the generated `uniffi/moq/moq.kt`: + - Every call allocates a JNA `Structure` (`UniffiRustCallStatus`, which + nests a `RustBuffer.ByValue`), and JNA builds and syncs those by + reflection. + - The async continuation is a plain JNA `Callback` with no + `CallbackThreadInitializer`, so each wakeup from a Rust worker attaches + that thread to the JVM and detaches it again. +- **Fix in [the fork](/quest/m1/uniffi-fork.md)'s Kotlin generator**, not by + patching generated code. Both costs are one quest: same root, same + generator, same proof. Prefer a shape that keeps the wrapper's API + unchanged. +- If [the BoltFFI spike](/quest/m1/boltffi.md) picks BoltFFI, re-plan this + against its Kotlin backend, or delete it if that backend already avoids JNA. + +Public API: none expected. Wire: none. + +## Required + +- [Hard fork](/quest/m1/uniffi-fork.md) - the generator this patches diff --git a/quest/m1/uniffi-fork.md b/quest/m1/uniffi-fork.md index 95369cccb57..588bba9bfaf 100644 --- a/quest/m1/uniffi-fork.md +++ b/quest/m1/uniffi-fork.md @@ -21,7 +21,8 @@ UniFFI: with the quests that name UniFFI internals: [Payload bytes](/quest/m1/ffi-bytes.md), [Poll-native async](/quest/m1/ffi-poll.md), - [Zero-copy byte returns](/quest/m2/ffi-bytes-return.md), and + [Zero-copy byte returns](/quest/m2/ffi-bytes-return.md), + [Kotlin calls](/quest/m1/kt-ffi-calls.md), and [Namespaced external types](/quest/m2/uniffi-namespaces.md). - **Why**: every UniFFI minor release breaks external generators (0.31 @@ -72,4 +73,5 @@ Public API: none at the move; moq-ffi's dependency names change. Wire: none. ## Related - [Payload bytes](/quest/m1/ffi-bytes.md) - the first patch the fork carries +- [Kotlin calls](/quest/m1/kt-ffi-calls.md) - a Kotlin generator patch the fork carries - [Namespaced external types](/quest/m2/uniffi-namespaces.md) - fixes mozilla/uniffi-rs#3026 in the fork From c4a23f3cd07dc11e4b07a08a31938dea95094d54 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 13:05:08 -0700 Subject: [PATCH 4/7] bench: check payload bytes in every runner, run Rust through nextest Every received frame is compared against a non-uniform pattern, so a corrupt or zero-filled copy fails instead of reading as a speedup. The Rust baselines run through `just rs test`, so a wedged read hits the slow-test timeout. Quest wording: narrow the Kotlin cause attribution. Co-Authored-By: Claude Opus 5.5 --- bench/README.md | 2 +- bench/ffi.sh | 3 ++- go/wrapper/runtime_bench_test.go | 10 ++++--- kt/moq/src/jvmTest/kotlin/dev/moq/FfiBench.kt | 6 +++-- py/moq-rs/benches/ffi.py | 9 ++++--- quest/m1/boltffi.md | 5 ++-- quest/m1/kt-ffi-calls.md | 8 +++--- rs/moq-ffi/src/bench.rs | 26 +++++++++++-------- 8 files changed, 42 insertions(+), 27 deletions(-) diff --git a/bench/README.md b/bench/README.md index 358a0c8c94b..5c8323a2ab1 100644 --- a/bench/README.md +++ b/bench/README.md @@ -144,7 +144,7 @@ sweeps frame sizes from 100 B to 1 MiB at one publisher and subscriber, then 1/4 publishers by 1/4 subscribers at 1 KiB, and times two sync calls (`sequence`, `info`). Each case reports delivered MB/s and process CPU (user + system, every thread) per MB. `MOQ_BENCH_SECONDS` sets each case's time budget -(default 1). +(default 1); every received frame is checked byte for byte. Each runner writes `target/bench/ffi/.json`; `bench/ffi.py` combines them into a table and `report.json`. Ratios are against `rust-ffi`, moq-ffi's diff --git a/bench/ffi.sh b/bench/ffi.sh index ccfd9badd77..4d456b83b38 100755 --- a/bench/ffi.sh +++ b/bench/ffi.sh @@ -8,7 +8,8 @@ export MOQ_BENCH_DIR="${MOQ_BENCH_DIR:-$root/target/bench/ffi}" rm -rf "$MOQ_BENCH_DIR" cd "$root" -cargo test --locked --release -p moq-ffi --lib bench::benchmark_ffi -- --ignored --nocapture +# Through nextest, so a wedged read hits the slow-test timeout instead of the job's. +just rs test -p moq-ffi --cargo-profile release -E 'test(bench::benchmark_ffi)' --run-ignored only just go bench -benchtime="${MOQ_BENCH_SECONDS:-1}s" just py bench just kt bench diff --git a/go/wrapper/runtime_bench_test.go b/go/wrapper/runtime_bench_test.go index 7971a345a4e..a922b261eb2 100644 --- a/go/wrapper/runtime_bench_test.go +++ b/go/wrapper/runtime_bench_test.go @@ -156,7 +156,11 @@ func benchThroughput(b *testing.B, c benchCase) { consumers = append(consumers, consumer) } } - payload := bytes.Repeat([]byte{0x5a}, c.size) + // A non-uniform payload, so a zero-filled or truncated copy fails the check. + payload := make([]byte, c.size) + for i := range payload { + payload[i] = byte(i % 251) + } delivered := int64(len(consumers) * frames * c.size) b.SetBytes(delivered) @@ -180,8 +184,8 @@ func benchThroughput(b *testing.B, c benchCase) { return } want := time.Duration(n*frames+f) * time.Microsecond - if frame == nil || len(frame.Payload) != c.size || frame.Timestamp == nil || *frame.Timestamp != want { - done <- fmt.Errorf("missing, reordered, or truncated frame %d of round %d", f, n) + if frame == nil || !bytes.Equal(frame.Payload, payload) || frame.Timestamp == nil || *frame.Timestamp != want { + done <- fmt.Errorf("missing, reordered, or corrupt frame %d of round %d", f, n) return } } diff --git a/kt/moq/src/jvmTest/kotlin/dev/moq/FfiBench.kt b/kt/moq/src/jvmTest/kotlin/dev/moq/FfiBench.kt index ab16fc0b89d..185650d9f86 100644 --- a/kt/moq/src/jvmTest/kotlin/dev/moq/FfiBench.kt +++ b/kt/moq/src/jvmTest/kotlin/dev/moq/FfiBench.kt @@ -11,6 +11,7 @@ import kotlinx.serialization.json.addJsonObject import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertNotNull +import kotlin.test.assertTrue /** One throughput case: [publishers] tracks, each read by [subscribers] consumers. */ private data class Case(val size: Int, val publishers: Int, val subscribers: Int) { @@ -68,7 +69,8 @@ class FfiBench { private suspend fun throughput(case: Case, results: MutableList) = kotlinx.coroutines.coroutineScope { val tracks = (0 until case.publishers).map { BroadcastProducer().publishTrack("$it", null) } val consumers = tracks.flatMap { track -> (0 until case.subscribers).map { track.consume(null) } } - val payload = ByteArray(case.size) { 0x5a } + // A non-uniform payload, so a zero-filled or truncated copy fails the check. + val payload = ByteArray(case.size) { (it % 251).toByte() } // Each consumer reads on its own coroutine while this one writes each group in turn. suspend fun step(n: Long) { @@ -77,7 +79,7 @@ class FfiBench { val group = assertNotNull(consumer.recvGroup(), "track ended") repeat(case.frames) { f -> val frame = assertNotNull(group.readFrame(), "group ended") - assertEquals(case.size, frame.payload.size, "frame $f of round $n") + assertTrue(frame.payload.contentEquals(payload), "corrupt frame $f of round $n") assertEquals((n * case.frames + f).toULong(), frame.timestampUs) } } diff --git a/py/moq-rs/benches/ffi.py b/py/moq-rs/benches/ffi.py index b1d69e5f5f2..45caff6d212 100644 --- a/py/moq-rs/benches/ffi.py +++ b/py/moq-rs/benches/ffi.py @@ -54,24 +54,25 @@ def cpu() -> float: return usage.ru_utime + usage.ru_stime -async def read(case: Case, consumer: moq.TrackConsumer, n: int) -> None: +async def read(case: Case, consumer: moq.TrackConsumer, payload: bytes, n: int) -> None: group = await consumer.recv_group() assert group is not None, "track ended" for f in range(case.frames): frame = await group.read_frame() assert frame is not None, "group ended" - assert len(frame.payload) == case.size, f"frame {f} of round {n}" + assert frame.payload == payload, f"corrupt frame {f} of round {n}" assert frame.timestamp == timedelta(microseconds=n * case.frames + f) async def throughput(case: Case, results: list[dict]) -> None: tracks = [moq.BroadcastProducer().publish_track(str(p)) for p in range(case.publishers)] consumers = [track.consume() for track in tracks for _ in range(case.subscribers)] - payload = b"\x5a" * case.size + # A non-uniform payload, so a zero-filled or truncated copy fails the check. + payload = bytes(i % 251 for i in range(case.size)) async def step(n: int) -> None: # Each consumer reads on its own task while this one writes each group in turn. - readers = [asyncio.create_task(read(case, consumer, n)) for consumer in consumers] + readers = [asyncio.create_task(read(case, consumer, payload, n)) for consumer in consumers] for track in tracks: group = track.append_group() for f in range(case.frames): diff --git a/quest/m1/boltffi.md b/quest/m1/boltffi.md index b6b3d852817..c0c55e3b6aa 100644 --- a/quest/m1/boltffi.md +++ b/quest/m1/boltffi.md @@ -45,8 +45,9 @@ Decided with the maintainer on 2026-10-13: Re-measure UniFFI rather than trusting BoltFFI's numbers for it. `just bench-ffi` gives UniFFI's numbers through the real moq-ffi. A known UniFFI cost to weigh: Kotlin sync calls measured 11-22 us against 1.3 us - from Python on 2026-10-15, from JNA per-call structures and per-wakeup - thread attach ([Kotlin calls](/quest/m1/kt-ffi-calls.md)). + from Python on 2026-10-15, possibly from JNA's per-call structures + ([Kotlin calls](/quest/m1/kt-ffi-calls.md)); async wakeups may add a JVM + thread attach each, a separate suspect. - **Size table**: native library, generated bindings, and wasm plus JS glue (gzip), for both generators. Measure the native library with moq-ffi's default features on and off; the toggle changes nothing on wasm32, where diff --git a/quest/m1/kt-ffi-calls.md b/quest/m1/kt-ffi-calls.md index 2f0db3cfad7..a176380d903 100644 --- a/quest/m1/kt-ffi-calls.md +++ b/quest/m1/kt-ffi-calls.md @@ -4,12 +4,14 @@ A Kotlin call into moq-ffi costs about what the same call costs from Python or Go, sync and async alike, proven on the -[binding benchmark](/bench/README.md). Measured 2026-10-15: a sync -`sequence()` took 11-22 us from Kotlin against 1.3 us from Python and 0.12 us -from Go, and the 100 B throughput case kept about 5.8 cores busy for 2 MB/s. +[binding benchmark](/bench/README.md). ## Plan +Measured 2026-10-15: a sync `sequence()` took 11-22 us from Kotlin against +1.3 us from Python and 0.12 us from Go, and the 100 B throughput case kept +about 5.8 cores busy for 2 MB/s. + Decided with the maintainer on 2026-10-15: - **Profile first** (JFR or async-profiler) and confirm each cause before diff --git a/rs/moq-ffi/src/bench.rs b/rs/moq-ffi/src/bench.rs index 048e2143cae..eacbf8acdfd 100644 --- a/rs/moq-ffi/src/bench.rs +++ b/rs/moq-ffi/src/bench.rs @@ -47,6 +47,11 @@ impl Case { (self.publishers * self.subscribers * self.frames() * self.size) as u64 } + /// A non-uniform payload, so a zero-filled or truncated copy fails the check. + fn payload(&self) -> Vec { + (0..self.size).map(|i| (i % 251) as u8).collect() + } + fn name(&self) -> String { format!("size={}/pub={}/sub={}", self.size, self.publishers, self.subscribers) } @@ -129,8 +134,8 @@ impl Report { } } -fn check(case: &Case, round: u64, frame: usize, payload: &[u8], timestamp_us: Option) { - assert_eq!(payload.len(), case.size, "frame {frame} of round {round}"); +fn check(case: &Case, round: u64, frame: usize, want: &[u8], payload: &[u8], timestamp_us: Option) { + assert!(payload == want, "corrupt frame {frame} of round {round}"); assert_eq!(timestamp_us, Some(round * case.frames() as u64 + frame as u64)); } @@ -138,20 +143,19 @@ async fn ffi_round(case: &Case, tracks: &[Arc], consumers: &[A let readers: Vec<_> = consumers .iter() .map(|consumer| { - let (consumer, frames, size) = (consumer.clone(), case.frames(), case.size); + let (consumer, frames) = (consumer.clone(), case.frames()); tokio::spawn(async move { let group = consumer.recv_group().await.unwrap().expect("track ended"); let mut read = Vec::with_capacity(frames); for _ in 0..frames { let frame = group.read_frame().await.unwrap().expect("group ended"); - read.push((frame.payload.len(), frame.timestamp_us)); + read.push((frame.payload, frame.timestamp_us)); } - assert!(read.iter().all(|(len, _)| *len == size)); read }) }) .collect(); - let payload = vec![0x5a; case.size]; + let payload = case.payload(); for track in tracks { let group = track.append_group().unwrap(); for f in 0..case.frames() { @@ -164,8 +168,8 @@ async fn ffi_round(case: &Case, tracks: &[Arc], consumers: &[A group.finish().unwrap(); } for reader in readers { - for (f, (len, timestamp_us)) in reader.await.unwrap().into_iter().enumerate() { - check(case, n, f, &payload[..len], timestamp_us); + for (f, (read, timestamp_us)) in reader.await.unwrap().into_iter().enumerate() { + check(case, n, f, &payload, &read, timestamp_us); } } } @@ -192,7 +196,7 @@ async fn net_round( }) }) .collect(); - let payload = bytes::Bytes::from(vec![0x5a; case.size]); + let payload = bytes::Bytes::from(case.payload()); for track in &mut tracks { let mut group = track.append_group().unwrap(); for f in 0..case.frames() { @@ -205,8 +209,8 @@ async fn net_round( let mut consumers = Vec::with_capacity(readers.len()); for reader in readers { let (consumer, read) = reader.await.unwrap(); - for (f, (payload, timestamp_us)) in read.into_iter().enumerate() { - check(case, n, f, &payload, timestamp_us); + for (f, (read, timestamp_us)) in read.into_iter().enumerate() { + check(case, n, f, &payload, &read, timestamp_us); } consumers.push(consumer); } From 55cf29410c78dbec86e17af94d5a9a22eb97ac08 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 13:30:48 -0700 Subject: [PATCH 5/7] bench: bound every runner's cache and close Kotlin handles per case Each throughput case publishes under an origin with a 64 MiB cache, so eviction keeps memory flat instead of caching every published byte. Kotlin closes its per-round group handles and per-case handles inside the case, so native teardown never lands in a later measurement. The Go runner builds only on unix, where getrusage exists. Co-Authored-By: Claude Opus 5.5 --- go/wrapper/runtime_bench_test.go | 21 ++++++++++- kt/moq/src/jvmTest/kotlin/dev/moq/FfiBench.kt | 37 +++++++++++++------ py/moq-rs/benches/ffi.py | 7 +++- rs/moq-ffi/src/bench.rs | 20 +++++++++- 4 files changed, 69 insertions(+), 16 deletions(-) diff --git a/go/wrapper/runtime_bench_test.go b/go/wrapper/runtime_bench_test.go index a922b261eb2..6c793e0351d 100644 --- a/go/wrapper/runtime_bench_test.go +++ b/go/wrapper/runtime_bench_test.go @@ -1,3 +1,5 @@ +//go:build unix + package moq_test import ( @@ -7,6 +9,7 @@ import ( "fmt" "os" "path/filepath" + "runtime" "sort" "strings" "sync" @@ -27,6 +30,10 @@ var benchCases = []benchCase{ {1024, 1, 4}, {1024, 4, 1}, {1024, 4, 4}, } +// benchCacheBytes is each case's origin cache budget: far above one round's groups, so +// eviction only reclaims groups every subscriber has read, and memory stays flat. +const benchCacheBytes uint64 = 64 << 20 + // frames per group, holding a group under moq-net's 32 MB group cache. func (c benchCase) frames() int { return min(max((8<<20)/c.size, 1), 64) } @@ -143,10 +150,22 @@ func benchThroughput(b *testing.B, c benchCase) { ctx, cancel := context.WithCancel(context.Background()) b.Cleanup(cancel) frames := c.frames() + cacheBytes := benchCacheBytes + origin := moq.NewOriginProducerWithConfig(moq.OriginConfig{CacheCapacityBytes: &cacheBytes}) + // The origin owns the cache policy; keep it until the case ends. + b.Cleanup(func() { runtime.KeepAlive(origin) }) + broadcast, err := origin.CreateBroadcast("bench") + if err != nil { + b.Fatal(err) + } + b.Cleanup(func() { _ = broadcast.Close() }) tracks := make([]*moq.TrackProducer, c.publishers) consumers := make([]*moq.TrackConsumer, 0, c.publishers*c.subscribers) for p := range tracks { - tracks[p] = benchTrack(b, fmt.Sprint(p)) + tracks[p], err = broadcast.PublishTrack(fmt.Sprint(p), nil) + if err != nil { + b.Fatal(err) + } for range c.subscribers { consumer, err := tracks[p].Consume(nil) if err != nil { diff --git a/kt/moq/src/jvmTest/kotlin/dev/moq/FfiBench.kt b/kt/moq/src/jvmTest/kotlin/dev/moq/FfiBench.kt index 185650d9f86..92f612b6bf5 100644 --- a/kt/moq/src/jvmTest/kotlin/dev/moq/FfiBench.kt +++ b/kt/moq/src/jvmTest/kotlin/dev/moq/FfiBench.kt @@ -26,6 +26,10 @@ private val cases = listOf(100, 1024, 64 * 1024, 1024 * 1024).map { Case(it, 1, listOf(1 to 4, 4 to 1, 4 to 4).map { (p, s) -> Case(1024, p, s) } private const val WARMUP = 3 + +// Each case's origin cache budget: far above one round's groups, so eviction only +// reclaims groups every subscriber has read, and memory stays flat over a long run. +private const val CACHE_BYTES: ULong = 67_108_864uL // 64 MiB private val budgetNs = ((System.getenv("MOQ_BENCH_SECONDS")?.toDouble() ?: 1.0) * 1e9).toLong() /** Process CPU time, user plus system, across every thread. */ @@ -47,11 +51,14 @@ class FfiBench { val results = mutableListOf() runBlocking { cases.forEach { throughput(it, results) } } - val track = BroadcastProducer().publishTrack("calls", null) - val group = track.appendGroup().consume() + val broadcast = BroadcastProducer() + val track = broadcast.publishTrack("calls", null) + val producer = track.appendGroup() + val group = producer.consume() call("sequence", results) { group.sequence() } val consumer = track.consume(null) call("info", results) { consumer.info() } + listOf(consumer, group, producer, track, broadcast).forEach { it.close() } val json = buildJsonArray { results.forEach { @@ -67,7 +74,10 @@ class FfiBench { } private suspend fun throughput(case: Case, results: MutableList) = kotlinx.coroutines.coroutineScope { - val tracks = (0 until case.publishers).map { BroadcastProducer().publishTrack("$it", null) } + // Close every handle inside the case, so native teardown never lands in a later one. + val origin = OriginProducer(OriginConfig(cacheCapacityBytes = CACHE_BYTES)) + val broadcast = origin.createBroadcast("bench") + val tracks = (0 until case.publishers).map { broadcast.publishTrack("$it", null) } val consumers = tracks.flatMap { track -> (0 until case.subscribers).map { track.consume(null) } } // A non-uniform payload, so a zero-filled or truncated copy fails the check. val payload = ByteArray(case.size) { (it % 251).toByte() } @@ -76,20 +86,22 @@ class FfiBench { suspend fun step(n: Long) { val readers = consumers.map { consumer -> async { - val group = assertNotNull(consumer.recvGroup(), "track ended") - repeat(case.frames) { f -> - val frame = assertNotNull(group.readFrame(), "group ended") - assertTrue(frame.payload.contentEquals(payload), "corrupt frame $f of round $n") - assertEquals((n * case.frames + f).toULong(), frame.timestampUs) + assertNotNull(consumer.recvGroup(), "track ended").use { group -> + repeat(case.frames) { f -> + val frame = assertNotNull(group.readFrame(), "group ended") + assertTrue(frame.payload.contentEquals(payload), "corrupt frame $f of round $n") + assertEquals((n * case.frames + f).toULong(), frame.timestampUs) + } } } } tracks.forEach { track -> - val group = track.appendGroup() - repeat(case.frames) { f -> - group.writeFrame(Frame(payload = payload, timestampUs = (n * case.frames + f).toULong())) + track.appendGroup().use { group -> + repeat(case.frames) { f -> + group.writeFrame(Frame(payload = payload, timestampUs = (n * case.frames + f).toULong())) + } + group.finish() } - group.finish() } readers.awaitAll() } @@ -105,6 +117,7 @@ class FfiBench { val mb = (n - WARMUP) * case.delivered / 1e6 results += Result("throughput/${case.name}", "MB/s", mb / (wallNs / 1e9)) results += Result("cpu/${case.name}", "ms/MB", cpuUsedNs / 1e6 / mb) + (consumers + tracks + listOf(broadcast, origin)).forEach { it.close() } } private inline fun call(name: String, results: MutableList, fn: () -> Unit) { diff --git a/py/moq-rs/benches/ffi.py b/py/moq-rs/benches/ffi.py index 45caff6d212..47030d92d02 100644 --- a/py/moq-rs/benches/ffi.py +++ b/py/moq-rs/benches/ffi.py @@ -17,6 +17,9 @@ import moq WARMUP = 3 +# Each case's origin cache budget: far above one round's groups, so eviction only +# reclaims groups every subscriber has read, and memory stays flat over a long run. +CACHE_BYTES = 64 << 20 BUDGET = float(os.environ.get("MOQ_BENCH_SECONDS", "1")) @@ -65,7 +68,9 @@ async def read(case: Case, consumer: moq.TrackConsumer, payload: bytes, n: int) async def throughput(case: Case, results: list[dict]) -> None: - tracks = [moq.BroadcastProducer().publish_track(str(p)) for p in range(case.publishers)] + origin = moq.OriginProducer(cache_capacity_bytes=CACHE_BYTES) + broadcast = origin.create_broadcast("bench") + tracks = [broadcast.publish_track(str(p)) for p in range(case.publishers)] consumers = [track.consume() for track in tracks for _ in range(case.subscribers)] # A non-uniform payload, so a zero-filled or truncated copy fails the check. payload = bytes(i % 251 for i in range(case.size)) diff --git a/rs/moq-ffi/src/bench.rs b/rs/moq-ffi/src/bench.rs index eacbf8acdfd..0376de6e619 100644 --- a/rs/moq-ffi/src/bench.rs +++ b/rs/moq-ffi/src/bench.rs @@ -6,6 +6,7 @@ use crate::consumer::MoqTrackConsumer; use crate::media::MoqFrame; +use crate::origin::{MoqOriginConfig, MoqOriginProducer}; use crate::producer::{MoqBroadcastProducer, MoqTrackProducer}; use std::future::Future; @@ -15,6 +16,10 @@ use std::time::{Duration, Instant}; const WARMUP: u64 = 3; +/// Each case's origin cache budget: far above one round's groups, so eviction only +/// reclaims groups every subscriber has read, and memory stays flat over a long run. +const CACHE_BYTES: u64 = 64 << 20; + /// One throughput case: `publishers` tracks, each read by `subscribers` consumers. struct Case { size: usize, @@ -220,7 +225,10 @@ async fn net_round( async fn rust_ffi() { let mut report = Report::default(); for case in Case::all() { - let broadcast = MoqBroadcastProducer::new().unwrap(); + let origin = MoqOriginProducer::new(MoqOriginConfig { + cache_capacity_bytes: Some(CACHE_BYTES), + }); + let broadcast = origin.create_broadcast("bench".into()).unwrap(); let tracks: Vec<_> = (0..case.publishers) .map(|p| broadcast.publish_track(p.to_string(), None).unwrap()) .collect(); @@ -249,7 +257,14 @@ async fn rust_ffi() { async fn rust_net() { let mut report = Report::default(); for case in Case::all() { - let broadcast = moq_net::broadcast::Info::new().produce(); + let mut config = moq_net::origin::Config::default(); + let cache = moq_net::cache::Config::default() + .with_capacity(CACHE_BYTES) + .with_expiry(config.pool.expiry()); + config.pool = moq_net::cache::Pool::new(cache); + let (origin, driver) = moq_net::origin::Producer::new(config); + let driver = tokio::spawn(moq_net::time::run(driver)); + let broadcast = origin.create_broadcast("bench").unwrap(); let info = moq_net::track::Info::default().with_timescale(moq_net::Timescale::MICRO); let tracks: Vec<_> = (0..case.publishers) .map(|p| broadcast.create_track(p.to_string(), Some(info.clone())).unwrap()) @@ -268,6 +283,7 @@ async fn rust_net() { consumers.replace(returned); }) .await; + driver.abort(); } report.write("rust-net"); } From 404c00a1b628c60f4d05eabb8ef41bfffd2bf35d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 13:48:17 -0700 Subject: [PATCH 6/7] bench: guard each Rust round against a wedge, build the payload once per case The Rust baselines run about 18 budgets in one test, so nextest's kill at a minute cut off any MOQ_BENCH_SECONDS of 4 or more. Each round now times out on its own and the test is exempt from termination. The reference payload is built per case, as the wrapper runners do, instead of in every timed round. Co-Authored-By: Claude Opus 5.5 --- .config/nextest.toml | 6 ++++++ bench/ffi.sh | 2 +- rs/moq-ffi/src/bench.rs | 34 +++++++++++++++++++++++----------- 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index cd588f2899a..a6077fa7e64 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -31,6 +31,12 @@ default-filter = "not kind(bench)" filter = "package(moq-uring) | binary_id(moq-relay::runtime_uring) | (binary_id(moq-relay::embed) & test(/uring/))" test-group = "io-uring" +# The binding benchmark runs many `MOQ_BENCH_SECONDS` budgets back to back, so its +# length scales with the budget. It guards each round against a wedge itself. +[[profile.default.overrides]] +filter = "package(moq-ffi) & test(=bench::benchmark_ffi)" +slow-timeout = { period = "60s" } + [test-groups] io-uring = { max-threads = 4 } diff --git a/bench/ffi.sh b/bench/ffi.sh index 4d456b83b38..f4e760af8f0 100755 --- a/bench/ffi.sh +++ b/bench/ffi.sh @@ -8,7 +8,7 @@ export MOQ_BENCH_DIR="${MOQ_BENCH_DIR:-$root/target/bench/ffi}" rm -rf "$MOQ_BENCH_DIR" cd "$root" -# Through nextest, so a wedged read hits the slow-test timeout instead of the job's. +# Through nextest like every Rust test; each round guards itself against a wedged read. just rs test -p moq-ffi --cargo-profile release -E 'test(bench::benchmark_ffi)' --run-ignored only just go bench -benchtime="${MOQ_BENCH_SECONDS:-1}s" just py bench diff --git a/rs/moq-ffi/src/bench.rs b/rs/moq-ffi/src/bench.rs index 0376de6e619..5bda2707be0 100644 --- a/rs/moq-ffi/src/bench.rs +++ b/rs/moq-ffi/src/bench.rs @@ -16,6 +16,9 @@ use std::time::{Duration, Instant}; const WARMUP: u64 = 3; +/// Far longer than any honest round, so only a wedged read trips it. +const ROUND_TIMEOUT: Duration = Duration::from_secs(30); + /// Each case's origin cache budget: far above one round's groups, so eviction only /// reclaims groups every subscriber has read, and memory stays flat over a long run. const CACHE_BYTES: u64 = 64 << 20; @@ -94,13 +97,15 @@ impl Report { F: FnMut(u64) -> Fut, Fut: Future, { + // The test runs many budgets back to back, so a wedged read is caught per round, not by nextest. + let mut run = |n| tokio::time::timeout(ROUND_TIMEOUT, round(n)); for n in 0..WARMUP { - round(n).await; + run(n).await.expect("round wedged"); } let (start, cpu_start, budget) = (Instant::now(), cpu(), budget()); let mut n = WARMUP; while start.elapsed() < budget { - round(n).await; + run(n).await.expect("round wedged"); n += 1; } let (wall, cpu) = (start.elapsed(), cpu() - cpu_start); @@ -144,7 +149,13 @@ fn check(case: &Case, round: u64, frame: usize, want: &[u8], payload: &[u8], tim assert_eq!(timestamp_us, Some(round * case.frames() as u64 + frame as u64)); } -async fn ffi_round(case: &Case, tracks: &[Arc], consumers: &[Arc], n: u64) { +async fn ffi_round( + case: &Case, + payload: &[u8], + tracks: &[Arc], + consumers: &[Arc], + n: u64, +) { let readers: Vec<_> = consumers .iter() .map(|consumer| { @@ -160,12 +171,11 @@ async fn ffi_round(case: &Case, tracks: &[Arc], consumers: &[A }) }) .collect(); - let payload = case.payload(); for track in tracks { let group = track.append_group().unwrap(); for f in 0..case.frames() { let frame = MoqFrame { - payload: payload.clone(), + payload: payload.to_vec(), timestamp_us: Some(n * case.frames() as u64 + f as u64), }; group.write_frame(frame).unwrap(); @@ -174,7 +184,7 @@ async fn ffi_round(case: &Case, tracks: &[Arc], consumers: &[A } for reader in readers { for (f, (read, timestamp_us)) in reader.await.unwrap().into_iter().enumerate() { - check(case, n, f, &payload, &read, timestamp_us); + check(case, n, f, payload, &read, timestamp_us); } } } @@ -182,6 +192,7 @@ async fn ffi_round(case: &Case, tracks: &[Arc], consumers: &[A /// Subscribers are not shared handles, so each reader task borrows one for the round and returns it. async fn net_round( case: &Case, + payload: &[u8], mut tracks: Vec, consumers: Vec, n: u64, @@ -201,7 +212,6 @@ async fn net_round( }) }) .collect(); - let payload = bytes::Bytes::from(case.payload()); for track in &mut tracks { let mut group = track.append_group().unwrap(); for f in 0..case.frames() { @@ -215,7 +225,7 @@ async fn net_round( for reader in readers { let (consumer, read) = reader.await.unwrap(); for (f, (read, timestamp_us)) in read.into_iter().enumerate() { - check(case, n, f, &payload, &read, timestamp_us); + check(case, n, f, payload, &read, timestamp_us); } consumers.push(consumer); } @@ -236,8 +246,9 @@ async fn rust_ffi() { .iter() .flat_map(|track| (0..case.subscribers).map(|_| track.consume(None).unwrap())) .collect(); + let payload = case.payload(); report - .throughput(&case, |n| ffi_round(&case, &tracks, &consumers, n)) + .throughput(&case, |n| ffi_round(&case, &payload, &tracks, &consumers, n)) .await; } @@ -275,11 +286,12 @@ async fn rust_net() { .flat_map(|track| (0..case.subscribers).map(|_| track.subscribe(None))) .collect::>(), ); + let payload = case.payload(); // Producers are shared handles, so each round appends through its own clones. - let (case, tracks, consumers) = (&case, &tracks, &consumers); + let (case, payload, tracks, consumers) = (&case, &payload, &tracks, &consumers); report .throughput(case, |n| async move { - let returned = net_round(case, tracks.clone(), consumers.take(), n).await; + let returned = net_round(case, payload, tracks.clone(), consumers.take(), n).await; consumers.replace(returned); }) .await; From 186a571e164bd050be525d85ba40ad73fbffd836 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 13:58:59 -0700 Subject: [PATCH 7/7] bench: bound each Python and Kotlin round against a lost wakeup Their readers had no deadline, so a wakeup regression would hold the nightly job for its full timeout instead of failing the round, as Rust and Go do. Co-Authored-By: Claude Opus 5.5 --- kt/moq/src/jvmTest/kotlin/dev/moq/FfiBench.kt | 6 +++++- py/moq-rs/benches/ffi.py | 4 +++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/kt/moq/src/jvmTest/kotlin/dev/moq/FfiBench.kt b/kt/moq/src/jvmTest/kotlin/dev/moq/FfiBench.kt index 92f612b6bf5..6c638fe37e9 100644 --- a/kt/moq/src/jvmTest/kotlin/dev/moq/FfiBench.kt +++ b/kt/moq/src/jvmTest/kotlin/dev/moq/FfiBench.kt @@ -5,6 +5,7 @@ import java.lang.management.ManagementFactory import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeout import kotlinx.serialization.json.buildJsonArray import kotlinx.serialization.json.put import kotlinx.serialization.json.addJsonObject @@ -32,6 +33,9 @@ private const val WARMUP = 3 private const val CACHE_BYTES: ULong = 67_108_864uL // 64 MiB private val budgetNs = ((System.getenv("MOQ_BENCH_SECONDS")?.toDouble() ?: 1.0) * 1e9).toLong() +// Far longer than any honest round, so only a lost wakeup trips it. +private const val ROUND_TIMEOUT_MS = 30_000L + /** Process CPU time, user plus system, across every thread. */ private fun cpuNs(): Long = (ManagementFactory.getOperatingSystemMXBean() as com.sun.management.OperatingSystemMXBean).processCpuTime @@ -103,7 +107,7 @@ class FfiBench { group.finish() } } - readers.awaitAll() + withTimeout(ROUND_TIMEOUT_MS) { readers.awaitAll() } } repeat(WARMUP) { step(it.toLong()) } diff --git a/py/moq-rs/benches/ffi.py b/py/moq-rs/benches/ffi.py index 47030d92d02..4e22fb66a43 100644 --- a/py/moq-rs/benches/ffi.py +++ b/py/moq-rs/benches/ffi.py @@ -21,6 +21,8 @@ # reclaims groups every subscriber has read, and memory stays flat over a long run. CACHE_BYTES = 64 << 20 BUDGET = float(os.environ.get("MOQ_BENCH_SECONDS", "1")) +# Far longer than any honest round, so only a lost wakeup trips it. +ROUND_TIMEOUT = 30 @dataclass(frozen=True) @@ -83,7 +85,7 @@ async def step(n: int) -> None: for f in range(case.frames): group.write_frame(payload, timedelta(microseconds=n * case.frames + f)) group.finish() - await asyncio.gather(*readers) + await asyncio.wait_for(asyncio.gather(*readers), ROUND_TIMEOUT) for n in range(WARMUP): await step(n)