diff --git a/.config/nextest.toml b/.config/nextest.toml index cd588f2899a..a6077fa7e64 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -31,6 +31,12 @@ default-filter = "not kind(bench)" filter = "package(moq-uring) | binary_id(moq-relay::runtime_uring) | (binary_id(moq-relay::embed) & test(/uring/))" test-group = "io-uring" +# The binding benchmark runs many `MOQ_BENCH_SECONDS` budgets back to back, so its +# length scales with the budget. It guards each round against a wedge itself. +[[profile.default.overrides]] +filter = "package(moq-ffi) & test(=bench::benchmark_ffi)" +slow-timeout = { period = "60s" } + [test-groups] io-uring = { max-threads = 4 } diff --git a/.github/justfile b/.github/justfile index 7df3cb98190..979945d743f 100644 --- a/.github/justfile +++ b/.github/justfile @@ -15,6 +15,7 @@ test: python3 bench/summary.test.py python3 bench/relay.test.py python3 bench/profile.test.py + python3 bench/ffi.test.py python3 sh/gh/gst-formula.test.py python3 sh/changed.test.py python3 sh/dispatch-query.test.py diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index c39b95e165b..3b6b5606dcd 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -62,8 +62,10 @@ jobs: - name: Features recipe: rs features profile: ci - - name: Go publisher runtime benchmark - recipe: go bench -benchtime=1x + - name: Binding benchmark + recipe: bench-ffi + artifact: bench-ffi + artifact-path: target/bench/ffi - name: Audio grouping relay benchmark recipe: bench-audio # Smoke-tests the capture path: exact off-CPU waits, resolved worker stacks and CPU. @@ -127,6 +129,14 @@ jobs: env: NEXTEST_PROFILE: ${{ matrix.profile || 'default' }} + - name: Preserve results + if: ${{ matrix.artifact && !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ${{ matrix.artifact }} + path: ${{ matrix.artifact-path }} + retention-days: 45 + tests: name: Tests (${{ matrix.recipe }}) runs-on: ubuntu-24.04-arm diff --git a/Cargo.lock b/Cargo.lock index 3bf788a746d..83fa3150ceb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4461,6 +4461,7 @@ dependencies = [ "getrandom 0.4.3", "hang", "kio 0.6.2", + "libc", "moq-audio", "moq-json", "moq-mux", diff --git a/bench/README.md b/bench/README.md index 65ce0278517..5c8323a2ab1 100644 --- a/bench/README.md +++ b/bench/README.md @@ -131,16 +131,26 @@ hide a relay improvement behind generator saturation. CPU efficiency remains withheld until offered load and delivery/loss are comparable. Zero-baseline metrics retain absolute deltas when their percentage ratio is undefined. -Measure delivered payload through the Go bindings and a local QUIC relay: +Measure what each binding adds over Rust: ```bash -nix develop --command just go bench -benchtime=3s -count=3 +nix develop --command just bench-ffi ``` -This independently sweeps 1/4 publishers and 1/4 subscribers per publisher, -checks every received frame's payload and timestamp, and acknowledges each batch before publishing the -next. Reported MB/s counts delivery to every subscriber. The optimized native -library, relay and clients share the process, so pin the benchmark process with -`taskset` to compare one CPU with multiple CPUs. Compare identical CPU sets and -traffic shapes across revisions; the nightly smoke runs each shape once without -a timing threshold. +Every runner (Rust, Go, Python, Kotlin) runs the same in-process rounds with no +network: each publisher writes one group from the driving thread while every +subscriber reads it on its own task, and a round waits for every subscriber. It +sweeps frame sizes from 100 B to 1 MiB at one publisher and subscriber, then +1/4 publishers by 1/4 subscribers at 1 KiB, and times two sync calls +(`sequence`, `info`). Each case reports delivered MB/s and process CPU (user + +system, every thread) per MB. `MOQ_BENCH_SECONDS` sets each case's time budget +(default 1); every received frame is checked byte for byte. + +Each runner writes `target/bench/ffi/.json`; `bench/ffi.py` combines +them into a table and `report.json`. Ratios are against `rust-ffi`, moq-ffi's +API called from Rust without crossing the FFI, so a wrapper's ratio is what its +boundary and wrapper cost. `rust-net` runs the same rounds on moq-net, so its +ratio is what moq-ffi's handles and runtime cost. Ratios hold only within one +machine. The nightly runs it, prints the table in the job summary, and keeps the +results as the `bench-ffi` artifact. Each language also has its own recipe +(`just go bench`, `just py bench`, `just kt bench`). diff --git a/bench/ffi.py b/bench/ffi.py new file mode 100644 index 00000000000..7f7703aa6c4 --- /dev/null +++ b/bench/ffi.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Combine the binding benchmark's per-runner results into one table and report. + +Each runner writes `/.json`, a list of {name, unit, value}. The table shows +every runner beside its ratio to the `rust-ffi` baseline, run on the same machine, which +cancels most of the runner's own speed. `report.json` holds every value and ratio, named +`/`, for the trend history. +""" +import argparse +import json +import os +from pathlib import Path + +BASELINE = 'rust-ffi' +# Baselines first, then the wrappers in a stable order. +ORDER = ('rust-net', 'rust-ffi') +SMALLER_IS_BETTER = {'ms/MB', 'ns'} + + +def load(directory): + runners = {} + for path in sorted(Path(directory).glob('*.json')): + if path.name == 'report.json': + continue + runners[path.stem] = {entry['name']: entry for entry in json.loads(path.read_text())} + if BASELINE not in runners: + raise ValueError(f'missing the {BASELINE} baseline in {directory}') + return runners + + +def order(runners): + return sorted(runners, key=lambda r: (ORDER.index(r) if r in ORDER else len(ORDER), r)) + + +def ratio(entry, baseline): + """How many times worse than the baseline: above 1 is slower or costlier.""" + if baseline is None or not baseline['value'] or not entry['value']: + return None + if entry['unit'] in SMALLER_IS_BETTER: + return entry['value'] / baseline['value'] + return baseline['value'] / entry['value'] + + +def report(runners): + entries = [] + baseline = runners[BASELINE] + for runner in order(runners): + for name, entry in runners[runner].items(): + entries.append({'name': f'{runner}/{name}', 'unit': entry['unit'], 'value': entry['value']}) + if runner != BASELINE and (r := ratio(entry, baseline.get(name))) is not None: + entries.append({'name': f'{runner}/{name} vs {BASELINE}', 'unit': 'x', 'value': r}) + return entries + + +def table(runners): + columns = order(runners) + names = list(dict.fromkeys(name for runner in columns for name in runners[runner])) + baseline = runners[BASELINE] + lines = [ + '| benchmark | unit | ' + ' | '.join(columns) + ' |', + '|---|---|' + '---|' * len(columns), + ] + for name in names: + unit = next(runners[r][name]['unit'] for r in columns if name in runners[r]) + cells = [] + for runner in columns: + entry = runners[runner].get(name) + if entry is None: + cells.append('') + continue + cell = f"{entry['value']:.4g}" + if runner != BASELINE and (r := ratio(entry, baseline.get(name))) is not None: + cell += f' ({r:.2f}x)' + cells.append(cell) + lines.append(f'| {name} | {unit} | ' + ' | '.join(cells) + ' |') + return '\n'.join(lines) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('directory') + args = parser.parse_args() + runners = load(args.directory) + (Path(args.directory) / 'report.json').write_text(json.dumps(report(runners), indent=2)) + markdown = ( + f'## Binding benchmark\n\nRatios are against `{BASELINE}`; above 1x is slower or costlier.\n\n' + + table(runners) + + '\n' + ) + print(markdown) + if summary := os.environ.get('GITHUB_STEP_SUMMARY'): + with open(summary, 'a') as f: + f.write(markdown) + + +if __name__ == '__main__': + main() diff --git a/bench/ffi.sh b/bench/ffi.sh new file mode 100755 index 00000000000..f4e760af8f0 --- /dev/null +++ b/bench/ffi.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Run the binding benchmark: the Rust baselines and every wrapper's runner on this +# machine, then the ratio table. Ratios only hold within one machine, so stale +# results from an earlier run are cleared first. +set -euo pipefail +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +export MOQ_BENCH_DIR="${MOQ_BENCH_DIR:-$root/target/bench/ffi}" +rm -rf "$MOQ_BENCH_DIR" + +cd "$root" +# Through nextest like every Rust test; each round guards itself against a wedged read. +just rs test -p moq-ffi --cargo-profile release -E 'test(bench::benchmark_ffi)' --run-ignored only +just go bench -benchtime="${MOQ_BENCH_SECONDS:-1}s" +just py bench +just kt bench +python3 bench/ffi.py "$MOQ_BENCH_DIR" diff --git a/bench/ffi.test.py b/bench/ffi.test.py new file mode 100644 index 00000000000..029d5a23824 --- /dev/null +++ b/bench/ffi.test.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path + +spec = importlib.util.spec_from_file_location('ffi', Path(__file__).with_name('ffi.py')) +ffi = importlib.util.module_from_spec(spec) +spec.loader.exec_module(ffi) + + +def write(directory, runner, entries): + (Path(directory) / f'{runner}.json').write_text( + json.dumps([{'name': n, 'unit': u, 'value': v} for n, u, v in entries]) + ) + + +class FfiTest(unittest.TestCase): + def runners(self, directory): + write(directory, 'rust-ffi', [('throughput/a', 'MB/s', 100.0), ('cpu/a', 'ms/MB', 2.0)]) + write(directory, 'rust-net', [('throughput/a', 'MB/s', 200.0)]) + write(directory, 'python', [('throughput/a', 'MB/s', 25.0), ('cpu/a', 'ms/MB', 8.0)]) + return ffi.load(directory) + + def test_ratio_is_worse_above_one(self): + with tempfile.TemporaryDirectory() as directory: + report = {e['name']: e['value'] for e in ffi.report(self.runners(directory))} + self.assertEqual(report['python/throughput/a vs rust-ffi'], 4.0) + self.assertEqual(report['python/cpu/a vs rust-ffi'], 4.0) + self.assertEqual(report['rust-net/throughput/a vs rust-ffi'], 0.5) + self.assertNotIn('rust-ffi/throughput/a vs rust-ffi', report) + + def test_table_orders_baselines_first(self): + with tempfile.TemporaryDirectory() as directory: + table = ffi.table(self.runners(directory)) + self.assertTrue(table.splitlines()[0].startswith('| benchmark | unit | rust-net | rust-ffi | python |')) + self.assertIn('| throughput/a | MB/s | 200 (0.50x) | 100 | 25 (4.00x) |', table) + + def test_missing_baseline_fails(self): + with tempfile.TemporaryDirectory() as directory: + write(directory, 'python', [('throughput/a', 'MB/s', 25.0)]) + with self.assertRaises(ValueError): + ffi.load(directory) + + def test_skips_its_own_report(self): + with tempfile.TemporaryDirectory() as directory: + self.runners(directory) + (Path(directory) / 'report.json').write_text('[]') + self.assertNotIn('report', ffi.load(directory)) + + +if __name__ == '__main__': + unittest.main() diff --git a/go/wrapper/runtime_bench_test.go b/go/wrapper/runtime_bench_test.go index 20a733cfa80..bbccac66c9b 100644 --- a/go/wrapper/runtime_bench_test.go +++ b/go/wrapper/runtime_bench_test.go @@ -1,139 +1,252 @@ +//go:build unix + package moq_test import ( "bytes" "context" + "encoding/json" "fmt" + "os" + "path/filepath" + "runtime" + "sort" + "strings" + "sync" + "syscall" "testing" "time" "moq.dev/moq" ) -// BenchmarkPublisher measures delivered payload through real QUIC sessions and -// generated Go callbacks. Each round waits for every subscriber, so dropping -// groups or merely buffering writes cannot improve the reported throughput. -func BenchmarkPublisher(b *testing.B) { - for _, publishers := range []int{1, 4} { - for _, subscribers := range []int{1, 4} { - b.Run(fmt.Sprintf("publishers=%d/subscribers=%d", publishers, subscribers), func(b *testing.B) { - ctx, cancel := context.WithCancel(context.Background()) - server, err := moq.Listen(ctx, "127.0.0.1:0", moq.WithTLSGenerate("localhost")) - if err != nil { - cancel() - b.Fatal(err) +// benchCase is one throughput case of the binding benchmark (see bench/README.md): +// publishers tracks, each read by subscribers consumers. +type benchCase struct{ size, publishers, subscribers int } + +// Frame sizes at one publisher and subscriber, then fan-out at 1 KiB. +var benchCases = []benchCase{ + {100, 1, 1}, {1024, 1, 1}, {64 * 1024, 1, 1}, {1024 * 1024, 1, 1}, + {1024, 1, 4}, {1024, 4, 1}, {1024, 4, 4}, +} + +// benchCacheBytes is each case's origin cache budget: far above one round's groups, so +// eviction only reclaims groups every subscriber has read, and memory stays flat. +const benchCacheBytes uint64 = 64 << 20 + +// frames per group, holding a group under moq-net's 32 MB group cache. +func (c benchCase) frames() int { return min(max((8<<20)/c.size, 1), 64) } + +func (c benchCase) name() string { + return fmt.Sprintf("size=%d/pub=%d/sub=%d", c.size, c.publishers, c.subscribers) +} + +// benchResult is one entry of the shared JSON report. +type benchResult struct { + Name string `json:"name"` + Unit string `json:"unit"` + Value float64 `json:"value"` +} + +// benchResults holds each benchmark's latest run: testing.B reruns a benchmark with a +// growing b.N, and the last run is the one it reports. +var benchResults sync.Map + +func benchRecord(name, unit string, value float64) { + benchResults.Store(name, benchResult{name, unit, value}) +} + +func cpuTime() time.Duration { + var usage syscall.Rusage + if err := syscall.Getrusage(syscall.RUSAGE_SELF, &usage); err != nil { + panic(err) + } + return time.Duration(usage.Utime.Nano() + usage.Stime.Nano()) +} + +// TestMain writes BenchmarkFFI's results to $MOQ_BENCH_DIR/go.json after the run. +func TestMain(m *testing.M) { + code := m.Run() + if dir := os.Getenv("MOQ_BENCH_DIR"); dir != "" && code == 0 { + var results []benchResult + benchResults.Range(func(_, value any) bool { + results = append(results, value.(benchResult)) + return true + }) + sort.Slice(results, func(i, j int) bool { return results[i].Name < results[j].Name }) + data, err := json.MarshalIndent(results, "", " ") + if err == nil { + err = os.MkdirAll(dir, 0o755) + } + if err == nil { + err = os.WriteFile(filepath.Join(dir, "go.json"), data, 0o644) + } + if err != nil { + fmt.Fprintln(os.Stderr, err) + code = 1 + } + } + os.Exit(code) +} + +// BenchmarkFFI is the Go runner of the binding benchmark: the same in-process rounds as +// every wrapper, so dividing by the Rust baselines isolates what Go adds. +func BenchmarkFFI(b *testing.B) { + for _, c := range benchCases { + b.Run("throughput/"+c.name(), func(b *testing.B) { benchThroughput(b, c) }) + } + b.Run("call/sequence", func(b *testing.B) { + track := benchTrack(b, "calls") + producer, err := track.AppendGroup() + if err != nil { + b.Fatal(err) + } + group, err := producer.Consume() + if err != nil { + b.Fatal(err) + } + b.Cleanup(group.Cancel) + benchCall(b, func() { group.Sequence() }) + }) + b.Run("call/info", func(b *testing.B) { + consumer, err := benchTrack(b, "calls").Consume(nil) + if err != nil { + b.Fatal(err) + } + b.Cleanup(consumer.Cancel) + benchCall(b, func() { + if _, err := consumer.Info(); err != nil { + b.Fatal(err) + } + }) + }) +} + +func benchTrack(b *testing.B, name string) *moq.TrackProducer { + broadcast, err := moq.NewBroadcastProducer() + if err != nil { + b.Fatal(err) + } + b.Cleanup(func() { _ = broadcast.Close() }) + track, err := broadcast.PublishTrack(name, nil) + if err != nil { + b.Fatal(err) + } + return track +} + +func benchCall(b *testing.B, call func()) { + b.ResetTimer() + for range b.N { + call() + } + benchRecord(strings.TrimPrefix(b.Name(), "BenchmarkFFI/"), "ns", float64(b.Elapsed().Nanoseconds())/float64(b.N)) +} + +// benchThroughput runs one round per iteration: each publisher writes a group from this +// goroutine in turn while every consumer reads it on its own goroutine. A round waits for +// every consumer, so buffering writes cannot improve the reported throughput. +func benchThroughput(b *testing.B, c benchCase) { + ctx, cancel := context.WithCancel(context.Background()) + b.Cleanup(cancel) + frames := c.frames() + cacheBytes := benchCacheBytes + origin := moq.NewOriginProducerWithConfig(moq.OriginConfig{CacheCapacityBytes: &cacheBytes}) + // The origin owns the cache policy; keep it until the case ends. + b.Cleanup(func() { runtime.KeepAlive(origin) }) + broadcast, err := origin.CreateBroadcast("bench") + if err != nil { + b.Fatal(err) + } + b.Cleanup(func() { _ = broadcast.Close() }) + tracks := make([]*moq.TrackProducer, c.publishers) + consumers := make([]*moq.TrackConsumer, 0, c.publishers*c.subscribers) + for p := range tracks { + tracks[p], err = broadcast.PublishTrack(fmt.Sprint(p), timedInfo()) + if err != nil { + b.Fatal(err) + } + for range c.subscribers { + consumer, err := tracks[p].Consume(nil) + if err != nil { + b.Fatal(err) + } + b.Cleanup(consumer.Cancel) + consumers = append(consumers, consumer) + } + } + // A non-uniform payload, so a zero-filled or truncated copy fails the check. + payload := make([]byte, c.size) + for i := range payload { + payload[i] = byte(i % 251) + } + delivered := int64(len(consumers) * frames * c.size) + b.SetBytes(delivered) + + round := func(n int) error { + done := make(chan error, len(consumers)) + for _, consumer := range consumers { + go func() { + group, err := consumer.RecvGroup(ctx) + if err == nil && group == nil { + err = fmt.Errorf("track ended") } - served := make(chan error, 1) - go func() { served <- server.Serve(ctx) }() - b.Cleanup(func() { cancel(); _ = server.Close(); <-served }) - url := "https://" + server.LocalAddr() - dial := func() *moq.Client { - client, err := moq.Dial(ctx, url, moq.WithTLSVerify(false)) - if err != nil { - b.Fatal(err) - } - b.Cleanup(func() { _ = client.Close() }) - return client + if err != nil { + done <- err + return } - tracks := make([]*moq.TrackProducer, publishers) - consumers := make([]*moq.TrackConsumer, 0, publishers*subscribers) - for p := range tracks { - client := dial() - name := fmt.Sprintf("runtime/%d", p) - broadcast, err := client.CreateBroadcast(name) - if err != nil { - b.Fatal(err) - } - b.Cleanup(func() { _ = broadcast.Close() }) - tracks[p], err = broadcast.PublishTrack("payload", timedInfo()) + defer group.Cancel() + for f := range frames { + frame, err := group.ReadFrame(ctx) if err != nil { - b.Fatal(err) + done <- err + return } - if err := broadcast.Announce(moq.Route{}); err != nil { - b.Fatal(err) - } - for s := 0; s < subscribers; s++ { - subscriber := dial() - announced, err := subscriber.AnnouncedBroadcast(name) - if err != nil { - b.Fatal(err) - } - b.Cleanup(announced.Cancel) - remote, err := announced.Available(ctx) - if err != nil { - b.Fatal(err) - } - consumer, err := remote.SubscribeTrack(ctx, "payload", nil) - if err != nil { - b.Fatal(err) - } - b.Cleanup(consumer.Cancel) - consumers = append(consumers, consumer) + want := time.Duration(n*frames+f) * time.Microsecond + if frame == nil || !bytes.Equal(frame.Payload, payload) || frame.Timestamp == nil || *frame.Timestamp != want { + done <- fmt.Errorf("missing, reordered, or corrupt frame %d of round %d", f, n) + return } } - const frames = 64 - payload := bytes.Repeat([]byte{0x5a}, 1024) - b.SetBytes(int64(publishers * subscribers * frames * len(payload))) - b.ResetTimer() - for n := 0; n < b.N; n++ { - done := make(chan error, len(tracks)+len(consumers)) - for _, consumer := range consumers { - go func(consumer *moq.TrackConsumer) { - group, err := consumer.RecvGroup(ctx) - if err != nil { - done <- err - return - } - if group == nil { - done <- fmt.Errorf("track ended before batch") - return - } - defer group.Cancel() - for f := 0; f < frames; f++ { - frame, err := group.ReadFrame(ctx) - if err != nil { - done <- err - return - } - want := time.Duration(n*frames+f) * time.Millisecond - if frame == nil || frame.Timestamp == nil || *frame.Timestamp != want || !bytes.Equal(frame.Payload, payload) { - done <- fmt.Errorf("missing, reordered, or corrupt frame %d", f) - return - } - } - done <- nil - }(consumer) - } - for _, track := range tracks { - go func(track *moq.TrackProducer) { - group, err := track.AppendGroup() - if err != nil { - done <- err - return - } - for f := 0; f < frames; f++ { - if err := group.WriteFrame(moq.Frame{Payload: payload, Timestamp: ts(time.Duration(n*frames+f) * time.Millisecond)}); err != nil { - done <- err - return - } - } - done <- group.Finish() - }(track) - } - var failure error - for range len(tracks) + len(consumers) { - if err := <-done; err != nil { - if failure == nil { - failure = err - } - cancel() - } - } - if failure != nil { - b.Fatal(failure) - } + done <- nil + }() + } + for _, track := range tracks { + group, err := track.AppendGroup() + if err != nil { + return err + } + for f := range frames { + timestamp := time.Duration(n*frames+f) * time.Microsecond + if err := group.WriteFrame(moq.Frame{Payload: payload, Timestamp: ×tamp}); err != nil { + return err } - }) + } + if err := group.Finish(); err != nil { + return err + } + } + var failure error + for range consumers { + if err := <-done; err != nil && failure == nil { + failure = err + cancel() + } + } + return failure + } + + b.ResetTimer() + cpu := cpuTime() + for n := range b.N { + if err := round(n); err != nil { + b.Fatal(err) } } + mb := float64(int64(b.N)*delivered) / 1e6 + cpuPerMB := (cpuTime() - cpu).Seconds() * 1e3 / mb + b.ReportMetric(cpuPerMB, "cpu-ms/MB") + name := c.name() + benchRecord("throughput/"+name, "MB/s", mb/b.Elapsed().Seconds()) + benchRecord("cpu/"+name, "ms/MB", cpuPerMB) } diff --git a/justfile b/justfile index 55111153e6f..1b926cd1fc8 100644 --- a/justfile +++ b/justfile @@ -46,6 +46,10 @@ dev: bench $BASE="": bench/run.sh "$BASE" +# Benchmark the Rust, Go, Python, and Kotlin bindings against two Rust baselines. +bench-ffi: + bench/ffi.sh + # Measure audio group sizes across relay publisher/subscriber and fanout shapes. bench-audio: bench/run.sh --audio diff --git a/kt/justfile b/kt/justfile index 2a3994710a6..5ef639e0768 100644 --- a/kt/justfile +++ b/kt/justfile @@ -21,6 +21,10 @@ check: generate: bash ../sh/kt/generate.sh +# Run the binding benchmark's Kotlin runner (see bench/README.md) on a release build. +bench: + bash ../sh/kt/bench.sh + # Runs in the caller's directory, so a relative `--lib-dir` or `--output` means # what it would to the script itself. See kt/README.md for the --lib-dir layout. diff --git a/kt/moq/src/jvmTest/kotlin/dev/moq/FfiBench.kt b/kt/moq/src/jvmTest/kotlin/dev/moq/FfiBench.kt new file mode 100644 index 00000000000..ace8b0b44d6 --- /dev/null +++ b/kt/moq/src/jvmTest/kotlin/dev/moq/FfiBench.kt @@ -0,0 +1,136 @@ +package dev.moq + +import java.io.File +import java.lang.management.ManagementFactory +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeout +import kotlinx.serialization.json.buildJsonArray +import kotlinx.serialization.json.put +import kotlinx.serialization.json.addJsonObject +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import kotlin.test.assertTrue + +/** One throughput case: [publishers] tracks, each read by [subscribers] consumers. */ +private data class Case(val size: Int, val publishers: Int, val subscribers: Int) { + /** Frames per group, holding a group under moq-net's 32 MB group cache. */ + val frames = ((8 shl 20) / size).coerceIn(1, 64) + val delivered = publishers.toLong() * subscribers * frames * size + val name = "size=$size/pub=$publishers/sub=$subscribers" +} + +// Frame sizes at one publisher and subscriber, then fan-out at 1 KiB. +private val cases = listOf(100, 1024, 64 * 1024, 1024 * 1024).map { Case(it, 1, 1) } + + listOf(1 to 4, 4 to 1, 4 to 4).map { (p, s) -> Case(1024, p, s) } + +private const val WARMUP = 3 + +// Each case's origin cache budget: far above one round's groups, so eviction only +// reclaims groups every subscriber has read, and memory stays flat over a long run. +private const val CACHE_BYTES: ULong = 67_108_864uL // 64 MiB +private val budgetNs = ((System.getenv("MOQ_BENCH_SECONDS")?.toDouble() ?: 1.0) * 1e9).toLong() + +// Far longer than any honest round, so only a lost wakeup trips it. +private const val ROUND_TIMEOUT_MS = 30_000L + +/** Process CPU time, user plus system, across every thread. */ +private fun cpuNs(): Long = + (ManagementFactory.getOperatingSystemMXBean() as com.sun.management.OperatingSystemMXBean).processCpuTime + +private class Result(val name: String, val unit: String, val value: Double) + +/** + * The Kotlin runner of the binding benchmark (`just bench-ffi`, see bench/README.md). + * + * The same in-process rounds as every wrapper, so dividing by the Rust baselines isolates + * what Kotlin adds. Runs only when `MOQ_BENCH_DIR` is set, so `just kt check` skips it. + */ +class FfiBench { + @Test + fun benchmark() { + val dir = System.getenv("MOQ_BENCH_DIR") ?: return + val results = mutableListOf() + runBlocking { cases.forEach { throughput(it, results) } } + + val broadcast = BroadcastProducer() + val track = broadcast.publishTrack("calls", null) + val producer = track.appendGroup() + val group = producer.consume() + call("sequence", results) { group.sequence() } + val consumer = track.consume(null) + call("info", results) { consumer.info() } + listOf(consumer, group, producer, track, broadcast).forEach { it.close() } + + val json = buildJsonArray { + results.forEach { + addJsonObject { + put("name", it.name) + put("unit", it.unit) + put("value", it.value) + } + } + } + File(dir).mkdirs() + File(dir, "kotlin.json").writeText(json.toString()) + } + + private suspend fun throughput(case: Case, results: MutableList) = kotlinx.coroutines.coroutineScope { + // Close every handle inside the case, so native teardown never lands in a later one. + val origin = OriginProducer(OriginConfig(cacheCapacityBytes = CACHE_BYTES)) + val broadcast = origin.createBroadcast("bench") + val tracks = (0 until case.publishers).map { broadcast.publishTrack("$it", TrackInfo(timescale = 1_000_000uL)) } + val consumers = tracks.flatMap { track -> (0 until case.subscribers).map { track.consume(null) } } + // A non-uniform payload, so a zero-filled or truncated copy fails the check. + val payload = ByteArray(case.size) { (it % 251).toByte() } + + // Each consumer reads on its own coroutine while this one writes each group in turn. + suspend fun step(n: Long) { + val readers = consumers.map { consumer -> + async { + assertNotNull(consumer.recvGroup(), "track ended").use { group -> + repeat(case.frames) { f -> + val frame = assertNotNull(group.readFrame(), "group ended") + assertTrue(frame.payload.contentEquals(payload), "corrupt frame $f of round $n") + assertEquals((n * case.frames + f).toULong(), frame.timestampUs) + } + } + } + } + tracks.forEach { track -> + track.appendGroup().use { group -> + repeat(case.frames) { f -> + group.writeFrame(Frame(payload = payload, timestampUs = (n * case.frames + f).toULong())) + } + group.finish() + } + } + withTimeout(ROUND_TIMEOUT_MS) { readers.awaitAll() } + } + + repeat(WARMUP) { step(it.toLong()) } + val (start, cpuStart) = System.nanoTime() to cpuNs() + var n = WARMUP.toLong() + while (System.nanoTime() - start < budgetNs) { + step(n) + n++ + } + val (wallNs, cpuUsedNs) = (System.nanoTime() - start) to (cpuNs() - cpuStart) + val mb = (n - WARMUP) * case.delivered / 1e6 + results += Result("throughput/${case.name}", "MB/s", mb / (wallNs / 1e9)) + results += Result("cpu/${case.name}", "ms/MB", cpuUsedNs / 1e6 / mb) + (consumers + tracks + listOf(broadcast, origin)).forEach { it.close() } + } + + private inline fun call(name: String, results: MutableList, fn: () -> Unit) { + val start = System.nanoTime() + var calls = 0L + while (System.nanoTime() - start < budgetNs) { + repeat(1000) { fn() } + calls += 1000 + } + results += Result("call/$name", "ns", (System.nanoTime() - start).toDouble() / calls) + } +} diff --git a/py/justfile b/py/justfile index da3fc8cfbf7..fd5bde95d3c 100644 --- a/py/justfile +++ b/py/justfile @@ -62,6 +62,13 @@ build: uv sync --locked --no-install-workspace just _develop +# Run the binding benchmark's Python runner (see bench/README.md) on a release build. +bench: + uv sync --locked --no-install-workspace + cd moq-ffi && uv run --no-sync maturin develop --locked --uv --release + uv pip install --no-deps -e moq-rs + uv run --no-sync python moq-rs/benches/ffi.py + # Build the pure-python moq-rs wrapper sdist + wheel into py/dist (for release). # moq-ffi is built separately by maturin (see release-py.yml); the wrapper is diff --git a/py/moq-rs/benches/ffi.py b/py/moq-rs/benches/ffi.py new file mode 100644 index 00000000000..b7a3ec4f0d2 --- /dev/null +++ b/py/moq-rs/benches/ffi.py @@ -0,0 +1,131 @@ +"""Python runner of the binding benchmark (`just bench-ffi`, see bench/README.md). + +The same in-process rounds as every wrapper, so dividing by the Rust baselines +isolates what Python adds: the FFI boundary, asyncio bridging, and the wrapper. +""" + +import asyncio +import json +import os +import resource +import time +from dataclasses import dataclass +from datetime import timedelta +from pathlib import Path +from typing import Callable + +import moq + +WARMUP = 3 +# Each case's origin cache budget: far above one round's groups, so eviction only +# reclaims groups every subscriber has read, and memory stays flat over a long run. +CACHE_BYTES = 64 << 20 +BUDGET = float(os.environ.get("MOQ_BENCH_SECONDS", "1")) +# Far longer than any honest round, so only a lost wakeup trips it. +ROUND_TIMEOUT = 30 + + +@dataclass(frozen=True) +class Case: + """One throughput case: `publishers` tracks, each read by `subscribers` consumers.""" + + size: int + publishers: int + subscribers: int + + @property + def frames(self) -> int: + """Frames per group, holding a group under moq-net's 32 MB group cache.""" + return min(max((8 << 20) // self.size, 1), 64) + + @property + def delivered(self) -> int: + return self.publishers * self.subscribers * self.frames * self.size + + @property + def name(self) -> str: + return f"size={self.size}/pub={self.publishers}/sub={self.subscribers}" + + +# Frame sizes at one publisher and subscriber, then fan-out at 1 KiB. +CASES = [Case(size, 1, 1) for size in (100, 1024, 64 * 1024, 1024 * 1024)] + [ + Case(1024, p, s) for p, s in ((1, 4), (4, 1), (4, 4)) +] + + +def cpu() -> float: + """Process CPU seconds, user plus system, across every thread.""" + usage = resource.getrusage(resource.RUSAGE_SELF) + return usage.ru_utime + usage.ru_stime + + +async def read(case: Case, consumer: moq.TrackConsumer, payload: bytes, n: int) -> None: + group = await consumer.recv_group() + assert group is not None, "track ended" + for f in range(case.frames): + frame = await group.read_frame() + assert frame is not None, "group ended" + assert frame.payload == payload, f"corrupt frame {f} of round {n}" + assert frame.timestamp == timedelta(microseconds=n * case.frames + f) + + +async def throughput(case: Case, results: list[dict]) -> None: + origin = moq.OriginProducer(cache_capacity_bytes=CACHE_BYTES) + broadcast = origin.create_broadcast("bench") + tracks = [broadcast.publish_track(str(p), moq.TrackInfo(timescale=1_000_000)) for p in range(case.publishers)] + consumers = [track.consume() for track in tracks for _ in range(case.subscribers)] + # A non-uniform payload, so a zero-filled or truncated copy fails the check. + payload = bytes(i % 251 for i in range(case.size)) + + async def step(n: int) -> None: + # Each consumer reads on its own task while this one writes each group in turn. + readers = [asyncio.create_task(read(case, consumer, payload, n)) for consumer in consumers] + for track in tracks: + group = track.append_group() + for f in range(case.frames): + group.write_frame(payload, timedelta(microseconds=n * case.frames + f)) + group.finish() + await asyncio.wait_for(asyncio.gather(*readers), ROUND_TIMEOUT) + + for n in range(WARMUP): + await step(n) + start, cpu_start, n = time.perf_counter(), cpu(), WARMUP + while time.perf_counter() - start < BUDGET: + await step(n) + n += 1 + wall, cpu_used = time.perf_counter() - start, cpu() - cpu_start + mb = (n - WARMUP) * case.delivered / 1e6 + results.append({"name": f"throughput/{case.name}", "unit": "MB/s", "value": mb / wall}) + results.append({"name": f"cpu/{case.name}", "unit": "ms/MB", "value": cpu_used * 1e3 / mb}) + + +def call(name: str, fn: Callable[[], object], results: list[dict]) -> None: + start, calls = time.perf_counter(), 0 + while time.perf_counter() - start < BUDGET: + for _ in range(1000): + fn() + calls += 1000 + results.append({"name": f"call/{name}", "unit": "ns", "value": (time.perf_counter() - start) * 1e9 / calls}) + + +async def main() -> None: + results: list[dict] = [] + for case in CASES: + await throughput(case, results) + + track = moq.BroadcastProducer().publish_track("calls") + group = track.append_group().consume() + call("sequence", lambda: group.sequence, results) + consumer = track.consume() + call("info", consumer.info, results) + + report = json.dumps(results, indent=2) + if out := os.environ.get("MOQ_BENCH_DIR"): + Path(out).mkdir(parents=True, exist_ok=True) + (Path(out) / "python.json").write_text(report) + else: + print(report) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/quest/m1/README.md b/quest/m1/README.md index 05c0803f7f7..df3c7a9beaa 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -115,11 +115,11 @@ above them. - [Generated C bindings](/quest/m1/c/README.md) - C generated from moq-ffi ships as `moq-c` 0.8.0 and replaces the hand-written libmoq - [BoltFFI or UniFFI](/quest/m1/boltffi.md) - BoltFFI vs UniFFI on a moq-ffi slice in Python, Kotlin, and Swift decides which one the moq-dev fork starts from - [Hard-fork the binding generator](/quest/m1/uniffi-fork.md) - one moq-dev repo holds the chosen generator's core and every language backend, released under one tag and one pin -- [Binding benchmark harness](/quest/m1/ffi-bench.md) - Python, Kotlin, and Go benchmark their wrappers nightly against two Rust baselines, so FFI overhead is tracked - [Benchmark trend site](/quest/m1/bench-site.md) - a GitHub Pages site charts the binding, Rust, and JS nightly benchmarks over time - [Link the benchmark trends](/quest/m1/bench-site-doc.md) - the docs point language choosers at the trend site - [Swift, Dart, and C++ benchmarks](/quest/m1/ffi-bench-wrappers.md) - the other three wrappers join the nightly binding benchmark - [Payload bytes](/quest/m1/ffi-bytes.md) - payloads cross the FFI in one bulk copy, proven on the binding benchmark in every wrapper +- [Kotlin calls](/quest/m1/kt-ffi-calls.md) - Kotlin calls into moq-ffi cost about what Python's and Go's do, sync and async, once the fork's Kotlin generator drops JNA's per-call overhead - [Poll-native async spike](/quest/m1/ffi-poll.md) - measure kio poll-native exports and foreign poll against `ffi::Task`, then pick the shape - [Rendition preference](/quest/m1/rendition-preference.md) - automatic selection by ``, `Video::ranked`, and WHEP keeps the highest `preference` that decodes, so a compatibility transcode is only picked when nothing preferred decodes - [Rust PROBE on demand](/quest/m1/probe-on-demand-rs.md) - a Rust client opens PROBE only while a `recv_bandwidth()` consumer is held, and `Stats` drops `estimated_recv_rate` diff --git a/quest/m1/bench-site.md b/quest/m1/bench-site.md index 8800cf00429..4d9e56f7228 100644 --- a/quest/m1/bench-site.md +++ b/quest/m1/bench-site.md @@ -4,7 +4,7 @@ A public GitHub Pages site charts nightly benchmarks over time, so a regression or an optimization shows as a step in the trend. It carries the -[binding benchmarks](/quest/m1/ffi-bench.md) and the existing nightly Rust +[binding benchmarks](/bench/README.md) and the existing nightly Rust Criterion and JS benchmarks, and other sources can join without a new site. @@ -37,7 +37,3 @@ Decided with the maintainer on 2026-10-15: `bench/README.md`. Public API: none. Wire: none. - -## Required - -- [Binding benchmark harness](/quest/m1/ffi-bench.md) - the first source the site charts diff --git a/quest/m1/boltffi.md b/quest/m1/boltffi.md index af7b681290a..c0c55e3b6aa 100644 --- a/quest/m1/boltffi.md +++ b/quest/m1/boltffi.md @@ -37,12 +37,17 @@ Decided with the maintainer on 2026-10-13: `read_frame`, and one async read loop. Bind them with both generators side by side via `cfg_attr` in a spike crate, not in moq-ffi. The spike crate carries its own benchmarks, so it does not wait on the - [binding benchmark](/quest/m1/ffi-bench.md). It also builds for wasm32: + [binding benchmark](/bench/README.md). It also builds for wasm32: the browser comparison in #5478 reuses this slice for per-call numbers, so tell that quest if the crate's location or shape changes. - **Measure**: no-op and record call overhead, payload throughput across frame sizes from 100 B to 1 MB, and the cost of each async await. Re-measure UniFFI rather than trusting BoltFFI's numbers for it. + `just bench-ffi` gives UniFFI's numbers through the real moq-ffi. A known + UniFFI cost to weigh: Kotlin sync calls measured 11-22 us against 1.3 us + from Python on 2026-10-15, possibly from JNA's per-call structures + ([Kotlin calls](/quest/m1/kt-ffi-calls.md)); async wakeups may add a JVM + thread attach each, a separate suspect. - **Size table**: native library, generated bindings, and wasm plus JS glue (gzip), for both generators. Measure the native library with moq-ffi's default features on and off; the toggle changes nothing on wasm32, where diff --git a/quest/m1/ffi-bench-wrappers.md b/quest/m1/ffi-bench-wrappers.md index f5c4de5096a..98678e30c1c 100644 --- a/quest/m1/ffi-bench-wrappers.md +++ b/quest/m1/ffi-bench-wrappers.md @@ -2,7 +2,7 @@ ## Goal -Swift, Dart, and C++ run the [binding benchmark](/quest/m1/ffi-bench.md) +Swift, Dart, and C++ run the [binding benchmark](/bench/README.md) nightly with the same shape, metrics, and output as Python, Kotlin, and Go, so every wrapper's FFI cost is tracked and shows on the trend site. @@ -16,11 +16,7 @@ all six to prove its fix. - Swift has no Linux toolchain in the dev shell; run it in the macOS job in `platform.yml`. Its numbers come from a different runner, so report its ratio against a Rust baseline measured in that same job. -- Follow whatever the harness quest settled on for shape and output; extend - it rather than fork it. +- Follow the shape and output in `bench/README.md` (`bench/ffi.sh` runs + each runner, `bench/ffi.py` combines them); extend it rather than fork it. Public API: none. Wire: none. - -## Required - -- [Binding benchmark harness](/quest/m1/ffi-bench.md) - the shape, baselines, and output these join diff --git a/quest/m1/ffi-bench.md b/quest/m1/ffi-bench.md deleted file mode 100644 index 62eb81b3ad0..00000000000 --- a/quest/m1/ffi-bench.md +++ /dev/null @@ -1,58 +0,0 @@ -# [L] Binding benchmark harness - -## Goal - -Python, Kotlin (JVM), and Go each run a benchmark of their wrapper's -publish and subscribe path every night, beside two Rust baselines measured -on the same runner, so FFI overhead is tracked and optimizations are -measured instead of guessed. The nightly job summary shows each binding's -numbers and its ratio to the baselines. This is the shared harness later -binding performance quests measure on. - -Swift, Dart, and C++ join in [their own quest](/quest/m1/ffi-bench-wrappers.md). -The trend website is [the site quest](/quest/m1/bench-site.md). - -## Plan - -Decided with the maintainer on 2026-10-15: - -- **Split from [Payload bytes](/quest/m1/ffi-bytes.md)**, which planned this - benchmark but is blocked behind the UniFFI fork. The harness needs no - fork, so it starts now and the fork's patches are measured on it. -- **Shape**: an in-process publisher and subscriber, no network - (`MoqTrackProducer::consume` connects them without a session). - `write_frame` and `read_frame` throughput over frame sizes from 100 B to - 1 MB, a no-op call, and a record round trip. Sweep publisher and - subscriber counts independently, as the Go bench it replaces does, so a - cost that grows with either axis shows as a slope. -- **Metrics**: throughput and per-call time, plus process CPU (user + sys) - per MB. Async bridging (asyncio, coroutines, goroutines) burns CPU on - other threads that wall time hides. -- **Layer**: each language benches its idiomatic wrapper (`py/moq-rs`, - `kt/moq`, `go/wrapper`), which is what users feel. The no-op call goes - through the raw generated binding to isolate the generator. -- **Two Rust baselines**: the same loop against moq-ffi's Rust API called - directly (binding / moq-ffi is the boundary's cost) and against moq-net - (moq-ffi / moq-net is the `ffi::Task` and mutex machinery that - [the poll spike](/quest/m1/ffi-poll.md) targets). Criterion, like the - other Rust benches. -- **Tracking**: report ratios to the same-job Rust baselines, which cancel - most hosted-runner noise. No thresholds. Emit every result in one JSON - shape (github-action-benchmark's custom format is a good fit) as a nightly - artifact; [the site](/quest/m1/bench-site.md) publishes the history. -- **Go**: replace the loopback-QUIC `BenchmarkPublisher` in - `go/wrapper/runtime_bench_test.go` with the in-process harness; the - nightly "Go publisher runtime benchmark" slot runs the new one. Delete the - QUIC version. -- Use each language's idiomatic bench tooling where it gives sound - statistics (Go `testing.B`, Criterion), but avoid heavy new build - plumbing just for a framework. A plain timed loop is fine. -- Local recipes per language plus one that runs them all, documented in - `bench/README.md`. - -Public API: none. Wire: none. - -## Related - -- [Payload bytes](/quest/m1/ffi-bytes.md) - the first optimization measured on this harness -- [Benchmark regressions in CI](/quest/m2/bench-ci.md) - the Criterion side of benchmark tracking diff --git a/quest/m1/ffi-bytes.md b/quest/m1/ffi-bytes.md index 1d5d817d08f..9777380d7e0 100644 --- a/quest/m1/ffi-bytes.md +++ b/quest/m1/ffi-bytes.md @@ -4,17 +4,16 @@ A binding moves a frame payload across the FFI with no per-byte loop: every copy on the path is a bulk copy, and the report counts them per direction. -Proven on the nightly [binding benchmark](/quest/m1/ffi-bench.md) in every +Proven on the nightly [binding benchmark](/bench/README.md) in every wrapper (Python, Go, Kotlin, Swift, Dart, and C++). ## Plan Decided with the maintainer on 2026-10-13: -- **Benchmark first.** The per-wrapper benchmark moved to - [its own quest](/quest/m1/ffi-bench.md) on 2026-10-15 so it could start - before the fork. Add a Rust microbenchmark of UniFFI's lift and lower for - `Vec` here, beside the patch it proves. +- **Benchmark first.** `just bench-ffi` ([bench/README.md](/bench/README.md)) + measures every wrapper before and after. Add a Rust microbenchmark of + UniFFI's lift and lower for `Vec` here, beside the patch it proves. - **Bulk copy in the fork's core.** UniFFI lifts and lowers `Vec` through the generic `Vec` path, one byte at a time with a bounds check each (`uniffi_core` `ffi_converter_impls.rs`), in both directions. That @@ -43,6 +42,5 @@ type. Wire: none. ## Required -- [Binding benchmark harness](/quest/m1/ffi-bench.md) - Python, Kotlin, and Go numbers before and after - [Swift, Dart, and C++ benchmarks](/quest/m1/ffi-bench-wrappers.md) - the other three wrappers' numbers - [Hard fork](/quest/m1/uniffi-fork.md) - the repository the core patch lands in diff --git a/quest/m1/ffi-poll.md b/quest/m1/ffi-poll.md index e3fbb57c9b1..5f54bbc37ec 100644 --- a/quest/m1/ffi-poll.md +++ b/quest/m1/ffi-poll.md @@ -32,7 +32,7 @@ Decided with the maintainer on 2026-10-13: the numbers justify it. - **Hot path**: track `next` and group `read_frame`, prototyped in both shapes beside the existing `Task` version. -- **Measure** in Python and Go, on the [binding benchmark](/quest/m1/ffi-bench.md): +- **Measure** in Python and Go, on the [binding benchmark](/bench/README.md): per-call latency, wakeups per frame, throughput, and CPU. Sweep subscriber count so a cost that grows with the fan-out shows up. Test cancellation semantics too: a cancelled read must not swallow the next @@ -42,10 +42,6 @@ Decided with the maintainer on 2026-10-13: Public API: none. Wire: none. -## Required - -- [Binding benchmark harness](/quest/m1/ffi-bench.md) - the per-wrapper benchmark this measures on - ## Related - [Runtime trait](/quest/m2/runtime-trait.md) - replaces web-async; moq-ffi's tokio runtime may move behind it diff --git a/quest/m1/kt-ffi-calls.md b/quest/m1/kt-ffi-calls.md new file mode 100644 index 00000000000..a176380d903 --- /dev/null +++ b/quest/m1/kt-ffi-calls.md @@ -0,0 +1,36 @@ +# [M] Kotlin binding calls without JNA overhead + +## Goal + +A Kotlin call into moq-ffi costs about what the same call costs from Python or +Go, sync and async alike, proven on the +[binding benchmark](/bench/README.md). + +## Plan + +Measured 2026-10-15: a sync `sequence()` took 11-22 us from Kotlin against +1.3 us from Python and 0.12 us from Go, and the 100 B throughput case kept +about 5.8 cores busy for 2 MB/s. + +Decided with the maintainer on 2026-10-15: + +- **Profile first** (JFR or async-profiler) and confirm each cause before + changing anything. The two suspects in the generated `uniffi/moq/moq.kt`: + - Every call allocates a JNA `Structure` (`UniffiRustCallStatus`, which + nests a `RustBuffer.ByValue`), and JNA builds and syncs those by + reflection. + - The async continuation is a plain JNA `Callback` with no + `CallbackThreadInitializer`, so each wakeup from a Rust worker attaches + that thread to the JVM and detaches it again. +- **Fix in [the fork](/quest/m1/uniffi-fork.md)'s Kotlin generator**, not by + patching generated code. Both costs are one quest: same root, same + generator, same proof. Prefer a shape that keeps the wrapper's API + unchanged. +- If [the BoltFFI spike](/quest/m1/boltffi.md) picks BoltFFI, re-plan this + against its Kotlin backend, or delete it if that backend already avoids JNA. + +Public API: none expected. Wire: none. + +## Required + +- [Hard fork](/quest/m1/uniffi-fork.md) - the generator this patches diff --git a/quest/m1/uniffi-fork.md b/quest/m1/uniffi-fork.md index 95369cccb57..588bba9bfaf 100644 --- a/quest/m1/uniffi-fork.md +++ b/quest/m1/uniffi-fork.md @@ -21,7 +21,8 @@ UniFFI: with the quests that name UniFFI internals: [Payload bytes](/quest/m1/ffi-bytes.md), [Poll-native async](/quest/m1/ffi-poll.md), - [Zero-copy byte returns](/quest/m2/ffi-bytes-return.md), and + [Zero-copy byte returns](/quest/m2/ffi-bytes-return.md), + [Kotlin calls](/quest/m1/kt-ffi-calls.md), and [Namespaced external types](/quest/m2/uniffi-namespaces.md). - **Why**: every UniFFI minor release breaks external generators (0.31 @@ -72,4 +73,5 @@ Public API: none at the move; moq-ffi's dependency names change. Wire: none. ## Related - [Payload bytes](/quest/m1/ffi-bytes.md) - the first patch the fork carries +- [Kotlin calls](/quest/m1/kt-ffi-calls.md) - a Kotlin generator patch the fork carries - [Namespaced external types](/quest/m2/uniffi-namespaces.md) - fixes mozilla/uniffi-rs#3026 in the fork diff --git a/quest/m2/ffi-bytes-return.md b/quest/m2/ffi-bytes-return.md index b7ebf86d163..ee9f05ee6ac 100644 --- a/quest/m2/ffi-bytes-return.md +++ b/quest/m2/ffi-bytes-return.md @@ -5,7 +5,7 @@ A frame payload returned to a binding (`read_frame`, `recv_datagram`, decoded media) reaches the host without a copy, through a Rust-owned buffer the host releases, in every generator of [moq-dev/uniffi](/quest/m1/uniffi-fork.md). -Proven on the [binding benchmark](/quest/m1/ffi-bench.md). +Proven on the [binding benchmark](/bench/README.md). ## Plan diff --git a/quest/m2/ffi-size-profile.md b/quest/m2/ffi-size-profile.md index b192e1e36ea..665b0069e2b 100644 --- a/quest/m2/ffi-size-profile.md +++ b/quest/m2/ffi-size-profile.md @@ -19,13 +19,9 @@ Guidance: opt-level 3 with per-package overrides, and compare against building everything at "s". - Measure what the bindings actually do: publish and subscribe throughput - through moq-ffi on the [binding benchmark](/quest/m1/ffi-bench.md), plus + through moq-ffi on the [binding benchmark](/bench/README.md), plus audio and video encode and decode. Wire the benchmark into CI, at least nightly. - If it's adopted, use a named profile (for example `[profile.ffi]`, inheriting release) that every binding build path selects, so the relay and self-builders keep opt-level 3. - -## Required - -- [Binding benchmark harness](/quest/m1/ffi-bench.md) - the per-wrapper benchmark this measures with diff --git a/rs/moq-ffi/Cargo.toml b/rs/moq-ffi/Cargo.toml index ccf3dbcae80..1820ca2612c 100644 --- a/rs/moq-ffi/Cargo.toml +++ b/rs/moq-ffi/Cargo.toml @@ -86,3 +86,7 @@ uniffi = { workspace = true, features = ["build"] } pollster = { workspace = true } tokio = { workspace = true, features = ["test-util"] } web-transport-trait = { workspace = true } + +[target.'cfg(unix)'.dev-dependencies] +# Process CPU time for the benchmark's baselines. +libc = { workspace = true } diff --git a/rs/moq-ffi/src/bench.rs b/rs/moq-ffi/src/bench.rs new file mode 100644 index 00000000000..3c4d3910d95 --- /dev/null +++ b/rs/moq-ffi/src/bench.rs @@ -0,0 +1,313 @@ +//! The Rust baselines for the binding benchmark (`just bench-ffi`, see `bench/README.md`). +//! +//! Every wrapper runs the same loop, so dividing a wrapper's numbers by these isolates +//! what it adds: `rust-ffi` calls the exported API without crossing the FFI, and `rust-net` +//! runs the same rounds on moq-net, below moq-ffi's handles and runtime. + +use crate::consumer::MoqTrackConsumer; +use crate::media::MoqFrame; +use crate::origin::{MoqOriginConfig, MoqOriginProducer}; +use crate::producer::{MoqBroadcastProducer, MoqTrackInfo, MoqTrackProducer}; + +use std::future::Future; +use std::hint::black_box; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +const WARMUP: u64 = 3; + +/// Far longer than any honest round, so only a wedged read trips it. +const ROUND_TIMEOUT: Duration = Duration::from_secs(30); + +/// Each case's origin cache budget: far above one round's groups, so eviction only +/// reclaims groups every subscriber has read, and memory stays flat over a long run. +const CACHE_BYTES: u64 = 64 << 20; + +/// One throughput case: `publishers` tracks, each read by `subscribers` consumers. +struct Case { + size: usize, + publishers: usize, + subscribers: usize, +} + +impl Case { + /// Frame sizes at one publisher and subscriber, then fan-out at 1 KiB. + fn all() -> Vec { + let sizes = [100, 1024, 64 * 1024, 1024 * 1024].map(|size| Case { + size, + publishers: 1, + subscribers: 1, + }); + let fanout = [(1, 4), (4, 1), (4, 4)].map(|(publishers, subscribers)| Case { + size: 1024, + publishers, + subscribers, + }); + sizes.into_iter().chain(fanout).collect() + } + + /// Frames per group, holding a group under moq-net's 32 MB group cache. + fn frames(&self) -> usize { + ((8 << 20) / self.size).clamp(1, 64) + } + + fn delivered(&self) -> u64 { + (self.publishers * self.subscribers * self.frames() * self.size) as u64 + } + + /// A non-uniform payload, so a zero-filled or truncated copy fails the check. + fn payload(&self) -> Vec { + (0..self.size).map(|i| (i % 251) as u8).collect() + } + + fn name(&self) -> String { + format!("size={}/pub={}/sub={}", self.size, self.publishers, self.subscribers) + } +} + +fn budget() -> Duration { + let seconds = std::env::var("MOQ_BENCH_SECONDS").map_or(1.0, |s| s.parse().expect("MOQ_BENCH_SECONDS")); + Duration::from_secs_f64(seconds) +} + +/// Process CPU time, user plus system, across every thread. +fn cpu() -> Duration { + let mut usage = std::mem::MaybeUninit::::uninit(); + // SAFETY: getrusage fills the struct it is handed and returns non-zero only for a bad `who`. + let usage = unsafe { + assert_eq!(libc::getrusage(libc::RUSAGE_SELF, usage.as_mut_ptr()), 0); + usage.assume_init() + }; + let time = |t: libc::timeval| Duration::new(t.tv_sec as u64, t.tv_usec as u32 * 1000); + time(usage.ru_utime) + time(usage.ru_stime) +} + +#[derive(Default)] +struct Report(Vec); + +impl Report { + fn push(&mut self, name: String, unit: &str, value: f64) { + self.0 + .push(serde_json::json!({ "name": name, "unit": unit, "value": value })); + } + + /// Run `round` for the time budget after a warmup, recording throughput and CPU per MB. + async fn throughput(&mut self, case: &Case, mut round: F) + where + F: FnMut(u64) -> Fut, + Fut: Future, + { + // The test runs many budgets back to back, so a wedged read is caught per round, not by nextest. + let mut run = |n| tokio::time::timeout(ROUND_TIMEOUT, round(n)); + for n in 0..WARMUP { + run(n).await.expect("round wedged"); + } + let (start, cpu_start, budget) = (Instant::now(), cpu(), budget()); + let mut n = WARMUP; + while start.elapsed() < budget { + run(n).await.expect("round wedged"); + n += 1; + } + let (wall, cpu) = (start.elapsed(), cpu() - cpu_start); + let mb = ((n - WARMUP) * case.delivered()) as f64 / 1e6; + self.push(format!("throughput/{}", case.name()), "MB/s", mb / wall.as_secs_f64()); + self.push(format!("cpu/{}", case.name()), "ms/MB", cpu.as_secs_f64() * 1e3 / mb); + } + + /// Time `call` over the budget, recording nanoseconds per call. + fn call(&mut self, name: &str, mut call: impl FnMut()) { + let (start, budget) = (Instant::now(), budget()); + let mut calls = 0u64; + while start.elapsed() < budget { + for _ in 0..1000 { + call(); + } + calls += 1000; + } + self.push( + format!("call/{name}"), + "ns", + start.elapsed().as_nanos() as f64 / calls as f64, + ); + } + + /// Write the report to `$MOQ_BENCH_DIR/.json`, or print it without one. + fn write(self, runner: &str) { + let json = serde_json::to_string_pretty(&self.0).unwrap(); + match std::env::var_os("MOQ_BENCH_DIR") { + Some(dir) => { + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(std::path::Path::new(&dir).join(format!("{runner}.json")), json).unwrap(); + } + None => println!("{runner}: {json}"), + } + } +} + +fn check(case: &Case, round: u64, frame: usize, want: &[u8], payload: &[u8], timestamp_us: Option) { + assert!(payload == want, "corrupt frame {frame} of round {round}"); + assert_eq!(timestamp_us, Some(round * case.frames() as u64 + frame as u64)); +} + +async fn ffi_round( + case: &Case, + payload: &[u8], + tracks: &[Arc], + consumers: &[Arc], + n: u64, +) { + let readers: Vec<_> = consumers + .iter() + .map(|consumer| { + let (consumer, frames) = (consumer.clone(), case.frames()); + tokio::spawn(async move { + let group = consumer.recv_group().await.unwrap().expect("track ended"); + let mut read = Vec::with_capacity(frames); + for _ in 0..frames { + let frame = group.read_frame().await.unwrap().expect("group ended"); + read.push((frame.payload, frame.timestamp_us)); + } + read + }) + }) + .collect(); + for track in tracks { + let group = track.append_group().unwrap(); + for f in 0..case.frames() { + let frame = MoqFrame { + payload: payload.to_vec(), + timestamp_us: Some(n * case.frames() as u64 + f as u64), + }; + group.write_frame(frame).unwrap(); + } + group.finish().unwrap(); + } + for reader in readers { + for (f, (read, timestamp_us)) in reader.await.unwrap().into_iter().enumerate() { + check(case, n, f, payload, &read, timestamp_us); + } + } +} + +/// Subscribers are not shared handles, so each reader task borrows one for the round and returns it. +async fn net_round( + case: &Case, + payload: &[u8], + mut tracks: Vec, + consumers: Vec, + n: u64, +) -> Vec { + let readers: Vec<_> = consumers + .into_iter() + .map(|mut consumer| { + let frames = case.frames(); + tokio::spawn(async move { + let mut group = consumer.recv_group().await.unwrap().expect("track ended"); + let mut read = Vec::with_capacity(frames); + for _ in 0..frames { + let frame = group.read_frame().await.unwrap().expect("group ended"); + read.push((frame.payload, frame.timestamp.map(|t| t.as_micros() as u64))); + } + (consumer, read) + }) + }) + .collect(); + for track in &mut tracks { + let mut group = track.append_group().unwrap(); + for f in 0..case.frames() { + let timestamp = moq_net::Timestamp::from_micros(n * case.frames() as u64 + f as u64).unwrap(); + // Copy, as a binding's write does, so the baseline does not get zero-copy for free. + group.write_frame(timestamp, payload.to_vec()).unwrap(); + } + group.finish().unwrap(); + } + let mut consumers = Vec::with_capacity(readers.len()); + for reader in readers { + let (consumer, read) = reader.await.unwrap(); + for (f, (read, timestamp_us)) in read.into_iter().enumerate() { + check(case, n, f, payload, &read, timestamp_us); + } + consumers.push(consumer); + } + consumers +} + +async fn rust_ffi() { + let mut report = Report::default(); + for case in Case::all() { + let origin = MoqOriginProducer::new(MoqOriginConfig { + cache_capacity_bytes: Some(CACHE_BYTES), + }); + let broadcast = origin.create_broadcast("bench".into()).unwrap(); + let timed = MoqTrackInfo { + priority: 127, + max_age_us: None, + timescale: Some(1_000_000), + }; + let tracks: Vec<_> = (0..case.publishers) + .map(|p| broadcast.publish_track(p.to_string(), Some(timed.clone())).unwrap()) + .collect(); + let consumers: Vec<_> = tracks + .iter() + .flat_map(|track| (0..case.subscribers).map(|_| track.consume(None).unwrap())) + .collect(); + let payload = case.payload(); + report + .throughput(&case, |n| ffi_round(&case, &payload, &tracks, &consumers, n)) + .await; + } + + let broadcast = MoqBroadcastProducer::new().unwrap(); + let track = broadcast.publish_track("calls".into(), None).unwrap(); + let group = track.append_group().unwrap().consume().unwrap(); + report.call("sequence", || { + black_box(black_box(&group).sequence()); + }); + let consumer = track.consume(None).unwrap(); + report.call("info", || { + black_box(black_box(&consumer).info().unwrap()); + }); + report.write("rust-ffi"); +} + +async fn rust_net() { + let mut report = Report::default(); + for case in Case::all() { + let mut config = moq_net::origin::Config::default(); + let cache = moq_net::cache::Config::default() + .with_capacity(CACHE_BYTES) + .with_expiry(config.pool.expiry()); + config.pool = moq_net::cache::Pool::new(cache); + let (origin, driver) = moq_net::origin::Producer::new(config); + let driver = tokio::spawn(moq_net::time::run(driver)); + let broadcast = origin.create_broadcast("bench").unwrap(); + let info = moq_net::track::Info::default().with_timescale(moq_net::Timescale::MICRO); + let tracks: Vec<_> = (0..case.publishers) + .map(|p| broadcast.create_track(p.to_string(), Some(info.clone())).unwrap()) + .collect(); + let consumers = std::cell::RefCell::new( + tracks + .iter() + .flat_map(|track| (0..case.subscribers).map(|_| track.subscribe(None))) + .collect::>(), + ); + let payload = case.payload(); + // Producers are shared handles, so each round appends through its own clones. + let (case, payload, tracks, consumers) = (&case, &payload, &tracks, &consumers); + report + .throughput(case, |n| async move { + let returned = net_round(case, payload, tracks.clone(), consumers.take(), n).await; + consumers.replace(returned); + }) + .await; + driver.abort(); + } + report.write("rust-net"); +} + +#[tokio::test] +#[ignore = "benchmark: run by `just bench-ffi`"] +async fn benchmark_ffi() { + rust_ffi().await; + rust_net().await; +} diff --git a/rs/moq-ffi/src/lib.rs b/rs/moq-ffi/src/lib.rs index e0c2a5f12e7..090bd368e06 100644 --- a/rs/moq-ffi/src/lib.rs +++ b/rs/moq-ffi/src/lib.rs @@ -60,3 +60,7 @@ pub extern "C" fn moq_ffi_shutdown() { // The test suite drives a native relay over a tokio runtime. #[cfg(all(test, not(target_arch = "wasm32")))] mod test; + +// The binding benchmark's Rust baselines; ignored unless `just bench-ffi` asks for them. +#[cfg(all(test, unix))] +mod bench; diff --git a/sh/go/bench.sh b/sh/go/bench.sh index af274ae4292..c155bbf04c0 100755 --- a/sh/go/bench.sh +++ b/sh/go/bench.sh @@ -6,4 +6,4 @@ export MOQ_FFI_PROFILE=release staged=$(bash "$here/stage.sh" --output "$root/dist/runtime-bench") wrapper=$(printf '%s\n' "$staged" | sed -n 2p) cd "$wrapper" -CGO_ENABLED=1 GOFLAGS=-mod=mod go test -run '^$' -bench '^BenchmarkPublisher$' "$@" +CGO_ENABLED=1 GOFLAGS=-mod=mod go test -run '^$' -bench '^BenchmarkFFI$' "$@" diff --git a/sh/kt/bench.sh b/sh/kt/bench.sh new file mode 100755 index 00000000000..57e61a43cdd --- /dev/null +++ b/sh/kt/bench.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Run the binding benchmark's Kotlin runner (see bench/README.md) on a release +# moq-ffi. The runner is a JVM test that skips itself without MOQ_BENCH_DIR, so +# `just kt check` never pays for it. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +KT_DIR="$(cd "$SCRIPT_DIR/../../kt" && pwd)" +export MOQ_BENCH_DIR="${MOQ_BENCH_DIR:-$KT_DIR/../target/bench/ffi}" + +MOQ_FFI_PROFILE=release bash "$SCRIPT_DIR/generate.sh" +gradle -p "$KT_DIR" -Pmoqffi.version=0.0.0-dev :moq:jvmTest --tests dev.moq.FfiBench --rerun +echo "kt bench: wrote $MOQ_BENCH_DIR/kotlin.json"