Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .config/nextest.toml
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,12 @@ default-filter = "not kind(bench)"
filter = "package(moq-uring) | binary_id(moq-relay::runtime_uring) | (binary_id(moq-relay::embed) & test(/uring/))"
test-group = "io-uring"

# The binding benchmark runs many `MOQ_BENCH_SECONDS` budgets back to back, so its
# length scales with the budget. It guards each round against a wedge itself.
[[profile.default.overrides]]
filter = "package(moq-ffi) & test(=bench::benchmark_ffi)"
slow-timeout = { period = "60s" }

[test-groups]
io-uring = { max-threads = 4 }

Expand Down
1 change: 1 addition & 0 deletions .github/justfile
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ test:
python3 bench/summary.test.py
python3 bench/relay.test.py
python3 bench/profile.test.py
python3 bench/ffi.test.py
python3 sh/gh/gst-formula.test.py
python3 sh/changed.test.py
python3 sh/dispatch-query.test.py
Expand Down
14 changes: 12 additions & 2 deletions .github/workflows/nightly.yml
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,10 @@ jobs:
- name: Features
recipe: rs features
profile: ci
- name: Go publisher runtime benchmark
recipe: go bench -benchtime=1x
- name: Binding benchmark
recipe: bench-ffi
artifact: bench-ffi
artifact-path: target/bench/ffi
- name: Audio grouping relay benchmark
recipe: bench-audio
# Smoke-tests the capture path: exact off-CPU waits, resolved worker stacks and CPU.
Expand Down Expand Up @@ -127,6 +129,14 @@ jobs:
env:
NEXTEST_PROFILE: ${{ matrix.profile || 'default' }}

- name: Preserve results
if: ${{ matrix.artifact && !cancelled() }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: ${{ matrix.artifact }}
path: ${{ matrix.artifact-path }}
retention-days: 45

tests:
name: Tests (${{ matrix.recipe }})
runs-on: ubuntu-24.04-arm
Expand Down
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

28 changes: 19 additions & 9 deletions bench/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -131,16 +131,26 @@ hide a relay improvement behind generator saturation. CPU efficiency remains
withheld until offered load and delivery/loss are comparable. Zero-baseline
metrics retain absolute deltas when their percentage ratio is undefined.

Measure delivered payload through the Go bindings and a local QUIC relay:
Measure what each binding adds over Rust:

```bash
nix develop --command just go bench -benchtime=3s -count=3
nix develop --command just bench-ffi
```

This independently sweeps 1/4 publishers and 1/4 subscribers per publisher,
checks every received frame's payload and timestamp, and acknowledges each batch before publishing the
next. Reported MB/s counts delivery to every subscriber. The optimized native
library, relay and clients share the process, so pin the benchmark process with
`taskset` to compare one CPU with multiple CPUs. Compare identical CPU sets and
traffic shapes across revisions; the nightly smoke runs each shape once without
a timing threshold.
Every runner (Rust, Go, Python, Kotlin) runs the same in-process rounds with no
network: each publisher writes one group from the driving thread while every
subscriber reads it on its own task, and a round waits for every subscriber. It
sweeps frame sizes from 100 B to 1 MiB at one publisher and subscriber, then
1/4 publishers by 1/4 subscribers at 1 KiB, and times two sync calls
(`sequence`, `info`). Each case reports delivered MB/s and process CPU (user +
system, every thread) per MB. `MOQ_BENCH_SECONDS` sets each case's time budget
(default 1); every received frame is checked byte for byte.

Each runner writes `target/bench/ffi/<runner>.json`; `bench/ffi.py` combines
them into a table and `report.json`. Ratios are against `rust-ffi`, moq-ffi's
API called from Rust without crossing the FFI, so a wrapper's ratio is what its
boundary and wrapper cost. `rust-net` runs the same rounds on moq-net, so its
ratio is what moq-ffi's handles and runtime cost. Ratios hold only within one
machine. The nightly runs it, prints the table in the job summary, and keeps the
results as the `bench-ffi` artifact. Each language also has its own recipe
(`just go bench`, `just py bench`, `just kt bench`).
97 changes: 97 additions & 0 deletions bench/ffi.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
#!/usr/bin/env python3
"""Combine the binding benchmark's per-runner results into one table and report.

Each runner writes `<dir>/<runner>.json`, a list of {name, unit, value}. The table shows
every runner beside its ratio to the `rust-ffi` baseline, run on the same machine, which
cancels most of the runner's own speed. `report.json` holds every value and ratio, named
`<runner>/<name>`, for the trend history.
"""
import argparse
import json
import os
from pathlib import Path

BASELINE = 'rust-ffi'
# Baselines first, then the wrappers in a stable order.
ORDER = ('rust-net', 'rust-ffi')
SMALLER_IS_BETTER = {'ms/MB', 'ns'}


def load(directory):
runners = {}
for path in sorted(Path(directory).glob('*.json')):
if path.name == 'report.json':
continue
runners[path.stem] = {entry['name']: entry for entry in json.loads(path.read_text())}
if BASELINE not in runners:
raise ValueError(f'missing the {BASELINE} baseline in {directory}')
return runners


def order(runners):
return sorted(runners, key=lambda r: (ORDER.index(r) if r in ORDER else len(ORDER), r))


def ratio(entry, baseline):
"""How many times worse than the baseline: above 1 is slower or costlier."""
if baseline is None or not baseline['value'] or not entry['value']:
return None
if entry['unit'] in SMALLER_IS_BETTER:
return entry['value'] / baseline['value']
return baseline['value'] / entry['value']


def report(runners):
entries = []
baseline = runners[BASELINE]
for runner in order(runners):
for name, entry in runners[runner].items():
entries.append({'name': f'{runner}/{name}', 'unit': entry['unit'], 'value': entry['value']})
if runner != BASELINE and (r := ratio(entry, baseline.get(name))) is not None:
entries.append({'name': f'{runner}/{name} vs {BASELINE}', 'unit': 'x', 'value': r})
return entries


def table(runners):
columns = order(runners)
names = list(dict.fromkeys(name for runner in columns for name in runners[runner]))
baseline = runners[BASELINE]
lines = [
'| benchmark | unit | ' + ' | '.join(columns) + ' |',
'|---|---|' + '---|' * len(columns),
]
for name in names:
unit = next(runners[r][name]['unit'] for r in columns if name in runners[r])
cells = []
for runner in columns:
entry = runners[runner].get(name)
if entry is None:
cells.append('')
continue
cell = f"{entry['value']:.4g}"
if runner != BASELINE and (r := ratio(entry, baseline.get(name))) is not None:
cell += f' ({r:.2f}x)'
cells.append(cell)
lines.append(f'| {name} | {unit} | ' + ' | '.join(cells) + ' |')
return '\n'.join(lines)


def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('directory')
args = parser.parse_args()
runners = load(args.directory)
(Path(args.directory) / 'report.json').write_text(json.dumps(report(runners), indent=2))
markdown = (
f'## Binding benchmark\n\nRatios are against `{BASELINE}`; above 1x is slower or costlier.\n\n'
+ table(runners)
+ '\n'
)
print(markdown)
if summary := os.environ.get('GITHUB_STEP_SUMMARY'):
with open(summary, 'a') as f:
f.write(markdown)


if __name__ == '__main__':
main()
16 changes: 16 additions & 0 deletions bench/ffi.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
#!/usr/bin/env bash
# Run the binding benchmark: the Rust baselines and every wrapper's runner on this
# machine, then the ratio table. Ratios only hold within one machine, so stale
# results from an earlier run are cleared first.
set -euo pipefail
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
export MOQ_BENCH_DIR="${MOQ_BENCH_DIR:-$root/target/bench/ffi}"
rm -rf "$MOQ_BENCH_DIR"

cd "$root"
# Through nextest like every Rust test; each round guards itself against a wedged read.
just rs test -p moq-ffi --cargo-profile release -E 'test(bench::benchmark_ffi)' --run-ignored only
Comment thread
kixelated marked this conversation as resolved.
Resolved
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
just go bench -benchtime="${MOQ_BENCH_SECONDS:-1}s"
just py bench
just kt bench
python3 bench/ffi.py "$MOQ_BENCH_DIR"
54 changes: 54 additions & 0 deletions bench/ffi.test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
#!/usr/bin/env python3
import importlib.util
import json
import tempfile
import unittest
from pathlib import Path

spec = importlib.util.spec_from_file_location('ffi', Path(__file__).with_name('ffi.py'))
ffi = importlib.util.module_from_spec(spec)
spec.loader.exec_module(ffi)


def write(directory, runner, entries):
(Path(directory) / f'{runner}.json').write_text(
json.dumps([{'name': n, 'unit': u, 'value': v} for n, u, v in entries])
)


class FfiTest(unittest.TestCase):
def runners(self, directory):
write(directory, 'rust-ffi', [('throughput/a', 'MB/s', 100.0), ('cpu/a', 'ms/MB', 2.0)])
write(directory, 'rust-net', [('throughput/a', 'MB/s', 200.0)])
write(directory, 'python', [('throughput/a', 'MB/s', 25.0), ('cpu/a', 'ms/MB', 8.0)])
return ffi.load(directory)

def test_ratio_is_worse_above_one(self):
with tempfile.TemporaryDirectory() as directory:
report = {e['name']: e['value'] for e in ffi.report(self.runners(directory))}
self.assertEqual(report['python/throughput/a vs rust-ffi'], 4.0)
self.assertEqual(report['python/cpu/a vs rust-ffi'], 4.0)
self.assertEqual(report['rust-net/throughput/a vs rust-ffi'], 0.5)
self.assertNotIn('rust-ffi/throughput/a vs rust-ffi', report)

def test_table_orders_baselines_first(self):
with tempfile.TemporaryDirectory() as directory:
table = ffi.table(self.runners(directory))
self.assertTrue(table.splitlines()[0].startswith('| benchmark | unit | rust-net | rust-ffi | python |'))
self.assertIn('| throughput/a | MB/s | 200 (0.50x) | 100 | 25 (4.00x) |', table)

def test_missing_baseline_fails(self):
with tempfile.TemporaryDirectory() as directory:
write(directory, 'python', [('throughput/a', 'MB/s', 25.0)])
with self.assertRaises(ValueError):
ffi.load(directory)

def test_skips_its_own_report(self):
with tempfile.TemporaryDirectory() as directory:
self.runners(directory)
(Path(directory) / 'report.json').write_text('[]')
self.assertNotIn('report', ffi.load(directory))


if __name__ == '__main__':
unittest.main()
Loading
Loading