Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
9b8f965
chore(quest): claim quest/m1/ffi-bench.md
kixelated Oct 24, 2026
03f4d92
feat(bench): move moq's binding benchmark here (wip)
kixelated Oct 24, 2026
ee17f0f
feat(bench): browser runner over ubrn, report from raw samples
kixelated Oct 24, 2026
6708672
Merge remote-tracking branch 'origin/main' into quest/m1/ffi-bench
kixelated Oct 24, 2026
353cde9
feat(bench): compare medians in the binding benchmark report
kixelated Oct 24, 2026
efb5866
ci(bench): run the build benchmark only for its own changes
kixelated Oct 24, 2026
d2d6977
quest: finish the binding benchmark quest
kixelated Oct 24, 2026
89a54bd
ci(bench): fetch the PR base for --base
kixelated Oct 24, 2026
f49ce5e
fix(bench): record the host on macOS
kixelated Oct 24, 2026
6546edf
fix(bench): give each runner's build its own target directory
kixelated Oct 24, 2026
a7fd9b6
Merge remote-tracking branch 'origin/main' into quest/m1/ffi-bench
kixelated Oct 24, 2026
073a651
fix(bench): run the Rust baselines without moq's clippy gate
kixelated Oct 24, 2026
851ae05
fix(bench): keep the C++ build out of Dart's target directory
kixelated Oct 24, 2026
cbffdd1
Merge remote-tracking branch 'origin/main' into quest/m1/ffi-bench
kixelated Oct 24, 2026
18b9cd4
feat(bench): compare trimmed means, as the build benchmark now does
kixelated Oct 24, 2026
81193f0
fix(bench): build from physical paths, which askama needs on macOS
kixelated Oct 24, 2026
c52e955
quest: plan the fork's symlinked-path build failure
kixelated Oct 24, 2026
305bf39
Merge remote-tracking branch 'origin/main' into quest/m1/ffi-bench
kixelated Oct 25, 2026
84c6575
fix(bench): serialize binding benchmark runs that share a work directory
kixelated Oct 25, 2026
9d7bbfb
Merge remote-tracking branch 'origin/main' into quest/m1/ffi-bench
kixelated Oct 25, 2026
e975390
Merge remote-tracking branch 'origin/main' into quest/m1/ffi-bench
kixelated Oct 25, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 68 additions & 0 deletions .github/workflows/bench-ffi.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
name: Bench FFI

permissions:
contents: read

on:
schedule:
- cron: "0 8 * * *"
workflow_dispatch:
# A change to the fork measures its effect against the PR's base, and a change to
# the benchmark or its toolchains reports its numbers, before either merges.
pull_request:
paths:
- uniffi/**
- Cargo.toml
- Cargo.lock
- bench/ffi/**
- .github/workflows/bench-ffi.yml
- flake.nix
- flake.lock
- rust-toolchain.toml

concurrency:
group: bench-ffi-${{ github.ref }}
cancel-in-progress: true

jobs:
bench:
name: Bench FFI (${{ matrix.name }})
runs-on: ${{ matrix.os }}
timeout-minutes: 330
strategy:
fail-fast: false
# Each job measures its own Rust baselines, so ratios never cross machines.
matrix:
include:
- name: Linux
os: ubuntu-24.04
- name: Swift
os: macos-latest
args: --runner swift

steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
# --base needs the PR's base commit.
fetch-depth: 0

- uses: ./.github/actions/nix

- name: Bench
run: nix develop --command just bench-ffi ${{ matrix.args }} ${{ github.event_name == 'pull_request' && format('--base {0}', github.event.pull_request.base.sha) || '' }} --out bench-results
env:
# The pinned moq's flake names its binary cache; without it the dev shell builds from source.
NIX_CONFIG: accept-flake-config = true

- name: Summary
run: cat bench-results/report.md >>"$GITHUB_STEP_SUMMARY"

- name: Upload
if: always()
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: bench-ffi-${{ strategy.job-index }}
path: bench-results
if-no-files-found: ignore
2 changes: 1 addition & 1 deletion .github/workflows/bench.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ on:
# A change to the benchmark or its toolchains reports its numbers before it merges.
pull_request:
paths:
- bench/**
- bench/build/**
- .github/workflows/bench.yml
- flake.nix
- flake.lock
Expand Down
120 changes: 120 additions & 0 deletions bench/ffi/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
# Binding benchmark

`just bench-ffi` measures what each of moq's binding wrappers adds over Rust, with moq-ffi at the revision pinned in [pins.toml](pins.toml) built against [the UniFFI fork](../../uniffi/README.md) in this tree.
It runs nightly in CI ([bench-ffi.yml](../../.github/workflows/bench-ffi.yml)), and every fork perf change states its effect with its numbers.

```bash
just bench-ffi # every wrapper this host runs, three rounds
just bench-ffi --runner py --rounds 1 # a quick smoke test
just bench-ffi --base origin/main # a fork change against its base
```

Results land in `target/bench/ffi/`, newest at `target/bench/ffi/latest/report.md`.
Checkouts and builds live in `/var/tmp/rust2-bench-<user>/ffi/`, with their own `CARGO_HOME` and none of the caller's cargo or rustc variables, as in the [build-speed benchmark](../build/README.md).
Runs on one host share that directory, so a second run waits for the first to finish.

## What runs

The native runners, their recipes, and the Rust baselines stay in moq beside the wrappers they measure (see moq's [bench/README.md](https://github.com/moq-dev/moq/blob/main/bench/README.md)); this harness drives them.
The browser runner lives here, since moq has no browser wrapper over moq-ffi yet.
Every runner runs the same in-process rounds with no network: each publisher writes one group while every subscriber reads it on its own task.
They sweep frame sizes from 100 B to 1 MiB at one publisher and subscriber, then 1 and 4 publishers by 1 and 4 subscribers at 1 KiB, and time two sync calls (`sequence`, `info`).
Each case reports delivered MB/s and process CPU per MB, and checks every received frame byte for byte.

- `rust-ffi` calls moq-ffi's API from Rust without crossing the FFI, so a wrapper's ratio to it is what its boundary and wrapper cost.
- `rust-net` runs the same rounds on moq-net, so its ratio is what moq-ffi's handles and runtime cost.
- `go`, `py`, `kt`, `cpp`, `dart`, and `browser` run on Linux and macOS; `swift` runs on macOS only.
- `browser` builds moq-ffi for wasm32, generates its TypeScript with ubrn's `wasm2` player, and runs [browser-page.ts](browser-page.ts) in headless Chromium through [browser.ts](browser.ts). Its CPU is every thread of the renderer, read over CDP. No ubrn release reads UniFFI 0.32 yet, so [pins.toml](pins.toml) pins the head of [jhugman/uniffi-bindgen-react-native#468](https://github.com/jhugman/uniffi-bindgen-react-native/pull/468), and the run builds its CLI and packs its `@ubjs/core` and `@ubjs/wasm` runtimes from that source.

The run enters the pinned moq's own dev shell, so every wrapper builds with the toolchains moq ships with.
moq's Go, Dart, and C++ scripts find their generators on `PATH`, where the run puts the fork's.
Its Kotlin, Python, and Swift generator and moq-ffi's runtime are the `uniffi` crate, which the run points at the fork's path in every manifest; the run fails if any UniFFI crate still resolves elsewhere.
For the browser, the run also links ubrn's `uniffi-runtime-wasm` helper crate into moq-ffi on wasm32, as ubrn's player requires.
Bump a pin in its own change, since numbers on either side of it do not compare.

## Variants

- `fork` builds against this tree.
- `control` reruns fork's builds, so its change against fork is the run's noise floor: an A/A control.
- `base`, with `--base REV`, builds against the fork at another rust2 revision, so fork's change against it is the effect of the change.

An untimed warm-up round builds every runner once per checkout.
The timed rounds interleave the variants and rotate which goes first each round, so drift over a long job spreads across all of them.
The report shows each case's trimmed mean over the rounds (without the fastest and slowest) with its cost against `rust-ffi`, taken within each round so drift between rounds cancels, then each comparison as a change in that cost, so one slow round on a busy host does not move it.
`results.jsonl` keeps every round's value, mean, and standard deviation.
The host, its load average before and after, and every toolchain's version head the report.
No threshold is set: the numbers inform a decision, they do not gate one.

## BoltFFI comparison

Before forking UniFFI, moq compared it with [BoltFFI](https://github.com/boltffi/boltffi) 0.31.1 on a slice of moq-ffi bound by each generator, measured from Python and Kotlin with a Rust floor.
Its harness is deleted; [moq-dev/moq@0f986d9](https://github.com/moq-dev/moq/tree/0f986d9fdb7c074b0bd4b34739f7199c3bf2d9e5/bench/ffi-gen) holds it if a decision needs fresh numbers.
BoltFFI's numbers are the targets for the fork's backends: its speed comes from calling conventions a UniFFI fork can adopt one backend at a time (a CPython extension instead of `ctypes`, JNI instead of JNA, and bulk byte copies), not from its core.

Measured 2026-10-16 on an Intel Core Ultra 7 270K Plus (24 threads), Linux, at moq `b4eab0db1`, on a shared host (load average 8 to 29), so cells are medians of 3 rounds.
Rust 1.98.1, CPython 3.14.7, OpenJDK 17 with Kotlin 2.4.20 and kotlinx-coroutines 1.11.0, JNA 5.19.1.
Gain is how many times better BoltFFI is than UniFFI.

### Calls (ns per call)

| call | Rust | Python UniFFI | Python BoltFFI | gain | Kotlin UniFFI | Kotlin BoltFFI | gain |
|---|---|---|---|---|---|---|---|
| `noop()` | 0.2 | 834 | 21.6 | 39x | 3,200 | 5.6 | 569x |
| `group.sequence()` | 0.4 | 1,920 | 63 | 30x | 7,440 | 5.8 | 1,290x |
| `track.info()`, a record return | 12 | 4,890 | 1,120 | 4.4x | 10,400 | 61 | 171x |
| `write_frame`, a 100 B record argument | 78 | 12,000 | 667 | 18x | 12,800 | 134 | 95x |
| `await noop_async()` | 0.3 | 7,190 | 5,330 | 1.3x | 8,320 | 342 | 24x |
| `await yield_once()`, one wakeup | 44 | 12,600 | 10,000 | 1.3x | 6,510 | 587 | 11x |

In Python, both generators sit at 5 to 13 us per await, which points at asyncio rather than the boundary.

### Throughput (MB/s, write plus async read)

| frame | Rust | Python UniFFI | Python BoltFFI | gain | Kotlin UniFFI | Kotlin BoltFFI | gain |
|---|---|---|---|---|---|---|---|
| 100 B | 473 | 3.8 | 12.1 | 3.2x | 2.9 | 105 | 36x |
| 1 KiB | 3,340 | 36 | 117 | 3.3x | 36 | 954 | 27x |
| 16 KiB | 10,400 | 323 | 1,410 | 4.4x | 286 | 2,730 | 9.6x |
| 64 KiB | 8,930 | 531 | 2,840 | 5.4x | 448 | 2,760 | 6.2x |
| 1 MiB | 6,120 | 605 | 3,220 | 5.3x | 562 | 1,560 | 2.8x |

### CPU (ms per MB)

| frame | Rust | Python UniFFI | Python BoltFFI | gain | Kotlin UniFFI | Kotlin BoltFFI | gain |
|---|---|---|---|---|---|---|---|
| 100 B | 2.1 | 261 | 83 | 3.2x | 1,290 | 15 | 85x |
| 1 KiB | 0.30 | 28 | 8.6 | 3.3x | 76 | 1.2 | 61x |
| 16 KiB | 0.10 | 3.1 | 0.71 | 4.4x | 10 | 0.39 | 27x |
| 64 KiB | 0.11 | 1.9 | 0.35 | 5.4x | 3.8 | 0.37 | 10x |
| 1 MiB | 0.16 | 1.7 | 0.31 | 5.3x | 2.3 | 0.66 | 3.4x |

At 100 B, Kotlin through UniFFI spends about 130 us of CPU per frame written and read, so 1,000 audio frames a second cost 13% of a core; BoltFFI spends 1.5 us.

### Size

Native libraries are `release` (fat LTO, `strip = "debuginfo"`, as moq-ffi ships); "stripped" is `strip --strip-all` on a copy.

| native library | as built | stripped |
|---|---|---|
| slice, UniFFI | 17,571 KiB | 14,553 KiB |
| slice, BoltFFI | 17,472 KiB | 14,482 KiB |
| BoltFFI's Python extension (`_native`), extra | 240 KiB | 64 KiB |
| BoltFFI's Kotlin JNI glue, extra | 52 KiB | 38 KiB |
| moq-ffi, default features | 34,100 KiB | 29,396 KiB |
| moq-ffi, no default features | 30,625 KiB | 26,416 KiB |

| generated bindings | files | raw | gzip |
|---|---|---|---|
| slice, Python: UniFFI / BoltFFI | 1 / 2 | 85 / 36 KiB | 12 / 7 KiB |
| slice, Kotlin: UniFFI / BoltFFI | 1 / 1 | 152 / 57 KiB | 17 / 9 KiB |
| slice, Swift: UniFFI / BoltFFI | 3 / 2 | 101 / 51 KiB | 14 / 8 KiB |
| moq-ffi, UniFFI: Python / Kotlin / Swift | 1 / 1 / 3 | 653 / 914 / 684 KiB | 66 / 73 / 72 KiB |

Browser builds use opt-level z after wasm-bindgen and no wasm-opt, as moq-wasm ships; the stripped columns drop the `name` section.

| browser module | .wasm as built | .wasm stripped | stripped, gzip | JS glue | JS glue, gzip |
|---|---|---|---|---|---|
| slice, UniFFI | 1,245 KiB | 515 KiB | 174 KiB | not generated | not generated |
| slice, BoltFFI | 1,148 KiB | 456 KiB | 155 KiB | 126 KiB | 25 KiB |
| moq-ffi, UniFFI | 6,218 KiB | 4,403 KiB | 1,575 KiB | not generated | not generated |
| moq-wasm, wasm-bindgen | 2,116 KiB | 2,115 KiB | 691 KiB | 38 KiB | 8 KiB |
161 changes: 161 additions & 0 deletions bench/ffi/browser-page.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,161 @@
/**
* The browser runner of the binding benchmark: moq-ffi bound by UniFFI over ubrn's
* `wasm2` player, running the same in-process rounds as moq's other runners.
*
* `browser.ts` bundles this page, serves it, and reads the renderer's CPU for it.
*/
import {
MoqBroadcastProducer,
MoqOriginConfig,
MoqOriginProducer,
MoqTrackInfo,
type MoqTrackConsumerLike,
uniffiInitAsync,
} from "./dist/index.ts";

interface Result {
name: string;
unit: string;
value: number;
}

declare global {
interface Window {
/** Renderer CPU seconds, read by the driver over CDP. */
rendererCpu(): Promise<number>;
runBench(seconds: number): Promise<Result[]>;
}
}

const WARMUP = 3;
// Each case's origin cache budget: far above one round's groups, so eviction only
// reclaims groups every subscriber has read, and memory stays flat over a long run.
const CACHE_BYTES = 64n << 20n;
// Far longer than any honest round, so only a lost wakeup trips it.
const ROUND_TIMEOUT_MS = 30_000;

/** One throughput case: `publishers` tracks, each read by `subscribers` consumers. */
class Case {
constructor(
readonly size: number,
readonly publishers: number,
readonly subscribers: number,
) {}

/** Frames per group, holding a group under moq-net's 32 MB group cache. */
get frames(): number {
return Math.min(Math.max(Math.floor((8 << 20) / this.size), 1), 64);
}

get delivered(): number {
return this.publishers * this.subscribers * this.frames * this.size;
}

get name(): string {
return `size=${this.size}/pub=${this.publishers}/sub=${this.subscribers}`;
}
}

// Frame sizes at one publisher and subscriber, then fan-out at 1 KiB.
const CASES = [
...[100, 1024, 64 * 1024, 1024 * 1024].map((size) => new Case(size, 1, 1)),
...[
[1, 4],
[4, 1],
[4, 4],
].map(([p, s]) => new Case(1024, p, s)),
];

/** A non-uniform payload, so a zero-filled or truncated copy fails the check. */
function payload(size: number): ArrayBuffer {
const bytes = new Uint8Array(size);
for (let i = 0; i < size; i++) bytes[i] = i % 251;
return bytes.buffer;
}

function same(a: ArrayBuffer, b: ArrayBuffer): boolean {
if (a.byteLength !== b.byteLength) return false;
const x = new Uint8Array(a);
const y = new Uint8Array(b);
for (let i = 0; i < x.length; i++) if (x[i] !== y[i]) return false;
return true;
}

async function read(c: Case, consumer: MoqTrackConsumerLike, want: ArrayBuffer, n: number): Promise<void> {
const group = await consumer.recvGroup();
if (!group) throw new Error("track ended");
for (let f = 0; f < c.frames; f++) {
const frame = await group.readFrame();
if (!frame) throw new Error("group ended");
if (!same(frame.payload, want)) throw new Error(`corrupt frame ${f} of round ${n}`);
if (frame.timestampUs !== BigInt(n * c.frames + f)) throw new Error(`wrong timestamp on frame ${f} of round ${n}`);
}
}

async function within<T>(promise: Promise<T>, ms: number, what: string): Promise<T> {
let timer: ReturnType<typeof setTimeout> | undefined;
try {
return await Promise.race([
promise,
new Promise<never>((_, reject) => {
timer = setTimeout(() => reject(new Error(`${what} did not finish in ${ms} ms`)), ms);
}),
]);
} finally {
clearTimeout(timer);
}
}

async function throughput(c: Case, budget: number, results: Result[]): Promise<void> {
const origin = new MoqOriginProducer(MoqOriginConfig.create({ cacheCapacityBytes: CACHE_BYTES }));
const broadcast = origin.createBroadcast("bench");
const info = MoqTrackInfo.create({ timescale: 1_000_000n });
const tracks = Array.from({ length: c.publishers }, (_, p) => broadcast.publishTrack(String(p), info));
const consumers = tracks.flatMap((track) => Array.from({ length: c.subscribers }, () => track.consume(undefined)));
const bytes = payload(c.size);

// Each consumer reads on its own task while this one writes each group in turn.
const step = async (n: number) => {
const readers = consumers.map((consumer) => read(c, consumer, bytes, n));
for (const track of tracks) {
const group = track.appendGroup();
for (let f = 0; f < c.frames; f++) group.writeFrame({ payload: bytes, timestampUs: BigInt(n * c.frames + f) });
group.finish();
}
await within(Promise.all(readers), ROUND_TIMEOUT_MS, `round ${n} of ${c.name}`);
};

for (let n = 0; n < WARMUP; n++) await step(n);
const cpuStart = await window.rendererCpu();
const start = performance.now();
let n = WARMUP;
while (performance.now() - start < budget * 1000) await step(n++);
const wall = (performance.now() - start) / 1000;
const cpu = (await window.rendererCpu()) - cpuStart;
const mb = ((n - WARMUP) * c.delivered) / 1e6;
results.push({ name: `throughput/${c.name}`, unit: "MB/s", value: mb / wall });
results.push({ name: `cpu/${c.name}`, unit: "ms/MB", value: (cpu * 1e3) / mb });
}

function call(name: string, budget: number, fn: () => unknown, results: Result[]): void {
const start = performance.now();
let calls = 0;
while (performance.now() - start < budget * 1000) {
for (let i = 0; i < 1000; i++) fn();
calls += 1000;
}
results.push({ name: `call/${name}`, unit: "ns", value: ((performance.now() - start) * 1e6) / calls });
}

window.runBench = async (budget: number): Promise<Result[]> => {
await uniffiInitAsync(new URL("/moq_ffi.wasm", location.href));
const results: Result[] = [];
for (const c of CASES) await throughput(c, budget, results);

const track = new MoqBroadcastProducer().publishTrack("calls", undefined);
const group = track.appendGroup().consume();
call("sequence", budget, () => group.sequence(), results);
const consumer = track.consume(undefined);
call("info", budget, () => consumer.info(), results);
return results;
};
Loading
Loading